1 //
   2 // Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211 
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // Class for all long registers (excluding RSP)
 321 reg_class long_reg_with_rbp(RAX, RAX_H,
 322                             RDX, RDX_H,
 323                             RBP, RBP_H,
 324                             RDI, RDI_H,
 325                             RSI, RSI_H,
 326                             RCX, RCX_H,
 327                             RBX, RBX_H,
 328                             R8,  R8_H,
 329                             R9,  R9_H,
 330                             R10, R10_H,
 331                             R11, R11_H,
 332                             R13, R13_H,
 333                             R14, R14_H);
 334 
 335 // Class for all long registers (excluding RSP and RBP)
 336 reg_class long_reg_no_rbp(RAX, RAX_H,
 337                           RDX, RDX_H,
 338                           RDI, RDI_H,
 339                           RSI, RSI_H,
 340                           RCX, RCX_H,
 341                           RBX, RBX_H,
 342                           R8,  R8_H,
 343                           R9,  R9_H,
 344                           R10, R10_H,
 345                           R11, R11_H,
 346                           R13, R13_H,
 347                           R14, R14_H);
 348 
 349 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 350 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 351 
 352 // Class for all long registers (excluding RAX, RDX and RSP)
 353 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 354                                        RDI, RDI_H,
 355                                        RSI, RSI_H,
 356                                        RCX, RCX_H,
 357                                        RBX, RBX_H,
 358                                        R8,  R8_H,
 359                                        R9,  R9_H,
 360                                        R10, R10_H,
 361                                        R11, R11_H,
 362                                        R13, R13_H,
 363                                        R14, R14_H);
 364 
 365 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 366 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 367                                      RSI, RSI_H,
 368                                      RCX, RCX_H,
 369                                      RBX, RBX_H,
 370                                      R8,  R8_H,
 371                                      R9,  R9_H,
 372                                      R10, R10_H,
 373                                      R11, R11_H,
 374                                      R13, R13_H,
 375                                      R14, R14_H);
 376 
 377 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 378 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 379 
 380 // Class for all long registers (excluding RCX and RSP)
 381 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 382                                    RDI, RDI_H,
 383                                    RSI, RSI_H,
 384                                    RAX, RAX_H,
 385                                    RDX, RDX_H,
 386                                    RBX, RBX_H,
 387                                    R8,  R8_H,
 388                                    R9,  R9_H,
 389                                    R10, R10_H,
 390                                    R11, R11_H,
 391                                    R13, R13_H,
 392                                    R14, R14_H);
 393 
 394 // Class for all long registers (excluding RCX, RSP, and RBP)
 395 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 396                                  RSI, RSI_H,
 397                                  RAX, RAX_H,
 398                                  RDX, RDX_H,
 399                                  RBX, RBX_H,
 400                                  R8,  R8_H,
 401                                  R9,  R9_H,
 402                                  R10, R10_H,
 403                                  R11, R11_H,
 404                                  R13, R13_H,
 405                                  R14, R14_H);
 406 
 407 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 408 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 409 
 410 // Singleton class for RAX long register
 411 reg_class long_rax_reg(RAX, RAX_H);
 412 
 413 // Singleton class for RCX long register
 414 reg_class long_rcx_reg(RCX, RCX_H);
 415 
 416 // Singleton class for RDX long register
 417 reg_class long_rdx_reg(RDX, RDX_H);
 418 
 419 // Class for all int registers (excluding RSP)
 420 reg_class int_reg_with_rbp(RAX,
 421                            RDX,
 422                            RBP,
 423                            RDI,
 424                            RSI,
 425                            RCX,
 426                            RBX,
 427                            R8,
 428                            R9,
 429                            R10,
 430                            R11,
 431                            R13,
 432                            R14);
 433 
 434 // Class for all int registers (excluding RSP and RBP)
 435 reg_class int_reg_no_rbp(RAX,
 436                          RDX,
 437                          RDI,
 438                          RSI,
 439                          RCX,
 440                          RBX,
 441                          R8,
 442                          R9,
 443                          R10,
 444                          R11,
 445                          R13,
 446                          R14);
 447 
 448 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 449 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 450 
 451 // Class for all int registers (excluding RCX and RSP)
 452 reg_class int_no_rcx_reg_with_rbp(RAX,
 453                                   RDX,
 454                                   RBP,
 455                                   RDI,
 456                                   RSI,
 457                                   RBX,
 458                                   R8,
 459                                   R9,
 460                                   R10,
 461                                   R11,
 462                                   R13,
 463                                   R14);
 464 
 465 // Class for all int registers (excluding RCX, RSP, and RBP)
 466 reg_class int_no_rcx_reg_no_rbp(RAX,
 467                                 RDX,
 468                                 RDI,
 469                                 RSI,
 470                                 RBX,
 471                                 R8,
 472                                 R9,
 473                                 R10,
 474                                 R11,
 475                                 R13,
 476                                 R14);
 477 
 478 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 479 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 480 
 481 // Class for all int registers (excluding RAX, RDX, and RSP)
 482 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 483                                       RDI,
 484                                       RSI,
 485                                       RCX,
 486                                       RBX,
 487                                       R8,
 488                                       R9,
 489                                       R10,
 490                                       R11,
 491                                       R13,
 492                                       R14);
 493 
 494 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 495 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 496                                     RSI,
 497                                     RCX,
 498                                     RBX,
 499                                     R8,
 500                                     R9,
 501                                     R10,
 502                                     R11,
 503                                     R13,
 504                                     R14);
 505 
 506 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 507 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 508 
 509 // Singleton class for RAX int register
 510 reg_class int_rax_reg(RAX);
 511 
 512 // Singleton class for RBX int register
 513 reg_class int_rbx_reg(RBX);
 514 
 515 // Singleton class for RCX int register
 516 reg_class int_rcx_reg(RCX);
 517 
 518 // Singleton class for RCX int register
 519 reg_class int_rdx_reg(RDX);
 520 
 521 // Singleton class for RCX int register
 522 reg_class int_rdi_reg(RDI);
 523 
 524 // Singleton class for instruction pointer
 525 // reg_class ip_reg(RIP);
 526 
 527 %}
 528 
 529 //----------SOURCE BLOCK-------------------------------------------------------
 530 // This is a block of C++ code which provides values, functions, and
 531 // definitions necessary in the rest of the architecture description
 532 source %{
 533 #define   RELOC_IMM64    Assembler::imm_operand
 534 #define   RELOC_DISP32   Assembler::disp32_operand
 535 
 536 #define __ _masm.
 537 
 538 static int clear_avx_size() {
 539   if(UseAVX > 2) {
 540     return 0; // vzeroupper is ignored
 541   } else {
 542     return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 543   }
 544 }
 545 
 546 // !!!!! Special hack to get all types of calls to specify the byte offset
 547 //       from the start of the call to the point where the return address
 548 //       will point.
 549 int MachCallStaticJavaNode::ret_addr_offset()
 550 {
 551   int offset = 5; // 5 bytes from start of call to where return address points
 552   offset += clear_avx_size();
 553   return offset;
 554 }
 555 
 556 int MachCallDynamicJavaNode::ret_addr_offset()
 557 {
 558   int offset = 15; // 15 bytes from start of call to where return address points
 559   offset += clear_avx_size();
 560   return offset;
 561 }
 562 
 563 int MachCallRuntimeNode::ret_addr_offset() {
 564   int offset = 13; // movq r10,#addr; callq (r10)
 565   offset += clear_avx_size();
 566   return offset;
 567 }
 568 
 569 // Indicate if the safepoint node needs the polling page as an input,
 570 // it does if the polling page is more than disp32 away.
 571 bool SafePointNode::needs_polling_address_input()
 572 {
 573   return Assembler::is_polling_page_far();
 574 }
 575 
 576 //
 577 // Compute padding required for nodes which need alignment
 578 //
 579 
 580 // The address of the call instruction needs to be 4-byte aligned to
 581 // ensure that it does not span a cache line so that it can be patched.
 582 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 583 {
 584   current_offset += clear_avx_size(); // skip vzeroupper
 585   current_offset += 1; // skip call opcode byte
 586   return round_to(current_offset, alignment_required()) - current_offset;
 587 }
 588 
 589 // The address of the call instruction needs to be 4-byte aligned to
 590 // ensure that it does not span a cache line so that it can be patched.
 591 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 592 {
 593   current_offset += clear_avx_size(); // skip vzeroupper
 594   current_offset += 11; // skip movq instruction + call opcode byte
 595   return round_to(current_offset, alignment_required()) - current_offset;
 596 }
 597 
 598 // EMIT_RM()
 599 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 600   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 601   cbuf.insts()->emit_int8(c);
 602 }
 603 
 604 // EMIT_CC()
 605 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 606   unsigned char c = (unsigned char) (f1 | f2);
 607   cbuf.insts()->emit_int8(c);
 608 }
 609 
 610 // EMIT_OPCODE()
 611 void emit_opcode(CodeBuffer &cbuf, int code) {
 612   cbuf.insts()->emit_int8((unsigned char) code);
 613 }
 614 
 615 // EMIT_OPCODE() w/ relocation information
 616 void emit_opcode(CodeBuffer &cbuf,
 617                  int code, relocInfo::relocType reloc, int offset, int format)
 618 {
 619   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 620   emit_opcode(cbuf, code);
 621 }
 622 
 623 // EMIT_D8()
 624 void emit_d8(CodeBuffer &cbuf, int d8) {
 625   cbuf.insts()->emit_int8((unsigned char) d8);
 626 }
 627 
 628 // EMIT_D16()
 629 void emit_d16(CodeBuffer &cbuf, int d16) {
 630   cbuf.insts()->emit_int16(d16);
 631 }
 632 
 633 // EMIT_D32()
 634 void emit_d32(CodeBuffer &cbuf, int d32) {
 635   cbuf.insts()->emit_int32(d32);
 636 }
 637 
 638 // EMIT_D64()
 639 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 640   cbuf.insts()->emit_int64(d64);
 641 }
 642 
 643 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 644 void emit_d32_reloc(CodeBuffer& cbuf,
 645                     int d32,
 646                     relocInfo::relocType reloc,
 647                     int format)
 648 {
 649   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 650   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 651   cbuf.insts()->emit_int32(d32);
 652 }
 653 
 654 // emit 32 bit value and construct relocation entry from RelocationHolder
 655 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 656 #ifdef ASSERT
 657   if (rspec.reloc()->type() == relocInfo::oop_type &&
 658       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 659     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 660     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 661   }
 662 #endif
 663   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 664   cbuf.insts()->emit_int32(d32);
 665 }
 666 
 667 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 668   address next_ip = cbuf.insts_end() + 4;
 669   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 670                  external_word_Relocation::spec(addr),
 671                  RELOC_DISP32);
 672 }
 673 
 674 
 675 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 676 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 677   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 678   cbuf.insts()->emit_int64(d64);
 679 }
 680 
 681 // emit 64 bit value and construct relocation entry from RelocationHolder
 682 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 683 #ifdef ASSERT
 684   if (rspec.reloc()->type() == relocInfo::oop_type &&
 685       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 686     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 687     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 688            "cannot embed scavengable oops in code");
 689   }
 690 #endif
 691   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 692   cbuf.insts()->emit_int64(d64);
 693 }
 694 
 695 // Access stack slot for load or store
 696 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 697 {
 698   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 699   if (-0x80 <= disp && disp < 0x80) {
 700     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 701     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 702     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 703   } else {
 704     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 705     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 706     emit_d32(cbuf, disp);     // Displacement // R/M byte
 707   }
 708 }
 709 
 710    // rRegI ereg, memory mem) %{    // emit_reg_mem
 711 void encode_RegMem(CodeBuffer &cbuf,
 712                    int reg,
 713                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 714 {
 715   assert(disp_reloc == relocInfo::none, "cannot have disp");
 716   int regenc = reg & 7;
 717   int baseenc = base & 7;
 718   int indexenc = index & 7;
 719 
 720   // There is no index & no scale, use form without SIB byte
 721   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 722     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 723     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 724       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 725     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 726       // If 8-bit displacement, mode 0x1
 727       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 728       emit_d8(cbuf, disp);
 729     } else {
 730       // If 32-bit displacement
 731       if (base == -1) { // Special flag for absolute address
 732         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 733         if (disp_reloc != relocInfo::none) {
 734           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 735         } else {
 736           emit_d32(cbuf, disp);
 737         }
 738       } else {
 739         // Normal base + offset
 740         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 741         if (disp_reloc != relocInfo::none) {
 742           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 743         } else {
 744           emit_d32(cbuf, disp);
 745         }
 746       }
 747     }
 748   } else {
 749     // Else, encode with the SIB byte
 750     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 751     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 752       // If no displacement
 753       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 754       emit_rm(cbuf, scale, indexenc, baseenc);
 755     } else {
 756       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 757         // If 8-bit displacement, mode 0x1
 758         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 759         emit_rm(cbuf, scale, indexenc, baseenc);
 760         emit_d8(cbuf, disp);
 761       } else {
 762         // If 32-bit displacement
 763         if (base == 0x04 ) {
 764           emit_rm(cbuf, 0x2, regenc, 0x4);
 765           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 766         } else {
 767           emit_rm(cbuf, 0x2, regenc, 0x4);
 768           emit_rm(cbuf, scale, indexenc, baseenc); // *
 769         }
 770         if (disp_reloc != relocInfo::none) {
 771           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 772         } else {
 773           emit_d32(cbuf, disp);
 774         }
 775       }
 776     }
 777   }
 778 }
 779 
 780 // This could be in MacroAssembler but it's fairly C2 specific
 781 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 782   Label exit;
 783   __ jccb(Assembler::noParity, exit);
 784   __ pushf();
 785   //
 786   // comiss/ucomiss instructions set ZF,PF,CF flags and
 787   // zero OF,AF,SF for NaN values.
 788   // Fixup flags by zeroing ZF,PF so that compare of NaN
 789   // values returns 'less than' result (CF is set).
 790   // Leave the rest of flags unchanged.
 791   //
 792   //    7 6 5 4 3 2 1 0
 793   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 794   //    0 0 1 0 1 0 1 1   (0x2B)
 795   //
 796   __ andq(Address(rsp, 0), 0xffffff2b);
 797   __ popf();
 798   __ bind(exit);
 799 }
 800 
 801 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 802   Label done;
 803   __ movl(dst, -1);
 804   __ jcc(Assembler::parity, done);
 805   __ jcc(Assembler::below, done);
 806   __ setb(Assembler::notEqual, dst);
 807   __ movzbl(dst, dst);
 808   __ bind(done);
 809 }
 810 
 811 
 812 //=============================================================================
 813 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 814 
 815 int Compile::ConstantTable::calculate_table_base_offset() const {
 816   return 0;  // absolute addressing, no offset
 817 }
 818 
 819 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 820 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 821   ShouldNotReachHere();
 822 }
 823 
 824 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 825   // Empty encoding
 826 }
 827 
 828 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 829   return 0;
 830 }
 831 
 832 #ifndef PRODUCT
 833 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 834   st->print("# MachConstantBaseNode (empty encoding)");
 835 }
 836 #endif
 837 
 838 
 839 //=============================================================================
 840 #ifndef PRODUCT
 841 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 842   Compile* C = ra_->C;
 843 
 844   int framesize = C->frame_size_in_bytes();
 845   int bangsize = C->bang_size_in_bytes();
 846   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 847   // Remove wordSize for return addr which is already pushed.
 848   framesize -= wordSize;
 849 
 850   if (C->need_stack_bang(bangsize)) {
 851     framesize -= wordSize;
 852     st->print("# stack bang (%d bytes)", bangsize);
 853     st->print("\n\t");
 854     st->print("pushq   rbp\t# Save rbp");
 855     if (PreserveFramePointer) {
 856         st->print("\n\t");
 857         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 858     }
 859     if (framesize) {
 860       st->print("\n\t");
 861       st->print("subq    rsp, #%d\t# Create frame",framesize);
 862     }
 863   } else {
 864     st->print("subq    rsp, #%d\t# Create frame",framesize);
 865     st->print("\n\t");
 866     framesize -= wordSize;
 867     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 868     if (PreserveFramePointer) {
 869       st->print("\n\t");
 870       st->print("movq    rbp, [rsp + #%d]\t# Save the caller's SP into rbp", (framesize + wordSize));
 871     }
 872   }
 873 
 874   if (VerifyStackAtCalls) {
 875     st->print("\n\t");
 876     framesize -= wordSize;
 877     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 878 #ifdef ASSERT
 879     st->print("\n\t");
 880     st->print("# stack alignment check");
 881 #endif
 882   }
 883   st->cr();
 884 }
 885 #endif
 886 
 887 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 888   Compile* C = ra_->C;
 889   MacroAssembler _masm(&cbuf);
 890 
 891   int framesize = C->frame_size_in_bytes();
 892   int bangsize = C->bang_size_in_bytes();
 893 
 894   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 895 
 896   C->set_frame_complete(cbuf.insts_size());
 897 
 898   if (C->has_mach_constant_base_node()) {
 899     // NOTE: We set the table base offset here because users might be
 900     // emitted before MachConstantBaseNode.
 901     Compile::ConstantTable& constant_table = C->constant_table();
 902     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 903   }
 904 }
 905 
 906 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 907 {
 908   return MachNode::size(ra_); // too many variables; just compute it
 909                               // the hard way
 910 }
 911 
 912 int MachPrologNode::reloc() const
 913 {
 914   return 0; // a large enough number
 915 }
 916 
 917 //=============================================================================
 918 #ifndef PRODUCT
 919 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 920 {
 921   Compile* C = ra_->C;
 922   if (C->max_vector_size() > 16) {
 923     st->print("vzeroupper");
 924     st->cr(); st->print("\t");
 925   }
 926 
 927   int framesize = C->frame_size_in_bytes();
 928   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 929   // Remove word for return adr already pushed
 930   // and RBP
 931   framesize -= 2*wordSize;
 932 
 933   if (framesize) {
 934     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 935     st->print("\t");
 936   }
 937 
 938   st->print_cr("popq   rbp");
 939   if (do_polling() && C->is_method_compilation()) {
 940     st->print("\t");
 941     if (Assembler::is_polling_page_far()) {
 942       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 943                    "testl  rax, [rscratch1]\t"
 944                    "# Safepoint: poll for GC");
 945     } else {
 946       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 947                    "# Safepoint: poll for GC");
 948     }
 949   }
 950 }
 951 #endif
 952 
 953 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 954 {
 955   Compile* C = ra_->C;
 956   if (C->max_vector_size() > 16) {
 957     // Clear upper bits of YMM registers when current compiled code uses
 958     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 959     MacroAssembler _masm(&cbuf);
 960     __ vzeroupper();
 961   }
 962 
 963   int framesize = C->frame_size_in_bytes();
 964   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 965   // Remove word for return adr already pushed
 966   // and RBP
 967   framesize -= 2*wordSize;
 968 
 969   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 970 
 971   if (framesize) {
 972     emit_opcode(cbuf, Assembler::REX_W);
 973     if (framesize < 0x80) {
 974       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 975       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 976       emit_d8(cbuf, framesize);
 977     } else {
 978       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 979       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 980       emit_d32(cbuf, framesize);
 981     }
 982   }
 983 
 984   // popq rbp
 985   emit_opcode(cbuf, 0x58 | RBP_enc);
 986 
 987   if (do_polling() && C->is_method_compilation()) {
 988     MacroAssembler _masm(&cbuf);
 989     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 990     if (Assembler::is_polling_page_far()) {
 991       __ lea(rscratch1, polling_page);
 992       __ relocate(relocInfo::poll_return_type);
 993       __ testl(rax, Address(rscratch1, 0));
 994     } else {
 995       __ testl(rax, polling_page);
 996     }
 997   }
 998 }
 999 
1000 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1001 {
1002   return MachNode::size(ra_); // too many variables; just compute it
1003                               // the hard way
1004 }
1005 
1006 int MachEpilogNode::reloc() const
1007 {
1008   return 2; // a large enough number
1009 }
1010 
1011 const Pipeline* MachEpilogNode::pipeline() const
1012 {
1013   return MachNode::pipeline_class();
1014 }
1015 
1016 int MachEpilogNode::safepoint_offset() const
1017 {
1018   return 0;
1019 }
1020 
1021 //=============================================================================
1022 
1023 enum RC {
1024   rc_bad,
1025   rc_int,
1026   rc_float,
1027   rc_stack
1028 };
1029 
1030 static enum RC rc_class(OptoReg::Name reg)
1031 {
1032   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1033 
1034   if (OptoReg::is_stack(reg)) return rc_stack;
1035 
1036   VMReg r = OptoReg::as_VMReg(reg);
1037 
1038   if (r->is_Register()) return rc_int;
1039 
1040   assert(r->is_XMMRegister(), "must be");
1041   return rc_float;
1042 }
1043 
1044 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1045 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1046                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1047 
1048 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1049                             int stack_offset, int reg, uint ireg, outputStream* st);
1050 
1051 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1052                                       int dst_offset, uint ireg, outputStream* st) {
1053   if (cbuf) {
1054     MacroAssembler _masm(cbuf);
1055     switch (ireg) {
1056     case Op_VecS:
1057       __ movq(Address(rsp, -8), rax);
1058       __ movl(rax, Address(rsp, src_offset));
1059       __ movl(Address(rsp, dst_offset), rax);
1060       __ movq(rax, Address(rsp, -8));
1061       break;
1062     case Op_VecD:
1063       __ pushq(Address(rsp, src_offset));
1064       __ popq (Address(rsp, dst_offset));
1065       break;
1066     case Op_VecX:
1067       __ pushq(Address(rsp, src_offset));
1068       __ popq (Address(rsp, dst_offset));
1069       __ pushq(Address(rsp, src_offset+8));
1070       __ popq (Address(rsp, dst_offset+8));
1071       break;
1072     case Op_VecY:
1073       __ vmovdqu(Address(rsp, -32), xmm0);
1074       __ vmovdqu(xmm0, Address(rsp, src_offset));
1075       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1076       __ vmovdqu(xmm0, Address(rsp, -32));
1077     case Op_VecZ:
1078       __ evmovdqul(Address(rsp, -64), xmm0, 2);
1079       __ evmovdqul(xmm0, Address(rsp, src_offset), 2);
1080       __ evmovdqul(Address(rsp, dst_offset), xmm0, 2);
1081       __ evmovdqul(xmm0, Address(rsp, -64), 2);
1082       break;
1083     default:
1084       ShouldNotReachHere();
1085     }
1086 #ifndef PRODUCT
1087   } else {
1088     switch (ireg) {
1089     case Op_VecS:
1090       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1091                 "movl    rax, [rsp + #%d]\n\t"
1092                 "movl    [rsp + #%d], rax\n\t"
1093                 "movq    rax, [rsp - #8]",
1094                 src_offset, dst_offset);
1095       break;
1096     case Op_VecD:
1097       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1098                 "popq    [rsp + #%d]",
1099                 src_offset, dst_offset);
1100       break;
1101      case Op_VecX:
1102       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1103                 "popq    [rsp + #%d]\n\t"
1104                 "pushq   [rsp + #%d]\n\t"
1105                 "popq    [rsp + #%d]",
1106                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1107       break;
1108     case Op_VecY:
1109       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1110                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1111                 "vmovdqu [rsp + #%d], xmm0\n\t"
1112                 "vmovdqu xmm0, [rsp - #32]",
1113                 src_offset, dst_offset);
1114       break;
1115     case Op_VecZ:
1116       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1117                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1118                 "vmovdqu [rsp + #%d], xmm0\n\t"
1119                 "vmovdqu xmm0, [rsp - #64]",
1120                 src_offset, dst_offset);
1121       break;
1122     default:
1123       ShouldNotReachHere();
1124     }
1125 #endif
1126   }
1127 }
1128 
1129 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1130                                        PhaseRegAlloc* ra_,
1131                                        bool do_size,
1132                                        outputStream* st) const {
1133   assert(cbuf != NULL || st  != NULL, "sanity");
1134   // Get registers to move
1135   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1136   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1137   OptoReg::Name dst_second = ra_->get_reg_second(this);
1138   OptoReg::Name dst_first = ra_->get_reg_first(this);
1139 
1140   enum RC src_second_rc = rc_class(src_second);
1141   enum RC src_first_rc = rc_class(src_first);
1142   enum RC dst_second_rc = rc_class(dst_second);
1143   enum RC dst_first_rc = rc_class(dst_first);
1144 
1145   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1146          "must move at least 1 register" );
1147 
1148   if (src_first == dst_first && src_second == dst_second) {
1149     // Self copy, no move
1150     return 0;
1151   }
1152   if (bottom_type()->isa_vect() != NULL) {
1153     uint ireg = ideal_reg();
1154     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1155     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1156     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1157       // mem -> mem
1158       int src_offset = ra_->reg2offset(src_first);
1159       int dst_offset = ra_->reg2offset(dst_first);
1160       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1161     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1162       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1163     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1164       int stack_offset = ra_->reg2offset(dst_first);
1165       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1166     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1167       int stack_offset = ra_->reg2offset(src_first);
1168       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1169     } else {
1170       ShouldNotReachHere();
1171     }
1172     return 0;
1173   }
1174   if (src_first_rc == rc_stack) {
1175     // mem ->
1176     if (dst_first_rc == rc_stack) {
1177       // mem -> mem
1178       assert(src_second != dst_first, "overlap");
1179       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1180           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1181         // 64-bit
1182         int src_offset = ra_->reg2offset(src_first);
1183         int dst_offset = ra_->reg2offset(dst_first);
1184         if (cbuf) {
1185           MacroAssembler _masm(cbuf);
1186           __ pushq(Address(rsp, src_offset));
1187           __ popq (Address(rsp, dst_offset));
1188 #ifndef PRODUCT
1189         } else {
1190           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1191                     "popq    [rsp + #%d]",
1192                      src_offset, dst_offset);
1193 #endif
1194         }
1195       } else {
1196         // 32-bit
1197         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1198         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1199         // No pushl/popl, so:
1200         int src_offset = ra_->reg2offset(src_first);
1201         int dst_offset = ra_->reg2offset(dst_first);
1202         if (cbuf) {
1203           MacroAssembler _masm(cbuf);
1204           __ movq(Address(rsp, -8), rax);
1205           __ movl(rax, Address(rsp, src_offset));
1206           __ movl(Address(rsp, dst_offset), rax);
1207           __ movq(rax, Address(rsp, -8));
1208 #ifndef PRODUCT
1209         } else {
1210           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1211                     "movl    rax, [rsp + #%d]\n\t"
1212                     "movl    [rsp + #%d], rax\n\t"
1213                     "movq    rax, [rsp - #8]",
1214                      src_offset, dst_offset);
1215 #endif
1216         }
1217       }
1218       return 0;
1219     } else if (dst_first_rc == rc_int) {
1220       // mem -> gpr
1221       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1222           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1223         // 64-bit
1224         int offset = ra_->reg2offset(src_first);
1225         if (cbuf) {
1226           MacroAssembler _masm(cbuf);
1227           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1228 #ifndef PRODUCT
1229         } else {
1230           st->print("movq    %s, [rsp + #%d]\t# spill",
1231                      Matcher::regName[dst_first],
1232                      offset);
1233 #endif
1234         }
1235       } else {
1236         // 32-bit
1237         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1238         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1239         int offset = ra_->reg2offset(src_first);
1240         if (cbuf) {
1241           MacroAssembler _masm(cbuf);
1242           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1243 #ifndef PRODUCT
1244         } else {
1245           st->print("movl    %s, [rsp + #%d]\t# spill",
1246                      Matcher::regName[dst_first],
1247                      offset);
1248 #endif
1249         }
1250       }
1251       return 0;
1252     } else if (dst_first_rc == rc_float) {
1253       // mem-> xmm
1254       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1255           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1256         // 64-bit
1257         int offset = ra_->reg2offset(src_first);
1258         if (cbuf) {
1259           MacroAssembler _masm(cbuf);
1260           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1261 #ifndef PRODUCT
1262         } else {
1263           st->print("%s  %s, [rsp + #%d]\t# spill",
1264                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1265                      Matcher::regName[dst_first],
1266                      offset);
1267 #endif
1268         }
1269       } else {
1270         // 32-bit
1271         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1272         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1273         int offset = ra_->reg2offset(src_first);
1274         if (cbuf) {
1275           MacroAssembler _masm(cbuf);
1276           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1277 #ifndef PRODUCT
1278         } else {
1279           st->print("movss   %s, [rsp + #%d]\t# spill",
1280                      Matcher::regName[dst_first],
1281                      offset);
1282 #endif
1283         }
1284       }
1285       return 0;
1286     }
1287   } else if (src_first_rc == rc_int) {
1288     // gpr ->
1289     if (dst_first_rc == rc_stack) {
1290       // gpr -> mem
1291       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1292           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1293         // 64-bit
1294         int offset = ra_->reg2offset(dst_first);
1295         if (cbuf) {
1296           MacroAssembler _masm(cbuf);
1297           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1298 #ifndef PRODUCT
1299         } else {
1300           st->print("movq    [rsp + #%d], %s\t# spill",
1301                      offset,
1302                      Matcher::regName[src_first]);
1303 #endif
1304         }
1305       } else {
1306         // 32-bit
1307         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1308         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1309         int offset = ra_->reg2offset(dst_first);
1310         if (cbuf) {
1311           MacroAssembler _masm(cbuf);
1312           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1313 #ifndef PRODUCT
1314         } else {
1315           st->print("movl    [rsp + #%d], %s\t# spill",
1316                      offset,
1317                      Matcher::regName[src_first]);
1318 #endif
1319         }
1320       }
1321       return 0;
1322     } else if (dst_first_rc == rc_int) {
1323       // gpr -> gpr
1324       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1325           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1326         // 64-bit
1327         if (cbuf) {
1328           MacroAssembler _masm(cbuf);
1329           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1330                   as_Register(Matcher::_regEncode[src_first]));
1331 #ifndef PRODUCT
1332         } else {
1333           st->print("movq    %s, %s\t# spill",
1334                      Matcher::regName[dst_first],
1335                      Matcher::regName[src_first]);
1336 #endif
1337         }
1338         return 0;
1339       } else {
1340         // 32-bit
1341         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1342         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1343         if (cbuf) {
1344           MacroAssembler _masm(cbuf);
1345           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1346                   as_Register(Matcher::_regEncode[src_first]));
1347 #ifndef PRODUCT
1348         } else {
1349           st->print("movl    %s, %s\t# spill",
1350                      Matcher::regName[dst_first],
1351                      Matcher::regName[src_first]);
1352 #endif
1353         }
1354         return 0;
1355       }
1356     } else if (dst_first_rc == rc_float) {
1357       // gpr -> xmm
1358       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1359           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1360         // 64-bit
1361         if (cbuf) {
1362           MacroAssembler _masm(cbuf);
1363           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1364 #ifndef PRODUCT
1365         } else {
1366           st->print("movdq   %s, %s\t# spill",
1367                      Matcher::regName[dst_first],
1368                      Matcher::regName[src_first]);
1369 #endif
1370         }
1371       } else {
1372         // 32-bit
1373         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1374         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1375         if (cbuf) {
1376           MacroAssembler _masm(cbuf);
1377           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1378 #ifndef PRODUCT
1379         } else {
1380           st->print("movdl   %s, %s\t# spill",
1381                      Matcher::regName[dst_first],
1382                      Matcher::regName[src_first]);
1383 #endif
1384         }
1385       }
1386       return 0;
1387     }
1388   } else if (src_first_rc == rc_float) {
1389     // xmm ->
1390     if (dst_first_rc == rc_stack) {
1391       // xmm -> mem
1392       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1393           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1394         // 64-bit
1395         int offset = ra_->reg2offset(dst_first);
1396         if (cbuf) {
1397           MacroAssembler _masm(cbuf);
1398           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1399 #ifndef PRODUCT
1400         } else {
1401           st->print("movsd   [rsp + #%d], %s\t# spill",
1402                      offset,
1403                      Matcher::regName[src_first]);
1404 #endif
1405         }
1406       } else {
1407         // 32-bit
1408         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1409         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1410         int offset = ra_->reg2offset(dst_first);
1411         if (cbuf) {
1412           MacroAssembler _masm(cbuf);
1413           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1414 #ifndef PRODUCT
1415         } else {
1416           st->print("movss   [rsp + #%d], %s\t# spill",
1417                      offset,
1418                      Matcher::regName[src_first]);
1419 #endif
1420         }
1421       }
1422       return 0;
1423     } else if (dst_first_rc == rc_int) {
1424       // xmm -> gpr
1425       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1426           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1427         // 64-bit
1428         if (cbuf) {
1429           MacroAssembler _masm(cbuf);
1430           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1431 #ifndef PRODUCT
1432         } else {
1433           st->print("movdq   %s, %s\t# spill",
1434                      Matcher::regName[dst_first],
1435                      Matcher::regName[src_first]);
1436 #endif
1437         }
1438       } else {
1439         // 32-bit
1440         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1441         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1442         if (cbuf) {
1443           MacroAssembler _masm(cbuf);
1444           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1445 #ifndef PRODUCT
1446         } else {
1447           st->print("movdl   %s, %s\t# spill",
1448                      Matcher::regName[dst_first],
1449                      Matcher::regName[src_first]);
1450 #endif
1451         }
1452       }
1453       return 0;
1454     } else if (dst_first_rc == rc_float) {
1455       // xmm -> xmm
1456       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1457           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1458         // 64-bit
1459         if (cbuf) {
1460           MacroAssembler _masm(cbuf);
1461           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1462 #ifndef PRODUCT
1463         } else {
1464           st->print("%s  %s, %s\t# spill",
1465                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1466                      Matcher::regName[dst_first],
1467                      Matcher::regName[src_first]);
1468 #endif
1469         }
1470       } else {
1471         // 32-bit
1472         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1473         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1474         if (cbuf) {
1475           MacroAssembler _masm(cbuf);
1476           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1477 #ifndef PRODUCT
1478         } else {
1479           st->print("%s  %s, %s\t# spill",
1480                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1481                      Matcher::regName[dst_first],
1482                      Matcher::regName[src_first]);
1483 #endif
1484         }
1485       }
1486       return 0;
1487     }
1488   }
1489 
1490   assert(0," foo ");
1491   Unimplemented();
1492   return 0;
1493 }
1494 
1495 #ifndef PRODUCT
1496 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1497   implementation(NULL, ra_, false, st);
1498 }
1499 #endif
1500 
1501 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1502   implementation(&cbuf, ra_, false, NULL);
1503 }
1504 
1505 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1506   return MachNode::size(ra_);
1507 }
1508 
1509 //=============================================================================
1510 #ifndef PRODUCT
1511 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1512 {
1513   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1514   int reg = ra_->get_reg_first(this);
1515   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1516             Matcher::regName[reg], offset);
1517 }
1518 #endif
1519 
1520 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1521 {
1522   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1523   int reg = ra_->get_encode(this);
1524   if (offset >= 0x80) {
1525     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1526     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1527     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1528     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1529     emit_d32(cbuf, offset);
1530   } else {
1531     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1532     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1533     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1534     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1535     emit_d8(cbuf, offset);
1536   }
1537 }
1538 
1539 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1540 {
1541   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1542   return (offset < 0x80) ? 5 : 8; // REX
1543 }
1544 
1545 //=============================================================================
1546 #ifndef PRODUCT
1547 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1548 {
1549   if (UseCompressedClassPointers) {
1550     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1551     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1552     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1553   } else {
1554     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1555                  "# Inline cache check");
1556   }
1557   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1558   st->print_cr("\tnop\t# nops to align entry point");
1559 }
1560 #endif
1561 
1562 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1563 {
1564   MacroAssembler masm(&cbuf);
1565   uint insts_size = cbuf.insts_size();
1566   if (UseCompressedClassPointers) {
1567     masm.load_klass(rscratch1, j_rarg0);
1568     masm.cmpptr(rax, rscratch1);
1569   } else {
1570     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1571   }
1572 
1573   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1574 
1575   /* WARNING these NOPs are critical so that verified entry point is properly
1576      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1577   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1578   if (OptoBreakpoint) {
1579     // Leave space for int3
1580     nops_cnt -= 1;
1581   }
1582   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1583   if (nops_cnt > 0)
1584     masm.nop(nops_cnt);
1585 }
1586 
1587 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1588 {
1589   return MachNode::size(ra_); // too many variables; just compute it
1590                               // the hard way
1591 }
1592 
1593 
1594 //=============================================================================
1595 
1596 int Matcher::regnum_to_fpu_offset(int regnum)
1597 {
1598   return regnum - 32; // The FP registers are in the second chunk
1599 }
1600 
1601 // This is UltraSparc specific, true just means we have fast l2f conversion
1602 const bool Matcher::convL2FSupported(void) {
1603   return true;
1604 }
1605 
1606 // Is this branch offset short enough that a short branch can be used?
1607 //
1608 // NOTE: If the platform does not provide any short branch variants, then
1609 //       this method should return false for offset 0.
1610 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1611   // The passed offset is relative to address of the branch.
1612   // On 86 a branch displacement is calculated relative to address
1613   // of a next instruction.
1614   offset -= br_size;
1615 
1616   // the short version of jmpConUCF2 contains multiple branches,
1617   // making the reach slightly less
1618   if (rule == jmpConUCF2_rule)
1619     return (-126 <= offset && offset <= 125);
1620   return (-128 <= offset && offset <= 127);
1621 }
1622 
1623 const bool Matcher::isSimpleConstant64(jlong value) {
1624   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1625   //return value == (int) value;  // Cf. storeImmL and immL32.
1626 
1627   // Probably always true, even if a temp register is required.
1628   return true;
1629 }
1630 
1631 // The ecx parameter to rep stosq for the ClearArray node is in words.
1632 const bool Matcher::init_array_count_is_in_bytes = false;
1633 
1634 // Threshold size for cleararray.
1635 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1636 
1637 // No additional cost for CMOVL.
1638 const int Matcher::long_cmove_cost() { return 0; }
1639 
1640 // No CMOVF/CMOVD with SSE2
1641 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1642 
1643 // Does the CPU require late expand (see block.cpp for description of late expand)?
1644 const bool Matcher::require_postalloc_expand = false;
1645 
1646 // Should the Matcher clone shifts on addressing modes, expecting them
1647 // to be subsumed into complex addressing expressions or compute them
1648 // into registers?  True for Intel but false for most RISCs
1649 const bool Matcher::clone_shift_expressions = true;
1650 
1651 // Do we need to mask the count passed to shift instructions or does
1652 // the cpu only look at the lower 5/6 bits anyway?
1653 const bool Matcher::need_masked_shift_count = false;
1654 
1655 bool Matcher::narrow_oop_use_complex_address() {
1656   assert(UseCompressedOops, "only for compressed oops code");
1657   return (LogMinObjAlignmentInBytes <= 3);
1658 }
1659 
1660 bool Matcher::narrow_klass_use_complex_address() {
1661   assert(UseCompressedClassPointers, "only for compressed klass code");
1662   return (LogKlassAlignmentInBytes <= 3);
1663 }
1664 
1665 // Is it better to copy float constants, or load them directly from
1666 // memory?  Intel can load a float constant from a direct address,
1667 // requiring no extra registers.  Most RISCs will have to materialize
1668 // an address into a register first, so they would do better to copy
1669 // the constant from stack.
1670 const bool Matcher::rematerialize_float_constants = true; // XXX
1671 
1672 // If CPU can load and store mis-aligned doubles directly then no
1673 // fixup is needed.  Else we split the double into 2 integer pieces
1674 // and move it piece-by-piece.  Only happens when passing doubles into
1675 // C code as the Java calling convention forces doubles to be aligned.
1676 const bool Matcher::misaligned_doubles_ok = true;
1677 
1678 // No-op on amd64
1679 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1680 
1681 // Advertise here if the CPU requires explicit rounding operations to
1682 // implement the UseStrictFP mode.
1683 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1684 
1685 // Are floats conerted to double when stored to stack during deoptimization?
1686 // On x64 it is stored without convertion so we can use normal access.
1687 bool Matcher::float_in_double() { return false; }
1688 
1689 // Do ints take an entire long register or just half?
1690 const bool Matcher::int_in_long = true;
1691 
1692 // Return whether or not this register is ever used as an argument.
1693 // This function is used on startup to build the trampoline stubs in
1694 // generateOptoStub.  Registers not mentioned will be killed by the VM
1695 // call in the trampoline, and arguments in those registers not be
1696 // available to the callee.
1697 bool Matcher::can_be_java_arg(int reg)
1698 {
1699   return
1700     reg ==  RDI_num || reg == RDI_H_num ||
1701     reg ==  RSI_num || reg == RSI_H_num ||
1702     reg ==  RDX_num || reg == RDX_H_num ||
1703     reg ==  RCX_num || reg == RCX_H_num ||
1704     reg ==   R8_num || reg ==  R8_H_num ||
1705     reg ==   R9_num || reg ==  R9_H_num ||
1706     reg ==  R12_num || reg == R12_H_num ||
1707     reg == XMM0_num || reg == XMM0b_num ||
1708     reg == XMM1_num || reg == XMM1b_num ||
1709     reg == XMM2_num || reg == XMM2b_num ||
1710     reg == XMM3_num || reg == XMM3b_num ||
1711     reg == XMM4_num || reg == XMM4b_num ||
1712     reg == XMM5_num || reg == XMM5b_num ||
1713     reg == XMM6_num || reg == XMM6b_num ||
1714     reg == XMM7_num || reg == XMM7b_num;
1715 }
1716 
1717 bool Matcher::is_spillable_arg(int reg)
1718 {
1719   return can_be_java_arg(reg);
1720 }
1721 
1722 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1723   // In 64 bit mode a code which use multiply when
1724   // devisor is constant is faster than hardware
1725   // DIV instruction (it uses MulHiL).
1726   return false;
1727 }
1728 
1729 // Register for DIVI projection of divmodI
1730 RegMask Matcher::divI_proj_mask() {
1731   return INT_RAX_REG_mask();
1732 }
1733 
1734 // Register for MODI projection of divmodI
1735 RegMask Matcher::modI_proj_mask() {
1736   return INT_RDX_REG_mask();
1737 }
1738 
1739 // Register for DIVL projection of divmodL
1740 RegMask Matcher::divL_proj_mask() {
1741   return LONG_RAX_REG_mask();
1742 }
1743 
1744 // Register for MODL projection of divmodL
1745 RegMask Matcher::modL_proj_mask() {
1746   return LONG_RDX_REG_mask();
1747 }
1748 
1749 // Register for saving SP into on method handle invokes. Not used on x86_64.
1750 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1751     return NO_REG_mask();
1752 }
1753 
1754 %}
1755 
1756 //----------ENCODING BLOCK-----------------------------------------------------
1757 // This block specifies the encoding classes used by the compiler to
1758 // output byte streams.  Encoding classes are parameterized macros
1759 // used by Machine Instruction Nodes in order to generate the bit
1760 // encoding of the instruction.  Operands specify their base encoding
1761 // interface with the interface keyword.  There are currently
1762 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1763 // COND_INTER.  REG_INTER causes an operand to generate a function
1764 // which returns its register number when queried.  CONST_INTER causes
1765 // an operand to generate a function which returns the value of the
1766 // constant when queried.  MEMORY_INTER causes an operand to generate
1767 // four functions which return the Base Register, the Index Register,
1768 // the Scale Value, and the Offset Value of the operand when queried.
1769 // COND_INTER causes an operand to generate six functions which return
1770 // the encoding code (ie - encoding bits for the instruction)
1771 // associated with each basic boolean condition for a conditional
1772 // instruction.
1773 //
1774 // Instructions specify two basic values for encoding.  Again, a
1775 // function is available to check if the constant displacement is an
1776 // oop. They use the ins_encode keyword to specify their encoding
1777 // classes (which must be a sequence of enc_class names, and their
1778 // parameters, specified in the encoding block), and they use the
1779 // opcode keyword to specify, in order, their primary, secondary, and
1780 // tertiary opcode.  Only the opcode sections which a particular
1781 // instruction needs for encoding need to be specified.
1782 encode %{
1783   // Build emit functions for each basic byte or larger field in the
1784   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1785   // from C++ code in the enc_class source block.  Emit functions will
1786   // live in the main source block for now.  In future, we can
1787   // generalize this by adding a syntax that specifies the sizes of
1788   // fields in an order, so that the adlc can build the emit functions
1789   // automagically
1790 
1791   // Emit primary opcode
1792   enc_class OpcP
1793   %{
1794     emit_opcode(cbuf, $primary);
1795   %}
1796 
1797   // Emit secondary opcode
1798   enc_class OpcS
1799   %{
1800     emit_opcode(cbuf, $secondary);
1801   %}
1802 
1803   // Emit tertiary opcode
1804   enc_class OpcT
1805   %{
1806     emit_opcode(cbuf, $tertiary);
1807   %}
1808 
1809   // Emit opcode directly
1810   enc_class Opcode(immI d8)
1811   %{
1812     emit_opcode(cbuf, $d8$$constant);
1813   %}
1814 
1815   // Emit size prefix
1816   enc_class SizePrefix
1817   %{
1818     emit_opcode(cbuf, 0x66);
1819   %}
1820 
1821   enc_class reg(rRegI reg)
1822   %{
1823     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1824   %}
1825 
1826   enc_class reg_reg(rRegI dst, rRegI src)
1827   %{
1828     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1829   %}
1830 
1831   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1832   %{
1833     emit_opcode(cbuf, $opcode$$constant);
1834     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1835   %}
1836 
1837   enc_class cdql_enc(no_rax_rdx_RegI div)
1838   %{
1839     // Full implementation of Java idiv and irem; checks for
1840     // special case as described in JVM spec., p.243 & p.271.
1841     //
1842     //         normal case                           special case
1843     //
1844     // input : rax: dividend                         min_int
1845     //         reg: divisor                          -1
1846     //
1847     // output: rax: quotient  (= rax idiv reg)       min_int
1848     //         rdx: remainder (= rax irem reg)       0
1849     //
1850     //  Code sequnce:
1851     //
1852     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1853     //    5:   75 07/08                jne    e <normal>
1854     //    7:   33 d2                   xor    %edx,%edx
1855     //  [div >= 8 -> offset + 1]
1856     //  [REX_B]
1857     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1858     //    c:   74 03/04                je     11 <done>
1859     // 000000000000000e <normal>:
1860     //    e:   99                      cltd
1861     //  [div >= 8 -> offset + 1]
1862     //  [REX_B]
1863     //    f:   f7 f9                   idiv   $div
1864     // 0000000000000011 <done>:
1865 
1866     // cmp    $0x80000000,%eax
1867     emit_opcode(cbuf, 0x3d);
1868     emit_d8(cbuf, 0x00);
1869     emit_d8(cbuf, 0x00);
1870     emit_d8(cbuf, 0x00);
1871     emit_d8(cbuf, 0x80);
1872 
1873     // jne    e <normal>
1874     emit_opcode(cbuf, 0x75);
1875     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1876 
1877     // xor    %edx,%edx
1878     emit_opcode(cbuf, 0x33);
1879     emit_d8(cbuf, 0xD2);
1880 
1881     // cmp    $0xffffffffffffffff,%ecx
1882     if ($div$$reg >= 8) {
1883       emit_opcode(cbuf, Assembler::REX_B);
1884     }
1885     emit_opcode(cbuf, 0x83);
1886     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1887     emit_d8(cbuf, 0xFF);
1888 
1889     // je     11 <done>
1890     emit_opcode(cbuf, 0x74);
1891     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1892 
1893     // <normal>
1894     // cltd
1895     emit_opcode(cbuf, 0x99);
1896 
1897     // idivl (note: must be emitted by the user of this rule)
1898     // <done>
1899   %}
1900 
1901   enc_class cdqq_enc(no_rax_rdx_RegL div)
1902   %{
1903     // Full implementation of Java ldiv and lrem; checks for
1904     // special case as described in JVM spec., p.243 & p.271.
1905     //
1906     //         normal case                           special case
1907     //
1908     // input : rax: dividend                         min_long
1909     //         reg: divisor                          -1
1910     //
1911     // output: rax: quotient  (= rax idiv reg)       min_long
1912     //         rdx: remainder (= rax irem reg)       0
1913     //
1914     //  Code sequnce:
1915     //
1916     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1917     //    7:   00 00 80
1918     //    a:   48 39 d0                cmp    %rdx,%rax
1919     //    d:   75 08                   jne    17 <normal>
1920     //    f:   33 d2                   xor    %edx,%edx
1921     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1922     //   15:   74 05                   je     1c <done>
1923     // 0000000000000017 <normal>:
1924     //   17:   48 99                   cqto
1925     //   19:   48 f7 f9                idiv   $div
1926     // 000000000000001c <done>:
1927 
1928     // mov    $0x8000000000000000,%rdx
1929     emit_opcode(cbuf, Assembler::REX_W);
1930     emit_opcode(cbuf, 0xBA);
1931     emit_d8(cbuf, 0x00);
1932     emit_d8(cbuf, 0x00);
1933     emit_d8(cbuf, 0x00);
1934     emit_d8(cbuf, 0x00);
1935     emit_d8(cbuf, 0x00);
1936     emit_d8(cbuf, 0x00);
1937     emit_d8(cbuf, 0x00);
1938     emit_d8(cbuf, 0x80);
1939 
1940     // cmp    %rdx,%rax
1941     emit_opcode(cbuf, Assembler::REX_W);
1942     emit_opcode(cbuf, 0x39);
1943     emit_d8(cbuf, 0xD0);
1944 
1945     // jne    17 <normal>
1946     emit_opcode(cbuf, 0x75);
1947     emit_d8(cbuf, 0x08);
1948 
1949     // xor    %edx,%edx
1950     emit_opcode(cbuf, 0x33);
1951     emit_d8(cbuf, 0xD2);
1952 
1953     // cmp    $0xffffffffffffffff,$div
1954     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1955     emit_opcode(cbuf, 0x83);
1956     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1957     emit_d8(cbuf, 0xFF);
1958 
1959     // je     1e <done>
1960     emit_opcode(cbuf, 0x74);
1961     emit_d8(cbuf, 0x05);
1962 
1963     // <normal>
1964     // cqto
1965     emit_opcode(cbuf, Assembler::REX_W);
1966     emit_opcode(cbuf, 0x99);
1967 
1968     // idivq (note: must be emitted by the user of this rule)
1969     // <done>
1970   %}
1971 
1972   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1973   enc_class OpcSE(immI imm)
1974   %{
1975     // Emit primary opcode and set sign-extend bit
1976     // Check for 8-bit immediate, and set sign extend bit in opcode
1977     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1978       emit_opcode(cbuf, $primary | 0x02);
1979     } else {
1980       // 32-bit immediate
1981       emit_opcode(cbuf, $primary);
1982     }
1983   %}
1984 
1985   enc_class OpcSErm(rRegI dst, immI imm)
1986   %{
1987     // OpcSEr/m
1988     int dstenc = $dst$$reg;
1989     if (dstenc >= 8) {
1990       emit_opcode(cbuf, Assembler::REX_B);
1991       dstenc -= 8;
1992     }
1993     // Emit primary opcode and set sign-extend bit
1994     // Check for 8-bit immediate, and set sign extend bit in opcode
1995     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1996       emit_opcode(cbuf, $primary | 0x02);
1997     } else {
1998       // 32-bit immediate
1999       emit_opcode(cbuf, $primary);
2000     }
2001     // Emit r/m byte with secondary opcode, after primary opcode.
2002     emit_rm(cbuf, 0x3, $secondary, dstenc);
2003   %}
2004 
2005   enc_class OpcSErm_wide(rRegL dst, immI imm)
2006   %{
2007     // OpcSEr/m
2008     int dstenc = $dst$$reg;
2009     if (dstenc < 8) {
2010       emit_opcode(cbuf, Assembler::REX_W);
2011     } else {
2012       emit_opcode(cbuf, Assembler::REX_WB);
2013       dstenc -= 8;
2014     }
2015     // Emit primary opcode and set sign-extend bit
2016     // Check for 8-bit immediate, and set sign extend bit in opcode
2017     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2018       emit_opcode(cbuf, $primary | 0x02);
2019     } else {
2020       // 32-bit immediate
2021       emit_opcode(cbuf, $primary);
2022     }
2023     // Emit r/m byte with secondary opcode, after primary opcode.
2024     emit_rm(cbuf, 0x3, $secondary, dstenc);
2025   %}
2026 
2027   enc_class Con8or32(immI imm)
2028   %{
2029     // Check for 8-bit immediate, and set sign extend bit in opcode
2030     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2031       $$$emit8$imm$$constant;
2032     } else {
2033       // 32-bit immediate
2034       $$$emit32$imm$$constant;
2035     }
2036   %}
2037 
2038   enc_class opc2_reg(rRegI dst)
2039   %{
2040     // BSWAP
2041     emit_cc(cbuf, $secondary, $dst$$reg);
2042   %}
2043 
2044   enc_class opc3_reg(rRegI dst)
2045   %{
2046     // BSWAP
2047     emit_cc(cbuf, $tertiary, $dst$$reg);
2048   %}
2049 
2050   enc_class reg_opc(rRegI div)
2051   %{
2052     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2053     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2054   %}
2055 
2056   enc_class enc_cmov(cmpOp cop)
2057   %{
2058     // CMOV
2059     $$$emit8$primary;
2060     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2061   %}
2062 
2063   enc_class enc_PartialSubtypeCheck()
2064   %{
2065     Register Rrdi = as_Register(RDI_enc); // result register
2066     Register Rrax = as_Register(RAX_enc); // super class
2067     Register Rrcx = as_Register(RCX_enc); // killed
2068     Register Rrsi = as_Register(RSI_enc); // sub class
2069     Label miss;
2070     const bool set_cond_codes = true;
2071 
2072     MacroAssembler _masm(&cbuf);
2073     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2074                                      NULL, &miss,
2075                                      /*set_cond_codes:*/ true);
2076     if ($primary) {
2077       __ xorptr(Rrdi, Rrdi);
2078     }
2079     __ bind(miss);
2080   %}
2081 
2082   enc_class clear_avx %{
2083     debug_only(int off0 = cbuf.insts_size());
2084     if (ra_->C->max_vector_size() > 16) {
2085       // Clear upper bits of YMM registers when current compiled code uses
2086       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2087       MacroAssembler _masm(&cbuf);
2088       __ vzeroupper();
2089     }
2090     debug_only(int off1 = cbuf.insts_size());
2091     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2092   %}
2093 
2094   enc_class Java_To_Runtime(method meth) %{
2095     // No relocation needed
2096     MacroAssembler _masm(&cbuf);
2097     __ mov64(r10, (int64_t) $meth$$method);
2098     __ call(r10);
2099   %}
2100 
2101   enc_class Java_To_Interpreter(method meth)
2102   %{
2103     // CALL Java_To_Interpreter
2104     // This is the instruction starting address for relocation info.
2105     cbuf.set_insts_mark();
2106     $$$emit8$primary;
2107     // CALL directly to the runtime
2108     emit_d32_reloc(cbuf,
2109                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2110                    runtime_call_Relocation::spec(),
2111                    RELOC_DISP32);
2112   %}
2113 
2114   enc_class Java_Static_Call(method meth)
2115   %{
2116     // JAVA STATIC CALL
2117     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2118     // determine who we intended to call.
2119     cbuf.set_insts_mark();
2120     $$$emit8$primary;
2121 
2122     if (!_method) {
2123       emit_d32_reloc(cbuf,
2124                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2125                      runtime_call_Relocation::spec(),
2126                      RELOC_DISP32);
2127     } else if (_optimized_virtual) {
2128       emit_d32_reloc(cbuf,
2129                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2130                      opt_virtual_call_Relocation::spec(),
2131                      RELOC_DISP32);
2132     } else {
2133       emit_d32_reloc(cbuf,
2134                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2135                      static_call_Relocation::spec(),
2136                      RELOC_DISP32);
2137     }
2138     if (_method) {
2139       // Emit stubs for static call.
2140       address mark = cbuf.insts_mark();
2141       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2142       if (stub == NULL) {
2143         ciEnv::current()->record_failure("CodeCache is full");
2144         return;
2145       }
2146     }
2147   %}
2148 
2149   enc_class Java_Dynamic_Call(method meth) %{
2150     MacroAssembler _masm(&cbuf);
2151     __ ic_call((address)$meth$$method);
2152   %}
2153 
2154   enc_class Java_Compiled_Call(method meth)
2155   %{
2156     // JAVA COMPILED CALL
2157     int disp = in_bytes(Method:: from_compiled_offset());
2158 
2159     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2160     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2161 
2162     // callq *disp(%rax)
2163     cbuf.set_insts_mark();
2164     $$$emit8$primary;
2165     if (disp < 0x80) {
2166       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2167       emit_d8(cbuf, disp); // Displacement
2168     } else {
2169       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2170       emit_d32(cbuf, disp); // Displacement
2171     }
2172   %}
2173 
2174   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2175   %{
2176     // SAL, SAR, SHR
2177     int dstenc = $dst$$reg;
2178     if (dstenc >= 8) {
2179       emit_opcode(cbuf, Assembler::REX_B);
2180       dstenc -= 8;
2181     }
2182     $$$emit8$primary;
2183     emit_rm(cbuf, 0x3, $secondary, dstenc);
2184     $$$emit8$shift$$constant;
2185   %}
2186 
2187   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2188   %{
2189     // SAL, SAR, SHR
2190     int dstenc = $dst$$reg;
2191     if (dstenc < 8) {
2192       emit_opcode(cbuf, Assembler::REX_W);
2193     } else {
2194       emit_opcode(cbuf, Assembler::REX_WB);
2195       dstenc -= 8;
2196     }
2197     $$$emit8$primary;
2198     emit_rm(cbuf, 0x3, $secondary, dstenc);
2199     $$$emit8$shift$$constant;
2200   %}
2201 
2202   enc_class load_immI(rRegI dst, immI src)
2203   %{
2204     int dstenc = $dst$$reg;
2205     if (dstenc >= 8) {
2206       emit_opcode(cbuf, Assembler::REX_B);
2207       dstenc -= 8;
2208     }
2209     emit_opcode(cbuf, 0xB8 | dstenc);
2210     $$$emit32$src$$constant;
2211   %}
2212 
2213   enc_class load_immL(rRegL dst, immL src)
2214   %{
2215     int dstenc = $dst$$reg;
2216     if (dstenc < 8) {
2217       emit_opcode(cbuf, Assembler::REX_W);
2218     } else {
2219       emit_opcode(cbuf, Assembler::REX_WB);
2220       dstenc -= 8;
2221     }
2222     emit_opcode(cbuf, 0xB8 | dstenc);
2223     emit_d64(cbuf, $src$$constant);
2224   %}
2225 
2226   enc_class load_immUL32(rRegL dst, immUL32 src)
2227   %{
2228     // same as load_immI, but this time we care about zeroes in the high word
2229     int dstenc = $dst$$reg;
2230     if (dstenc >= 8) {
2231       emit_opcode(cbuf, Assembler::REX_B);
2232       dstenc -= 8;
2233     }
2234     emit_opcode(cbuf, 0xB8 | dstenc);
2235     $$$emit32$src$$constant;
2236   %}
2237 
2238   enc_class load_immL32(rRegL dst, immL32 src)
2239   %{
2240     int dstenc = $dst$$reg;
2241     if (dstenc < 8) {
2242       emit_opcode(cbuf, Assembler::REX_W);
2243     } else {
2244       emit_opcode(cbuf, Assembler::REX_WB);
2245       dstenc -= 8;
2246     }
2247     emit_opcode(cbuf, 0xC7);
2248     emit_rm(cbuf, 0x03, 0x00, dstenc);
2249     $$$emit32$src$$constant;
2250   %}
2251 
2252   enc_class load_immP31(rRegP dst, immP32 src)
2253   %{
2254     // same as load_immI, but this time we care about zeroes in the high word
2255     int dstenc = $dst$$reg;
2256     if (dstenc >= 8) {
2257       emit_opcode(cbuf, Assembler::REX_B);
2258       dstenc -= 8;
2259     }
2260     emit_opcode(cbuf, 0xB8 | dstenc);
2261     $$$emit32$src$$constant;
2262   %}
2263 
2264   enc_class load_immP(rRegP dst, immP src)
2265   %{
2266     int dstenc = $dst$$reg;
2267     if (dstenc < 8) {
2268       emit_opcode(cbuf, Assembler::REX_W);
2269     } else {
2270       emit_opcode(cbuf, Assembler::REX_WB);
2271       dstenc -= 8;
2272     }
2273     emit_opcode(cbuf, 0xB8 | dstenc);
2274     // This next line should be generated from ADLC
2275     if ($src->constant_reloc() != relocInfo::none) {
2276       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2277     } else {
2278       emit_d64(cbuf, $src$$constant);
2279     }
2280   %}
2281 
2282   enc_class Con32(immI src)
2283   %{
2284     // Output immediate
2285     $$$emit32$src$$constant;
2286   %}
2287 
2288   enc_class Con32F_as_bits(immF src)
2289   %{
2290     // Output Float immediate bits
2291     jfloat jf = $src$$constant;
2292     jint jf_as_bits = jint_cast(jf);
2293     emit_d32(cbuf, jf_as_bits);
2294   %}
2295 
2296   enc_class Con16(immI src)
2297   %{
2298     // Output immediate
2299     $$$emit16$src$$constant;
2300   %}
2301 
2302   // How is this different from Con32??? XXX
2303   enc_class Con_d32(immI src)
2304   %{
2305     emit_d32(cbuf,$src$$constant);
2306   %}
2307 
2308   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2309     // Output immediate memory reference
2310     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2311     emit_d32(cbuf, 0x00);
2312   %}
2313 
2314   enc_class lock_prefix()
2315   %{
2316     if (os::is_MP()) {
2317       emit_opcode(cbuf, 0xF0); // lock
2318     }
2319   %}
2320 
2321   enc_class REX_mem(memory mem)
2322   %{
2323     if ($mem$$base >= 8) {
2324       if ($mem$$index < 8) {
2325         emit_opcode(cbuf, Assembler::REX_B);
2326       } else {
2327         emit_opcode(cbuf, Assembler::REX_XB);
2328       }
2329     } else {
2330       if ($mem$$index >= 8) {
2331         emit_opcode(cbuf, Assembler::REX_X);
2332       }
2333     }
2334   %}
2335 
2336   enc_class REX_mem_wide(memory mem)
2337   %{
2338     if ($mem$$base >= 8) {
2339       if ($mem$$index < 8) {
2340         emit_opcode(cbuf, Assembler::REX_WB);
2341       } else {
2342         emit_opcode(cbuf, Assembler::REX_WXB);
2343       }
2344     } else {
2345       if ($mem$$index < 8) {
2346         emit_opcode(cbuf, Assembler::REX_W);
2347       } else {
2348         emit_opcode(cbuf, Assembler::REX_WX);
2349       }
2350     }
2351   %}
2352 
2353   // for byte regs
2354   enc_class REX_breg(rRegI reg)
2355   %{
2356     if ($reg$$reg >= 4) {
2357       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2358     }
2359   %}
2360 
2361   // for byte regs
2362   enc_class REX_reg_breg(rRegI dst, rRegI src)
2363   %{
2364     if ($dst$$reg < 8) {
2365       if ($src$$reg >= 4) {
2366         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2367       }
2368     } else {
2369       if ($src$$reg < 8) {
2370         emit_opcode(cbuf, Assembler::REX_R);
2371       } else {
2372         emit_opcode(cbuf, Assembler::REX_RB);
2373       }
2374     }
2375   %}
2376 
2377   // for byte regs
2378   enc_class REX_breg_mem(rRegI reg, memory mem)
2379   %{
2380     if ($reg$$reg < 8) {
2381       if ($mem$$base < 8) {
2382         if ($mem$$index >= 8) {
2383           emit_opcode(cbuf, Assembler::REX_X);
2384         } else if ($reg$$reg >= 4) {
2385           emit_opcode(cbuf, Assembler::REX);
2386         }
2387       } else {
2388         if ($mem$$index < 8) {
2389           emit_opcode(cbuf, Assembler::REX_B);
2390         } else {
2391           emit_opcode(cbuf, Assembler::REX_XB);
2392         }
2393       }
2394     } else {
2395       if ($mem$$base < 8) {
2396         if ($mem$$index < 8) {
2397           emit_opcode(cbuf, Assembler::REX_R);
2398         } else {
2399           emit_opcode(cbuf, Assembler::REX_RX);
2400         }
2401       } else {
2402         if ($mem$$index < 8) {
2403           emit_opcode(cbuf, Assembler::REX_RB);
2404         } else {
2405           emit_opcode(cbuf, Assembler::REX_RXB);
2406         }
2407       }
2408     }
2409   %}
2410 
2411   enc_class REX_reg(rRegI reg)
2412   %{
2413     if ($reg$$reg >= 8) {
2414       emit_opcode(cbuf, Assembler::REX_B);
2415     }
2416   %}
2417 
2418   enc_class REX_reg_wide(rRegI reg)
2419   %{
2420     if ($reg$$reg < 8) {
2421       emit_opcode(cbuf, Assembler::REX_W);
2422     } else {
2423       emit_opcode(cbuf, Assembler::REX_WB);
2424     }
2425   %}
2426 
2427   enc_class REX_reg_reg(rRegI dst, rRegI src)
2428   %{
2429     if ($dst$$reg < 8) {
2430       if ($src$$reg >= 8) {
2431         emit_opcode(cbuf, Assembler::REX_B);
2432       }
2433     } else {
2434       if ($src$$reg < 8) {
2435         emit_opcode(cbuf, Assembler::REX_R);
2436       } else {
2437         emit_opcode(cbuf, Assembler::REX_RB);
2438       }
2439     }
2440   %}
2441 
2442   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2443   %{
2444     if ($dst$$reg < 8) {
2445       if ($src$$reg < 8) {
2446         emit_opcode(cbuf, Assembler::REX_W);
2447       } else {
2448         emit_opcode(cbuf, Assembler::REX_WB);
2449       }
2450     } else {
2451       if ($src$$reg < 8) {
2452         emit_opcode(cbuf, Assembler::REX_WR);
2453       } else {
2454         emit_opcode(cbuf, Assembler::REX_WRB);
2455       }
2456     }
2457   %}
2458 
2459   enc_class REX_reg_mem(rRegI reg, memory mem)
2460   %{
2461     if ($reg$$reg < 8) {
2462       if ($mem$$base < 8) {
2463         if ($mem$$index >= 8) {
2464           emit_opcode(cbuf, Assembler::REX_X);
2465         }
2466       } else {
2467         if ($mem$$index < 8) {
2468           emit_opcode(cbuf, Assembler::REX_B);
2469         } else {
2470           emit_opcode(cbuf, Assembler::REX_XB);
2471         }
2472       }
2473     } else {
2474       if ($mem$$base < 8) {
2475         if ($mem$$index < 8) {
2476           emit_opcode(cbuf, Assembler::REX_R);
2477         } else {
2478           emit_opcode(cbuf, Assembler::REX_RX);
2479         }
2480       } else {
2481         if ($mem$$index < 8) {
2482           emit_opcode(cbuf, Assembler::REX_RB);
2483         } else {
2484           emit_opcode(cbuf, Assembler::REX_RXB);
2485         }
2486       }
2487     }
2488   %}
2489 
2490   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2491   %{
2492     if ($reg$$reg < 8) {
2493       if ($mem$$base < 8) {
2494         if ($mem$$index < 8) {
2495           emit_opcode(cbuf, Assembler::REX_W);
2496         } else {
2497           emit_opcode(cbuf, Assembler::REX_WX);
2498         }
2499       } else {
2500         if ($mem$$index < 8) {
2501           emit_opcode(cbuf, Assembler::REX_WB);
2502         } else {
2503           emit_opcode(cbuf, Assembler::REX_WXB);
2504         }
2505       }
2506     } else {
2507       if ($mem$$base < 8) {
2508         if ($mem$$index < 8) {
2509           emit_opcode(cbuf, Assembler::REX_WR);
2510         } else {
2511           emit_opcode(cbuf, Assembler::REX_WRX);
2512         }
2513       } else {
2514         if ($mem$$index < 8) {
2515           emit_opcode(cbuf, Assembler::REX_WRB);
2516         } else {
2517           emit_opcode(cbuf, Assembler::REX_WRXB);
2518         }
2519       }
2520     }
2521   %}
2522 
2523   enc_class reg_mem(rRegI ereg, memory mem)
2524   %{
2525     // High registers handle in encode_RegMem
2526     int reg = $ereg$$reg;
2527     int base = $mem$$base;
2528     int index = $mem$$index;
2529     int scale = $mem$$scale;
2530     int disp = $mem$$disp;
2531     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2532 
2533     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2534   %}
2535 
2536   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2537   %{
2538     int rm_byte_opcode = $rm_opcode$$constant;
2539 
2540     // High registers handle in encode_RegMem
2541     int base = $mem$$base;
2542     int index = $mem$$index;
2543     int scale = $mem$$scale;
2544     int displace = $mem$$disp;
2545 
2546     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2547                                             // working with static
2548                                             // globals
2549     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2550                   disp_reloc);
2551   %}
2552 
2553   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2554   %{
2555     int reg_encoding = $dst$$reg;
2556     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2557     int index        = 0x04;            // 0x04 indicates no index
2558     int scale        = 0x00;            // 0x00 indicates no scale
2559     int displace     = $src1$$constant; // 0x00 indicates no displacement
2560     relocInfo::relocType disp_reloc = relocInfo::none;
2561     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2562                   disp_reloc);
2563   %}
2564 
2565   enc_class neg_reg(rRegI dst)
2566   %{
2567     int dstenc = $dst$$reg;
2568     if (dstenc >= 8) {
2569       emit_opcode(cbuf, Assembler::REX_B);
2570       dstenc -= 8;
2571     }
2572     // NEG $dst
2573     emit_opcode(cbuf, 0xF7);
2574     emit_rm(cbuf, 0x3, 0x03, dstenc);
2575   %}
2576 
2577   enc_class neg_reg_wide(rRegI dst)
2578   %{
2579     int dstenc = $dst$$reg;
2580     if (dstenc < 8) {
2581       emit_opcode(cbuf, Assembler::REX_W);
2582     } else {
2583       emit_opcode(cbuf, Assembler::REX_WB);
2584       dstenc -= 8;
2585     }
2586     // NEG $dst
2587     emit_opcode(cbuf, 0xF7);
2588     emit_rm(cbuf, 0x3, 0x03, dstenc);
2589   %}
2590 
2591   enc_class setLT_reg(rRegI dst)
2592   %{
2593     int dstenc = $dst$$reg;
2594     if (dstenc >= 8) {
2595       emit_opcode(cbuf, Assembler::REX_B);
2596       dstenc -= 8;
2597     } else if (dstenc >= 4) {
2598       emit_opcode(cbuf, Assembler::REX);
2599     }
2600     // SETLT $dst
2601     emit_opcode(cbuf, 0x0F);
2602     emit_opcode(cbuf, 0x9C);
2603     emit_rm(cbuf, 0x3, 0x0, dstenc);
2604   %}
2605 
2606   enc_class setNZ_reg(rRegI dst)
2607   %{
2608     int dstenc = $dst$$reg;
2609     if (dstenc >= 8) {
2610       emit_opcode(cbuf, Assembler::REX_B);
2611       dstenc -= 8;
2612     } else if (dstenc >= 4) {
2613       emit_opcode(cbuf, Assembler::REX);
2614     }
2615     // SETNZ $dst
2616     emit_opcode(cbuf, 0x0F);
2617     emit_opcode(cbuf, 0x95);
2618     emit_rm(cbuf, 0x3, 0x0, dstenc);
2619   %}
2620 
2621 
2622   // Compare the lonogs and set -1, 0, or 1 into dst
2623   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2624   %{
2625     int src1enc = $src1$$reg;
2626     int src2enc = $src2$$reg;
2627     int dstenc = $dst$$reg;
2628 
2629     // cmpq $src1, $src2
2630     if (src1enc < 8) {
2631       if (src2enc < 8) {
2632         emit_opcode(cbuf, Assembler::REX_W);
2633       } else {
2634         emit_opcode(cbuf, Assembler::REX_WB);
2635       }
2636     } else {
2637       if (src2enc < 8) {
2638         emit_opcode(cbuf, Assembler::REX_WR);
2639       } else {
2640         emit_opcode(cbuf, Assembler::REX_WRB);
2641       }
2642     }
2643     emit_opcode(cbuf, 0x3B);
2644     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2645 
2646     // movl $dst, -1
2647     if (dstenc >= 8) {
2648       emit_opcode(cbuf, Assembler::REX_B);
2649     }
2650     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2651     emit_d32(cbuf, -1);
2652 
2653     // jl,s done
2654     emit_opcode(cbuf, 0x7C);
2655     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2656 
2657     // setne $dst
2658     if (dstenc >= 4) {
2659       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2660     }
2661     emit_opcode(cbuf, 0x0F);
2662     emit_opcode(cbuf, 0x95);
2663     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2664 
2665     // movzbl $dst, $dst
2666     if (dstenc >= 4) {
2667       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2668     }
2669     emit_opcode(cbuf, 0x0F);
2670     emit_opcode(cbuf, 0xB6);
2671     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2672   %}
2673 
2674   enc_class Push_ResultXD(regD dst) %{
2675     MacroAssembler _masm(&cbuf);
2676     __ fstp_d(Address(rsp, 0));
2677     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2678     __ addptr(rsp, 8);
2679   %}
2680 
2681   enc_class Push_SrcXD(regD src) %{
2682     MacroAssembler _masm(&cbuf);
2683     __ subptr(rsp, 8);
2684     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2685     __ fld_d(Address(rsp, 0));
2686   %}
2687 
2688 
2689   enc_class enc_rethrow()
2690   %{
2691     cbuf.set_insts_mark();
2692     emit_opcode(cbuf, 0xE9); // jmp entry
2693     emit_d32_reloc(cbuf,
2694                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2695                    runtime_call_Relocation::spec(),
2696                    RELOC_DISP32);
2697   %}
2698 
2699 %}
2700 
2701 
2702 
2703 //----------FRAME--------------------------------------------------------------
2704 // Definition of frame structure and management information.
2705 //
2706 //  S T A C K   L A Y O U T    Allocators stack-slot number
2707 //                             |   (to get allocators register number
2708 //  G  Owned by    |        |  v    add OptoReg::stack0())
2709 //  r   CALLER     |        |
2710 //  o     |        +--------+      pad to even-align allocators stack-slot
2711 //  w     V        |  pad0  |        numbers; owned by CALLER
2712 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2713 //  h     ^        |   in   |  5
2714 //        |        |  args  |  4   Holes in incoming args owned by SELF
2715 //  |     |        |        |  3
2716 //  |     |        +--------+
2717 //  V     |        | old out|      Empty on Intel, window on Sparc
2718 //        |    old |preserve|      Must be even aligned.
2719 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2720 //        |        |   in   |  3   area for Intel ret address
2721 //     Owned by    |preserve|      Empty on Sparc.
2722 //       SELF      +--------+
2723 //        |        |  pad2  |  2   pad to align old SP
2724 //        |        +--------+  1
2725 //        |        | locks  |  0
2726 //        |        +--------+----> OptoReg::stack0(), even aligned
2727 //        |        |  pad1  | 11   pad to align new SP
2728 //        |        +--------+
2729 //        |        |        | 10
2730 //        |        | spills |  9   spills
2731 //        V        |        |  8   (pad0 slot for callee)
2732 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2733 //        ^        |  out   |  7
2734 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2735 //     Owned by    +--------+
2736 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2737 //        |    new |preserve|      Must be even-aligned.
2738 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2739 //        |        |        |
2740 //
2741 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2742 //         known from SELF's arguments and the Java calling convention.
2743 //         Region 6-7 is determined per call site.
2744 // Note 2: If the calling convention leaves holes in the incoming argument
2745 //         area, those holes are owned by SELF.  Holes in the outgoing area
2746 //         are owned by the CALLEE.  Holes should not be nessecary in the
2747 //         incoming area, as the Java calling convention is completely under
2748 //         the control of the AD file.  Doubles can be sorted and packed to
2749 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2750 //         varargs C calling conventions.
2751 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2752 //         even aligned with pad0 as needed.
2753 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2754 //         region 6-11 is even aligned; it may be padded out more so that
2755 //         the region from SP to FP meets the minimum stack alignment.
2756 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2757 //         alignment.  Region 11, pad1, may be dynamically extended so that
2758 //         SP meets the minimum alignment.
2759 
2760 frame
2761 %{
2762   // What direction does stack grow in (assumed to be same for C & Java)
2763   stack_direction(TOWARDS_LOW);
2764 
2765   // These three registers define part of the calling convention
2766   // between compiled code and the interpreter.
2767   inline_cache_reg(RAX);                // Inline Cache Register
2768   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2769                                         // calling interpreter
2770 
2771   // Optional: name the operand used by cisc-spilling to access
2772   // [stack_pointer + offset]
2773   cisc_spilling_operand_name(indOffset32);
2774 
2775   // Number of stack slots consumed by locking an object
2776   sync_stack_slots(2);
2777 
2778   // Compiled code's Frame Pointer
2779   frame_pointer(RSP);
2780 
2781   // Interpreter stores its frame pointer in a register which is
2782   // stored to the stack by I2CAdaptors.
2783   // I2CAdaptors convert from interpreted java to compiled java.
2784   interpreter_frame_pointer(RBP);
2785 
2786   // Stack alignment requirement
2787   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2788 
2789   // Number of stack slots between incoming argument block and the start of
2790   // a new frame.  The PROLOG must add this many slots to the stack.  The
2791   // EPILOG must remove this many slots.  amd64 needs two slots for
2792   // return address.
2793   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2794 
2795   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2796   // for calls to C.  Supports the var-args backing area for register parms.
2797   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2798 
2799   // The after-PROLOG location of the return address.  Location of
2800   // return address specifies a type (REG or STACK) and a number
2801   // representing the register number (i.e. - use a register name) or
2802   // stack slot.
2803   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2804   // Otherwise, it is above the locks and verification slot and alignment word
2805   return_addr(STACK - 2 +
2806               round_to((Compile::current()->in_preserve_stack_slots() +
2807                         Compile::current()->fixed_slots()),
2808                        stack_alignment_in_slots()));
2809 
2810   // Body of function which returns an integer array locating
2811   // arguments either in registers or in stack slots.  Passed an array
2812   // of ideal registers called "sig" and a "length" count.  Stack-slot
2813   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2814   // arguments for a CALLEE.  Incoming stack arguments are
2815   // automatically biased by the preserve_stack_slots field above.
2816 
2817   calling_convention
2818   %{
2819     // No difference between ingoing/outgoing just pass false
2820     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2821   %}
2822 
2823   c_calling_convention
2824   %{
2825     // This is obviously always outgoing
2826     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2827   %}
2828 
2829   // Location of compiled Java return values.  Same as C for now.
2830   return_value
2831   %{
2832     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2833            "only return normal values");
2834 
2835     static const int lo[Op_RegL + 1] = {
2836       0,
2837       0,
2838       RAX_num,  // Op_RegN
2839       RAX_num,  // Op_RegI
2840       RAX_num,  // Op_RegP
2841       XMM0_num, // Op_RegF
2842       XMM0_num, // Op_RegD
2843       RAX_num   // Op_RegL
2844     };
2845     static const int hi[Op_RegL + 1] = {
2846       0,
2847       0,
2848       OptoReg::Bad, // Op_RegN
2849       OptoReg::Bad, // Op_RegI
2850       RAX_H_num,    // Op_RegP
2851       OptoReg::Bad, // Op_RegF
2852       XMM0b_num,    // Op_RegD
2853       RAX_H_num     // Op_RegL
2854     };
2855     // Excluded flags and vector registers.
2856     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2857     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2858   %}
2859 %}
2860 
2861 //----------ATTRIBUTES---------------------------------------------------------
2862 //----------Operand Attributes-------------------------------------------------
2863 op_attrib op_cost(0);        // Required cost attribute
2864 
2865 //----------Instruction Attributes---------------------------------------------
2866 ins_attrib ins_cost(100);       // Required cost attribute
2867 ins_attrib ins_size(8);         // Required size attribute (in bits)
2868 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2869                                 // a non-matching short branch variant
2870                                 // of some long branch?
2871 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2872                                 // be a power of 2) specifies the
2873                                 // alignment that some part of the
2874                                 // instruction (not necessarily the
2875                                 // start) requires.  If > 1, a
2876                                 // compute_padding() function must be
2877                                 // provided for the instruction
2878 
2879 //----------OPERANDS-----------------------------------------------------------
2880 // Operand definitions must precede instruction definitions for correct parsing
2881 // in the ADLC because operands constitute user defined types which are used in
2882 // instruction definitions.
2883 
2884 //----------Simple Operands----------------------------------------------------
2885 // Immediate Operands
2886 // Integer Immediate
2887 operand immI()
2888 %{
2889   match(ConI);
2890 
2891   op_cost(10);
2892   format %{ %}
2893   interface(CONST_INTER);
2894 %}
2895 
2896 // Constant for test vs zero
2897 operand immI0()
2898 %{
2899   predicate(n->get_int() == 0);
2900   match(ConI);
2901 
2902   op_cost(0);
2903   format %{ %}
2904   interface(CONST_INTER);
2905 %}
2906 
2907 // Constant for increment
2908 operand immI1()
2909 %{
2910   predicate(n->get_int() == 1);
2911   match(ConI);
2912 
2913   op_cost(0);
2914   format %{ %}
2915   interface(CONST_INTER);
2916 %}
2917 
2918 // Constant for decrement
2919 operand immI_M1()
2920 %{
2921   predicate(n->get_int() == -1);
2922   match(ConI);
2923 
2924   op_cost(0);
2925   format %{ %}
2926   interface(CONST_INTER);
2927 %}
2928 
2929 // Valid scale values for addressing modes
2930 operand immI2()
2931 %{
2932   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2933   match(ConI);
2934 
2935   format %{ %}
2936   interface(CONST_INTER);
2937 %}
2938 
2939 operand immI8()
2940 %{
2941   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2942   match(ConI);
2943 
2944   op_cost(5);
2945   format %{ %}
2946   interface(CONST_INTER);
2947 %}
2948 
2949 operand immI16()
2950 %{
2951   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2952   match(ConI);
2953 
2954   op_cost(10);
2955   format %{ %}
2956   interface(CONST_INTER);
2957 %}
2958 
2959 // Int Immediate non-negative
2960 operand immU31()
2961 %{
2962   predicate(n->get_int() >= 0);
2963   match(ConI);
2964 
2965   op_cost(0);
2966   format %{ %}
2967   interface(CONST_INTER);
2968 %}
2969 
2970 // Constant for long shifts
2971 operand immI_32()
2972 %{
2973   predicate( n->get_int() == 32 );
2974   match(ConI);
2975 
2976   op_cost(0);
2977   format %{ %}
2978   interface(CONST_INTER);
2979 %}
2980 
2981 // Constant for long shifts
2982 operand immI_64()
2983 %{
2984   predicate( n->get_int() == 64 );
2985   match(ConI);
2986 
2987   op_cost(0);
2988   format %{ %}
2989   interface(CONST_INTER);
2990 %}
2991 
2992 // Pointer Immediate
2993 operand immP()
2994 %{
2995   match(ConP);
2996 
2997   op_cost(10);
2998   format %{ %}
2999   interface(CONST_INTER);
3000 %}
3001 
3002 // NULL Pointer Immediate
3003 operand immP0()
3004 %{
3005   predicate(n->get_ptr() == 0);
3006   match(ConP);
3007 
3008   op_cost(5);
3009   format %{ %}
3010   interface(CONST_INTER);
3011 %}
3012 
3013 // Pointer Immediate
3014 operand immN() %{
3015   match(ConN);
3016 
3017   op_cost(10);
3018   format %{ %}
3019   interface(CONST_INTER);
3020 %}
3021 
3022 operand immNKlass() %{
3023   match(ConNKlass);
3024 
3025   op_cost(10);
3026   format %{ %}
3027   interface(CONST_INTER);
3028 %}
3029 
3030 // NULL Pointer Immediate
3031 operand immN0() %{
3032   predicate(n->get_narrowcon() == 0);
3033   match(ConN);
3034 
3035   op_cost(5);
3036   format %{ %}
3037   interface(CONST_INTER);
3038 %}
3039 
3040 operand immP31()
3041 %{
3042   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3043             && (n->get_ptr() >> 31) == 0);
3044   match(ConP);
3045 
3046   op_cost(5);
3047   format %{ %}
3048   interface(CONST_INTER);
3049 %}
3050 
3051 
3052 // Long Immediate
3053 operand immL()
3054 %{
3055   match(ConL);
3056 
3057   op_cost(20);
3058   format %{ %}
3059   interface(CONST_INTER);
3060 %}
3061 
3062 // Long Immediate 8-bit
3063 operand immL8()
3064 %{
3065   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3066   match(ConL);
3067 
3068   op_cost(5);
3069   format %{ %}
3070   interface(CONST_INTER);
3071 %}
3072 
3073 // Long Immediate 32-bit unsigned
3074 operand immUL32()
3075 %{
3076   predicate(n->get_long() == (unsigned int) (n->get_long()));
3077   match(ConL);
3078 
3079   op_cost(10);
3080   format %{ %}
3081   interface(CONST_INTER);
3082 %}
3083 
3084 // Long Immediate 32-bit signed
3085 operand immL32()
3086 %{
3087   predicate(n->get_long() == (int) (n->get_long()));
3088   match(ConL);
3089 
3090   op_cost(15);
3091   format %{ %}
3092   interface(CONST_INTER);
3093 %}
3094 
3095 // Long Immediate zero
3096 operand immL0()
3097 %{
3098   predicate(n->get_long() == 0L);
3099   match(ConL);
3100 
3101   op_cost(10);
3102   format %{ %}
3103   interface(CONST_INTER);
3104 %}
3105 
3106 // Constant for increment
3107 operand immL1()
3108 %{
3109   predicate(n->get_long() == 1);
3110   match(ConL);
3111 
3112   format %{ %}
3113   interface(CONST_INTER);
3114 %}
3115 
3116 // Constant for decrement
3117 operand immL_M1()
3118 %{
3119   predicate(n->get_long() == -1);
3120   match(ConL);
3121 
3122   format %{ %}
3123   interface(CONST_INTER);
3124 %}
3125 
3126 // Long Immediate: the value 10
3127 operand immL10()
3128 %{
3129   predicate(n->get_long() == 10);
3130   match(ConL);
3131 
3132   format %{ %}
3133   interface(CONST_INTER);
3134 %}
3135 
3136 // Long immediate from 0 to 127.
3137 // Used for a shorter form of long mul by 10.
3138 operand immL_127()
3139 %{
3140   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3141   match(ConL);
3142 
3143   op_cost(10);
3144   format %{ %}
3145   interface(CONST_INTER);
3146 %}
3147 
3148 // Long Immediate: low 32-bit mask
3149 operand immL_32bits()
3150 %{
3151   predicate(n->get_long() == 0xFFFFFFFFL);
3152   match(ConL);
3153   op_cost(20);
3154 
3155   format %{ %}
3156   interface(CONST_INTER);
3157 %}
3158 
3159 // Float Immediate zero
3160 operand immF0()
3161 %{
3162   predicate(jint_cast(n->getf()) == 0);
3163   match(ConF);
3164 
3165   op_cost(5);
3166   format %{ %}
3167   interface(CONST_INTER);
3168 %}
3169 
3170 // Float Immediate
3171 operand immF()
3172 %{
3173   match(ConF);
3174 
3175   op_cost(15);
3176   format %{ %}
3177   interface(CONST_INTER);
3178 %}
3179 
3180 // Double Immediate zero
3181 operand immD0()
3182 %{
3183   predicate(jlong_cast(n->getd()) == 0);
3184   match(ConD);
3185 
3186   op_cost(5);
3187   format %{ %}
3188   interface(CONST_INTER);
3189 %}
3190 
3191 // Double Immediate
3192 operand immD()
3193 %{
3194   match(ConD);
3195 
3196   op_cost(15);
3197   format %{ %}
3198   interface(CONST_INTER);
3199 %}
3200 
3201 // Immediates for special shifts (sign extend)
3202 
3203 // Constants for increment
3204 operand immI_16()
3205 %{
3206   predicate(n->get_int() == 16);
3207   match(ConI);
3208 
3209   format %{ %}
3210   interface(CONST_INTER);
3211 %}
3212 
3213 operand immI_24()
3214 %{
3215   predicate(n->get_int() == 24);
3216   match(ConI);
3217 
3218   format %{ %}
3219   interface(CONST_INTER);
3220 %}
3221 
3222 // Constant for byte-wide masking
3223 operand immI_255()
3224 %{
3225   predicate(n->get_int() == 255);
3226   match(ConI);
3227 
3228   format %{ %}
3229   interface(CONST_INTER);
3230 %}
3231 
3232 // Constant for short-wide masking
3233 operand immI_65535()
3234 %{
3235   predicate(n->get_int() == 65535);
3236   match(ConI);
3237 
3238   format %{ %}
3239   interface(CONST_INTER);
3240 %}
3241 
3242 // Constant for byte-wide masking
3243 operand immL_255()
3244 %{
3245   predicate(n->get_long() == 255);
3246   match(ConL);
3247 
3248   format %{ %}
3249   interface(CONST_INTER);
3250 %}
3251 
3252 // Constant for short-wide masking
3253 operand immL_65535()
3254 %{
3255   predicate(n->get_long() == 65535);
3256   match(ConL);
3257 
3258   format %{ %}
3259   interface(CONST_INTER);
3260 %}
3261 
3262 // Register Operands
3263 // Integer Register
3264 operand rRegI()
3265 %{
3266   constraint(ALLOC_IN_RC(int_reg));
3267   match(RegI);
3268 
3269   match(rax_RegI);
3270   match(rbx_RegI);
3271   match(rcx_RegI);
3272   match(rdx_RegI);
3273   match(rdi_RegI);
3274 
3275   format %{ %}
3276   interface(REG_INTER);
3277 %}
3278 
3279 // Special Registers
3280 operand rax_RegI()
3281 %{
3282   constraint(ALLOC_IN_RC(int_rax_reg));
3283   match(RegI);
3284   match(rRegI);
3285 
3286   format %{ "RAX" %}
3287   interface(REG_INTER);
3288 %}
3289 
3290 // Special Registers
3291 operand rbx_RegI()
3292 %{
3293   constraint(ALLOC_IN_RC(int_rbx_reg));
3294   match(RegI);
3295   match(rRegI);
3296 
3297   format %{ "RBX" %}
3298   interface(REG_INTER);
3299 %}
3300 
3301 operand rcx_RegI()
3302 %{
3303   constraint(ALLOC_IN_RC(int_rcx_reg));
3304   match(RegI);
3305   match(rRegI);
3306 
3307   format %{ "RCX" %}
3308   interface(REG_INTER);
3309 %}
3310 
3311 operand rdx_RegI()
3312 %{
3313   constraint(ALLOC_IN_RC(int_rdx_reg));
3314   match(RegI);
3315   match(rRegI);
3316 
3317   format %{ "RDX" %}
3318   interface(REG_INTER);
3319 %}
3320 
3321 operand rdi_RegI()
3322 %{
3323   constraint(ALLOC_IN_RC(int_rdi_reg));
3324   match(RegI);
3325   match(rRegI);
3326 
3327   format %{ "RDI" %}
3328   interface(REG_INTER);
3329 %}
3330 
3331 operand no_rcx_RegI()
3332 %{
3333   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3334   match(RegI);
3335   match(rax_RegI);
3336   match(rbx_RegI);
3337   match(rdx_RegI);
3338   match(rdi_RegI);
3339 
3340   format %{ %}
3341   interface(REG_INTER);
3342 %}
3343 
3344 operand no_rax_rdx_RegI()
3345 %{
3346   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3347   match(RegI);
3348   match(rbx_RegI);
3349   match(rcx_RegI);
3350   match(rdi_RegI);
3351 
3352   format %{ %}
3353   interface(REG_INTER);
3354 %}
3355 
3356 // Pointer Register
3357 operand any_RegP()
3358 %{
3359   constraint(ALLOC_IN_RC(any_reg));
3360   match(RegP);
3361   match(rax_RegP);
3362   match(rbx_RegP);
3363   match(rdi_RegP);
3364   match(rsi_RegP);
3365   match(rbp_RegP);
3366   match(r15_RegP);
3367   match(rRegP);
3368 
3369   format %{ %}
3370   interface(REG_INTER);
3371 %}
3372 
3373 operand rRegP()
3374 %{
3375   constraint(ALLOC_IN_RC(ptr_reg));
3376   match(RegP);
3377   match(rax_RegP);
3378   match(rbx_RegP);
3379   match(rdi_RegP);
3380   match(rsi_RegP);
3381   match(rbp_RegP);  // See Q&A below about
3382   match(r15_RegP);  // r15_RegP and rbp_RegP.
3383 
3384   format %{ %}
3385   interface(REG_INTER);
3386 %}
3387 
3388 operand rRegN() %{
3389   constraint(ALLOC_IN_RC(int_reg));
3390   match(RegN);
3391 
3392   format %{ %}
3393   interface(REG_INTER);
3394 %}
3395 
3396 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3397 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3398 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3399 // The output of an instruction is controlled by the allocator, which respects
3400 // register class masks, not match rules.  Unless an instruction mentions
3401 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3402 // by the allocator as an input.
3403 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3404 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3405 // result, RBP is not included in the output of the instruction either.
3406 
3407 operand no_rax_RegP()
3408 %{
3409   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3410   match(RegP);
3411   match(rbx_RegP);
3412   match(rsi_RegP);
3413   match(rdi_RegP);
3414 
3415   format %{ %}
3416   interface(REG_INTER);
3417 %}
3418 
3419 // This operand is not allowed to use RBP even if
3420 // RBP is not used to hold the frame pointer.
3421 operand no_rbp_RegP()
3422 %{
3423   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3424   match(RegP);
3425   match(rbx_RegP);
3426   match(rsi_RegP);
3427   match(rdi_RegP);
3428 
3429   format %{ %}
3430   interface(REG_INTER);
3431 %}
3432 
3433 operand no_rax_rbx_RegP()
3434 %{
3435   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3436   match(RegP);
3437   match(rsi_RegP);
3438   match(rdi_RegP);
3439 
3440   format %{ %}
3441   interface(REG_INTER);
3442 %}
3443 
3444 // Special Registers
3445 // Return a pointer value
3446 operand rax_RegP()
3447 %{
3448   constraint(ALLOC_IN_RC(ptr_rax_reg));
3449   match(RegP);
3450   match(rRegP);
3451 
3452   format %{ %}
3453   interface(REG_INTER);
3454 %}
3455 
3456 // Special Registers
3457 // Return a compressed pointer value
3458 operand rax_RegN()
3459 %{
3460   constraint(ALLOC_IN_RC(int_rax_reg));
3461   match(RegN);
3462   match(rRegN);
3463 
3464   format %{ %}
3465   interface(REG_INTER);
3466 %}
3467 
3468 // Used in AtomicAdd
3469 operand rbx_RegP()
3470 %{
3471   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3472   match(RegP);
3473   match(rRegP);
3474 
3475   format %{ %}
3476   interface(REG_INTER);
3477 %}
3478 
3479 operand rsi_RegP()
3480 %{
3481   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3482   match(RegP);
3483   match(rRegP);
3484 
3485   format %{ %}
3486   interface(REG_INTER);
3487 %}
3488 
3489 // Used in rep stosq
3490 operand rdi_RegP()
3491 %{
3492   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3493   match(RegP);
3494   match(rRegP);
3495 
3496   format %{ %}
3497   interface(REG_INTER);
3498 %}
3499 
3500 operand r15_RegP()
3501 %{
3502   constraint(ALLOC_IN_RC(ptr_r15_reg));
3503   match(RegP);
3504   match(rRegP);
3505 
3506   format %{ %}
3507   interface(REG_INTER);
3508 %}
3509 
3510 operand rRegL()
3511 %{
3512   constraint(ALLOC_IN_RC(long_reg));
3513   match(RegL);
3514   match(rax_RegL);
3515   match(rdx_RegL);
3516 
3517   format %{ %}
3518   interface(REG_INTER);
3519 %}
3520 
3521 // Special Registers
3522 operand no_rax_rdx_RegL()
3523 %{
3524   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3525   match(RegL);
3526   match(rRegL);
3527 
3528   format %{ %}
3529   interface(REG_INTER);
3530 %}
3531 
3532 operand no_rax_RegL()
3533 %{
3534   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3535   match(RegL);
3536   match(rRegL);
3537   match(rdx_RegL);
3538 
3539   format %{ %}
3540   interface(REG_INTER);
3541 %}
3542 
3543 operand no_rcx_RegL()
3544 %{
3545   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3546   match(RegL);
3547   match(rRegL);
3548 
3549   format %{ %}
3550   interface(REG_INTER);
3551 %}
3552 
3553 operand rax_RegL()
3554 %{
3555   constraint(ALLOC_IN_RC(long_rax_reg));
3556   match(RegL);
3557   match(rRegL);
3558 
3559   format %{ "RAX" %}
3560   interface(REG_INTER);
3561 %}
3562 
3563 operand rcx_RegL()
3564 %{
3565   constraint(ALLOC_IN_RC(long_rcx_reg));
3566   match(RegL);
3567   match(rRegL);
3568 
3569   format %{ %}
3570   interface(REG_INTER);
3571 %}
3572 
3573 operand rdx_RegL()
3574 %{
3575   constraint(ALLOC_IN_RC(long_rdx_reg));
3576   match(RegL);
3577   match(rRegL);
3578 
3579   format %{ %}
3580   interface(REG_INTER);
3581 %}
3582 
3583 // Flags register, used as output of compare instructions
3584 operand rFlagsReg()
3585 %{
3586   constraint(ALLOC_IN_RC(int_flags));
3587   match(RegFlags);
3588 
3589   format %{ "RFLAGS" %}
3590   interface(REG_INTER);
3591 %}
3592 
3593 // Flags register, used as output of FLOATING POINT compare instructions
3594 operand rFlagsRegU()
3595 %{
3596   constraint(ALLOC_IN_RC(int_flags));
3597   match(RegFlags);
3598 
3599   format %{ "RFLAGS_U" %}
3600   interface(REG_INTER);
3601 %}
3602 
3603 operand rFlagsRegUCF() %{
3604   constraint(ALLOC_IN_RC(int_flags));
3605   match(RegFlags);
3606   predicate(false);
3607 
3608   format %{ "RFLAGS_U_CF" %}
3609   interface(REG_INTER);
3610 %}
3611 
3612 // Float register operands
3613 operand regF() %{
3614    constraint(ALLOC_IN_RC(float_reg));
3615    match(RegF);
3616 
3617    format %{ %}
3618    interface(REG_INTER);
3619 %}
3620 
3621 // Double register operands
3622 operand regD() %{
3623    constraint(ALLOC_IN_RC(double_reg));
3624    match(RegD);
3625 
3626    format %{ %}
3627    interface(REG_INTER);
3628 %}
3629 
3630 // Vectors
3631 operand vecS() %{
3632   constraint(ALLOC_IN_RC(vectors_reg));
3633   match(VecS);
3634 
3635   format %{ %}
3636   interface(REG_INTER);
3637 %}
3638 
3639 operand vecD() %{
3640   constraint(ALLOC_IN_RC(vectord_reg));
3641   match(VecD);
3642 
3643   format %{ %}
3644   interface(REG_INTER);
3645 %}
3646 
3647 operand vecX() %{
3648   constraint(ALLOC_IN_RC(vectorx_reg));
3649   match(VecX);
3650 
3651   format %{ %}
3652   interface(REG_INTER);
3653 %}
3654 
3655 operand vecY() %{
3656   constraint(ALLOC_IN_RC(vectory_reg));
3657   match(VecY);
3658 
3659   format %{ %}
3660   interface(REG_INTER);
3661 %}
3662 
3663 //----------Memory Operands----------------------------------------------------
3664 // Direct Memory Operand
3665 // operand direct(immP addr)
3666 // %{
3667 //   match(addr);
3668 
3669 //   format %{ "[$addr]" %}
3670 //   interface(MEMORY_INTER) %{
3671 //     base(0xFFFFFFFF);
3672 //     index(0x4);
3673 //     scale(0x0);
3674 //     disp($addr);
3675 //   %}
3676 // %}
3677 
3678 // Indirect Memory Operand
3679 operand indirect(any_RegP reg)
3680 %{
3681   constraint(ALLOC_IN_RC(ptr_reg));
3682   match(reg);
3683 
3684   format %{ "[$reg]" %}
3685   interface(MEMORY_INTER) %{
3686     base($reg);
3687     index(0x4);
3688     scale(0x0);
3689     disp(0x0);
3690   %}
3691 %}
3692 
3693 // Indirect Memory Plus Short Offset Operand
3694 operand indOffset8(any_RegP reg, immL8 off)
3695 %{
3696   constraint(ALLOC_IN_RC(ptr_reg));
3697   match(AddP reg off);
3698 
3699   format %{ "[$reg + $off (8-bit)]" %}
3700   interface(MEMORY_INTER) %{
3701     base($reg);
3702     index(0x4);
3703     scale(0x0);
3704     disp($off);
3705   %}
3706 %}
3707 
3708 // Indirect Memory Plus Long Offset Operand
3709 operand indOffset32(any_RegP reg, immL32 off)
3710 %{
3711   constraint(ALLOC_IN_RC(ptr_reg));
3712   match(AddP reg off);
3713 
3714   format %{ "[$reg + $off (32-bit)]" %}
3715   interface(MEMORY_INTER) %{
3716     base($reg);
3717     index(0x4);
3718     scale(0x0);
3719     disp($off);
3720   %}
3721 %}
3722 
3723 // Indirect Memory Plus Index Register Plus Offset Operand
3724 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3725 %{
3726   constraint(ALLOC_IN_RC(ptr_reg));
3727   match(AddP (AddP reg lreg) off);
3728 
3729   op_cost(10);
3730   format %{"[$reg + $off + $lreg]" %}
3731   interface(MEMORY_INTER) %{
3732     base($reg);
3733     index($lreg);
3734     scale(0x0);
3735     disp($off);
3736   %}
3737 %}
3738 
3739 // Indirect Memory Plus Index Register Plus Offset Operand
3740 operand indIndex(any_RegP reg, rRegL lreg)
3741 %{
3742   constraint(ALLOC_IN_RC(ptr_reg));
3743   match(AddP reg lreg);
3744 
3745   op_cost(10);
3746   format %{"[$reg + $lreg]" %}
3747   interface(MEMORY_INTER) %{
3748     base($reg);
3749     index($lreg);
3750     scale(0x0);
3751     disp(0x0);
3752   %}
3753 %}
3754 
3755 // Indirect Memory Times Scale Plus Index Register
3756 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3757 %{
3758   constraint(ALLOC_IN_RC(ptr_reg));
3759   match(AddP reg (LShiftL lreg scale));
3760 
3761   op_cost(10);
3762   format %{"[$reg + $lreg << $scale]" %}
3763   interface(MEMORY_INTER) %{
3764     base($reg);
3765     index($lreg);
3766     scale($scale);
3767     disp(0x0);
3768   %}
3769 %}
3770 
3771 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3772 %{
3773   constraint(ALLOC_IN_RC(ptr_reg));
3774   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3775   match(AddP reg (LShiftL (ConvI2L idx) scale));
3776 
3777   op_cost(10);
3778   format %{"[$reg + pos $idx << $scale]" %}
3779   interface(MEMORY_INTER) %{
3780     base($reg);
3781     index($idx);
3782     scale($scale);
3783     disp(0x0);
3784   %}
3785 %}
3786 
3787 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3788 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3789 %{
3790   constraint(ALLOC_IN_RC(ptr_reg));
3791   match(AddP (AddP reg (LShiftL lreg scale)) off);
3792 
3793   op_cost(10);
3794   format %{"[$reg + $off + $lreg << $scale]" %}
3795   interface(MEMORY_INTER) %{
3796     base($reg);
3797     index($lreg);
3798     scale($scale);
3799     disp($off);
3800   %}
3801 %}
3802 
3803 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3804 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3805 %{
3806   constraint(ALLOC_IN_RC(ptr_reg));
3807   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3808   match(AddP (AddP reg (ConvI2L idx)) off);
3809 
3810   op_cost(10);
3811   format %{"[$reg + $off + $idx]" %}
3812   interface(MEMORY_INTER) %{
3813     base($reg);
3814     index($idx);
3815     scale(0x0);
3816     disp($off);
3817   %}
3818 %}
3819 
3820 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3821 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3822 %{
3823   constraint(ALLOC_IN_RC(ptr_reg));
3824   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3825   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3826 
3827   op_cost(10);
3828   format %{"[$reg + $off + $idx << $scale]" %}
3829   interface(MEMORY_INTER) %{
3830     base($reg);
3831     index($idx);
3832     scale($scale);
3833     disp($off);
3834   %}
3835 %}
3836 
3837 // Indirect Narrow Oop Plus Offset Operand
3838 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3839 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3840 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3841   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3842   constraint(ALLOC_IN_RC(ptr_reg));
3843   match(AddP (DecodeN reg) off);
3844 
3845   op_cost(10);
3846   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3847   interface(MEMORY_INTER) %{
3848     base(0xc); // R12
3849     index($reg);
3850     scale(0x3);
3851     disp($off);
3852   %}
3853 %}
3854 
3855 // Indirect Memory Operand
3856 operand indirectNarrow(rRegN reg)
3857 %{
3858   predicate(Universe::narrow_oop_shift() == 0);
3859   constraint(ALLOC_IN_RC(ptr_reg));
3860   match(DecodeN reg);
3861 
3862   format %{ "[$reg]" %}
3863   interface(MEMORY_INTER) %{
3864     base($reg);
3865     index(0x4);
3866     scale(0x0);
3867     disp(0x0);
3868   %}
3869 %}
3870 
3871 // Indirect Memory Plus Short Offset Operand
3872 operand indOffset8Narrow(rRegN reg, immL8 off)
3873 %{
3874   predicate(Universe::narrow_oop_shift() == 0);
3875   constraint(ALLOC_IN_RC(ptr_reg));
3876   match(AddP (DecodeN reg) off);
3877 
3878   format %{ "[$reg + $off (8-bit)]" %}
3879   interface(MEMORY_INTER) %{
3880     base($reg);
3881     index(0x4);
3882     scale(0x0);
3883     disp($off);
3884   %}
3885 %}
3886 
3887 // Indirect Memory Plus Long Offset Operand
3888 operand indOffset32Narrow(rRegN reg, immL32 off)
3889 %{
3890   predicate(Universe::narrow_oop_shift() == 0);
3891   constraint(ALLOC_IN_RC(ptr_reg));
3892   match(AddP (DecodeN reg) off);
3893 
3894   format %{ "[$reg + $off (32-bit)]" %}
3895   interface(MEMORY_INTER) %{
3896     base($reg);
3897     index(0x4);
3898     scale(0x0);
3899     disp($off);
3900   %}
3901 %}
3902 
3903 // Indirect Memory Plus Index Register Plus Offset Operand
3904 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3905 %{
3906   predicate(Universe::narrow_oop_shift() == 0);
3907   constraint(ALLOC_IN_RC(ptr_reg));
3908   match(AddP (AddP (DecodeN reg) lreg) off);
3909 
3910   op_cost(10);
3911   format %{"[$reg + $off + $lreg]" %}
3912   interface(MEMORY_INTER) %{
3913     base($reg);
3914     index($lreg);
3915     scale(0x0);
3916     disp($off);
3917   %}
3918 %}
3919 
3920 // Indirect Memory Plus Index Register Plus Offset Operand
3921 operand indIndexNarrow(rRegN reg, rRegL lreg)
3922 %{
3923   predicate(Universe::narrow_oop_shift() == 0);
3924   constraint(ALLOC_IN_RC(ptr_reg));
3925   match(AddP (DecodeN reg) lreg);
3926 
3927   op_cost(10);
3928   format %{"[$reg + $lreg]" %}
3929   interface(MEMORY_INTER) %{
3930     base($reg);
3931     index($lreg);
3932     scale(0x0);
3933     disp(0x0);
3934   %}
3935 %}
3936 
3937 // Indirect Memory Times Scale Plus Index Register
3938 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3939 %{
3940   predicate(Universe::narrow_oop_shift() == 0);
3941   constraint(ALLOC_IN_RC(ptr_reg));
3942   match(AddP (DecodeN reg) (LShiftL lreg scale));
3943 
3944   op_cost(10);
3945   format %{"[$reg + $lreg << $scale]" %}
3946   interface(MEMORY_INTER) %{
3947     base($reg);
3948     index($lreg);
3949     scale($scale);
3950     disp(0x0);
3951   %}
3952 %}
3953 
3954 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3955 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3956 %{
3957   predicate(Universe::narrow_oop_shift() == 0);
3958   constraint(ALLOC_IN_RC(ptr_reg));
3959   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3960 
3961   op_cost(10);
3962   format %{"[$reg + $off + $lreg << $scale]" %}
3963   interface(MEMORY_INTER) %{
3964     base($reg);
3965     index($lreg);
3966     scale($scale);
3967     disp($off);
3968   %}
3969 %}
3970 
3971 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
3972 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
3973 %{
3974   constraint(ALLOC_IN_RC(ptr_reg));
3975   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3976   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
3977 
3978   op_cost(10);
3979   format %{"[$reg + $off + $idx]" %}
3980   interface(MEMORY_INTER) %{
3981     base($reg);
3982     index($idx);
3983     scale(0x0);
3984     disp($off);
3985   %}
3986 %}
3987 
3988 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3989 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3990 %{
3991   constraint(ALLOC_IN_RC(ptr_reg));
3992   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3993   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3994 
3995   op_cost(10);
3996   format %{"[$reg + $off + $idx << $scale]" %}
3997   interface(MEMORY_INTER) %{
3998     base($reg);
3999     index($idx);
4000     scale($scale);
4001     disp($off);
4002   %}
4003 %}
4004 
4005 //----------Special Memory Operands--------------------------------------------
4006 // Stack Slot Operand - This operand is used for loading and storing temporary
4007 //                      values on the stack where a match requires a value to
4008 //                      flow through memory.
4009 operand stackSlotP(sRegP reg)
4010 %{
4011   constraint(ALLOC_IN_RC(stack_slots));
4012   // No match rule because this operand is only generated in matching
4013 
4014   format %{ "[$reg]" %}
4015   interface(MEMORY_INTER) %{
4016     base(0x4);   // RSP
4017     index(0x4);  // No Index
4018     scale(0x0);  // No Scale
4019     disp($reg);  // Stack Offset
4020   %}
4021 %}
4022 
4023 operand stackSlotI(sRegI reg)
4024 %{
4025   constraint(ALLOC_IN_RC(stack_slots));
4026   // No match rule because this operand is only generated in matching
4027 
4028   format %{ "[$reg]" %}
4029   interface(MEMORY_INTER) %{
4030     base(0x4);   // RSP
4031     index(0x4);  // No Index
4032     scale(0x0);  // No Scale
4033     disp($reg);  // Stack Offset
4034   %}
4035 %}
4036 
4037 operand stackSlotF(sRegF reg)
4038 %{
4039   constraint(ALLOC_IN_RC(stack_slots));
4040   // No match rule because this operand is only generated in matching
4041 
4042   format %{ "[$reg]" %}
4043   interface(MEMORY_INTER) %{
4044     base(0x4);   // RSP
4045     index(0x4);  // No Index
4046     scale(0x0);  // No Scale
4047     disp($reg);  // Stack Offset
4048   %}
4049 %}
4050 
4051 operand stackSlotD(sRegD reg)
4052 %{
4053   constraint(ALLOC_IN_RC(stack_slots));
4054   // No match rule because this operand is only generated in matching
4055 
4056   format %{ "[$reg]" %}
4057   interface(MEMORY_INTER) %{
4058     base(0x4);   // RSP
4059     index(0x4);  // No Index
4060     scale(0x0);  // No Scale
4061     disp($reg);  // Stack Offset
4062   %}
4063 %}
4064 operand stackSlotL(sRegL reg)
4065 %{
4066   constraint(ALLOC_IN_RC(stack_slots));
4067   // No match rule because this operand is only generated in matching
4068 
4069   format %{ "[$reg]" %}
4070   interface(MEMORY_INTER) %{
4071     base(0x4);   // RSP
4072     index(0x4);  // No Index
4073     scale(0x0);  // No Scale
4074     disp($reg);  // Stack Offset
4075   %}
4076 %}
4077 
4078 //----------Conditional Branch Operands----------------------------------------
4079 // Comparison Op  - This is the operation of the comparison, and is limited to
4080 //                  the following set of codes:
4081 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4082 //
4083 // Other attributes of the comparison, such as unsignedness, are specified
4084 // by the comparison instruction that sets a condition code flags register.
4085 // That result is represented by a flags operand whose subtype is appropriate
4086 // to the unsignedness (etc.) of the comparison.
4087 //
4088 // Later, the instruction which matches both the Comparison Op (a Bool) and
4089 // the flags (produced by the Cmp) specifies the coding of the comparison op
4090 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4091 
4092 // Comparision Code
4093 operand cmpOp()
4094 %{
4095   match(Bool);
4096 
4097   format %{ "" %}
4098   interface(COND_INTER) %{
4099     equal(0x4, "e");
4100     not_equal(0x5, "ne");
4101     less(0xC, "l");
4102     greater_equal(0xD, "ge");
4103     less_equal(0xE, "le");
4104     greater(0xF, "g");
4105     overflow(0x0, "o");
4106     no_overflow(0x1, "no");
4107   %}
4108 %}
4109 
4110 // Comparison Code, unsigned compare.  Used by FP also, with
4111 // C2 (unordered) turned into GT or LT already.  The other bits
4112 // C0 and C3 are turned into Carry & Zero flags.
4113 operand cmpOpU()
4114 %{
4115   match(Bool);
4116 
4117   format %{ "" %}
4118   interface(COND_INTER) %{
4119     equal(0x4, "e");
4120     not_equal(0x5, "ne");
4121     less(0x2, "b");
4122     greater_equal(0x3, "nb");
4123     less_equal(0x6, "be");
4124     greater(0x7, "nbe");
4125     overflow(0x0, "o");
4126     no_overflow(0x1, "no");
4127   %}
4128 %}
4129 
4130 
4131 // Floating comparisons that don't require any fixup for the unordered case
4132 operand cmpOpUCF() %{
4133   match(Bool);
4134   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4135             n->as_Bool()->_test._test == BoolTest::ge ||
4136             n->as_Bool()->_test._test == BoolTest::le ||
4137             n->as_Bool()->_test._test == BoolTest::gt);
4138   format %{ "" %}
4139   interface(COND_INTER) %{
4140     equal(0x4, "e");
4141     not_equal(0x5, "ne");
4142     less(0x2, "b");
4143     greater_equal(0x3, "nb");
4144     less_equal(0x6, "be");
4145     greater(0x7, "nbe");
4146     overflow(0x0, "o");
4147     no_overflow(0x1, "no");
4148   %}
4149 %}
4150 
4151 
4152 // Floating comparisons that can be fixed up with extra conditional jumps
4153 operand cmpOpUCF2() %{
4154   match(Bool);
4155   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4156             n->as_Bool()->_test._test == BoolTest::eq);
4157   format %{ "" %}
4158   interface(COND_INTER) %{
4159     equal(0x4, "e");
4160     not_equal(0x5, "ne");
4161     less(0x2, "b");
4162     greater_equal(0x3, "nb");
4163     less_equal(0x6, "be");
4164     greater(0x7, "nbe");
4165     overflow(0x0, "o");
4166     no_overflow(0x1, "no");
4167   %}
4168 %}
4169 
4170 
4171 //----------OPERAND CLASSES----------------------------------------------------
4172 // Operand Classes are groups of operands that are used as to simplify
4173 // instruction definitions by not requiring the AD writer to specify separate
4174 // instructions for every form of operand when the instruction accepts
4175 // multiple operand types with the same basic encoding and format.  The classic
4176 // case of this is memory operands.
4177 
4178 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4179                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4180                indCompressedOopOffset,
4181                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4182                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4183                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4184 
4185 //----------PIPELINE-----------------------------------------------------------
4186 // Rules which define the behavior of the target architectures pipeline.
4187 pipeline %{
4188 
4189 //----------ATTRIBUTES---------------------------------------------------------
4190 attributes %{
4191   variable_size_instructions;        // Fixed size instructions
4192   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4193   instruction_unit_size = 1;         // An instruction is 1 bytes long
4194   instruction_fetch_unit_size = 16;  // The processor fetches one line
4195   instruction_fetch_units = 1;       // of 16 bytes
4196 
4197   // List of nop instructions
4198   nops( MachNop );
4199 %}
4200 
4201 //----------RESOURCES----------------------------------------------------------
4202 // Resources are the functional units available to the machine
4203 
4204 // Generic P2/P3 pipeline
4205 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4206 // 3 instructions decoded per cycle.
4207 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4208 // 3 ALU op, only ALU0 handles mul instructions.
4209 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4210            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4211            BR, FPU,
4212            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4213 
4214 //----------PIPELINE DESCRIPTION-----------------------------------------------
4215 // Pipeline Description specifies the stages in the machine's pipeline
4216 
4217 // Generic P2/P3 pipeline
4218 pipe_desc(S0, S1, S2, S3, S4, S5);
4219 
4220 //----------PIPELINE CLASSES---------------------------------------------------
4221 // Pipeline Classes describe the stages in which input and output are
4222 // referenced by the hardware pipeline.
4223 
4224 // Naming convention: ialu or fpu
4225 // Then: _reg
4226 // Then: _reg if there is a 2nd register
4227 // Then: _long if it's a pair of instructions implementing a long
4228 // Then: _fat if it requires the big decoder
4229 //   Or: _mem if it requires the big decoder and a memory unit.
4230 
4231 // Integer ALU reg operation
4232 pipe_class ialu_reg(rRegI dst)
4233 %{
4234     single_instruction;
4235     dst    : S4(write);
4236     dst    : S3(read);
4237     DECODE : S0;        // any decoder
4238     ALU    : S3;        // any alu
4239 %}
4240 
4241 // Long ALU reg operation
4242 pipe_class ialu_reg_long(rRegL dst)
4243 %{
4244     instruction_count(2);
4245     dst    : S4(write);
4246     dst    : S3(read);
4247     DECODE : S0(2);     // any 2 decoders
4248     ALU    : S3(2);     // both alus
4249 %}
4250 
4251 // Integer ALU reg operation using big decoder
4252 pipe_class ialu_reg_fat(rRegI dst)
4253 %{
4254     single_instruction;
4255     dst    : S4(write);
4256     dst    : S3(read);
4257     D0     : S0;        // big decoder only
4258     ALU    : S3;        // any alu
4259 %}
4260 
4261 // Long ALU reg operation using big decoder
4262 pipe_class ialu_reg_long_fat(rRegL dst)
4263 %{
4264     instruction_count(2);
4265     dst    : S4(write);
4266     dst    : S3(read);
4267     D0     : S0(2);     // big decoder only; twice
4268     ALU    : S3(2);     // any 2 alus
4269 %}
4270 
4271 // Integer ALU reg-reg operation
4272 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4273 %{
4274     single_instruction;
4275     dst    : S4(write);
4276     src    : S3(read);
4277     DECODE : S0;        // any decoder
4278     ALU    : S3;        // any alu
4279 %}
4280 
4281 // Long ALU reg-reg operation
4282 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4283 %{
4284     instruction_count(2);
4285     dst    : S4(write);
4286     src    : S3(read);
4287     DECODE : S0(2);     // any 2 decoders
4288     ALU    : S3(2);     // both alus
4289 %}
4290 
4291 // Integer ALU reg-reg operation
4292 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4293 %{
4294     single_instruction;
4295     dst    : S4(write);
4296     src    : S3(read);
4297     D0     : S0;        // big decoder only
4298     ALU    : S3;        // any alu
4299 %}
4300 
4301 // Long ALU reg-reg operation
4302 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4303 %{
4304     instruction_count(2);
4305     dst    : S4(write);
4306     src    : S3(read);
4307     D0     : S0(2);     // big decoder only; twice
4308     ALU    : S3(2);     // both alus
4309 %}
4310 
4311 // Integer ALU reg-mem operation
4312 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4313 %{
4314     single_instruction;
4315     dst    : S5(write);
4316     mem    : S3(read);
4317     D0     : S0;        // big decoder only
4318     ALU    : S4;        // any alu
4319     MEM    : S3;        // any mem
4320 %}
4321 
4322 // Integer mem operation (prefetch)
4323 pipe_class ialu_mem(memory mem)
4324 %{
4325     single_instruction;
4326     mem    : S3(read);
4327     D0     : S0;        // big decoder only
4328     MEM    : S3;        // any mem
4329 %}
4330 
4331 // Integer Store to Memory
4332 pipe_class ialu_mem_reg(memory mem, rRegI src)
4333 %{
4334     single_instruction;
4335     mem    : S3(read);
4336     src    : S5(read);
4337     D0     : S0;        // big decoder only
4338     ALU    : S4;        // any alu
4339     MEM    : S3;
4340 %}
4341 
4342 // // Long Store to Memory
4343 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4344 // %{
4345 //     instruction_count(2);
4346 //     mem    : S3(read);
4347 //     src    : S5(read);
4348 //     D0     : S0(2);          // big decoder only; twice
4349 //     ALU    : S4(2);     // any 2 alus
4350 //     MEM    : S3(2);  // Both mems
4351 // %}
4352 
4353 // Integer Store to Memory
4354 pipe_class ialu_mem_imm(memory mem)
4355 %{
4356     single_instruction;
4357     mem    : S3(read);
4358     D0     : S0;        // big decoder only
4359     ALU    : S4;        // any alu
4360     MEM    : S3;
4361 %}
4362 
4363 // Integer ALU0 reg-reg operation
4364 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4365 %{
4366     single_instruction;
4367     dst    : S4(write);
4368     src    : S3(read);
4369     D0     : S0;        // Big decoder only
4370     ALU0   : S3;        // only alu0
4371 %}
4372 
4373 // Integer ALU0 reg-mem operation
4374 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4375 %{
4376     single_instruction;
4377     dst    : S5(write);
4378     mem    : S3(read);
4379     D0     : S0;        // big decoder only
4380     ALU0   : S4;        // ALU0 only
4381     MEM    : S3;        // any mem
4382 %}
4383 
4384 // Integer ALU reg-reg operation
4385 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4386 %{
4387     single_instruction;
4388     cr     : S4(write);
4389     src1   : S3(read);
4390     src2   : S3(read);
4391     DECODE : S0;        // any decoder
4392     ALU    : S3;        // any alu
4393 %}
4394 
4395 // Integer ALU reg-imm operation
4396 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4397 %{
4398     single_instruction;
4399     cr     : S4(write);
4400     src1   : S3(read);
4401     DECODE : S0;        // any decoder
4402     ALU    : S3;        // any alu
4403 %}
4404 
4405 // Integer ALU reg-mem operation
4406 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4407 %{
4408     single_instruction;
4409     cr     : S4(write);
4410     src1   : S3(read);
4411     src2   : S3(read);
4412     D0     : S0;        // big decoder only
4413     ALU    : S4;        // any alu
4414     MEM    : S3;
4415 %}
4416 
4417 // Conditional move reg-reg
4418 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4419 %{
4420     instruction_count(4);
4421     y      : S4(read);
4422     q      : S3(read);
4423     p      : S3(read);
4424     DECODE : S0(4);     // any decoder
4425 %}
4426 
4427 // Conditional move reg-reg
4428 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4429 %{
4430     single_instruction;
4431     dst    : S4(write);
4432     src    : S3(read);
4433     cr     : S3(read);
4434     DECODE : S0;        // any decoder
4435 %}
4436 
4437 // Conditional move reg-mem
4438 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4439 %{
4440     single_instruction;
4441     dst    : S4(write);
4442     src    : S3(read);
4443     cr     : S3(read);
4444     DECODE : S0;        // any decoder
4445     MEM    : S3;
4446 %}
4447 
4448 // Conditional move reg-reg long
4449 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4450 %{
4451     single_instruction;
4452     dst    : S4(write);
4453     src    : S3(read);
4454     cr     : S3(read);
4455     DECODE : S0(2);     // any 2 decoders
4456 %}
4457 
4458 // XXX
4459 // // Conditional move double reg-reg
4460 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4461 // %{
4462 //     single_instruction;
4463 //     dst    : S4(write);
4464 //     src    : S3(read);
4465 //     cr     : S3(read);
4466 //     DECODE : S0;     // any decoder
4467 // %}
4468 
4469 // Float reg-reg operation
4470 pipe_class fpu_reg(regD dst)
4471 %{
4472     instruction_count(2);
4473     dst    : S3(read);
4474     DECODE : S0(2);     // any 2 decoders
4475     FPU    : S3;
4476 %}
4477 
4478 // Float reg-reg operation
4479 pipe_class fpu_reg_reg(regD dst, regD src)
4480 %{
4481     instruction_count(2);
4482     dst    : S4(write);
4483     src    : S3(read);
4484     DECODE : S0(2);     // any 2 decoders
4485     FPU    : S3;
4486 %}
4487 
4488 // Float reg-reg operation
4489 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4490 %{
4491     instruction_count(3);
4492     dst    : S4(write);
4493     src1   : S3(read);
4494     src2   : S3(read);
4495     DECODE : S0(3);     // any 3 decoders
4496     FPU    : S3(2);
4497 %}
4498 
4499 // Float reg-reg operation
4500 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4501 %{
4502     instruction_count(4);
4503     dst    : S4(write);
4504     src1   : S3(read);
4505     src2   : S3(read);
4506     src3   : S3(read);
4507     DECODE : S0(4);     // any 3 decoders
4508     FPU    : S3(2);
4509 %}
4510 
4511 // Float reg-reg operation
4512 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4513 %{
4514     instruction_count(4);
4515     dst    : S4(write);
4516     src1   : S3(read);
4517     src2   : S3(read);
4518     src3   : S3(read);
4519     DECODE : S1(3);     // any 3 decoders
4520     D0     : S0;        // Big decoder only
4521     FPU    : S3(2);
4522     MEM    : S3;
4523 %}
4524 
4525 // Float reg-mem operation
4526 pipe_class fpu_reg_mem(regD dst, memory mem)
4527 %{
4528     instruction_count(2);
4529     dst    : S5(write);
4530     mem    : S3(read);
4531     D0     : S0;        // big decoder only
4532     DECODE : S1;        // any decoder for FPU POP
4533     FPU    : S4;
4534     MEM    : S3;        // any mem
4535 %}
4536 
4537 // Float reg-mem operation
4538 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4539 %{
4540     instruction_count(3);
4541     dst    : S5(write);
4542     src1   : S3(read);
4543     mem    : S3(read);
4544     D0     : S0;        // big decoder only
4545     DECODE : S1(2);     // any decoder for FPU POP
4546     FPU    : S4;
4547     MEM    : S3;        // any mem
4548 %}
4549 
4550 // Float mem-reg operation
4551 pipe_class fpu_mem_reg(memory mem, regD src)
4552 %{
4553     instruction_count(2);
4554     src    : S5(read);
4555     mem    : S3(read);
4556     DECODE : S0;        // any decoder for FPU PUSH
4557     D0     : S1;        // big decoder only
4558     FPU    : S4;
4559     MEM    : S3;        // any mem
4560 %}
4561 
4562 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4563 %{
4564     instruction_count(3);
4565     src1   : S3(read);
4566     src2   : S3(read);
4567     mem    : S3(read);
4568     DECODE : S0(2);     // any decoder for FPU PUSH
4569     D0     : S1;        // big decoder only
4570     FPU    : S4;
4571     MEM    : S3;        // any mem
4572 %}
4573 
4574 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4575 %{
4576     instruction_count(3);
4577     src1   : S3(read);
4578     src2   : S3(read);
4579     mem    : S4(read);
4580     DECODE : S0;        // any decoder for FPU PUSH
4581     D0     : S0(2);     // big decoder only
4582     FPU    : S4;
4583     MEM    : S3(2);     // any mem
4584 %}
4585 
4586 pipe_class fpu_mem_mem(memory dst, memory src1)
4587 %{
4588     instruction_count(2);
4589     src1   : S3(read);
4590     dst    : S4(read);
4591     D0     : S0(2);     // big decoder only
4592     MEM    : S3(2);     // any mem
4593 %}
4594 
4595 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4596 %{
4597     instruction_count(3);
4598     src1   : S3(read);
4599     src2   : S3(read);
4600     dst    : S4(read);
4601     D0     : S0(3);     // big decoder only
4602     FPU    : S4;
4603     MEM    : S3(3);     // any mem
4604 %}
4605 
4606 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4607 %{
4608     instruction_count(3);
4609     src1   : S4(read);
4610     mem    : S4(read);
4611     DECODE : S0;        // any decoder for FPU PUSH
4612     D0     : S0(2);     // big decoder only
4613     FPU    : S4;
4614     MEM    : S3(2);     // any mem
4615 %}
4616 
4617 // Float load constant
4618 pipe_class fpu_reg_con(regD dst)
4619 %{
4620     instruction_count(2);
4621     dst    : S5(write);
4622     D0     : S0;        // big decoder only for the load
4623     DECODE : S1;        // any decoder for FPU POP
4624     FPU    : S4;
4625     MEM    : S3;        // any mem
4626 %}
4627 
4628 // Float load constant
4629 pipe_class fpu_reg_reg_con(regD dst, regD src)
4630 %{
4631     instruction_count(3);
4632     dst    : S5(write);
4633     src    : S3(read);
4634     D0     : S0;        // big decoder only for the load
4635     DECODE : S1(2);     // any decoder for FPU POP
4636     FPU    : S4;
4637     MEM    : S3;        // any mem
4638 %}
4639 
4640 // UnConditional branch
4641 pipe_class pipe_jmp(label labl)
4642 %{
4643     single_instruction;
4644     BR   : S3;
4645 %}
4646 
4647 // Conditional branch
4648 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4649 %{
4650     single_instruction;
4651     cr    : S1(read);
4652     BR    : S3;
4653 %}
4654 
4655 // Allocation idiom
4656 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4657 %{
4658     instruction_count(1); force_serialization;
4659     fixed_latency(6);
4660     heap_ptr : S3(read);
4661     DECODE   : S0(3);
4662     D0       : S2;
4663     MEM      : S3;
4664     ALU      : S3(2);
4665     dst      : S5(write);
4666     BR       : S5;
4667 %}
4668 
4669 // Generic big/slow expanded idiom
4670 pipe_class pipe_slow()
4671 %{
4672     instruction_count(10); multiple_bundles; force_serialization;
4673     fixed_latency(100);
4674     D0  : S0(2);
4675     MEM : S3(2);
4676 %}
4677 
4678 // The real do-nothing guy
4679 pipe_class empty()
4680 %{
4681     instruction_count(0);
4682 %}
4683 
4684 // Define the class for the Nop node
4685 define
4686 %{
4687    MachNop = empty;
4688 %}
4689 
4690 %}
4691 
4692 //----------INSTRUCTIONS-------------------------------------------------------
4693 //
4694 // match      -- States which machine-independent subtree may be replaced
4695 //               by this instruction.
4696 // ins_cost   -- The estimated cost of this instruction is used by instruction
4697 //               selection to identify a minimum cost tree of machine
4698 //               instructions that matches a tree of machine-independent
4699 //               instructions.
4700 // format     -- A string providing the disassembly for this instruction.
4701 //               The value of an instruction's operand may be inserted
4702 //               by referring to it with a '$' prefix.
4703 // opcode     -- Three instruction opcodes may be provided.  These are referred
4704 //               to within an encode class as $primary, $secondary, and $tertiary
4705 //               rrspectively.  The primary opcode is commonly used to
4706 //               indicate the type of machine instruction, while secondary
4707 //               and tertiary are often used for prefix options or addressing
4708 //               modes.
4709 // ins_encode -- A list of encode classes with parameters. The encode class
4710 //               name must have been defined in an 'enc_class' specification
4711 //               in the encode section of the architecture description.
4712 
4713 
4714 //----------Load/Store/Move Instructions---------------------------------------
4715 //----------Load Instructions--------------------------------------------------
4716 
4717 // Load Byte (8 bit signed)
4718 instruct loadB(rRegI dst, memory mem)
4719 %{
4720   match(Set dst (LoadB mem));
4721 
4722   ins_cost(125);
4723   format %{ "movsbl  $dst, $mem\t# byte" %}
4724 
4725   ins_encode %{
4726     __ movsbl($dst$$Register, $mem$$Address);
4727   %}
4728 
4729   ins_pipe(ialu_reg_mem);
4730 %}
4731 
4732 // Load Byte (8 bit signed) into Long Register
4733 instruct loadB2L(rRegL dst, memory mem)
4734 %{
4735   match(Set dst (ConvI2L (LoadB mem)));
4736 
4737   ins_cost(125);
4738   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4739 
4740   ins_encode %{
4741     __ movsbq($dst$$Register, $mem$$Address);
4742   %}
4743 
4744   ins_pipe(ialu_reg_mem);
4745 %}
4746 
4747 // Load Unsigned Byte (8 bit UNsigned)
4748 instruct loadUB(rRegI dst, memory mem)
4749 %{
4750   match(Set dst (LoadUB mem));
4751 
4752   ins_cost(125);
4753   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4754 
4755   ins_encode %{
4756     __ movzbl($dst$$Register, $mem$$Address);
4757   %}
4758 
4759   ins_pipe(ialu_reg_mem);
4760 %}
4761 
4762 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4763 instruct loadUB2L(rRegL dst, memory mem)
4764 %{
4765   match(Set dst (ConvI2L (LoadUB mem)));
4766 
4767   ins_cost(125);
4768   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4769 
4770   ins_encode %{
4771     __ movzbq($dst$$Register, $mem$$Address);
4772   %}
4773 
4774   ins_pipe(ialu_reg_mem);
4775 %}
4776 
4777 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
4778 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4779   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4780   effect(KILL cr);
4781 
4782   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
4783             "andl    $dst, right_n_bits($mask, 8)" %}
4784   ins_encode %{
4785     Register Rdst = $dst$$Register;
4786     __ movzbq(Rdst, $mem$$Address);
4787     __ andl(Rdst, $mask$$constant & right_n_bits(8));
4788   %}
4789   ins_pipe(ialu_reg_mem);
4790 %}
4791 
4792 // Load Short (16 bit signed)
4793 instruct loadS(rRegI dst, memory mem)
4794 %{
4795   match(Set dst (LoadS mem));
4796 
4797   ins_cost(125);
4798   format %{ "movswl $dst, $mem\t# short" %}
4799 
4800   ins_encode %{
4801     __ movswl($dst$$Register, $mem$$Address);
4802   %}
4803 
4804   ins_pipe(ialu_reg_mem);
4805 %}
4806 
4807 // Load Short (16 bit signed) to Byte (8 bit signed)
4808 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4809   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4810 
4811   ins_cost(125);
4812   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4813   ins_encode %{
4814     __ movsbl($dst$$Register, $mem$$Address);
4815   %}
4816   ins_pipe(ialu_reg_mem);
4817 %}
4818 
4819 // Load Short (16 bit signed) into Long Register
4820 instruct loadS2L(rRegL dst, memory mem)
4821 %{
4822   match(Set dst (ConvI2L (LoadS mem)));
4823 
4824   ins_cost(125);
4825   format %{ "movswq $dst, $mem\t# short -> long" %}
4826 
4827   ins_encode %{
4828     __ movswq($dst$$Register, $mem$$Address);
4829   %}
4830 
4831   ins_pipe(ialu_reg_mem);
4832 %}
4833 
4834 // Load Unsigned Short/Char (16 bit UNsigned)
4835 instruct loadUS(rRegI dst, memory mem)
4836 %{
4837   match(Set dst (LoadUS mem));
4838 
4839   ins_cost(125);
4840   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4841 
4842   ins_encode %{
4843     __ movzwl($dst$$Register, $mem$$Address);
4844   %}
4845 
4846   ins_pipe(ialu_reg_mem);
4847 %}
4848 
4849 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4850 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4851   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4852 
4853   ins_cost(125);
4854   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4855   ins_encode %{
4856     __ movsbl($dst$$Register, $mem$$Address);
4857   %}
4858   ins_pipe(ialu_reg_mem);
4859 %}
4860 
4861 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4862 instruct loadUS2L(rRegL dst, memory mem)
4863 %{
4864   match(Set dst (ConvI2L (LoadUS mem)));
4865 
4866   ins_cost(125);
4867   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4868 
4869   ins_encode %{
4870     __ movzwq($dst$$Register, $mem$$Address);
4871   %}
4872 
4873   ins_pipe(ialu_reg_mem);
4874 %}
4875 
4876 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4877 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4878   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4879 
4880   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4881   ins_encode %{
4882     __ movzbq($dst$$Register, $mem$$Address);
4883   %}
4884   ins_pipe(ialu_reg_mem);
4885 %}
4886 
4887 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
4888 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4889   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4890   effect(KILL cr);
4891 
4892   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
4893             "andl    $dst, right_n_bits($mask, 16)" %}
4894   ins_encode %{
4895     Register Rdst = $dst$$Register;
4896     __ movzwq(Rdst, $mem$$Address);
4897     __ andl(Rdst, $mask$$constant & right_n_bits(16));
4898   %}
4899   ins_pipe(ialu_reg_mem);
4900 %}
4901 
4902 // Load Integer
4903 instruct loadI(rRegI dst, memory mem)
4904 %{
4905   match(Set dst (LoadI mem));
4906 
4907   ins_cost(125);
4908   format %{ "movl    $dst, $mem\t# int" %}
4909 
4910   ins_encode %{
4911     __ movl($dst$$Register, $mem$$Address);
4912   %}
4913 
4914   ins_pipe(ialu_reg_mem);
4915 %}
4916 
4917 // Load Integer (32 bit signed) to Byte (8 bit signed)
4918 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4919   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4920 
4921   ins_cost(125);
4922   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4923   ins_encode %{
4924     __ movsbl($dst$$Register, $mem$$Address);
4925   %}
4926   ins_pipe(ialu_reg_mem);
4927 %}
4928 
4929 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4930 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4931   match(Set dst (AndI (LoadI mem) mask));
4932 
4933   ins_cost(125);
4934   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4935   ins_encode %{
4936     __ movzbl($dst$$Register, $mem$$Address);
4937   %}
4938   ins_pipe(ialu_reg_mem);
4939 %}
4940 
4941 // Load Integer (32 bit signed) to Short (16 bit signed)
4942 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4943   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4944 
4945   ins_cost(125);
4946   format %{ "movswl  $dst, $mem\t# int -> short" %}
4947   ins_encode %{
4948     __ movswl($dst$$Register, $mem$$Address);
4949   %}
4950   ins_pipe(ialu_reg_mem);
4951 %}
4952 
4953 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4954 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4955   match(Set dst (AndI (LoadI mem) mask));
4956 
4957   ins_cost(125);
4958   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
4959   ins_encode %{
4960     __ movzwl($dst$$Register, $mem$$Address);
4961   %}
4962   ins_pipe(ialu_reg_mem);
4963 %}
4964 
4965 // Load Integer into Long Register
4966 instruct loadI2L(rRegL dst, memory mem)
4967 %{
4968   match(Set dst (ConvI2L (LoadI mem)));
4969 
4970   ins_cost(125);
4971   format %{ "movslq  $dst, $mem\t# int -> long" %}
4972 
4973   ins_encode %{
4974     __ movslq($dst$$Register, $mem$$Address);
4975   %}
4976 
4977   ins_pipe(ialu_reg_mem);
4978 %}
4979 
4980 // Load Integer with mask 0xFF into Long Register
4981 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4982   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4983 
4984   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
4985   ins_encode %{
4986     __ movzbq($dst$$Register, $mem$$Address);
4987   %}
4988   ins_pipe(ialu_reg_mem);
4989 %}
4990 
4991 // Load Integer with mask 0xFFFF into Long Register
4992 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4993   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4994 
4995   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
4996   ins_encode %{
4997     __ movzwq($dst$$Register, $mem$$Address);
4998   %}
4999   ins_pipe(ialu_reg_mem);
5000 %}
5001 
5002 // Load Integer with a 31-bit mask into Long Register
5003 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5004   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5005   effect(KILL cr);
5006 
5007   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5008             "andl    $dst, $mask" %}
5009   ins_encode %{
5010     Register Rdst = $dst$$Register;
5011     __ movl(Rdst, $mem$$Address);
5012     __ andl(Rdst, $mask$$constant);
5013   %}
5014   ins_pipe(ialu_reg_mem);
5015 %}
5016 
5017 // Load Unsigned Integer into Long Register
5018 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5019 %{
5020   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5021 
5022   ins_cost(125);
5023   format %{ "movl    $dst, $mem\t# uint -> long" %}
5024 
5025   ins_encode %{
5026     __ movl($dst$$Register, $mem$$Address);
5027   %}
5028 
5029   ins_pipe(ialu_reg_mem);
5030 %}
5031 
5032 // Load Long
5033 instruct loadL(rRegL dst, memory mem)
5034 %{
5035   match(Set dst (LoadL mem));
5036 
5037   ins_cost(125);
5038   format %{ "movq    $dst, $mem\t# long" %}
5039 
5040   ins_encode %{
5041     __ movq($dst$$Register, $mem$$Address);
5042   %}
5043 
5044   ins_pipe(ialu_reg_mem); // XXX
5045 %}
5046 
5047 // Load Range
5048 instruct loadRange(rRegI dst, memory mem)
5049 %{
5050   match(Set dst (LoadRange mem));
5051 
5052   ins_cost(125); // XXX
5053   format %{ "movl    $dst, $mem\t# range" %}
5054   opcode(0x8B);
5055   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5056   ins_pipe(ialu_reg_mem);
5057 %}
5058 
5059 // Load Pointer
5060 instruct loadP(rRegP dst, memory mem)
5061 %{
5062   match(Set dst (LoadP mem));
5063 
5064   ins_cost(125); // XXX
5065   format %{ "movq    $dst, $mem\t# ptr" %}
5066   opcode(0x8B);
5067   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5068   ins_pipe(ialu_reg_mem); // XXX
5069 %}
5070 
5071 // Load Compressed Pointer
5072 instruct loadN(rRegN dst, memory mem)
5073 %{
5074    match(Set dst (LoadN mem));
5075 
5076    ins_cost(125); // XXX
5077    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5078    ins_encode %{
5079      __ movl($dst$$Register, $mem$$Address);
5080    %}
5081    ins_pipe(ialu_reg_mem); // XXX
5082 %}
5083 
5084 
5085 // Load Klass Pointer
5086 instruct loadKlass(rRegP dst, memory mem)
5087 %{
5088   match(Set dst (LoadKlass mem));
5089 
5090   ins_cost(125); // XXX
5091   format %{ "movq    $dst, $mem\t# class" %}
5092   opcode(0x8B);
5093   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5094   ins_pipe(ialu_reg_mem); // XXX
5095 %}
5096 
5097 // Load narrow Klass Pointer
5098 instruct loadNKlass(rRegN dst, memory mem)
5099 %{
5100   match(Set dst (LoadNKlass mem));
5101 
5102   ins_cost(125); // XXX
5103   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5104   ins_encode %{
5105     __ movl($dst$$Register, $mem$$Address);
5106   %}
5107   ins_pipe(ialu_reg_mem); // XXX
5108 %}
5109 
5110 // Load Float
5111 instruct loadF(regF dst, memory mem)
5112 %{
5113   match(Set dst (LoadF mem));
5114 
5115   ins_cost(145); // XXX
5116   format %{ "movss   $dst, $mem\t# float" %}
5117   ins_encode %{
5118     __ movflt($dst$$XMMRegister, $mem$$Address);
5119   %}
5120   ins_pipe(pipe_slow); // XXX
5121 %}
5122 
5123 // Load Double
5124 instruct loadD_partial(regD dst, memory mem)
5125 %{
5126   predicate(!UseXmmLoadAndClearUpper);
5127   match(Set dst (LoadD mem));
5128 
5129   ins_cost(145); // XXX
5130   format %{ "movlpd  $dst, $mem\t# double" %}
5131   ins_encode %{
5132     __ movdbl($dst$$XMMRegister, $mem$$Address);
5133   %}
5134   ins_pipe(pipe_slow); // XXX
5135 %}
5136 
5137 instruct loadD(regD dst, memory mem)
5138 %{
5139   predicate(UseXmmLoadAndClearUpper);
5140   match(Set dst (LoadD mem));
5141 
5142   ins_cost(145); // XXX
5143   format %{ "movsd   $dst, $mem\t# double" %}
5144   ins_encode %{
5145     __ movdbl($dst$$XMMRegister, $mem$$Address);
5146   %}
5147   ins_pipe(pipe_slow); // XXX
5148 %}
5149 
5150 // Load Effective Address
5151 instruct leaP8(rRegP dst, indOffset8 mem)
5152 %{
5153   match(Set dst mem);
5154 
5155   ins_cost(110); // XXX
5156   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5157   opcode(0x8D);
5158   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5159   ins_pipe(ialu_reg_reg_fat);
5160 %}
5161 
5162 instruct leaP32(rRegP dst, indOffset32 mem)
5163 %{
5164   match(Set dst mem);
5165 
5166   ins_cost(110);
5167   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5168   opcode(0x8D);
5169   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5170   ins_pipe(ialu_reg_reg_fat);
5171 %}
5172 
5173 // instruct leaPIdx(rRegP dst, indIndex mem)
5174 // %{
5175 //   match(Set dst mem);
5176 
5177 //   ins_cost(110);
5178 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5179 //   opcode(0x8D);
5180 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5181 //   ins_pipe(ialu_reg_reg_fat);
5182 // %}
5183 
5184 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5185 %{
5186   match(Set dst mem);
5187 
5188   ins_cost(110);
5189   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5190   opcode(0x8D);
5191   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5192   ins_pipe(ialu_reg_reg_fat);
5193 %}
5194 
5195 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5196 %{
5197   match(Set dst mem);
5198 
5199   ins_cost(110);
5200   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5201   opcode(0x8D);
5202   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5203   ins_pipe(ialu_reg_reg_fat);
5204 %}
5205 
5206 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5207 %{
5208   match(Set dst mem);
5209 
5210   ins_cost(110);
5211   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5212   opcode(0x8D);
5213   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5214   ins_pipe(ialu_reg_reg_fat);
5215 %}
5216 
5217 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5218 %{
5219   match(Set dst mem);
5220 
5221   ins_cost(110);
5222   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5223   opcode(0x8D);
5224   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5225   ins_pipe(ialu_reg_reg_fat);
5226 %}
5227 
5228 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5229 %{
5230   match(Set dst mem);
5231 
5232   ins_cost(110);
5233   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5234   opcode(0x8D);
5235   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5236   ins_pipe(ialu_reg_reg_fat);
5237 %}
5238 
5239 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5240 %{
5241   match(Set dst mem);
5242 
5243   ins_cost(110);
5244   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5245   opcode(0x8D);
5246   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5247   ins_pipe(ialu_reg_reg_fat);
5248 %}
5249 
5250 // Load Effective Address which uses Narrow (32-bits) oop
5251 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5252 %{
5253   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5254   match(Set dst mem);
5255 
5256   ins_cost(110);
5257   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5258   opcode(0x8D);
5259   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5260   ins_pipe(ialu_reg_reg_fat);
5261 %}
5262 
5263 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5264 %{
5265   predicate(Universe::narrow_oop_shift() == 0);
5266   match(Set dst mem);
5267 
5268   ins_cost(110); // XXX
5269   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5270   opcode(0x8D);
5271   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5272   ins_pipe(ialu_reg_reg_fat);
5273 %}
5274 
5275 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5276 %{
5277   predicate(Universe::narrow_oop_shift() == 0);
5278   match(Set dst mem);
5279 
5280   ins_cost(110);
5281   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5282   opcode(0x8D);
5283   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5284   ins_pipe(ialu_reg_reg_fat);
5285 %}
5286 
5287 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5288 %{
5289   predicate(Universe::narrow_oop_shift() == 0);
5290   match(Set dst mem);
5291 
5292   ins_cost(110);
5293   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5294   opcode(0x8D);
5295   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5296   ins_pipe(ialu_reg_reg_fat);
5297 %}
5298 
5299 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5300 %{
5301   predicate(Universe::narrow_oop_shift() == 0);
5302   match(Set dst mem);
5303 
5304   ins_cost(110);
5305   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5306   opcode(0x8D);
5307   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5308   ins_pipe(ialu_reg_reg_fat);
5309 %}
5310 
5311 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5312 %{
5313   predicate(Universe::narrow_oop_shift() == 0);
5314   match(Set dst mem);
5315 
5316   ins_cost(110);
5317   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5318   opcode(0x8D);
5319   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5320   ins_pipe(ialu_reg_reg_fat);
5321 %}
5322 
5323 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5324 %{
5325   predicate(Universe::narrow_oop_shift() == 0);
5326   match(Set dst mem);
5327 
5328   ins_cost(110);
5329   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5330   opcode(0x8D);
5331   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5332   ins_pipe(ialu_reg_reg_fat);
5333 %}
5334 
5335 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5336 %{
5337   predicate(Universe::narrow_oop_shift() == 0);
5338   match(Set dst mem);
5339 
5340   ins_cost(110);
5341   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5342   opcode(0x8D);
5343   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5344   ins_pipe(ialu_reg_reg_fat);
5345 %}
5346 
5347 instruct loadConI(rRegI dst, immI src)
5348 %{
5349   match(Set dst src);
5350 
5351   format %{ "movl    $dst, $src\t# int" %}
5352   ins_encode(load_immI(dst, src));
5353   ins_pipe(ialu_reg_fat); // XXX
5354 %}
5355 
5356 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5357 %{
5358   match(Set dst src);
5359   effect(KILL cr);
5360 
5361   ins_cost(50);
5362   format %{ "xorl    $dst, $dst\t# int" %}
5363   opcode(0x33); /* + rd */
5364   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5365   ins_pipe(ialu_reg);
5366 %}
5367 
5368 instruct loadConL(rRegL dst, immL src)
5369 %{
5370   match(Set dst src);
5371 
5372   ins_cost(150);
5373   format %{ "movq    $dst, $src\t# long" %}
5374   ins_encode(load_immL(dst, src));
5375   ins_pipe(ialu_reg);
5376 %}
5377 
5378 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5379 %{
5380   match(Set dst src);
5381   effect(KILL cr);
5382 
5383   ins_cost(50);
5384   format %{ "xorl    $dst, $dst\t# long" %}
5385   opcode(0x33); /* + rd */
5386   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5387   ins_pipe(ialu_reg); // XXX
5388 %}
5389 
5390 instruct loadConUL32(rRegL dst, immUL32 src)
5391 %{
5392   match(Set dst src);
5393 
5394   ins_cost(60);
5395   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5396   ins_encode(load_immUL32(dst, src));
5397   ins_pipe(ialu_reg);
5398 %}
5399 
5400 instruct loadConL32(rRegL dst, immL32 src)
5401 %{
5402   match(Set dst src);
5403 
5404   ins_cost(70);
5405   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5406   ins_encode(load_immL32(dst, src));
5407   ins_pipe(ialu_reg);
5408 %}
5409 
5410 instruct loadConP(rRegP dst, immP con) %{
5411   match(Set dst con);
5412 
5413   format %{ "movq    $dst, $con\t# ptr" %}
5414   ins_encode(load_immP(dst, con));
5415   ins_pipe(ialu_reg_fat); // XXX
5416 %}
5417 
5418 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5419 %{
5420   match(Set dst src);
5421   effect(KILL cr);
5422 
5423   ins_cost(50);
5424   format %{ "xorl    $dst, $dst\t# ptr" %}
5425   opcode(0x33); /* + rd */
5426   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5427   ins_pipe(ialu_reg);
5428 %}
5429 
5430 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5431 %{
5432   match(Set dst src);
5433   effect(KILL cr);
5434 
5435   ins_cost(60);
5436   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5437   ins_encode(load_immP31(dst, src));
5438   ins_pipe(ialu_reg);
5439 %}
5440 
5441 instruct loadConF(regF dst, immF con) %{
5442   match(Set dst con);
5443   ins_cost(125);
5444   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5445   ins_encode %{
5446     __ movflt($dst$$XMMRegister, $constantaddress($con));
5447   %}
5448   ins_pipe(pipe_slow);
5449 %}
5450 
5451 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5452   match(Set dst src);
5453   effect(KILL cr);
5454   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5455   ins_encode %{
5456     __ xorq($dst$$Register, $dst$$Register);
5457   %}
5458   ins_pipe(ialu_reg);
5459 %}
5460 
5461 instruct loadConN(rRegN dst, immN src) %{
5462   match(Set dst src);
5463 
5464   ins_cost(125);
5465   format %{ "movl    $dst, $src\t# compressed ptr" %}
5466   ins_encode %{
5467     address con = (address)$src$$constant;
5468     if (con == NULL) {
5469       ShouldNotReachHere();
5470     } else {
5471       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5472     }
5473   %}
5474   ins_pipe(ialu_reg_fat); // XXX
5475 %}
5476 
5477 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5478   match(Set dst src);
5479 
5480   ins_cost(125);
5481   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5482   ins_encode %{
5483     address con = (address)$src$$constant;
5484     if (con == NULL) {
5485       ShouldNotReachHere();
5486     } else {
5487       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5488     }
5489   %}
5490   ins_pipe(ialu_reg_fat); // XXX
5491 %}
5492 
5493 instruct loadConF0(regF dst, immF0 src)
5494 %{
5495   match(Set dst src);
5496   ins_cost(100);
5497 
5498   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5499   ins_encode %{
5500     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5501   %}
5502   ins_pipe(pipe_slow);
5503 %}
5504 
5505 // Use the same format since predicate() can not be used here.
5506 instruct loadConD(regD dst, immD con) %{
5507   match(Set dst con);
5508   ins_cost(125);
5509   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5510   ins_encode %{
5511     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5512   %}
5513   ins_pipe(pipe_slow);
5514 %}
5515 
5516 instruct loadConD0(regD dst, immD0 src)
5517 %{
5518   match(Set dst src);
5519   ins_cost(100);
5520 
5521   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5522   ins_encode %{
5523     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5524   %}
5525   ins_pipe(pipe_slow);
5526 %}
5527 
5528 instruct loadSSI(rRegI dst, stackSlotI src)
5529 %{
5530   match(Set dst src);
5531 
5532   ins_cost(125);
5533   format %{ "movl    $dst, $src\t# int stk" %}
5534   opcode(0x8B);
5535   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5536   ins_pipe(ialu_reg_mem);
5537 %}
5538 
5539 instruct loadSSL(rRegL dst, stackSlotL src)
5540 %{
5541   match(Set dst src);
5542 
5543   ins_cost(125);
5544   format %{ "movq    $dst, $src\t# long stk" %}
5545   opcode(0x8B);
5546   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5547   ins_pipe(ialu_reg_mem);
5548 %}
5549 
5550 instruct loadSSP(rRegP dst, stackSlotP src)
5551 %{
5552   match(Set dst src);
5553 
5554   ins_cost(125);
5555   format %{ "movq    $dst, $src\t# ptr stk" %}
5556   opcode(0x8B);
5557   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5558   ins_pipe(ialu_reg_mem);
5559 %}
5560 
5561 instruct loadSSF(regF dst, stackSlotF src)
5562 %{
5563   match(Set dst src);
5564 
5565   ins_cost(125);
5566   format %{ "movss   $dst, $src\t# float stk" %}
5567   ins_encode %{
5568     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5569   %}
5570   ins_pipe(pipe_slow); // XXX
5571 %}
5572 
5573 // Use the same format since predicate() can not be used here.
5574 instruct loadSSD(regD dst, stackSlotD src)
5575 %{
5576   match(Set dst src);
5577 
5578   ins_cost(125);
5579   format %{ "movsd   $dst, $src\t# double stk" %}
5580   ins_encode  %{
5581     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5582   %}
5583   ins_pipe(pipe_slow); // XXX
5584 %}
5585 
5586 // Prefetch instructions for allocation.
5587 // Must be safe to execute with invalid address (cannot fault).
5588 
5589 instruct prefetchAlloc( memory mem ) %{
5590   predicate(AllocatePrefetchInstr==3);
5591   match(PrefetchAllocation mem);
5592   ins_cost(125);
5593 
5594   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5595   ins_encode %{
5596     __ prefetchw($mem$$Address);
5597   %}
5598   ins_pipe(ialu_mem);
5599 %}
5600 
5601 instruct prefetchAllocNTA( memory mem ) %{
5602   predicate(AllocatePrefetchInstr==0);
5603   match(PrefetchAllocation mem);
5604   ins_cost(125);
5605 
5606   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5607   ins_encode %{
5608     __ prefetchnta($mem$$Address);
5609   %}
5610   ins_pipe(ialu_mem);
5611 %}
5612 
5613 instruct prefetchAllocT0( memory mem ) %{
5614   predicate(AllocatePrefetchInstr==1);
5615   match(PrefetchAllocation mem);
5616   ins_cost(125);
5617 
5618   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5619   ins_encode %{
5620     __ prefetcht0($mem$$Address);
5621   %}
5622   ins_pipe(ialu_mem);
5623 %}
5624 
5625 instruct prefetchAllocT2( memory mem ) %{
5626   predicate(AllocatePrefetchInstr==2);
5627   match(PrefetchAllocation mem);
5628   ins_cost(125);
5629 
5630   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5631   ins_encode %{
5632     __ prefetcht2($mem$$Address);
5633   %}
5634   ins_pipe(ialu_mem);
5635 %}
5636 
5637 //----------Store Instructions-------------------------------------------------
5638 
5639 // Store Byte
5640 instruct storeB(memory mem, rRegI src)
5641 %{
5642   match(Set mem (StoreB mem src));
5643 
5644   ins_cost(125); // XXX
5645   format %{ "movb    $mem, $src\t# byte" %}
5646   opcode(0x88);
5647   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5648   ins_pipe(ialu_mem_reg);
5649 %}
5650 
5651 // Store Char/Short
5652 instruct storeC(memory mem, rRegI src)
5653 %{
5654   match(Set mem (StoreC mem src));
5655 
5656   ins_cost(125); // XXX
5657   format %{ "movw    $mem, $src\t# char/short" %}
5658   opcode(0x89);
5659   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5660   ins_pipe(ialu_mem_reg);
5661 %}
5662 
5663 // Store Integer
5664 instruct storeI(memory mem, rRegI src)
5665 %{
5666   match(Set mem (StoreI mem src));
5667 
5668   ins_cost(125); // XXX
5669   format %{ "movl    $mem, $src\t# int" %}
5670   opcode(0x89);
5671   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5672   ins_pipe(ialu_mem_reg);
5673 %}
5674 
5675 // Store Long
5676 instruct storeL(memory mem, rRegL src)
5677 %{
5678   match(Set mem (StoreL mem src));
5679 
5680   ins_cost(125); // XXX
5681   format %{ "movq    $mem, $src\t# long" %}
5682   opcode(0x89);
5683   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5684   ins_pipe(ialu_mem_reg); // XXX
5685 %}
5686 
5687 // Store Pointer
5688 instruct storeP(memory mem, any_RegP src)
5689 %{
5690   match(Set mem (StoreP mem src));
5691 
5692   ins_cost(125); // XXX
5693   format %{ "movq    $mem, $src\t# ptr" %}
5694   opcode(0x89);
5695   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5696   ins_pipe(ialu_mem_reg);
5697 %}
5698 
5699 instruct storeImmP0(memory mem, immP0 zero)
5700 %{
5701   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5702   match(Set mem (StoreP mem zero));
5703 
5704   ins_cost(125); // XXX
5705   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5706   ins_encode %{
5707     __ movq($mem$$Address, r12);
5708   %}
5709   ins_pipe(ialu_mem_reg);
5710 %}
5711 
5712 // Store NULL Pointer, mark word, or other simple pointer constant.
5713 instruct storeImmP(memory mem, immP31 src)
5714 %{
5715   match(Set mem (StoreP mem src));
5716 
5717   ins_cost(150); // XXX
5718   format %{ "movq    $mem, $src\t# ptr" %}
5719   opcode(0xC7); /* C7 /0 */
5720   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5721   ins_pipe(ialu_mem_imm);
5722 %}
5723 
5724 // Store Compressed Pointer
5725 instruct storeN(memory mem, rRegN src)
5726 %{
5727   match(Set mem (StoreN mem src));
5728 
5729   ins_cost(125); // XXX
5730   format %{ "movl    $mem, $src\t# compressed ptr" %}
5731   ins_encode %{
5732     __ movl($mem$$Address, $src$$Register);
5733   %}
5734   ins_pipe(ialu_mem_reg);
5735 %}
5736 
5737 instruct storeNKlass(memory mem, rRegN src)
5738 %{
5739   match(Set mem (StoreNKlass mem src));
5740 
5741   ins_cost(125); // XXX
5742   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5743   ins_encode %{
5744     __ movl($mem$$Address, $src$$Register);
5745   %}
5746   ins_pipe(ialu_mem_reg);
5747 %}
5748 
5749 instruct storeImmN0(memory mem, immN0 zero)
5750 %{
5751   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5752   match(Set mem (StoreN mem zero));
5753 
5754   ins_cost(125); // XXX
5755   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5756   ins_encode %{
5757     __ movl($mem$$Address, r12);
5758   %}
5759   ins_pipe(ialu_mem_reg);
5760 %}
5761 
5762 instruct storeImmN(memory mem, immN src)
5763 %{
5764   match(Set mem (StoreN mem src));
5765 
5766   ins_cost(150); // XXX
5767   format %{ "movl    $mem, $src\t# compressed ptr" %}
5768   ins_encode %{
5769     address con = (address)$src$$constant;
5770     if (con == NULL) {
5771       __ movl($mem$$Address, (int32_t)0);
5772     } else {
5773       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5774     }
5775   %}
5776   ins_pipe(ialu_mem_imm);
5777 %}
5778 
5779 instruct storeImmNKlass(memory mem, immNKlass src)
5780 %{
5781   match(Set mem (StoreNKlass mem src));
5782 
5783   ins_cost(150); // XXX
5784   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5785   ins_encode %{
5786     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5787   %}
5788   ins_pipe(ialu_mem_imm);
5789 %}
5790 
5791 // Store Integer Immediate
5792 instruct storeImmI0(memory mem, immI0 zero)
5793 %{
5794   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5795   match(Set mem (StoreI mem zero));
5796 
5797   ins_cost(125); // XXX
5798   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5799   ins_encode %{
5800     __ movl($mem$$Address, r12);
5801   %}
5802   ins_pipe(ialu_mem_reg);
5803 %}
5804 
5805 instruct storeImmI(memory mem, immI src)
5806 %{
5807   match(Set mem (StoreI mem src));
5808 
5809   ins_cost(150);
5810   format %{ "movl    $mem, $src\t# int" %}
5811   opcode(0xC7); /* C7 /0 */
5812   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5813   ins_pipe(ialu_mem_imm);
5814 %}
5815 
5816 // Store Long Immediate
5817 instruct storeImmL0(memory mem, immL0 zero)
5818 %{
5819   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5820   match(Set mem (StoreL mem zero));
5821 
5822   ins_cost(125); // XXX
5823   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5824   ins_encode %{
5825     __ movq($mem$$Address, r12);
5826   %}
5827   ins_pipe(ialu_mem_reg);
5828 %}
5829 
5830 instruct storeImmL(memory mem, immL32 src)
5831 %{
5832   match(Set mem (StoreL mem src));
5833 
5834   ins_cost(150);
5835   format %{ "movq    $mem, $src\t# long" %}
5836   opcode(0xC7); /* C7 /0 */
5837   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5838   ins_pipe(ialu_mem_imm);
5839 %}
5840 
5841 // Store Short/Char Immediate
5842 instruct storeImmC0(memory mem, immI0 zero)
5843 %{
5844   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5845   match(Set mem (StoreC mem zero));
5846 
5847   ins_cost(125); // XXX
5848   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5849   ins_encode %{
5850     __ movw($mem$$Address, r12);
5851   %}
5852   ins_pipe(ialu_mem_reg);
5853 %}
5854 
5855 instruct storeImmI16(memory mem, immI16 src)
5856 %{
5857   predicate(UseStoreImmI16);
5858   match(Set mem (StoreC mem src));
5859 
5860   ins_cost(150);
5861   format %{ "movw    $mem, $src\t# short/char" %}
5862   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5863   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5864   ins_pipe(ialu_mem_imm);
5865 %}
5866 
5867 // Store Byte Immediate
5868 instruct storeImmB0(memory mem, immI0 zero)
5869 %{
5870   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5871   match(Set mem (StoreB mem zero));
5872 
5873   ins_cost(125); // XXX
5874   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5875   ins_encode %{
5876     __ movb($mem$$Address, r12);
5877   %}
5878   ins_pipe(ialu_mem_reg);
5879 %}
5880 
5881 instruct storeImmB(memory mem, immI8 src)
5882 %{
5883   match(Set mem (StoreB mem src));
5884 
5885   ins_cost(150); // XXX
5886   format %{ "movb    $mem, $src\t# byte" %}
5887   opcode(0xC6); /* C6 /0 */
5888   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5889   ins_pipe(ialu_mem_imm);
5890 %}
5891 
5892 // Store CMS card-mark Immediate
5893 instruct storeImmCM0_reg(memory mem, immI0 zero)
5894 %{
5895   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5896   match(Set mem (StoreCM mem zero));
5897 
5898   ins_cost(125); // XXX
5899   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5900   ins_encode %{
5901     __ movb($mem$$Address, r12);
5902   %}
5903   ins_pipe(ialu_mem_reg);
5904 %}
5905 
5906 instruct storeImmCM0(memory mem, immI0 src)
5907 %{
5908   match(Set mem (StoreCM mem src));
5909 
5910   ins_cost(150); // XXX
5911   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5912   opcode(0xC6); /* C6 /0 */
5913   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5914   ins_pipe(ialu_mem_imm);
5915 %}
5916 
5917 // Store Float
5918 instruct storeF(memory mem, regF src)
5919 %{
5920   match(Set mem (StoreF mem src));
5921 
5922   ins_cost(95); // XXX
5923   format %{ "movss   $mem, $src\t# float" %}
5924   ins_encode %{
5925     __ movflt($mem$$Address, $src$$XMMRegister);
5926   %}
5927   ins_pipe(pipe_slow); // XXX
5928 %}
5929 
5930 // Store immediate Float value (it is faster than store from XMM register)
5931 instruct storeF0(memory mem, immF0 zero)
5932 %{
5933   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5934   match(Set mem (StoreF mem zero));
5935 
5936   ins_cost(25); // XXX
5937   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5938   ins_encode %{
5939     __ movl($mem$$Address, r12);
5940   %}
5941   ins_pipe(ialu_mem_reg);
5942 %}
5943 
5944 instruct storeF_imm(memory mem, immF src)
5945 %{
5946   match(Set mem (StoreF mem src));
5947 
5948   ins_cost(50);
5949   format %{ "movl    $mem, $src\t# float" %}
5950   opcode(0xC7); /* C7 /0 */
5951   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5952   ins_pipe(ialu_mem_imm);
5953 %}
5954 
5955 // Store Double
5956 instruct storeD(memory mem, regD src)
5957 %{
5958   match(Set mem (StoreD mem src));
5959 
5960   ins_cost(95); // XXX
5961   format %{ "movsd   $mem, $src\t# double" %}
5962   ins_encode %{
5963     __ movdbl($mem$$Address, $src$$XMMRegister);
5964   %}
5965   ins_pipe(pipe_slow); // XXX
5966 %}
5967 
5968 // Store immediate double 0.0 (it is faster than store from XMM register)
5969 instruct storeD0_imm(memory mem, immD0 src)
5970 %{
5971   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
5972   match(Set mem (StoreD mem src));
5973 
5974   ins_cost(50);
5975   format %{ "movq    $mem, $src\t# double 0." %}
5976   opcode(0xC7); /* C7 /0 */
5977   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5978   ins_pipe(ialu_mem_imm);
5979 %}
5980 
5981 instruct storeD0(memory mem, immD0 zero)
5982 %{
5983   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5984   match(Set mem (StoreD mem zero));
5985 
5986   ins_cost(25); // XXX
5987   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
5988   ins_encode %{
5989     __ movq($mem$$Address, r12);
5990   %}
5991   ins_pipe(ialu_mem_reg);
5992 %}
5993 
5994 instruct storeSSI(stackSlotI dst, rRegI src)
5995 %{
5996   match(Set dst src);
5997 
5998   ins_cost(100);
5999   format %{ "movl    $dst, $src\t# int stk" %}
6000   opcode(0x89);
6001   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6002   ins_pipe( ialu_mem_reg );
6003 %}
6004 
6005 instruct storeSSL(stackSlotL dst, rRegL src)
6006 %{
6007   match(Set dst src);
6008 
6009   ins_cost(100);
6010   format %{ "movq    $dst, $src\t# long stk" %}
6011   opcode(0x89);
6012   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6013   ins_pipe(ialu_mem_reg);
6014 %}
6015 
6016 instruct storeSSP(stackSlotP dst, rRegP src)
6017 %{
6018   match(Set dst src);
6019 
6020   ins_cost(100);
6021   format %{ "movq    $dst, $src\t# ptr stk" %}
6022   opcode(0x89);
6023   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6024   ins_pipe(ialu_mem_reg);
6025 %}
6026 
6027 instruct storeSSF(stackSlotF dst, regF src)
6028 %{
6029   match(Set dst src);
6030 
6031   ins_cost(95); // XXX
6032   format %{ "movss   $dst, $src\t# float stk" %}
6033   ins_encode %{
6034     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6035   %}
6036   ins_pipe(pipe_slow); // XXX
6037 %}
6038 
6039 instruct storeSSD(stackSlotD dst, regD src)
6040 %{
6041   match(Set dst src);
6042 
6043   ins_cost(95); // XXX
6044   format %{ "movsd   $dst, $src\t# double stk" %}
6045   ins_encode %{
6046     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6047   %}
6048   ins_pipe(pipe_slow); // XXX
6049 %}
6050 
6051 //----------BSWAP Instructions-------------------------------------------------
6052 instruct bytes_reverse_int(rRegI dst) %{
6053   match(Set dst (ReverseBytesI dst));
6054 
6055   format %{ "bswapl  $dst" %}
6056   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6057   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6058   ins_pipe( ialu_reg );
6059 %}
6060 
6061 instruct bytes_reverse_long(rRegL dst) %{
6062   match(Set dst (ReverseBytesL dst));
6063 
6064   format %{ "bswapq  $dst" %}
6065   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6066   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6067   ins_pipe( ialu_reg);
6068 %}
6069 
6070 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6071   match(Set dst (ReverseBytesUS dst));
6072   effect(KILL cr);
6073 
6074   format %{ "bswapl  $dst\n\t"
6075             "shrl    $dst,16\n\t" %}
6076   ins_encode %{
6077     __ bswapl($dst$$Register);
6078     __ shrl($dst$$Register, 16);
6079   %}
6080   ins_pipe( ialu_reg );
6081 %}
6082 
6083 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6084   match(Set dst (ReverseBytesS dst));
6085   effect(KILL cr);
6086 
6087   format %{ "bswapl  $dst\n\t"
6088             "sar     $dst,16\n\t" %}
6089   ins_encode %{
6090     __ bswapl($dst$$Register);
6091     __ sarl($dst$$Register, 16);
6092   %}
6093   ins_pipe( ialu_reg );
6094 %}
6095 
6096 //---------- Zeros Count Instructions ------------------------------------------
6097 
6098 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6099   predicate(UseCountLeadingZerosInstruction);
6100   match(Set dst (CountLeadingZerosI src));
6101   effect(KILL cr);
6102 
6103   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6104   ins_encode %{
6105     __ lzcntl($dst$$Register, $src$$Register);
6106   %}
6107   ins_pipe(ialu_reg);
6108 %}
6109 
6110 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6111   predicate(!UseCountLeadingZerosInstruction);
6112   match(Set dst (CountLeadingZerosI src));
6113   effect(KILL cr);
6114 
6115   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6116             "jnz     skip\n\t"
6117             "movl    $dst, -1\n"
6118       "skip:\n\t"
6119             "negl    $dst\n\t"
6120             "addl    $dst, 31" %}
6121   ins_encode %{
6122     Register Rdst = $dst$$Register;
6123     Register Rsrc = $src$$Register;
6124     Label skip;
6125     __ bsrl(Rdst, Rsrc);
6126     __ jccb(Assembler::notZero, skip);
6127     __ movl(Rdst, -1);
6128     __ bind(skip);
6129     __ negl(Rdst);
6130     __ addl(Rdst, BitsPerInt - 1);
6131   %}
6132   ins_pipe(ialu_reg);
6133 %}
6134 
6135 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6136   predicate(UseCountLeadingZerosInstruction);
6137   match(Set dst (CountLeadingZerosL src));
6138   effect(KILL cr);
6139 
6140   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6141   ins_encode %{
6142     __ lzcntq($dst$$Register, $src$$Register);
6143   %}
6144   ins_pipe(ialu_reg);
6145 %}
6146 
6147 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6148   predicate(!UseCountLeadingZerosInstruction);
6149   match(Set dst (CountLeadingZerosL src));
6150   effect(KILL cr);
6151 
6152   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6153             "jnz     skip\n\t"
6154             "movl    $dst, -1\n"
6155       "skip:\n\t"
6156             "negl    $dst\n\t"
6157             "addl    $dst, 63" %}
6158   ins_encode %{
6159     Register Rdst = $dst$$Register;
6160     Register Rsrc = $src$$Register;
6161     Label skip;
6162     __ bsrq(Rdst, Rsrc);
6163     __ jccb(Assembler::notZero, skip);
6164     __ movl(Rdst, -1);
6165     __ bind(skip);
6166     __ negl(Rdst);
6167     __ addl(Rdst, BitsPerLong - 1);
6168   %}
6169   ins_pipe(ialu_reg);
6170 %}
6171 
6172 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6173   predicate(UseCountTrailingZerosInstruction);
6174   match(Set dst (CountTrailingZerosI src));
6175   effect(KILL cr);
6176 
6177   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6178   ins_encode %{
6179     __ tzcntl($dst$$Register, $src$$Register);
6180   %}
6181   ins_pipe(ialu_reg);
6182 %}
6183 
6184 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6185   predicate(!UseCountTrailingZerosInstruction);
6186   match(Set dst (CountTrailingZerosI src));
6187   effect(KILL cr);
6188 
6189   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6190             "jnz     done\n\t"
6191             "movl    $dst, 32\n"
6192       "done:" %}
6193   ins_encode %{
6194     Register Rdst = $dst$$Register;
6195     Label done;
6196     __ bsfl(Rdst, $src$$Register);
6197     __ jccb(Assembler::notZero, done);
6198     __ movl(Rdst, BitsPerInt);
6199     __ bind(done);
6200   %}
6201   ins_pipe(ialu_reg);
6202 %}
6203 
6204 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6205   predicate(UseCountTrailingZerosInstruction);
6206   match(Set dst (CountTrailingZerosL src));
6207   effect(KILL cr);
6208 
6209   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6210   ins_encode %{
6211     __ tzcntq($dst$$Register, $src$$Register);
6212   %}
6213   ins_pipe(ialu_reg);
6214 %}
6215 
6216 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6217   predicate(!UseCountTrailingZerosInstruction);
6218   match(Set dst (CountTrailingZerosL src));
6219   effect(KILL cr);
6220 
6221   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6222             "jnz     done\n\t"
6223             "movl    $dst, 64\n"
6224       "done:" %}
6225   ins_encode %{
6226     Register Rdst = $dst$$Register;
6227     Label done;
6228     __ bsfq(Rdst, $src$$Register);
6229     __ jccb(Assembler::notZero, done);
6230     __ movl(Rdst, BitsPerLong);
6231     __ bind(done);
6232   %}
6233   ins_pipe(ialu_reg);
6234 %}
6235 
6236 
6237 //---------- Population Count Instructions -------------------------------------
6238 
6239 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6240   predicate(UsePopCountInstruction);
6241   match(Set dst (PopCountI src));
6242   effect(KILL cr);
6243 
6244   format %{ "popcnt  $dst, $src" %}
6245   ins_encode %{
6246     __ popcntl($dst$$Register, $src$$Register);
6247   %}
6248   ins_pipe(ialu_reg);
6249 %}
6250 
6251 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6252   predicate(UsePopCountInstruction);
6253   match(Set dst (PopCountI (LoadI mem)));
6254   effect(KILL cr);
6255 
6256   format %{ "popcnt  $dst, $mem" %}
6257   ins_encode %{
6258     __ popcntl($dst$$Register, $mem$$Address);
6259   %}
6260   ins_pipe(ialu_reg);
6261 %}
6262 
6263 // Note: Long.bitCount(long) returns an int.
6264 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6265   predicate(UsePopCountInstruction);
6266   match(Set dst (PopCountL src));
6267   effect(KILL cr);
6268 
6269   format %{ "popcnt  $dst, $src" %}
6270   ins_encode %{
6271     __ popcntq($dst$$Register, $src$$Register);
6272   %}
6273   ins_pipe(ialu_reg);
6274 %}
6275 
6276 // Note: Long.bitCount(long) returns an int.
6277 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6278   predicate(UsePopCountInstruction);
6279   match(Set dst (PopCountL (LoadL mem)));
6280   effect(KILL cr);
6281 
6282   format %{ "popcnt  $dst, $mem" %}
6283   ins_encode %{
6284     __ popcntq($dst$$Register, $mem$$Address);
6285   %}
6286   ins_pipe(ialu_reg);
6287 %}
6288 
6289 
6290 //----------MemBar Instructions-----------------------------------------------
6291 // Memory barrier flavors
6292 
6293 instruct membar_acquire()
6294 %{
6295   match(MemBarAcquire);
6296   match(LoadFence);
6297   ins_cost(0);
6298 
6299   size(0);
6300   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6301   ins_encode();
6302   ins_pipe(empty);
6303 %}
6304 
6305 instruct membar_acquire_lock()
6306 %{
6307   match(MemBarAcquireLock);
6308   ins_cost(0);
6309 
6310   size(0);
6311   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6312   ins_encode();
6313   ins_pipe(empty);
6314 %}
6315 
6316 instruct membar_release()
6317 %{
6318   match(MemBarRelease);
6319   match(StoreFence);
6320   ins_cost(0);
6321 
6322   size(0);
6323   format %{ "MEMBAR-release ! (empty encoding)" %}
6324   ins_encode();
6325   ins_pipe(empty);
6326 %}
6327 
6328 instruct membar_release_lock()
6329 %{
6330   match(MemBarReleaseLock);
6331   ins_cost(0);
6332 
6333   size(0);
6334   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6335   ins_encode();
6336   ins_pipe(empty);
6337 %}
6338 
6339 instruct membar_volatile(rFlagsReg cr) %{
6340   match(MemBarVolatile);
6341   effect(KILL cr);
6342   ins_cost(400);
6343 
6344   format %{
6345     $$template
6346     if (os::is_MP()) {
6347       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6348     } else {
6349       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6350     }
6351   %}
6352   ins_encode %{
6353     __ membar(Assembler::StoreLoad);
6354   %}
6355   ins_pipe(pipe_slow);
6356 %}
6357 
6358 instruct unnecessary_membar_volatile()
6359 %{
6360   match(MemBarVolatile);
6361   predicate(Matcher::post_store_load_barrier(n));
6362   ins_cost(0);
6363 
6364   size(0);
6365   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6366   ins_encode();
6367   ins_pipe(empty);
6368 %}
6369 
6370 instruct membar_storestore() %{
6371   match(MemBarStoreStore);
6372   ins_cost(0);
6373 
6374   size(0);
6375   format %{ "MEMBAR-storestore (empty encoding)" %}
6376   ins_encode( );
6377   ins_pipe(empty);
6378 %}
6379 
6380 //----------Move Instructions--------------------------------------------------
6381 
6382 instruct castX2P(rRegP dst, rRegL src)
6383 %{
6384   match(Set dst (CastX2P src));
6385 
6386   format %{ "movq    $dst, $src\t# long->ptr" %}
6387   ins_encode %{
6388     if ($dst$$reg != $src$$reg) {
6389       __ movptr($dst$$Register, $src$$Register);
6390     }
6391   %}
6392   ins_pipe(ialu_reg_reg); // XXX
6393 %}
6394 
6395 instruct castP2X(rRegL dst, rRegP src)
6396 %{
6397   match(Set dst (CastP2X src));
6398 
6399   format %{ "movq    $dst, $src\t# ptr -> long" %}
6400   ins_encode %{
6401     if ($dst$$reg != $src$$reg) {
6402       __ movptr($dst$$Register, $src$$Register);
6403     }
6404   %}
6405   ins_pipe(ialu_reg_reg); // XXX
6406 %}
6407 
6408 // Convert oop into int for vectors alignment masking
6409 instruct convP2I(rRegI dst, rRegP src)
6410 %{
6411   match(Set dst (ConvL2I (CastP2X src)));
6412 
6413   format %{ "movl    $dst, $src\t# ptr -> int" %}
6414   ins_encode %{
6415     __ movl($dst$$Register, $src$$Register);
6416   %}
6417   ins_pipe(ialu_reg_reg); // XXX
6418 %}
6419 
6420 // Convert compressed oop into int for vectors alignment masking
6421 // in case of 32bit oops (heap < 4Gb).
6422 instruct convN2I(rRegI dst, rRegN src)
6423 %{
6424   predicate(Universe::narrow_oop_shift() == 0);
6425   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6426 
6427   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6428   ins_encode %{
6429     __ movl($dst$$Register, $src$$Register);
6430   %}
6431   ins_pipe(ialu_reg_reg); // XXX
6432 %}
6433 
6434 // Convert oop pointer into compressed form
6435 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6436   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6437   match(Set dst (EncodeP src));
6438   effect(KILL cr);
6439   format %{ "encode_heap_oop $dst,$src" %}
6440   ins_encode %{
6441     Register s = $src$$Register;
6442     Register d = $dst$$Register;
6443     if (s != d) {
6444       __ movq(d, s);
6445     }
6446     __ encode_heap_oop(d);
6447   %}
6448   ins_pipe(ialu_reg_long);
6449 %}
6450 
6451 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6452   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6453   match(Set dst (EncodeP src));
6454   effect(KILL cr);
6455   format %{ "encode_heap_oop_not_null $dst,$src" %}
6456   ins_encode %{
6457     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6458   %}
6459   ins_pipe(ialu_reg_long);
6460 %}
6461 
6462 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6463   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6464             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6465   match(Set dst (DecodeN src));
6466   effect(KILL cr);
6467   format %{ "decode_heap_oop $dst,$src" %}
6468   ins_encode %{
6469     Register s = $src$$Register;
6470     Register d = $dst$$Register;
6471     if (s != d) {
6472       __ movq(d, s);
6473     }
6474     __ decode_heap_oop(d);
6475   %}
6476   ins_pipe(ialu_reg_long);
6477 %}
6478 
6479 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6480   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6481             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6482   match(Set dst (DecodeN src));
6483   effect(KILL cr);
6484   format %{ "decode_heap_oop_not_null $dst,$src" %}
6485   ins_encode %{
6486     Register s = $src$$Register;
6487     Register d = $dst$$Register;
6488     if (s != d) {
6489       __ decode_heap_oop_not_null(d, s);
6490     } else {
6491       __ decode_heap_oop_not_null(d);
6492     }
6493   %}
6494   ins_pipe(ialu_reg_long);
6495 %}
6496 
6497 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6498   match(Set dst (EncodePKlass src));
6499   effect(KILL cr);
6500   format %{ "encode_klass_not_null $dst,$src" %}
6501   ins_encode %{
6502     __ encode_klass_not_null($dst$$Register, $src$$Register);
6503   %}
6504   ins_pipe(ialu_reg_long);
6505 %}
6506 
6507 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6508   match(Set dst (DecodeNKlass src));
6509   effect(KILL cr);
6510   format %{ "decode_klass_not_null $dst,$src" %}
6511   ins_encode %{
6512     Register s = $src$$Register;
6513     Register d = $dst$$Register;
6514     if (s != d) {
6515       __ decode_klass_not_null(d, s);
6516     } else {
6517       __ decode_klass_not_null(d);
6518     }
6519   %}
6520   ins_pipe(ialu_reg_long);
6521 %}
6522 
6523 
6524 //----------Conditional Move---------------------------------------------------
6525 // Jump
6526 // dummy instruction for generating temp registers
6527 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6528   match(Jump (LShiftL switch_val shift));
6529   ins_cost(350);
6530   predicate(false);
6531   effect(TEMP dest);
6532 
6533   format %{ "leaq    $dest, [$constantaddress]\n\t"
6534             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6535   ins_encode %{
6536     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6537     // to do that and the compiler is using that register as one it can allocate.
6538     // So we build it all by hand.
6539     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6540     // ArrayAddress dispatch(table, index);
6541     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6542     __ lea($dest$$Register, $constantaddress);
6543     __ jmp(dispatch);
6544   %}
6545   ins_pipe(pipe_jmp);
6546 %}
6547 
6548 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6549   match(Jump (AddL (LShiftL switch_val shift) offset));
6550   ins_cost(350);
6551   effect(TEMP dest);
6552 
6553   format %{ "leaq    $dest, [$constantaddress]\n\t"
6554             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6555   ins_encode %{
6556     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6557     // to do that and the compiler is using that register as one it can allocate.
6558     // So we build it all by hand.
6559     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6560     // ArrayAddress dispatch(table, index);
6561     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6562     __ lea($dest$$Register, $constantaddress);
6563     __ jmp(dispatch);
6564   %}
6565   ins_pipe(pipe_jmp);
6566 %}
6567 
6568 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6569   match(Jump switch_val);
6570   ins_cost(350);
6571   effect(TEMP dest);
6572 
6573   format %{ "leaq    $dest, [$constantaddress]\n\t"
6574             "jmp     [$dest + $switch_val]\n\t" %}
6575   ins_encode %{
6576     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6577     // to do that and the compiler is using that register as one it can allocate.
6578     // So we build it all by hand.
6579     // Address index(noreg, switch_reg, Address::times_1);
6580     // ArrayAddress dispatch(table, index);
6581     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6582     __ lea($dest$$Register, $constantaddress);
6583     __ jmp(dispatch);
6584   %}
6585   ins_pipe(pipe_jmp);
6586 %}
6587 
6588 // Conditional move
6589 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6590 %{
6591   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6592 
6593   ins_cost(200); // XXX
6594   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6595   opcode(0x0F, 0x40);
6596   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6597   ins_pipe(pipe_cmov_reg);
6598 %}
6599 
6600 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6601   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6602 
6603   ins_cost(200); // XXX
6604   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6605   opcode(0x0F, 0x40);
6606   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6607   ins_pipe(pipe_cmov_reg);
6608 %}
6609 
6610 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6611   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6612   ins_cost(200);
6613   expand %{
6614     cmovI_regU(cop, cr, dst, src);
6615   %}
6616 %}
6617 
6618 // Conditional move
6619 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6620   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6621 
6622   ins_cost(250); // XXX
6623   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6624   opcode(0x0F, 0x40);
6625   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6626   ins_pipe(pipe_cmov_mem);
6627 %}
6628 
6629 // Conditional move
6630 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6631 %{
6632   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6633 
6634   ins_cost(250); // XXX
6635   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6636   opcode(0x0F, 0x40);
6637   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6638   ins_pipe(pipe_cmov_mem);
6639 %}
6640 
6641 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6642   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6643   ins_cost(250);
6644   expand %{
6645     cmovI_memU(cop, cr, dst, src);
6646   %}
6647 %}
6648 
6649 // Conditional move
6650 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6651 %{
6652   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6653 
6654   ins_cost(200); // XXX
6655   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6656   opcode(0x0F, 0x40);
6657   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6658   ins_pipe(pipe_cmov_reg);
6659 %}
6660 
6661 // Conditional move
6662 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6663 %{
6664   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6665 
6666   ins_cost(200); // XXX
6667   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6668   opcode(0x0F, 0x40);
6669   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6670   ins_pipe(pipe_cmov_reg);
6671 %}
6672 
6673 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6674   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6675   ins_cost(200);
6676   expand %{
6677     cmovN_regU(cop, cr, dst, src);
6678   %}
6679 %}
6680 
6681 // Conditional move
6682 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6683 %{
6684   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6685 
6686   ins_cost(200); // XXX
6687   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6688   opcode(0x0F, 0x40);
6689   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6690   ins_pipe(pipe_cmov_reg);  // XXX
6691 %}
6692 
6693 // Conditional move
6694 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6695 %{
6696   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6697 
6698   ins_cost(200); // XXX
6699   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6700   opcode(0x0F, 0x40);
6701   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6702   ins_pipe(pipe_cmov_reg); // XXX
6703 %}
6704 
6705 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6706   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6707   ins_cost(200);
6708   expand %{
6709     cmovP_regU(cop, cr, dst, src);
6710   %}
6711 %}
6712 
6713 // DISABLED: Requires the ADLC to emit a bottom_type call that
6714 // correctly meets the two pointer arguments; one is an incoming
6715 // register but the other is a memory operand.  ALSO appears to
6716 // be buggy with implicit null checks.
6717 //
6718 //// Conditional move
6719 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6720 //%{
6721 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6722 //  ins_cost(250);
6723 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6724 //  opcode(0x0F,0x40);
6725 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6726 //  ins_pipe( pipe_cmov_mem );
6727 //%}
6728 //
6729 //// Conditional move
6730 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6731 //%{
6732 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6733 //  ins_cost(250);
6734 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6735 //  opcode(0x0F,0x40);
6736 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6737 //  ins_pipe( pipe_cmov_mem );
6738 //%}
6739 
6740 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6741 %{
6742   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6743 
6744   ins_cost(200); // XXX
6745   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6746   opcode(0x0F, 0x40);
6747   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6748   ins_pipe(pipe_cmov_reg);  // XXX
6749 %}
6750 
6751 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6752 %{
6753   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6754 
6755   ins_cost(200); // XXX
6756   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6757   opcode(0x0F, 0x40);
6758   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6759   ins_pipe(pipe_cmov_mem);  // XXX
6760 %}
6761 
6762 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6763 %{
6764   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6765 
6766   ins_cost(200); // XXX
6767   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6768   opcode(0x0F, 0x40);
6769   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6770   ins_pipe(pipe_cmov_reg); // XXX
6771 %}
6772 
6773 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6774   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6775   ins_cost(200);
6776   expand %{
6777     cmovL_regU(cop, cr, dst, src);
6778   %}
6779 %}
6780 
6781 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6782 %{
6783   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6784 
6785   ins_cost(200); // XXX
6786   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6787   opcode(0x0F, 0x40);
6788   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6789   ins_pipe(pipe_cmov_mem); // XXX
6790 %}
6791 
6792 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6793   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6794   ins_cost(200);
6795   expand %{
6796     cmovL_memU(cop, cr, dst, src);
6797   %}
6798 %}
6799 
6800 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6801 %{
6802   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6803 
6804   ins_cost(200); // XXX
6805   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6806             "movss     $dst, $src\n"
6807     "skip:" %}
6808   ins_encode %{
6809     Label Lskip;
6810     // Invert sense of branch from sense of CMOV
6811     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6812     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6813     __ bind(Lskip);
6814   %}
6815   ins_pipe(pipe_slow);
6816 %}
6817 
6818 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6819 // %{
6820 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6821 
6822 //   ins_cost(200); // XXX
6823 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6824 //             "movss     $dst, $src\n"
6825 //     "skip:" %}
6826 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6827 //   ins_pipe(pipe_slow);
6828 // %}
6829 
6830 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6831 %{
6832   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6833 
6834   ins_cost(200); // XXX
6835   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6836             "movss     $dst, $src\n"
6837     "skip:" %}
6838   ins_encode %{
6839     Label Lskip;
6840     // Invert sense of branch from sense of CMOV
6841     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6842     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6843     __ bind(Lskip);
6844   %}
6845   ins_pipe(pipe_slow);
6846 %}
6847 
6848 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6849   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6850   ins_cost(200);
6851   expand %{
6852     cmovF_regU(cop, cr, dst, src);
6853   %}
6854 %}
6855 
6856 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6857 %{
6858   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6859 
6860   ins_cost(200); // XXX
6861   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6862             "movsd     $dst, $src\n"
6863     "skip:" %}
6864   ins_encode %{
6865     Label Lskip;
6866     // Invert sense of branch from sense of CMOV
6867     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6868     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6869     __ bind(Lskip);
6870   %}
6871   ins_pipe(pipe_slow);
6872 %}
6873 
6874 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6875 %{
6876   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6877 
6878   ins_cost(200); // XXX
6879   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6880             "movsd     $dst, $src\n"
6881     "skip:" %}
6882   ins_encode %{
6883     Label Lskip;
6884     // Invert sense of branch from sense of CMOV
6885     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6886     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6887     __ bind(Lskip);
6888   %}
6889   ins_pipe(pipe_slow);
6890 %}
6891 
6892 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6893   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6894   ins_cost(200);
6895   expand %{
6896     cmovD_regU(cop, cr, dst, src);
6897   %}
6898 %}
6899 
6900 //----------Arithmetic Instructions--------------------------------------------
6901 //----------Addition Instructions----------------------------------------------
6902 
6903 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6904 %{
6905   match(Set dst (AddI dst src));
6906   effect(KILL cr);
6907 
6908   format %{ "addl    $dst, $src\t# int" %}
6909   opcode(0x03);
6910   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6911   ins_pipe(ialu_reg_reg);
6912 %}
6913 
6914 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6915 %{
6916   match(Set dst (AddI dst src));
6917   effect(KILL cr);
6918 
6919   format %{ "addl    $dst, $src\t# int" %}
6920   opcode(0x81, 0x00); /* /0 id */
6921   ins_encode(OpcSErm(dst, src), Con8or32(src));
6922   ins_pipe( ialu_reg );
6923 %}
6924 
6925 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6926 %{
6927   match(Set dst (AddI dst (LoadI src)));
6928   effect(KILL cr);
6929 
6930   ins_cost(125); // XXX
6931   format %{ "addl    $dst, $src\t# int" %}
6932   opcode(0x03);
6933   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6934   ins_pipe(ialu_reg_mem);
6935 %}
6936 
6937 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6938 %{
6939   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6940   effect(KILL cr);
6941 
6942   ins_cost(150); // XXX
6943   format %{ "addl    $dst, $src\t# int" %}
6944   opcode(0x01); /* Opcode 01 /r */
6945   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6946   ins_pipe(ialu_mem_reg);
6947 %}
6948 
6949 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6950 %{
6951   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6952   effect(KILL cr);
6953 
6954   ins_cost(125); // XXX
6955   format %{ "addl    $dst, $src\t# int" %}
6956   opcode(0x81); /* Opcode 81 /0 id */
6957   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6958   ins_pipe(ialu_mem_imm);
6959 %}
6960 
6961 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
6962 %{
6963   predicate(UseIncDec);
6964   match(Set dst (AddI dst src));
6965   effect(KILL cr);
6966 
6967   format %{ "incl    $dst\t# int" %}
6968   opcode(0xFF, 0x00); // FF /0
6969   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6970   ins_pipe(ialu_reg);
6971 %}
6972 
6973 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
6974 %{
6975   predicate(UseIncDec);
6976   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6977   effect(KILL cr);
6978 
6979   ins_cost(125); // XXX
6980   format %{ "incl    $dst\t# int" %}
6981   opcode(0xFF); /* Opcode FF /0 */
6982   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
6983   ins_pipe(ialu_mem_imm);
6984 %}
6985 
6986 // XXX why does that use AddI
6987 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
6988 %{
6989   predicate(UseIncDec);
6990   match(Set dst (AddI dst src));
6991   effect(KILL cr);
6992 
6993   format %{ "decl    $dst\t# int" %}
6994   opcode(0xFF, 0x01); // FF /1
6995   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6996   ins_pipe(ialu_reg);
6997 %}
6998 
6999 // XXX why does that use AddI
7000 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7001 %{
7002   predicate(UseIncDec);
7003   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7004   effect(KILL cr);
7005 
7006   ins_cost(125); // XXX
7007   format %{ "decl    $dst\t# int" %}
7008   opcode(0xFF); /* Opcode FF /1 */
7009   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7010   ins_pipe(ialu_mem_imm);
7011 %}
7012 
7013 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7014 %{
7015   match(Set dst (AddI src0 src1));
7016 
7017   ins_cost(110);
7018   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7019   opcode(0x8D); /* 0x8D /r */
7020   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7021   ins_pipe(ialu_reg_reg);
7022 %}
7023 
7024 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7025 %{
7026   match(Set dst (AddL dst src));
7027   effect(KILL cr);
7028 
7029   format %{ "addq    $dst, $src\t# long" %}
7030   opcode(0x03);
7031   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7032   ins_pipe(ialu_reg_reg);
7033 %}
7034 
7035 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7036 %{
7037   match(Set dst (AddL dst src));
7038   effect(KILL cr);
7039 
7040   format %{ "addq    $dst, $src\t# long" %}
7041   opcode(0x81, 0x00); /* /0 id */
7042   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7043   ins_pipe( ialu_reg );
7044 %}
7045 
7046 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7047 %{
7048   match(Set dst (AddL dst (LoadL src)));
7049   effect(KILL cr);
7050 
7051   ins_cost(125); // XXX
7052   format %{ "addq    $dst, $src\t# long" %}
7053   opcode(0x03);
7054   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7055   ins_pipe(ialu_reg_mem);
7056 %}
7057 
7058 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7059 %{
7060   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7061   effect(KILL cr);
7062 
7063   ins_cost(150); // XXX
7064   format %{ "addq    $dst, $src\t# long" %}
7065   opcode(0x01); /* Opcode 01 /r */
7066   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7067   ins_pipe(ialu_mem_reg);
7068 %}
7069 
7070 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7071 %{
7072   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7073   effect(KILL cr);
7074 
7075   ins_cost(125); // XXX
7076   format %{ "addq    $dst, $src\t# long" %}
7077   opcode(0x81); /* Opcode 81 /0 id */
7078   ins_encode(REX_mem_wide(dst),
7079              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7080   ins_pipe(ialu_mem_imm);
7081 %}
7082 
7083 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7084 %{
7085   predicate(UseIncDec);
7086   match(Set dst (AddL dst src));
7087   effect(KILL cr);
7088 
7089   format %{ "incq    $dst\t# long" %}
7090   opcode(0xFF, 0x00); // FF /0
7091   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7092   ins_pipe(ialu_reg);
7093 %}
7094 
7095 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7096 %{
7097   predicate(UseIncDec);
7098   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7099   effect(KILL cr);
7100 
7101   ins_cost(125); // XXX
7102   format %{ "incq    $dst\t# long" %}
7103   opcode(0xFF); /* Opcode FF /0 */
7104   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7105   ins_pipe(ialu_mem_imm);
7106 %}
7107 
7108 // XXX why does that use AddL
7109 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7110 %{
7111   predicate(UseIncDec);
7112   match(Set dst (AddL dst src));
7113   effect(KILL cr);
7114 
7115   format %{ "decq    $dst\t# long" %}
7116   opcode(0xFF, 0x01); // FF /1
7117   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7118   ins_pipe(ialu_reg);
7119 %}
7120 
7121 // XXX why does that use AddL
7122 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7123 %{
7124   predicate(UseIncDec);
7125   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7126   effect(KILL cr);
7127 
7128   ins_cost(125); // XXX
7129   format %{ "decq    $dst\t# long" %}
7130   opcode(0xFF); /* Opcode FF /1 */
7131   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7132   ins_pipe(ialu_mem_imm);
7133 %}
7134 
7135 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7136 %{
7137   match(Set dst (AddL src0 src1));
7138 
7139   ins_cost(110);
7140   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7141   opcode(0x8D); /* 0x8D /r */
7142   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7143   ins_pipe(ialu_reg_reg);
7144 %}
7145 
7146 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7147 %{
7148   match(Set dst (AddP dst src));
7149   effect(KILL cr);
7150 
7151   format %{ "addq    $dst, $src\t# ptr" %}
7152   opcode(0x03);
7153   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7154   ins_pipe(ialu_reg_reg);
7155 %}
7156 
7157 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7158 %{
7159   match(Set dst (AddP dst src));
7160   effect(KILL cr);
7161 
7162   format %{ "addq    $dst, $src\t# ptr" %}
7163   opcode(0x81, 0x00); /* /0 id */
7164   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7165   ins_pipe( ialu_reg );
7166 %}
7167 
7168 // XXX addP mem ops ????
7169 
7170 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7171 %{
7172   match(Set dst (AddP src0 src1));
7173 
7174   ins_cost(110);
7175   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7176   opcode(0x8D); /* 0x8D /r */
7177   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7178   ins_pipe(ialu_reg_reg);
7179 %}
7180 
7181 instruct checkCastPP(rRegP dst)
7182 %{
7183   match(Set dst (CheckCastPP dst));
7184 
7185   size(0);
7186   format %{ "# checkcastPP of $dst" %}
7187   ins_encode(/* empty encoding */);
7188   ins_pipe(empty);
7189 %}
7190 
7191 instruct castPP(rRegP dst)
7192 %{
7193   match(Set dst (CastPP dst));
7194 
7195   size(0);
7196   format %{ "# castPP of $dst" %}
7197   ins_encode(/* empty encoding */);
7198   ins_pipe(empty);
7199 %}
7200 
7201 instruct castII(rRegI dst)
7202 %{
7203   match(Set dst (CastII dst));
7204 
7205   size(0);
7206   format %{ "# castII of $dst" %}
7207   ins_encode(/* empty encoding */);
7208   ins_cost(0);
7209   ins_pipe(empty);
7210 %}
7211 
7212 // LoadP-locked same as a regular LoadP when used with compare-swap
7213 instruct loadPLocked(rRegP dst, memory mem)
7214 %{
7215   match(Set dst (LoadPLocked mem));
7216 
7217   ins_cost(125); // XXX
7218   format %{ "movq    $dst, $mem\t# ptr locked" %}
7219   opcode(0x8B);
7220   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7221   ins_pipe(ialu_reg_mem); // XXX
7222 %}
7223 
7224 // Conditional-store of the updated heap-top.
7225 // Used during allocation of the shared heap.
7226 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7227 
7228 instruct storePConditional(memory heap_top_ptr,
7229                            rax_RegP oldval, rRegP newval,
7230                            rFlagsReg cr)
7231 %{
7232   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7233 
7234   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7235             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7236   opcode(0x0F, 0xB1);
7237   ins_encode(lock_prefix,
7238              REX_reg_mem_wide(newval, heap_top_ptr),
7239              OpcP, OpcS,
7240              reg_mem(newval, heap_top_ptr));
7241   ins_pipe(pipe_cmpxchg);
7242 %}
7243 
7244 // Conditional-store of an int value.
7245 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7246 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7247 %{
7248   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7249   effect(KILL oldval);
7250 
7251   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7252   opcode(0x0F, 0xB1);
7253   ins_encode(lock_prefix,
7254              REX_reg_mem(newval, mem),
7255              OpcP, OpcS,
7256              reg_mem(newval, mem));
7257   ins_pipe(pipe_cmpxchg);
7258 %}
7259 
7260 // Conditional-store of a long value.
7261 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7262 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7263 %{
7264   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7265   effect(KILL oldval);
7266 
7267   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7268   opcode(0x0F, 0xB1);
7269   ins_encode(lock_prefix,
7270              REX_reg_mem_wide(newval, mem),
7271              OpcP, OpcS,
7272              reg_mem(newval, mem));
7273   ins_pipe(pipe_cmpxchg);
7274 %}
7275 
7276 
7277 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7278 instruct compareAndSwapP(rRegI res,
7279                          memory mem_ptr,
7280                          rax_RegP oldval, rRegP newval,
7281                          rFlagsReg cr)
7282 %{
7283   predicate(VM_Version::supports_cx8());
7284   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7285   effect(KILL cr, KILL oldval);
7286 
7287   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7288             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7289             "sete    $res\n\t"
7290             "movzbl  $res, $res" %}
7291   opcode(0x0F, 0xB1);
7292   ins_encode(lock_prefix,
7293              REX_reg_mem_wide(newval, mem_ptr),
7294              OpcP, OpcS,
7295              reg_mem(newval, mem_ptr),
7296              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7297              REX_reg_breg(res, res), // movzbl
7298              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7299   ins_pipe( pipe_cmpxchg );
7300 %}
7301 
7302 instruct compareAndSwapL(rRegI res,
7303                          memory mem_ptr,
7304                          rax_RegL oldval, rRegL newval,
7305                          rFlagsReg cr)
7306 %{
7307   predicate(VM_Version::supports_cx8());
7308   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7309   effect(KILL cr, KILL oldval);
7310 
7311   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7312             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7313             "sete    $res\n\t"
7314             "movzbl  $res, $res" %}
7315   opcode(0x0F, 0xB1);
7316   ins_encode(lock_prefix,
7317              REX_reg_mem_wide(newval, mem_ptr),
7318              OpcP, OpcS,
7319              reg_mem(newval, mem_ptr),
7320              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7321              REX_reg_breg(res, res), // movzbl
7322              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7323   ins_pipe( pipe_cmpxchg );
7324 %}
7325 
7326 instruct compareAndSwapI(rRegI res,
7327                          memory mem_ptr,
7328                          rax_RegI oldval, rRegI newval,
7329                          rFlagsReg cr)
7330 %{
7331   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7332   effect(KILL cr, KILL oldval);
7333 
7334   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7335             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7336             "sete    $res\n\t"
7337             "movzbl  $res, $res" %}
7338   opcode(0x0F, 0xB1);
7339   ins_encode(lock_prefix,
7340              REX_reg_mem(newval, mem_ptr),
7341              OpcP, OpcS,
7342              reg_mem(newval, mem_ptr),
7343              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7344              REX_reg_breg(res, res), // movzbl
7345              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7346   ins_pipe( pipe_cmpxchg );
7347 %}
7348 
7349 
7350 instruct compareAndSwapN(rRegI res,
7351                           memory mem_ptr,
7352                           rax_RegN oldval, rRegN newval,
7353                           rFlagsReg cr) %{
7354   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7355   effect(KILL cr, KILL oldval);
7356 
7357   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7358             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7359             "sete    $res\n\t"
7360             "movzbl  $res, $res" %}
7361   opcode(0x0F, 0xB1);
7362   ins_encode(lock_prefix,
7363              REX_reg_mem(newval, mem_ptr),
7364              OpcP, OpcS,
7365              reg_mem(newval, mem_ptr),
7366              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7367              REX_reg_breg(res, res), // movzbl
7368              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7369   ins_pipe( pipe_cmpxchg );
7370 %}
7371 
7372 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7373   predicate(n->as_LoadStore()->result_not_used());
7374   match(Set dummy (GetAndAddI mem add));
7375   effect(KILL cr);
7376   format %{ "ADDL  [$mem],$add" %}
7377   ins_encode %{
7378     if (os::is_MP()) { __ lock(); }
7379     __ addl($mem$$Address, $add$$constant);
7380   %}
7381   ins_pipe( pipe_cmpxchg );
7382 %}
7383 
7384 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7385   match(Set newval (GetAndAddI mem newval));
7386   effect(KILL cr);
7387   format %{ "XADDL  [$mem],$newval" %}
7388   ins_encode %{
7389     if (os::is_MP()) { __ lock(); }
7390     __ xaddl($mem$$Address, $newval$$Register);
7391   %}
7392   ins_pipe( pipe_cmpxchg );
7393 %}
7394 
7395 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7396   predicate(n->as_LoadStore()->result_not_used());
7397   match(Set dummy (GetAndAddL mem add));
7398   effect(KILL cr);
7399   format %{ "ADDQ  [$mem],$add" %}
7400   ins_encode %{
7401     if (os::is_MP()) { __ lock(); }
7402     __ addq($mem$$Address, $add$$constant);
7403   %}
7404   ins_pipe( pipe_cmpxchg );
7405 %}
7406 
7407 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7408   match(Set newval (GetAndAddL mem newval));
7409   effect(KILL cr);
7410   format %{ "XADDQ  [$mem],$newval" %}
7411   ins_encode %{
7412     if (os::is_MP()) { __ lock(); }
7413     __ xaddq($mem$$Address, $newval$$Register);
7414   %}
7415   ins_pipe( pipe_cmpxchg );
7416 %}
7417 
7418 instruct xchgI( memory mem, rRegI newval) %{
7419   match(Set newval (GetAndSetI mem newval));
7420   format %{ "XCHGL  $newval,[$mem]" %}
7421   ins_encode %{
7422     __ xchgl($newval$$Register, $mem$$Address);
7423   %}
7424   ins_pipe( pipe_cmpxchg );
7425 %}
7426 
7427 instruct xchgL( memory mem, rRegL newval) %{
7428   match(Set newval (GetAndSetL mem newval));
7429   format %{ "XCHGL  $newval,[$mem]" %}
7430   ins_encode %{
7431     __ xchgq($newval$$Register, $mem$$Address);
7432   %}
7433   ins_pipe( pipe_cmpxchg );
7434 %}
7435 
7436 instruct xchgP( memory mem, rRegP newval) %{
7437   match(Set newval (GetAndSetP mem newval));
7438   format %{ "XCHGQ  $newval,[$mem]" %}
7439   ins_encode %{
7440     __ xchgq($newval$$Register, $mem$$Address);
7441   %}
7442   ins_pipe( pipe_cmpxchg );
7443 %}
7444 
7445 instruct xchgN( memory mem, rRegN newval) %{
7446   match(Set newval (GetAndSetN mem newval));
7447   format %{ "XCHGL  $newval,$mem]" %}
7448   ins_encode %{
7449     __ xchgl($newval$$Register, $mem$$Address);
7450   %}
7451   ins_pipe( pipe_cmpxchg );
7452 %}
7453 
7454 //----------Subtraction Instructions-------------------------------------------
7455 
7456 // Integer Subtraction Instructions
7457 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7458 %{
7459   match(Set dst (SubI dst src));
7460   effect(KILL cr);
7461 
7462   format %{ "subl    $dst, $src\t# int" %}
7463   opcode(0x2B);
7464   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7465   ins_pipe(ialu_reg_reg);
7466 %}
7467 
7468 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7469 %{
7470   match(Set dst (SubI dst src));
7471   effect(KILL cr);
7472 
7473   format %{ "subl    $dst, $src\t# int" %}
7474   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7475   ins_encode(OpcSErm(dst, src), Con8or32(src));
7476   ins_pipe(ialu_reg);
7477 %}
7478 
7479 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7480 %{
7481   match(Set dst (SubI dst (LoadI src)));
7482   effect(KILL cr);
7483 
7484   ins_cost(125);
7485   format %{ "subl    $dst, $src\t# int" %}
7486   opcode(0x2B);
7487   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7488   ins_pipe(ialu_reg_mem);
7489 %}
7490 
7491 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7492 %{
7493   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7494   effect(KILL cr);
7495 
7496   ins_cost(150);
7497   format %{ "subl    $dst, $src\t# int" %}
7498   opcode(0x29); /* Opcode 29 /r */
7499   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7500   ins_pipe(ialu_mem_reg);
7501 %}
7502 
7503 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7504 %{
7505   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7506   effect(KILL cr);
7507 
7508   ins_cost(125); // XXX
7509   format %{ "subl    $dst, $src\t# int" %}
7510   opcode(0x81); /* Opcode 81 /5 id */
7511   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7512   ins_pipe(ialu_mem_imm);
7513 %}
7514 
7515 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7516 %{
7517   match(Set dst (SubL dst src));
7518   effect(KILL cr);
7519 
7520   format %{ "subq    $dst, $src\t# long" %}
7521   opcode(0x2B);
7522   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7523   ins_pipe(ialu_reg_reg);
7524 %}
7525 
7526 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7527 %{
7528   match(Set dst (SubL dst src));
7529   effect(KILL cr);
7530 
7531   format %{ "subq    $dst, $src\t# long" %}
7532   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7533   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7534   ins_pipe(ialu_reg);
7535 %}
7536 
7537 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7538 %{
7539   match(Set dst (SubL dst (LoadL src)));
7540   effect(KILL cr);
7541 
7542   ins_cost(125);
7543   format %{ "subq    $dst, $src\t# long" %}
7544   opcode(0x2B);
7545   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7546   ins_pipe(ialu_reg_mem);
7547 %}
7548 
7549 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7550 %{
7551   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7552   effect(KILL cr);
7553 
7554   ins_cost(150);
7555   format %{ "subq    $dst, $src\t# long" %}
7556   opcode(0x29); /* Opcode 29 /r */
7557   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7558   ins_pipe(ialu_mem_reg);
7559 %}
7560 
7561 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7562 %{
7563   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7564   effect(KILL cr);
7565 
7566   ins_cost(125); // XXX
7567   format %{ "subq    $dst, $src\t# long" %}
7568   opcode(0x81); /* Opcode 81 /5 id */
7569   ins_encode(REX_mem_wide(dst),
7570              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7571   ins_pipe(ialu_mem_imm);
7572 %}
7573 
7574 // Subtract from a pointer
7575 // XXX hmpf???
7576 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7577 %{
7578   match(Set dst (AddP dst (SubI zero src)));
7579   effect(KILL cr);
7580 
7581   format %{ "subq    $dst, $src\t# ptr - int" %}
7582   opcode(0x2B);
7583   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7584   ins_pipe(ialu_reg_reg);
7585 %}
7586 
7587 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7588 %{
7589   match(Set dst (SubI zero dst));
7590   effect(KILL cr);
7591 
7592   format %{ "negl    $dst\t# int" %}
7593   opcode(0xF7, 0x03);  // Opcode F7 /3
7594   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7595   ins_pipe(ialu_reg);
7596 %}
7597 
7598 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7599 %{
7600   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7601   effect(KILL cr);
7602 
7603   format %{ "negl    $dst\t# int" %}
7604   opcode(0xF7, 0x03);  // Opcode F7 /3
7605   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7606   ins_pipe(ialu_reg);
7607 %}
7608 
7609 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7610 %{
7611   match(Set dst (SubL zero dst));
7612   effect(KILL cr);
7613 
7614   format %{ "negq    $dst\t# long" %}
7615   opcode(0xF7, 0x03);  // Opcode F7 /3
7616   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7617   ins_pipe(ialu_reg);
7618 %}
7619 
7620 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7621 %{
7622   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7623   effect(KILL cr);
7624 
7625   format %{ "negq    $dst\t# long" %}
7626   opcode(0xF7, 0x03);  // Opcode F7 /3
7627   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7628   ins_pipe(ialu_reg);
7629 %}
7630 
7631 //----------Multiplication/Division Instructions-------------------------------
7632 // Integer Multiplication Instructions
7633 // Multiply Register
7634 
7635 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7636 %{
7637   match(Set dst (MulI dst src));
7638   effect(KILL cr);
7639 
7640   ins_cost(300);
7641   format %{ "imull   $dst, $src\t# int" %}
7642   opcode(0x0F, 0xAF);
7643   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7644   ins_pipe(ialu_reg_reg_alu0);
7645 %}
7646 
7647 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7648 %{
7649   match(Set dst (MulI src imm));
7650   effect(KILL cr);
7651 
7652   ins_cost(300);
7653   format %{ "imull   $dst, $src, $imm\t# int" %}
7654   opcode(0x69); /* 69 /r id */
7655   ins_encode(REX_reg_reg(dst, src),
7656              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7657   ins_pipe(ialu_reg_reg_alu0);
7658 %}
7659 
7660 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7661 %{
7662   match(Set dst (MulI dst (LoadI src)));
7663   effect(KILL cr);
7664 
7665   ins_cost(350);
7666   format %{ "imull   $dst, $src\t# int" %}
7667   opcode(0x0F, 0xAF);
7668   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7669   ins_pipe(ialu_reg_mem_alu0);
7670 %}
7671 
7672 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7673 %{
7674   match(Set dst (MulI (LoadI src) imm));
7675   effect(KILL cr);
7676 
7677   ins_cost(300);
7678   format %{ "imull   $dst, $src, $imm\t# int" %}
7679   opcode(0x69); /* 69 /r id */
7680   ins_encode(REX_reg_mem(dst, src),
7681              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7682   ins_pipe(ialu_reg_mem_alu0);
7683 %}
7684 
7685 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7686 %{
7687   match(Set dst (MulL dst src));
7688   effect(KILL cr);
7689 
7690   ins_cost(300);
7691   format %{ "imulq   $dst, $src\t# long" %}
7692   opcode(0x0F, 0xAF);
7693   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7694   ins_pipe(ialu_reg_reg_alu0);
7695 %}
7696 
7697 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7698 %{
7699   match(Set dst (MulL src imm));
7700   effect(KILL cr);
7701 
7702   ins_cost(300);
7703   format %{ "imulq   $dst, $src, $imm\t# long" %}
7704   opcode(0x69); /* 69 /r id */
7705   ins_encode(REX_reg_reg_wide(dst, src),
7706              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7707   ins_pipe(ialu_reg_reg_alu0);
7708 %}
7709 
7710 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7711 %{
7712   match(Set dst (MulL dst (LoadL src)));
7713   effect(KILL cr);
7714 
7715   ins_cost(350);
7716   format %{ "imulq   $dst, $src\t# long" %}
7717   opcode(0x0F, 0xAF);
7718   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7719   ins_pipe(ialu_reg_mem_alu0);
7720 %}
7721 
7722 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7723 %{
7724   match(Set dst (MulL (LoadL src) imm));
7725   effect(KILL cr);
7726 
7727   ins_cost(300);
7728   format %{ "imulq   $dst, $src, $imm\t# long" %}
7729   opcode(0x69); /* 69 /r id */
7730   ins_encode(REX_reg_mem_wide(dst, src),
7731              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7732   ins_pipe(ialu_reg_mem_alu0);
7733 %}
7734 
7735 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7736 %{
7737   match(Set dst (MulHiL src rax));
7738   effect(USE_KILL rax, KILL cr);
7739 
7740   ins_cost(300);
7741   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7742   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7743   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7744   ins_pipe(ialu_reg_reg_alu0);
7745 %}
7746 
7747 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7748                    rFlagsReg cr)
7749 %{
7750   match(Set rax (DivI rax div));
7751   effect(KILL rdx, KILL cr);
7752 
7753   ins_cost(30*100+10*100); // XXX
7754   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7755             "jne,s   normal\n\t"
7756             "xorl    rdx, rdx\n\t"
7757             "cmpl    $div, -1\n\t"
7758             "je,s    done\n"
7759     "normal: cdql\n\t"
7760             "idivl   $div\n"
7761     "done:"        %}
7762   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7763   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7764   ins_pipe(ialu_reg_reg_alu0);
7765 %}
7766 
7767 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7768                    rFlagsReg cr)
7769 %{
7770   match(Set rax (DivL rax div));
7771   effect(KILL rdx, KILL cr);
7772 
7773   ins_cost(30*100+10*100); // XXX
7774   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7775             "cmpq    rax, rdx\n\t"
7776             "jne,s   normal\n\t"
7777             "xorl    rdx, rdx\n\t"
7778             "cmpq    $div, -1\n\t"
7779             "je,s    done\n"
7780     "normal: cdqq\n\t"
7781             "idivq   $div\n"
7782     "done:"        %}
7783   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7784   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7785   ins_pipe(ialu_reg_reg_alu0);
7786 %}
7787 
7788 // Integer DIVMOD with Register, both quotient and mod results
7789 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7790                              rFlagsReg cr)
7791 %{
7792   match(DivModI rax div);
7793   effect(KILL cr);
7794 
7795   ins_cost(30*100+10*100); // XXX
7796   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7797             "jne,s   normal\n\t"
7798             "xorl    rdx, rdx\n\t"
7799             "cmpl    $div, -1\n\t"
7800             "je,s    done\n"
7801     "normal: cdql\n\t"
7802             "idivl   $div\n"
7803     "done:"        %}
7804   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7805   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7806   ins_pipe(pipe_slow);
7807 %}
7808 
7809 // Long DIVMOD with Register, both quotient and mod results
7810 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7811                              rFlagsReg cr)
7812 %{
7813   match(DivModL rax div);
7814   effect(KILL cr);
7815 
7816   ins_cost(30*100+10*100); // XXX
7817   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7818             "cmpq    rax, rdx\n\t"
7819             "jne,s   normal\n\t"
7820             "xorl    rdx, rdx\n\t"
7821             "cmpq    $div, -1\n\t"
7822             "je,s    done\n"
7823     "normal: cdqq\n\t"
7824             "idivq   $div\n"
7825     "done:"        %}
7826   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7827   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7828   ins_pipe(pipe_slow);
7829 %}
7830 
7831 //----------- DivL-By-Constant-Expansions--------------------------------------
7832 // DivI cases are handled by the compiler
7833 
7834 // Magic constant, reciprocal of 10
7835 instruct loadConL_0x6666666666666667(rRegL dst)
7836 %{
7837   effect(DEF dst);
7838 
7839   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
7840   ins_encode(load_immL(dst, 0x6666666666666667));
7841   ins_pipe(ialu_reg);
7842 %}
7843 
7844 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7845 %{
7846   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
7847 
7848   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
7849   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7850   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7851   ins_pipe(ialu_reg_reg_alu0);
7852 %}
7853 
7854 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
7855 %{
7856   effect(USE_DEF dst, KILL cr);
7857 
7858   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
7859   opcode(0xC1, 0x7); /* C1 /7 ib */
7860   ins_encode(reg_opc_imm_wide(dst, 0x3F));
7861   ins_pipe(ialu_reg);
7862 %}
7863 
7864 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
7865 %{
7866   effect(USE_DEF dst, KILL cr);
7867 
7868   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
7869   opcode(0xC1, 0x7); /* C1 /7 ib */
7870   ins_encode(reg_opc_imm_wide(dst, 0x2));
7871   ins_pipe(ialu_reg);
7872 %}
7873 
7874 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
7875 %{
7876   match(Set dst (DivL src div));
7877 
7878   ins_cost((5+8)*100);
7879   expand %{
7880     rax_RegL rax;                     // Killed temp
7881     rFlagsReg cr;                     // Killed
7882     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
7883     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
7884     sarL_rReg_63(src, cr);            // sarq  src, 63
7885     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
7886     subL_rReg(dst, src, cr);          // subl  rdx, src
7887   %}
7888 %}
7889 
7890 //-----------------------------------------------------------------------------
7891 
7892 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
7893                    rFlagsReg cr)
7894 %{
7895   match(Set rdx (ModI rax div));
7896   effect(KILL rax, KILL cr);
7897 
7898   ins_cost(300); // XXX
7899   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
7900             "jne,s   normal\n\t"
7901             "xorl    rdx, rdx\n\t"
7902             "cmpl    $div, -1\n\t"
7903             "je,s    done\n"
7904     "normal: cdql\n\t"
7905             "idivl   $div\n"
7906     "done:"        %}
7907   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7908   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7909   ins_pipe(ialu_reg_reg_alu0);
7910 %}
7911 
7912 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
7913                    rFlagsReg cr)
7914 %{
7915   match(Set rdx (ModL rax div));
7916   effect(KILL rax, KILL cr);
7917 
7918   ins_cost(300); // XXX
7919   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
7920             "cmpq    rax, rdx\n\t"
7921             "jne,s   normal\n\t"
7922             "xorl    rdx, rdx\n\t"
7923             "cmpq    $div, -1\n\t"
7924             "je,s    done\n"
7925     "normal: cdqq\n\t"
7926             "idivq   $div\n"
7927     "done:"        %}
7928   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7929   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7930   ins_pipe(ialu_reg_reg_alu0);
7931 %}
7932 
7933 // Integer Shift Instructions
7934 // Shift Left by one
7935 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7936 %{
7937   match(Set dst (LShiftI dst shift));
7938   effect(KILL cr);
7939 
7940   format %{ "sall    $dst, $shift" %}
7941   opcode(0xD1, 0x4); /* D1 /4 */
7942   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7943   ins_pipe(ialu_reg);
7944 %}
7945 
7946 // Shift Left by one
7947 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7948 %{
7949   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7950   effect(KILL cr);
7951 
7952   format %{ "sall    $dst, $shift\t" %}
7953   opcode(0xD1, 0x4); /* D1 /4 */
7954   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7955   ins_pipe(ialu_mem_imm);
7956 %}
7957 
7958 // Shift Left by 8-bit immediate
7959 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7960 %{
7961   match(Set dst (LShiftI dst shift));
7962   effect(KILL cr);
7963 
7964   format %{ "sall    $dst, $shift" %}
7965   opcode(0xC1, 0x4); /* C1 /4 ib */
7966   ins_encode(reg_opc_imm(dst, shift));
7967   ins_pipe(ialu_reg);
7968 %}
7969 
7970 // Shift Left by 8-bit immediate
7971 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7972 %{
7973   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7974   effect(KILL cr);
7975 
7976   format %{ "sall    $dst, $shift" %}
7977   opcode(0xC1, 0x4); /* C1 /4 ib */
7978   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7979   ins_pipe(ialu_mem_imm);
7980 %}
7981 
7982 // Shift Left by variable
7983 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7984 %{
7985   match(Set dst (LShiftI dst shift));
7986   effect(KILL cr);
7987 
7988   format %{ "sall    $dst, $shift" %}
7989   opcode(0xD3, 0x4); /* D3 /4 */
7990   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7991   ins_pipe(ialu_reg_reg);
7992 %}
7993 
7994 // Shift Left by variable
7995 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7996 %{
7997   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7998   effect(KILL cr);
7999 
8000   format %{ "sall    $dst, $shift" %}
8001   opcode(0xD3, 0x4); /* D3 /4 */
8002   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8003   ins_pipe(ialu_mem_reg);
8004 %}
8005 
8006 // Arithmetic shift right by one
8007 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8008 %{
8009   match(Set dst (RShiftI dst shift));
8010   effect(KILL cr);
8011 
8012   format %{ "sarl    $dst, $shift" %}
8013   opcode(0xD1, 0x7); /* D1 /7 */
8014   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8015   ins_pipe(ialu_reg);
8016 %}
8017 
8018 // Arithmetic shift right by one
8019 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8020 %{
8021   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8022   effect(KILL cr);
8023 
8024   format %{ "sarl    $dst, $shift" %}
8025   opcode(0xD1, 0x7); /* D1 /7 */
8026   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8027   ins_pipe(ialu_mem_imm);
8028 %}
8029 
8030 // Arithmetic Shift Right by 8-bit immediate
8031 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8032 %{
8033   match(Set dst (RShiftI dst shift));
8034   effect(KILL cr);
8035 
8036   format %{ "sarl    $dst, $shift" %}
8037   opcode(0xC1, 0x7); /* C1 /7 ib */
8038   ins_encode(reg_opc_imm(dst, shift));
8039   ins_pipe(ialu_mem_imm);
8040 %}
8041 
8042 // Arithmetic Shift Right by 8-bit immediate
8043 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8044 %{
8045   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8046   effect(KILL cr);
8047 
8048   format %{ "sarl    $dst, $shift" %}
8049   opcode(0xC1, 0x7); /* C1 /7 ib */
8050   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8051   ins_pipe(ialu_mem_imm);
8052 %}
8053 
8054 // Arithmetic Shift Right by variable
8055 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8056 %{
8057   match(Set dst (RShiftI dst shift));
8058   effect(KILL cr);
8059 
8060   format %{ "sarl    $dst, $shift" %}
8061   opcode(0xD3, 0x7); /* D3 /7 */
8062   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8063   ins_pipe(ialu_reg_reg);
8064 %}
8065 
8066 // Arithmetic Shift Right by variable
8067 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8068 %{
8069   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8070   effect(KILL cr);
8071 
8072   format %{ "sarl    $dst, $shift" %}
8073   opcode(0xD3, 0x7); /* D3 /7 */
8074   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8075   ins_pipe(ialu_mem_reg);
8076 %}
8077 
8078 // Logical shift right by one
8079 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8080 %{
8081   match(Set dst (URShiftI dst shift));
8082   effect(KILL cr);
8083 
8084   format %{ "shrl    $dst, $shift" %}
8085   opcode(0xD1, 0x5); /* D1 /5 */
8086   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8087   ins_pipe(ialu_reg);
8088 %}
8089 
8090 // Logical shift right by one
8091 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8092 %{
8093   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8094   effect(KILL cr);
8095 
8096   format %{ "shrl    $dst, $shift" %}
8097   opcode(0xD1, 0x5); /* D1 /5 */
8098   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8099   ins_pipe(ialu_mem_imm);
8100 %}
8101 
8102 // Logical Shift Right by 8-bit immediate
8103 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8104 %{
8105   match(Set dst (URShiftI dst shift));
8106   effect(KILL cr);
8107 
8108   format %{ "shrl    $dst, $shift" %}
8109   opcode(0xC1, 0x5); /* C1 /5 ib */
8110   ins_encode(reg_opc_imm(dst, shift));
8111   ins_pipe(ialu_reg);
8112 %}
8113 
8114 // Logical Shift Right by 8-bit immediate
8115 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8116 %{
8117   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8118   effect(KILL cr);
8119 
8120   format %{ "shrl    $dst, $shift" %}
8121   opcode(0xC1, 0x5); /* C1 /5 ib */
8122   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8123   ins_pipe(ialu_mem_imm);
8124 %}
8125 
8126 // Logical Shift Right by variable
8127 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8128 %{
8129   match(Set dst (URShiftI dst shift));
8130   effect(KILL cr);
8131 
8132   format %{ "shrl    $dst, $shift" %}
8133   opcode(0xD3, 0x5); /* D3 /5 */
8134   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8135   ins_pipe(ialu_reg_reg);
8136 %}
8137 
8138 // Logical Shift Right by variable
8139 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8140 %{
8141   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8142   effect(KILL cr);
8143 
8144   format %{ "shrl    $dst, $shift" %}
8145   opcode(0xD3, 0x5); /* D3 /5 */
8146   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8147   ins_pipe(ialu_mem_reg);
8148 %}
8149 
8150 // Long Shift Instructions
8151 // Shift Left by one
8152 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8153 %{
8154   match(Set dst (LShiftL dst shift));
8155   effect(KILL cr);
8156 
8157   format %{ "salq    $dst, $shift" %}
8158   opcode(0xD1, 0x4); /* D1 /4 */
8159   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8160   ins_pipe(ialu_reg);
8161 %}
8162 
8163 // Shift Left by one
8164 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8165 %{
8166   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8167   effect(KILL cr);
8168 
8169   format %{ "salq    $dst, $shift" %}
8170   opcode(0xD1, 0x4); /* D1 /4 */
8171   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8172   ins_pipe(ialu_mem_imm);
8173 %}
8174 
8175 // Shift Left by 8-bit immediate
8176 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8177 %{
8178   match(Set dst (LShiftL dst shift));
8179   effect(KILL cr);
8180 
8181   format %{ "salq    $dst, $shift" %}
8182   opcode(0xC1, 0x4); /* C1 /4 ib */
8183   ins_encode(reg_opc_imm_wide(dst, shift));
8184   ins_pipe(ialu_reg);
8185 %}
8186 
8187 // Shift Left by 8-bit immediate
8188 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8189 %{
8190   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8191   effect(KILL cr);
8192 
8193   format %{ "salq    $dst, $shift" %}
8194   opcode(0xC1, 0x4); /* C1 /4 ib */
8195   ins_encode(REX_mem_wide(dst), OpcP,
8196              RM_opc_mem(secondary, dst), Con8or32(shift));
8197   ins_pipe(ialu_mem_imm);
8198 %}
8199 
8200 // Shift Left by variable
8201 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8202 %{
8203   match(Set dst (LShiftL dst shift));
8204   effect(KILL cr);
8205 
8206   format %{ "salq    $dst, $shift" %}
8207   opcode(0xD3, 0x4); /* D3 /4 */
8208   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8209   ins_pipe(ialu_reg_reg);
8210 %}
8211 
8212 // Shift Left by variable
8213 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8214 %{
8215   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8216   effect(KILL cr);
8217 
8218   format %{ "salq    $dst, $shift" %}
8219   opcode(0xD3, 0x4); /* D3 /4 */
8220   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8221   ins_pipe(ialu_mem_reg);
8222 %}
8223 
8224 // Arithmetic shift right by one
8225 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8226 %{
8227   match(Set dst (RShiftL dst shift));
8228   effect(KILL cr);
8229 
8230   format %{ "sarq    $dst, $shift" %}
8231   opcode(0xD1, 0x7); /* D1 /7 */
8232   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8233   ins_pipe(ialu_reg);
8234 %}
8235 
8236 // Arithmetic shift right by one
8237 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8238 %{
8239   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8240   effect(KILL cr);
8241 
8242   format %{ "sarq    $dst, $shift" %}
8243   opcode(0xD1, 0x7); /* D1 /7 */
8244   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8245   ins_pipe(ialu_mem_imm);
8246 %}
8247 
8248 // Arithmetic Shift Right by 8-bit immediate
8249 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8250 %{
8251   match(Set dst (RShiftL dst shift));
8252   effect(KILL cr);
8253 
8254   format %{ "sarq    $dst, $shift" %}
8255   opcode(0xC1, 0x7); /* C1 /7 ib */
8256   ins_encode(reg_opc_imm_wide(dst, shift));
8257   ins_pipe(ialu_mem_imm);
8258 %}
8259 
8260 // Arithmetic Shift Right by 8-bit immediate
8261 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8262 %{
8263   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8264   effect(KILL cr);
8265 
8266   format %{ "sarq    $dst, $shift" %}
8267   opcode(0xC1, 0x7); /* C1 /7 ib */
8268   ins_encode(REX_mem_wide(dst), OpcP,
8269              RM_opc_mem(secondary, dst), Con8or32(shift));
8270   ins_pipe(ialu_mem_imm);
8271 %}
8272 
8273 // Arithmetic Shift Right by variable
8274 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8275 %{
8276   match(Set dst (RShiftL dst shift));
8277   effect(KILL cr);
8278 
8279   format %{ "sarq    $dst, $shift" %}
8280   opcode(0xD3, 0x7); /* D3 /7 */
8281   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8282   ins_pipe(ialu_reg_reg);
8283 %}
8284 
8285 // Arithmetic Shift Right by variable
8286 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8287 %{
8288   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8289   effect(KILL cr);
8290 
8291   format %{ "sarq    $dst, $shift" %}
8292   opcode(0xD3, 0x7); /* D3 /7 */
8293   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8294   ins_pipe(ialu_mem_reg);
8295 %}
8296 
8297 // Logical shift right by one
8298 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8299 %{
8300   match(Set dst (URShiftL dst shift));
8301   effect(KILL cr);
8302 
8303   format %{ "shrq    $dst, $shift" %}
8304   opcode(0xD1, 0x5); /* D1 /5 */
8305   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8306   ins_pipe(ialu_reg);
8307 %}
8308 
8309 // Logical shift right by one
8310 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8311 %{
8312   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8313   effect(KILL cr);
8314 
8315   format %{ "shrq    $dst, $shift" %}
8316   opcode(0xD1, 0x5); /* D1 /5 */
8317   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8318   ins_pipe(ialu_mem_imm);
8319 %}
8320 
8321 // Logical Shift Right by 8-bit immediate
8322 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8323 %{
8324   match(Set dst (URShiftL dst shift));
8325   effect(KILL cr);
8326 
8327   format %{ "shrq    $dst, $shift" %}
8328   opcode(0xC1, 0x5); /* C1 /5 ib */
8329   ins_encode(reg_opc_imm_wide(dst, shift));
8330   ins_pipe(ialu_reg);
8331 %}
8332 
8333 
8334 // Logical Shift Right by 8-bit immediate
8335 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8336 %{
8337   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8338   effect(KILL cr);
8339 
8340   format %{ "shrq    $dst, $shift" %}
8341   opcode(0xC1, 0x5); /* C1 /5 ib */
8342   ins_encode(REX_mem_wide(dst), OpcP,
8343              RM_opc_mem(secondary, dst), Con8or32(shift));
8344   ins_pipe(ialu_mem_imm);
8345 %}
8346 
8347 // Logical Shift Right by variable
8348 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8349 %{
8350   match(Set dst (URShiftL dst shift));
8351   effect(KILL cr);
8352 
8353   format %{ "shrq    $dst, $shift" %}
8354   opcode(0xD3, 0x5); /* D3 /5 */
8355   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8356   ins_pipe(ialu_reg_reg);
8357 %}
8358 
8359 // Logical Shift Right by variable
8360 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8361 %{
8362   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8363   effect(KILL cr);
8364 
8365   format %{ "shrq    $dst, $shift" %}
8366   opcode(0xD3, 0x5); /* D3 /5 */
8367   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8368   ins_pipe(ialu_mem_reg);
8369 %}
8370 
8371 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8372 // This idiom is used by the compiler for the i2b bytecode.
8373 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8374 %{
8375   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8376 
8377   format %{ "movsbl  $dst, $src\t# i2b" %}
8378   opcode(0x0F, 0xBE);
8379   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8380   ins_pipe(ialu_reg_reg);
8381 %}
8382 
8383 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8384 // This idiom is used by the compiler the i2s bytecode.
8385 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8386 %{
8387   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8388 
8389   format %{ "movswl  $dst, $src\t# i2s" %}
8390   opcode(0x0F, 0xBF);
8391   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8392   ins_pipe(ialu_reg_reg);
8393 %}
8394 
8395 // ROL/ROR instructions
8396 
8397 // ROL expand
8398 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8399   effect(KILL cr, USE_DEF dst);
8400 
8401   format %{ "roll    $dst" %}
8402   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8403   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8404   ins_pipe(ialu_reg);
8405 %}
8406 
8407 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8408   effect(USE_DEF dst, USE shift, KILL cr);
8409 
8410   format %{ "roll    $dst, $shift" %}
8411   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8412   ins_encode( reg_opc_imm(dst, shift) );
8413   ins_pipe(ialu_reg);
8414 %}
8415 
8416 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8417 %{
8418   effect(USE_DEF dst, USE shift, KILL cr);
8419 
8420   format %{ "roll    $dst, $shift" %}
8421   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8422   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8423   ins_pipe(ialu_reg_reg);
8424 %}
8425 // end of ROL expand
8426 
8427 // Rotate Left by one
8428 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8429 %{
8430   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8431 
8432   expand %{
8433     rolI_rReg_imm1(dst, cr);
8434   %}
8435 %}
8436 
8437 // Rotate Left by 8-bit immediate
8438 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8439 %{
8440   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8441   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8442 
8443   expand %{
8444     rolI_rReg_imm8(dst, lshift, cr);
8445   %}
8446 %}
8447 
8448 // Rotate Left by variable
8449 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8450 %{
8451   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8452 
8453   expand %{
8454     rolI_rReg_CL(dst, shift, cr);
8455   %}
8456 %}
8457 
8458 // Rotate Left by variable
8459 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8460 %{
8461   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8462 
8463   expand %{
8464     rolI_rReg_CL(dst, shift, cr);
8465   %}
8466 %}
8467 
8468 // ROR expand
8469 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8470 %{
8471   effect(USE_DEF dst, KILL cr);
8472 
8473   format %{ "rorl    $dst" %}
8474   opcode(0xD1, 0x1); /* D1 /1 */
8475   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8476   ins_pipe(ialu_reg);
8477 %}
8478 
8479 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8480 %{
8481   effect(USE_DEF dst, USE shift, KILL cr);
8482 
8483   format %{ "rorl    $dst, $shift" %}
8484   opcode(0xC1, 0x1); /* C1 /1 ib */
8485   ins_encode(reg_opc_imm(dst, shift));
8486   ins_pipe(ialu_reg);
8487 %}
8488 
8489 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8490 %{
8491   effect(USE_DEF dst, USE shift, KILL cr);
8492 
8493   format %{ "rorl    $dst, $shift" %}
8494   opcode(0xD3, 0x1); /* D3 /1 */
8495   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8496   ins_pipe(ialu_reg_reg);
8497 %}
8498 // end of ROR expand
8499 
8500 // Rotate Right by one
8501 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8502 %{
8503   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8504 
8505   expand %{
8506     rorI_rReg_imm1(dst, cr);
8507   %}
8508 %}
8509 
8510 // Rotate Right by 8-bit immediate
8511 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8512 %{
8513   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8514   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8515 
8516   expand %{
8517     rorI_rReg_imm8(dst, rshift, cr);
8518   %}
8519 %}
8520 
8521 // Rotate Right by variable
8522 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8523 %{
8524   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8525 
8526   expand %{
8527     rorI_rReg_CL(dst, shift, cr);
8528   %}
8529 %}
8530 
8531 // Rotate Right by variable
8532 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8533 %{
8534   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8535 
8536   expand %{
8537     rorI_rReg_CL(dst, shift, cr);
8538   %}
8539 %}
8540 
8541 // for long rotate
8542 // ROL expand
8543 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8544   effect(USE_DEF dst, KILL cr);
8545 
8546   format %{ "rolq    $dst" %}
8547   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8548   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8549   ins_pipe(ialu_reg);
8550 %}
8551 
8552 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8553   effect(USE_DEF dst, USE shift, KILL cr);
8554 
8555   format %{ "rolq    $dst, $shift" %}
8556   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8557   ins_encode( reg_opc_imm_wide(dst, shift) );
8558   ins_pipe(ialu_reg);
8559 %}
8560 
8561 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8562 %{
8563   effect(USE_DEF dst, USE shift, KILL cr);
8564 
8565   format %{ "rolq    $dst, $shift" %}
8566   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8567   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8568   ins_pipe(ialu_reg_reg);
8569 %}
8570 // end of ROL expand
8571 
8572 // Rotate Left by one
8573 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8574 %{
8575   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8576 
8577   expand %{
8578     rolL_rReg_imm1(dst, cr);
8579   %}
8580 %}
8581 
8582 // Rotate Left by 8-bit immediate
8583 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8584 %{
8585   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8586   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8587 
8588   expand %{
8589     rolL_rReg_imm8(dst, lshift, cr);
8590   %}
8591 %}
8592 
8593 // Rotate Left by variable
8594 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8595 %{
8596   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8597 
8598   expand %{
8599     rolL_rReg_CL(dst, shift, cr);
8600   %}
8601 %}
8602 
8603 // Rotate Left by variable
8604 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8605 %{
8606   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8607 
8608   expand %{
8609     rolL_rReg_CL(dst, shift, cr);
8610   %}
8611 %}
8612 
8613 // ROR expand
8614 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8615 %{
8616   effect(USE_DEF dst, KILL cr);
8617 
8618   format %{ "rorq    $dst" %}
8619   opcode(0xD1, 0x1); /* D1 /1 */
8620   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8621   ins_pipe(ialu_reg);
8622 %}
8623 
8624 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8625 %{
8626   effect(USE_DEF dst, USE shift, KILL cr);
8627 
8628   format %{ "rorq    $dst, $shift" %}
8629   opcode(0xC1, 0x1); /* C1 /1 ib */
8630   ins_encode(reg_opc_imm_wide(dst, shift));
8631   ins_pipe(ialu_reg);
8632 %}
8633 
8634 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8635 %{
8636   effect(USE_DEF dst, USE shift, KILL cr);
8637 
8638   format %{ "rorq    $dst, $shift" %}
8639   opcode(0xD3, 0x1); /* D3 /1 */
8640   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8641   ins_pipe(ialu_reg_reg);
8642 %}
8643 // end of ROR expand
8644 
8645 // Rotate Right by one
8646 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8647 %{
8648   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8649 
8650   expand %{
8651     rorL_rReg_imm1(dst, cr);
8652   %}
8653 %}
8654 
8655 // Rotate Right by 8-bit immediate
8656 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8657 %{
8658   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8659   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8660 
8661   expand %{
8662     rorL_rReg_imm8(dst, rshift, cr);
8663   %}
8664 %}
8665 
8666 // Rotate Right by variable
8667 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8668 %{
8669   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8670 
8671   expand %{
8672     rorL_rReg_CL(dst, shift, cr);
8673   %}
8674 %}
8675 
8676 // Rotate Right by variable
8677 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8678 %{
8679   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8680 
8681   expand %{
8682     rorL_rReg_CL(dst, shift, cr);
8683   %}
8684 %}
8685 
8686 // Logical Instructions
8687 
8688 // Integer Logical Instructions
8689 
8690 // And Instructions
8691 // And Register with Register
8692 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8693 %{
8694   match(Set dst (AndI dst src));
8695   effect(KILL cr);
8696 
8697   format %{ "andl    $dst, $src\t# int" %}
8698   opcode(0x23);
8699   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8700   ins_pipe(ialu_reg_reg);
8701 %}
8702 
8703 // And Register with Immediate 255
8704 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8705 %{
8706   match(Set dst (AndI dst src));
8707 
8708   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8709   opcode(0x0F, 0xB6);
8710   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8711   ins_pipe(ialu_reg);
8712 %}
8713 
8714 // And Register with Immediate 255 and promote to long
8715 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8716 %{
8717   match(Set dst (ConvI2L (AndI src mask)));
8718 
8719   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8720   opcode(0x0F, 0xB6);
8721   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8722   ins_pipe(ialu_reg);
8723 %}
8724 
8725 // And Register with Immediate 65535
8726 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
8727 %{
8728   match(Set dst (AndI dst src));
8729 
8730   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
8731   opcode(0x0F, 0xB7);
8732   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8733   ins_pipe(ialu_reg);
8734 %}
8735 
8736 // And Register with Immediate 65535 and promote to long
8737 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8738 %{
8739   match(Set dst (ConvI2L (AndI src mask)));
8740 
8741   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
8742   opcode(0x0F, 0xB7);
8743   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8744   ins_pipe(ialu_reg);
8745 %}
8746 
8747 // And Register with Immediate
8748 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8749 %{
8750   match(Set dst (AndI dst src));
8751   effect(KILL cr);
8752 
8753   format %{ "andl    $dst, $src\t# int" %}
8754   opcode(0x81, 0x04); /* Opcode 81 /4 */
8755   ins_encode(OpcSErm(dst, src), Con8or32(src));
8756   ins_pipe(ialu_reg);
8757 %}
8758 
8759 // And Register with Memory
8760 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8761 %{
8762   match(Set dst (AndI dst (LoadI src)));
8763   effect(KILL cr);
8764 
8765   ins_cost(125);
8766   format %{ "andl    $dst, $src\t# int" %}
8767   opcode(0x23);
8768   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8769   ins_pipe(ialu_reg_mem);
8770 %}
8771 
8772 // And Memory with Register
8773 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8774 %{
8775   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8776   effect(KILL cr);
8777 
8778   ins_cost(150);
8779   format %{ "andl    $dst, $src\t# int" %}
8780   opcode(0x21); /* Opcode 21 /r */
8781   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8782   ins_pipe(ialu_mem_reg);
8783 %}
8784 
8785 // And Memory with Immediate
8786 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
8787 %{
8788   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8789   effect(KILL cr);
8790 
8791   ins_cost(125);
8792   format %{ "andl    $dst, $src\t# int" %}
8793   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8794   ins_encode(REX_mem(dst), OpcSE(src),
8795              RM_opc_mem(secondary, dst), Con8or32(src));
8796   ins_pipe(ialu_mem_imm);
8797 %}
8798 
8799 // BMI1 instructions
8800 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
8801   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
8802   predicate(UseBMI1Instructions);
8803   effect(KILL cr);
8804 
8805   ins_cost(125);
8806   format %{ "andnl  $dst, $src1, $src2" %}
8807 
8808   ins_encode %{
8809     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
8810   %}
8811   ins_pipe(ialu_reg_mem);
8812 %}
8813 
8814 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
8815   match(Set dst (AndI (XorI src1 minus_1) src2));
8816   predicate(UseBMI1Instructions);
8817   effect(KILL cr);
8818 
8819   format %{ "andnl  $dst, $src1, $src2" %}
8820 
8821   ins_encode %{
8822     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
8823   %}
8824   ins_pipe(ialu_reg);
8825 %}
8826 
8827 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
8828   match(Set dst (AndI (SubI imm_zero src) src));
8829   predicate(UseBMI1Instructions);
8830   effect(KILL cr);
8831 
8832   format %{ "blsil  $dst, $src" %}
8833 
8834   ins_encode %{
8835     __ blsil($dst$$Register, $src$$Register);
8836   %}
8837   ins_pipe(ialu_reg);
8838 %}
8839 
8840 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
8841   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
8842   predicate(UseBMI1Instructions);
8843   effect(KILL cr);
8844 
8845   ins_cost(125);
8846   format %{ "blsil  $dst, $src" %}
8847 
8848   ins_encode %{
8849     __ blsil($dst$$Register, $src$$Address);
8850   %}
8851   ins_pipe(ialu_reg_mem);
8852 %}
8853 
8854 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8855 %{
8856   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
8857   predicate(UseBMI1Instructions);
8858   effect(KILL cr);
8859 
8860   ins_cost(125);
8861   format %{ "blsmskl $dst, $src" %}
8862 
8863   ins_encode %{
8864     __ blsmskl($dst$$Register, $src$$Address);
8865   %}
8866   ins_pipe(ialu_reg_mem);
8867 %}
8868 
8869 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8870 %{
8871   match(Set dst (XorI (AddI src minus_1) src));
8872   predicate(UseBMI1Instructions);
8873   effect(KILL cr);
8874 
8875   format %{ "blsmskl $dst, $src" %}
8876 
8877   ins_encode %{
8878     __ blsmskl($dst$$Register, $src$$Register);
8879   %}
8880 
8881   ins_pipe(ialu_reg);
8882 %}
8883 
8884 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8885 %{
8886   match(Set dst (AndI (AddI src minus_1) src) );
8887   predicate(UseBMI1Instructions);
8888   effect(KILL cr);
8889 
8890   format %{ "blsrl  $dst, $src" %}
8891 
8892   ins_encode %{
8893     __ blsrl($dst$$Register, $src$$Register);
8894   %}
8895 
8896   ins_pipe(ialu_reg_mem);
8897 %}
8898 
8899 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8900 %{
8901   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
8902   predicate(UseBMI1Instructions);
8903   effect(KILL cr);
8904 
8905   ins_cost(125);
8906   format %{ "blsrl  $dst, $src" %}
8907 
8908   ins_encode %{
8909     __ blsrl($dst$$Register, $src$$Address);
8910   %}
8911 
8912   ins_pipe(ialu_reg);
8913 %}
8914 
8915 // Or Instructions
8916 // Or Register with Register
8917 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8918 %{
8919   match(Set dst (OrI dst src));
8920   effect(KILL cr);
8921 
8922   format %{ "orl     $dst, $src\t# int" %}
8923   opcode(0x0B);
8924   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8925   ins_pipe(ialu_reg_reg);
8926 %}
8927 
8928 // Or Register with Immediate
8929 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8930 %{
8931   match(Set dst (OrI dst src));
8932   effect(KILL cr);
8933 
8934   format %{ "orl     $dst, $src\t# int" %}
8935   opcode(0x81, 0x01); /* Opcode 81 /1 id */
8936   ins_encode(OpcSErm(dst, src), Con8or32(src));
8937   ins_pipe(ialu_reg);
8938 %}
8939 
8940 // Or Register with Memory
8941 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8942 %{
8943   match(Set dst (OrI dst (LoadI src)));
8944   effect(KILL cr);
8945 
8946   ins_cost(125);
8947   format %{ "orl     $dst, $src\t# int" %}
8948   opcode(0x0B);
8949   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8950   ins_pipe(ialu_reg_mem);
8951 %}
8952 
8953 // Or Memory with Register
8954 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8955 %{
8956   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8957   effect(KILL cr);
8958 
8959   ins_cost(150);
8960   format %{ "orl     $dst, $src\t# int" %}
8961   opcode(0x09); /* Opcode 09 /r */
8962   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8963   ins_pipe(ialu_mem_reg);
8964 %}
8965 
8966 // Or Memory with Immediate
8967 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
8968 %{
8969   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8970   effect(KILL cr);
8971 
8972   ins_cost(125);
8973   format %{ "orl     $dst, $src\t# int" %}
8974   opcode(0x81, 0x1); /* Opcode 81 /1 id */
8975   ins_encode(REX_mem(dst), OpcSE(src),
8976              RM_opc_mem(secondary, dst), Con8or32(src));
8977   ins_pipe(ialu_mem_imm);
8978 %}
8979 
8980 // Xor Instructions
8981 // Xor Register with Register
8982 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8983 %{
8984   match(Set dst (XorI dst src));
8985   effect(KILL cr);
8986 
8987   format %{ "xorl    $dst, $src\t# int" %}
8988   opcode(0x33);
8989   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8990   ins_pipe(ialu_reg_reg);
8991 %}
8992 
8993 // Xor Register with Immediate -1
8994 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
8995   match(Set dst (XorI dst imm));
8996 
8997   format %{ "not    $dst" %}
8998   ins_encode %{
8999      __ notl($dst$$Register);
9000   %}
9001   ins_pipe(ialu_reg);
9002 %}
9003 
9004 // Xor Register with Immediate
9005 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9006 %{
9007   match(Set dst (XorI dst src));
9008   effect(KILL cr);
9009 
9010   format %{ "xorl    $dst, $src\t# int" %}
9011   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9012   ins_encode(OpcSErm(dst, src), Con8or32(src));
9013   ins_pipe(ialu_reg);
9014 %}
9015 
9016 // Xor Register with Memory
9017 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9018 %{
9019   match(Set dst (XorI dst (LoadI src)));
9020   effect(KILL cr);
9021 
9022   ins_cost(125);
9023   format %{ "xorl    $dst, $src\t# int" %}
9024   opcode(0x33);
9025   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9026   ins_pipe(ialu_reg_mem);
9027 %}
9028 
9029 // Xor Memory with Register
9030 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9031 %{
9032   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9033   effect(KILL cr);
9034 
9035   ins_cost(150);
9036   format %{ "xorl    $dst, $src\t# int" %}
9037   opcode(0x31); /* Opcode 31 /r */
9038   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9039   ins_pipe(ialu_mem_reg);
9040 %}
9041 
9042 // Xor Memory with Immediate
9043 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9044 %{
9045   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9046   effect(KILL cr);
9047 
9048   ins_cost(125);
9049   format %{ "xorl    $dst, $src\t# int" %}
9050   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9051   ins_encode(REX_mem(dst), OpcSE(src),
9052              RM_opc_mem(secondary, dst), Con8or32(src));
9053   ins_pipe(ialu_mem_imm);
9054 %}
9055 
9056 
9057 // Long Logical Instructions
9058 
9059 // And Instructions
9060 // And Register with Register
9061 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9062 %{
9063   match(Set dst (AndL dst src));
9064   effect(KILL cr);
9065 
9066   format %{ "andq    $dst, $src\t# long" %}
9067   opcode(0x23);
9068   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9069   ins_pipe(ialu_reg_reg);
9070 %}
9071 
9072 // And Register with Immediate 255
9073 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9074 %{
9075   match(Set dst (AndL dst src));
9076 
9077   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9078   opcode(0x0F, 0xB6);
9079   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9080   ins_pipe(ialu_reg);
9081 %}
9082 
9083 // And Register with Immediate 65535
9084 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9085 %{
9086   match(Set dst (AndL dst src));
9087 
9088   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9089   opcode(0x0F, 0xB7);
9090   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9091   ins_pipe(ialu_reg);
9092 %}
9093 
9094 // And Register with Immediate
9095 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9096 %{
9097   match(Set dst (AndL dst src));
9098   effect(KILL cr);
9099 
9100   format %{ "andq    $dst, $src\t# long" %}
9101   opcode(0x81, 0x04); /* Opcode 81 /4 */
9102   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9103   ins_pipe(ialu_reg);
9104 %}
9105 
9106 // And Register with Memory
9107 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9108 %{
9109   match(Set dst (AndL dst (LoadL src)));
9110   effect(KILL cr);
9111 
9112   ins_cost(125);
9113   format %{ "andq    $dst, $src\t# long" %}
9114   opcode(0x23);
9115   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9116   ins_pipe(ialu_reg_mem);
9117 %}
9118 
9119 // And Memory with Register
9120 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9121 %{
9122   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9123   effect(KILL cr);
9124 
9125   ins_cost(150);
9126   format %{ "andq    $dst, $src\t# long" %}
9127   opcode(0x21); /* Opcode 21 /r */
9128   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9129   ins_pipe(ialu_mem_reg);
9130 %}
9131 
9132 // And Memory with Immediate
9133 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9134 %{
9135   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9136   effect(KILL cr);
9137 
9138   ins_cost(125);
9139   format %{ "andq    $dst, $src\t# long" %}
9140   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9141   ins_encode(REX_mem_wide(dst), OpcSE(src),
9142              RM_opc_mem(secondary, dst), Con8or32(src));
9143   ins_pipe(ialu_mem_imm);
9144 %}
9145 
9146 // BMI1 instructions
9147 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9148   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9149   predicate(UseBMI1Instructions);
9150   effect(KILL cr);
9151 
9152   ins_cost(125);
9153   format %{ "andnq  $dst, $src1, $src2" %}
9154 
9155   ins_encode %{
9156     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9157   %}
9158   ins_pipe(ialu_reg_mem);
9159 %}
9160 
9161 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9162   match(Set dst (AndL (XorL src1 minus_1) src2));
9163   predicate(UseBMI1Instructions);
9164   effect(KILL cr);
9165 
9166   format %{ "andnq  $dst, $src1, $src2" %}
9167 
9168   ins_encode %{
9169   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9170   %}
9171   ins_pipe(ialu_reg_mem);
9172 %}
9173 
9174 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9175   match(Set dst (AndL (SubL imm_zero src) src));
9176   predicate(UseBMI1Instructions);
9177   effect(KILL cr);
9178 
9179   format %{ "blsiq  $dst, $src" %}
9180 
9181   ins_encode %{
9182     __ blsiq($dst$$Register, $src$$Register);
9183   %}
9184   ins_pipe(ialu_reg);
9185 %}
9186 
9187 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9188   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9189   predicate(UseBMI1Instructions);
9190   effect(KILL cr);
9191 
9192   ins_cost(125);
9193   format %{ "blsiq  $dst, $src" %}
9194 
9195   ins_encode %{
9196     __ blsiq($dst$$Register, $src$$Address);
9197   %}
9198   ins_pipe(ialu_reg_mem);
9199 %}
9200 
9201 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9202 %{
9203   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9204   predicate(UseBMI1Instructions);
9205   effect(KILL cr);
9206 
9207   ins_cost(125);
9208   format %{ "blsmskq $dst, $src" %}
9209 
9210   ins_encode %{
9211     __ blsmskq($dst$$Register, $src$$Address);
9212   %}
9213   ins_pipe(ialu_reg_mem);
9214 %}
9215 
9216 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9217 %{
9218   match(Set dst (XorL (AddL src minus_1) src));
9219   predicate(UseBMI1Instructions);
9220   effect(KILL cr);
9221 
9222   format %{ "blsmskq $dst, $src" %}
9223 
9224   ins_encode %{
9225     __ blsmskq($dst$$Register, $src$$Register);
9226   %}
9227 
9228   ins_pipe(ialu_reg);
9229 %}
9230 
9231 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9232 %{
9233   match(Set dst (AndL (AddL src minus_1) src) );
9234   predicate(UseBMI1Instructions);
9235   effect(KILL cr);
9236 
9237   format %{ "blsrq  $dst, $src" %}
9238 
9239   ins_encode %{
9240     __ blsrq($dst$$Register, $src$$Register);
9241   %}
9242 
9243   ins_pipe(ialu_reg);
9244 %}
9245 
9246 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9247 %{
9248   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9249   predicate(UseBMI1Instructions);
9250   effect(KILL cr);
9251 
9252   ins_cost(125);
9253   format %{ "blsrq  $dst, $src" %}
9254 
9255   ins_encode %{
9256     __ blsrq($dst$$Register, $src$$Address);
9257   %}
9258 
9259   ins_pipe(ialu_reg);
9260 %}
9261 
9262 // Or Instructions
9263 // Or Register with Register
9264 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9265 %{
9266   match(Set dst (OrL dst src));
9267   effect(KILL cr);
9268 
9269   format %{ "orq     $dst, $src\t# long" %}
9270   opcode(0x0B);
9271   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9272   ins_pipe(ialu_reg_reg);
9273 %}
9274 
9275 // Use any_RegP to match R15 (TLS register) without spilling.
9276 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9277   match(Set dst (OrL dst (CastP2X src)));
9278   effect(KILL cr);
9279 
9280   format %{ "orq     $dst, $src\t# long" %}
9281   opcode(0x0B);
9282   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9283   ins_pipe(ialu_reg_reg);
9284 %}
9285 
9286 
9287 // Or Register with Immediate
9288 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9289 %{
9290   match(Set dst (OrL dst src));
9291   effect(KILL cr);
9292 
9293   format %{ "orq     $dst, $src\t# long" %}
9294   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9295   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9296   ins_pipe(ialu_reg);
9297 %}
9298 
9299 // Or Register with Memory
9300 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9301 %{
9302   match(Set dst (OrL dst (LoadL src)));
9303   effect(KILL cr);
9304 
9305   ins_cost(125);
9306   format %{ "orq     $dst, $src\t# long" %}
9307   opcode(0x0B);
9308   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9309   ins_pipe(ialu_reg_mem);
9310 %}
9311 
9312 // Or Memory with Register
9313 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9314 %{
9315   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9316   effect(KILL cr);
9317 
9318   ins_cost(150);
9319   format %{ "orq     $dst, $src\t# long" %}
9320   opcode(0x09); /* Opcode 09 /r */
9321   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9322   ins_pipe(ialu_mem_reg);
9323 %}
9324 
9325 // Or Memory with Immediate
9326 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9327 %{
9328   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9329   effect(KILL cr);
9330 
9331   ins_cost(125);
9332   format %{ "orq     $dst, $src\t# long" %}
9333   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9334   ins_encode(REX_mem_wide(dst), OpcSE(src),
9335              RM_opc_mem(secondary, dst), Con8or32(src));
9336   ins_pipe(ialu_mem_imm);
9337 %}
9338 
9339 // Xor Instructions
9340 // Xor Register with Register
9341 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9342 %{
9343   match(Set dst (XorL dst src));
9344   effect(KILL cr);
9345 
9346   format %{ "xorq    $dst, $src\t# long" %}
9347   opcode(0x33);
9348   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9349   ins_pipe(ialu_reg_reg);
9350 %}
9351 
9352 // Xor Register with Immediate -1
9353 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9354   match(Set dst (XorL dst imm));
9355 
9356   format %{ "notq   $dst" %}
9357   ins_encode %{
9358      __ notq($dst$$Register);
9359   %}
9360   ins_pipe(ialu_reg);
9361 %}
9362 
9363 // Xor Register with Immediate
9364 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9365 %{
9366   match(Set dst (XorL dst src));
9367   effect(KILL cr);
9368 
9369   format %{ "xorq    $dst, $src\t# long" %}
9370   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9371   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9372   ins_pipe(ialu_reg);
9373 %}
9374 
9375 // Xor Register with Memory
9376 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9377 %{
9378   match(Set dst (XorL dst (LoadL src)));
9379   effect(KILL cr);
9380 
9381   ins_cost(125);
9382   format %{ "xorq    $dst, $src\t# long" %}
9383   opcode(0x33);
9384   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9385   ins_pipe(ialu_reg_mem);
9386 %}
9387 
9388 // Xor Memory with Register
9389 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9390 %{
9391   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9392   effect(KILL cr);
9393 
9394   ins_cost(150);
9395   format %{ "xorq    $dst, $src\t# long" %}
9396   opcode(0x31); /* Opcode 31 /r */
9397   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9398   ins_pipe(ialu_mem_reg);
9399 %}
9400 
9401 // Xor Memory with Immediate
9402 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9403 %{
9404   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9405   effect(KILL cr);
9406 
9407   ins_cost(125);
9408   format %{ "xorq    $dst, $src\t# long" %}
9409   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9410   ins_encode(REX_mem_wide(dst), OpcSE(src),
9411              RM_opc_mem(secondary, dst), Con8or32(src));
9412   ins_pipe(ialu_mem_imm);
9413 %}
9414 
9415 // Convert Int to Boolean
9416 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9417 %{
9418   match(Set dst (Conv2B src));
9419   effect(KILL cr);
9420 
9421   format %{ "testl   $src, $src\t# ci2b\n\t"
9422             "setnz   $dst\n\t"
9423             "movzbl  $dst, $dst" %}
9424   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9425              setNZ_reg(dst),
9426              REX_reg_breg(dst, dst), // movzbl
9427              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9428   ins_pipe(pipe_slow); // XXX
9429 %}
9430 
9431 // Convert Pointer to Boolean
9432 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9433 %{
9434   match(Set dst (Conv2B src));
9435   effect(KILL cr);
9436 
9437   format %{ "testq   $src, $src\t# cp2b\n\t"
9438             "setnz   $dst\n\t"
9439             "movzbl  $dst, $dst" %}
9440   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9441              setNZ_reg(dst),
9442              REX_reg_breg(dst, dst), // movzbl
9443              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9444   ins_pipe(pipe_slow); // XXX
9445 %}
9446 
9447 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9448 %{
9449   match(Set dst (CmpLTMask p q));
9450   effect(KILL cr);
9451 
9452   ins_cost(400);
9453   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9454             "setlt   $dst\n\t"
9455             "movzbl  $dst, $dst\n\t"
9456             "negl    $dst" %}
9457   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9458              setLT_reg(dst),
9459              REX_reg_breg(dst, dst), // movzbl
9460              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9461              neg_reg(dst));
9462   ins_pipe(pipe_slow);
9463 %}
9464 
9465 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9466 %{
9467   match(Set dst (CmpLTMask dst zero));
9468   effect(KILL cr);
9469 
9470   ins_cost(100);
9471   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9472   ins_encode %{
9473   __ sarl($dst$$Register, 31);
9474   %}
9475   ins_pipe(ialu_reg);
9476 %}
9477 
9478 /* Better to save a register than avoid a branch */
9479 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9480 %{
9481   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9482   effect(KILL cr);
9483   ins_cost(300);
9484   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9485             "jge    done\n\t"
9486             "addl   $p,$y\n"
9487             "done:  " %}
9488   ins_encode %{
9489     Register Rp = $p$$Register;
9490     Register Rq = $q$$Register;
9491     Register Ry = $y$$Register;
9492     Label done;
9493     __ subl(Rp, Rq);
9494     __ jccb(Assembler::greaterEqual, done);
9495     __ addl(Rp, Ry);
9496     __ bind(done);
9497   %}
9498   ins_pipe(pipe_cmplt);
9499 %}
9500 
9501 /* Better to save a register than avoid a branch */
9502 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9503 %{
9504   match(Set y (AndI (CmpLTMask p q) y));
9505   effect(KILL cr);
9506 
9507   ins_cost(300);
9508 
9509   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9510             "jlt      done\n\t"
9511             "xorl     $y, $y\n"
9512             "done:  " %}
9513   ins_encode %{
9514     Register Rp = $p$$Register;
9515     Register Rq = $q$$Register;
9516     Register Ry = $y$$Register;
9517     Label done;
9518     __ cmpl(Rp, Rq);
9519     __ jccb(Assembler::less, done);
9520     __ xorl(Ry, Ry);
9521     __ bind(done);
9522   %}
9523   ins_pipe(pipe_cmplt);
9524 %}
9525 
9526 
9527 //---------- FP Instructions------------------------------------------------
9528 
9529 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9530 %{
9531   match(Set cr (CmpF src1 src2));
9532 
9533   ins_cost(145);
9534   format %{ "ucomiss $src1, $src2\n\t"
9535             "jnp,s   exit\n\t"
9536             "pushfq\t# saw NaN, set CF\n\t"
9537             "andq    [rsp], #0xffffff2b\n\t"
9538             "popfq\n"
9539     "exit:" %}
9540   ins_encode %{
9541     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9542     emit_cmpfp_fixup(_masm);
9543   %}
9544   ins_pipe(pipe_slow);
9545 %}
9546 
9547 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9548   match(Set cr (CmpF src1 src2));
9549 
9550   ins_cost(100);
9551   format %{ "ucomiss $src1, $src2" %}
9552   ins_encode %{
9553     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9554   %}
9555   ins_pipe(pipe_slow);
9556 %}
9557 
9558 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9559 %{
9560   match(Set cr (CmpF src1 (LoadF src2)));
9561 
9562   ins_cost(145);
9563   format %{ "ucomiss $src1, $src2\n\t"
9564             "jnp,s   exit\n\t"
9565             "pushfq\t# saw NaN, set CF\n\t"
9566             "andq    [rsp], #0xffffff2b\n\t"
9567             "popfq\n"
9568     "exit:" %}
9569   ins_encode %{
9570     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9571     emit_cmpfp_fixup(_masm);
9572   %}
9573   ins_pipe(pipe_slow);
9574 %}
9575 
9576 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9577   match(Set cr (CmpF src1 (LoadF src2)));
9578 
9579   ins_cost(100);
9580   format %{ "ucomiss $src1, $src2" %}
9581   ins_encode %{
9582     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9583   %}
9584   ins_pipe(pipe_slow);
9585 %}
9586 
9587 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9588   match(Set cr (CmpF src con));
9589 
9590   ins_cost(145);
9591   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9592             "jnp,s   exit\n\t"
9593             "pushfq\t# saw NaN, set CF\n\t"
9594             "andq    [rsp], #0xffffff2b\n\t"
9595             "popfq\n"
9596     "exit:" %}
9597   ins_encode %{
9598     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9599     emit_cmpfp_fixup(_masm);
9600   %}
9601   ins_pipe(pipe_slow);
9602 %}
9603 
9604 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9605   match(Set cr (CmpF src con));
9606   ins_cost(100);
9607   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9608   ins_encode %{
9609     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9610   %}
9611   ins_pipe(pipe_slow);
9612 %}
9613 
9614 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9615 %{
9616   match(Set cr (CmpD src1 src2));
9617 
9618   ins_cost(145);
9619   format %{ "ucomisd $src1, $src2\n\t"
9620             "jnp,s   exit\n\t"
9621             "pushfq\t# saw NaN, set CF\n\t"
9622             "andq    [rsp], #0xffffff2b\n\t"
9623             "popfq\n"
9624     "exit:" %}
9625   ins_encode %{
9626     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9627     emit_cmpfp_fixup(_masm);
9628   %}
9629   ins_pipe(pipe_slow);
9630 %}
9631 
9632 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9633   match(Set cr (CmpD src1 src2));
9634 
9635   ins_cost(100);
9636   format %{ "ucomisd $src1, $src2 test" %}
9637   ins_encode %{
9638     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9639   %}
9640   ins_pipe(pipe_slow);
9641 %}
9642 
9643 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9644 %{
9645   match(Set cr (CmpD src1 (LoadD src2)));
9646 
9647   ins_cost(145);
9648   format %{ "ucomisd $src1, $src2\n\t"
9649             "jnp,s   exit\n\t"
9650             "pushfq\t# saw NaN, set CF\n\t"
9651             "andq    [rsp], #0xffffff2b\n\t"
9652             "popfq\n"
9653     "exit:" %}
9654   ins_encode %{
9655     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9656     emit_cmpfp_fixup(_masm);
9657   %}
9658   ins_pipe(pipe_slow);
9659 %}
9660 
9661 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9662   match(Set cr (CmpD src1 (LoadD src2)));
9663 
9664   ins_cost(100);
9665   format %{ "ucomisd $src1, $src2" %}
9666   ins_encode %{
9667     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9668   %}
9669   ins_pipe(pipe_slow);
9670 %}
9671 
9672 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9673   match(Set cr (CmpD src con));
9674 
9675   ins_cost(145);
9676   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9677             "jnp,s   exit\n\t"
9678             "pushfq\t# saw NaN, set CF\n\t"
9679             "andq    [rsp], #0xffffff2b\n\t"
9680             "popfq\n"
9681     "exit:" %}
9682   ins_encode %{
9683     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9684     emit_cmpfp_fixup(_masm);
9685   %}
9686   ins_pipe(pipe_slow);
9687 %}
9688 
9689 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9690   match(Set cr (CmpD src con));
9691   ins_cost(100);
9692   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9693   ins_encode %{
9694     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9695   %}
9696   ins_pipe(pipe_slow);
9697 %}
9698 
9699 // Compare into -1,0,1
9700 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9701 %{
9702   match(Set dst (CmpF3 src1 src2));
9703   effect(KILL cr);
9704 
9705   ins_cost(275);
9706   format %{ "ucomiss $src1, $src2\n\t"
9707             "movl    $dst, #-1\n\t"
9708             "jp,s    done\n\t"
9709             "jb,s    done\n\t"
9710             "setne   $dst\n\t"
9711             "movzbl  $dst, $dst\n"
9712     "done:" %}
9713   ins_encode %{
9714     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9715     emit_cmpfp3(_masm, $dst$$Register);
9716   %}
9717   ins_pipe(pipe_slow);
9718 %}
9719 
9720 // Compare into -1,0,1
9721 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9722 %{
9723   match(Set dst (CmpF3 src1 (LoadF src2)));
9724   effect(KILL cr);
9725 
9726   ins_cost(275);
9727   format %{ "ucomiss $src1, $src2\n\t"
9728             "movl    $dst, #-1\n\t"
9729             "jp,s    done\n\t"
9730             "jb,s    done\n\t"
9731             "setne   $dst\n\t"
9732             "movzbl  $dst, $dst\n"
9733     "done:" %}
9734   ins_encode %{
9735     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9736     emit_cmpfp3(_masm, $dst$$Register);
9737   %}
9738   ins_pipe(pipe_slow);
9739 %}
9740 
9741 // Compare into -1,0,1
9742 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9743   match(Set dst (CmpF3 src con));
9744   effect(KILL cr);
9745 
9746   ins_cost(275);
9747   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9748             "movl    $dst, #-1\n\t"
9749             "jp,s    done\n\t"
9750             "jb,s    done\n\t"
9751             "setne   $dst\n\t"
9752             "movzbl  $dst, $dst\n"
9753     "done:" %}
9754   ins_encode %{
9755     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9756     emit_cmpfp3(_masm, $dst$$Register);
9757   %}
9758   ins_pipe(pipe_slow);
9759 %}
9760 
9761 // Compare into -1,0,1
9762 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9763 %{
9764   match(Set dst (CmpD3 src1 src2));
9765   effect(KILL cr);
9766 
9767   ins_cost(275);
9768   format %{ "ucomisd $src1, $src2\n\t"
9769             "movl    $dst, #-1\n\t"
9770             "jp,s    done\n\t"
9771             "jb,s    done\n\t"
9772             "setne   $dst\n\t"
9773             "movzbl  $dst, $dst\n"
9774     "done:" %}
9775   ins_encode %{
9776     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9777     emit_cmpfp3(_masm, $dst$$Register);
9778   %}
9779   ins_pipe(pipe_slow);
9780 %}
9781 
9782 // Compare into -1,0,1
9783 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9784 %{
9785   match(Set dst (CmpD3 src1 (LoadD src2)));
9786   effect(KILL cr);
9787 
9788   ins_cost(275);
9789   format %{ "ucomisd $src1, $src2\n\t"
9790             "movl    $dst, #-1\n\t"
9791             "jp,s    done\n\t"
9792             "jb,s    done\n\t"
9793             "setne   $dst\n\t"
9794             "movzbl  $dst, $dst\n"
9795     "done:" %}
9796   ins_encode %{
9797     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9798     emit_cmpfp3(_masm, $dst$$Register);
9799   %}
9800   ins_pipe(pipe_slow);
9801 %}
9802 
9803 // Compare into -1,0,1
9804 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
9805   match(Set dst (CmpD3 src con));
9806   effect(KILL cr);
9807 
9808   ins_cost(275);
9809   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9810             "movl    $dst, #-1\n\t"
9811             "jp,s    done\n\t"
9812             "jb,s    done\n\t"
9813             "setne   $dst\n\t"
9814             "movzbl  $dst, $dst\n"
9815     "done:" %}
9816   ins_encode %{
9817     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9818     emit_cmpfp3(_masm, $dst$$Register);
9819   %}
9820   ins_pipe(pipe_slow);
9821 %}
9822 
9823 // -----------Trig and Trancendental Instructions------------------------------
9824 instruct cosD_reg(regD dst) %{
9825   match(Set dst (CosD dst));
9826 
9827   format %{ "dcos   $dst\n\t" %}
9828   opcode(0xD9, 0xFF);
9829   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9830   ins_pipe( pipe_slow );
9831 %}
9832 
9833 instruct sinD_reg(regD dst) %{
9834   match(Set dst (SinD dst));
9835 
9836   format %{ "dsin   $dst\n\t" %}
9837   opcode(0xD9, 0xFE);
9838   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9839   ins_pipe( pipe_slow );
9840 %}
9841 
9842 instruct tanD_reg(regD dst) %{
9843   match(Set dst (TanD dst));
9844 
9845   format %{ "dtan   $dst\n\t" %}
9846   ins_encode( Push_SrcXD(dst),
9847               Opcode(0xD9), Opcode(0xF2),   //fptan
9848               Opcode(0xDD), Opcode(0xD8),   //fstp st
9849               Push_ResultXD(dst) );
9850   ins_pipe( pipe_slow );
9851 %}
9852 
9853 instruct log10D_reg(regD dst) %{
9854   // The source and result Double operands in XMM registers
9855   match(Set dst (Log10D dst));
9856   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
9857   // fyl2x        ; compute log_10(2) * log_2(x)
9858   format %{ "fldlg2\t\t\t#Log10\n\t"
9859             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
9860          %}
9861    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
9862               Push_SrcXD(dst),
9863               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9864               Push_ResultXD(dst));
9865 
9866   ins_pipe( pipe_slow );
9867 %}
9868 
9869 instruct logD_reg(regD dst) %{
9870   // The source and result Double operands in XMM registers
9871   match(Set dst (LogD dst));
9872   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
9873   // fyl2x        ; compute log_e(2) * log_2(x)
9874   format %{ "fldln2\t\t\t#Log_e\n\t"
9875             "fyl2x\t\t\t# Q=Log_e*Log_2(x)\n\t"
9876          %}
9877   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
9878               Push_SrcXD(dst),
9879               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9880               Push_ResultXD(dst));
9881   ins_pipe( pipe_slow );
9882 %}
9883 
9884 instruct powD_reg(regD dst, regD src0, regD src1, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9885   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
9886   effect(KILL rax, KILL rdx, KILL rcx, KILL cr);
9887   format %{ "fast_pow $src0 $src1 -> $dst  // KILL $rax, $rcx, $rdx" %}
9888   ins_encode %{
9889     __ subptr(rsp, 8);
9890     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);
9891     __ fld_d(Address(rsp, 0));
9892     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);
9893     __ fld_d(Address(rsp, 0));
9894     __ fast_pow();
9895     __ fstp_d(Address(rsp, 0));
9896     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9897     __ addptr(rsp, 8);
9898   %}
9899   ins_pipe( pipe_slow );
9900 %}
9901 
9902 instruct expD_reg(regD dst, regD src, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9903   match(Set dst (ExpD src));
9904   effect(KILL rax, KILL rcx, KILL rdx, KILL cr);
9905   format %{ "fast_exp $dst -> $src  // KILL $rax, $rcx, $rdx" %}
9906   ins_encode %{
9907     __ subptr(rsp, 8);
9908     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9909     __ fld_d(Address(rsp, 0));
9910     __ fast_exp();
9911     __ fstp_d(Address(rsp, 0));
9912     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9913     __ addptr(rsp, 8);
9914   %}
9915   ins_pipe( pipe_slow );
9916 %}
9917 
9918 //----------Arithmetic Conversion Instructions---------------------------------
9919 
9920 instruct roundFloat_nop(regF dst)
9921 %{
9922   match(Set dst (RoundFloat dst));
9923 
9924   ins_cost(0);
9925   ins_encode();
9926   ins_pipe(empty);
9927 %}
9928 
9929 instruct roundDouble_nop(regD dst)
9930 %{
9931   match(Set dst (RoundDouble dst));
9932 
9933   ins_cost(0);
9934   ins_encode();
9935   ins_pipe(empty);
9936 %}
9937 
9938 instruct convF2D_reg_reg(regD dst, regF src)
9939 %{
9940   match(Set dst (ConvF2D src));
9941 
9942   format %{ "cvtss2sd $dst, $src" %}
9943   ins_encode %{
9944     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
9945   %}
9946   ins_pipe(pipe_slow); // XXX
9947 %}
9948 
9949 instruct convF2D_reg_mem(regD dst, memory src)
9950 %{
9951   match(Set dst (ConvF2D (LoadF src)));
9952 
9953   format %{ "cvtss2sd $dst, $src" %}
9954   ins_encode %{
9955     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
9956   %}
9957   ins_pipe(pipe_slow); // XXX
9958 %}
9959 
9960 instruct convD2F_reg_reg(regF dst, regD src)
9961 %{
9962   match(Set dst (ConvD2F src));
9963 
9964   format %{ "cvtsd2ss $dst, $src" %}
9965   ins_encode %{
9966     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
9967   %}
9968   ins_pipe(pipe_slow); // XXX
9969 %}
9970 
9971 instruct convD2F_reg_mem(regF dst, memory src)
9972 %{
9973   match(Set dst (ConvD2F (LoadD src)));
9974 
9975   format %{ "cvtsd2ss $dst, $src" %}
9976   ins_encode %{
9977     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
9978   %}
9979   ins_pipe(pipe_slow); // XXX
9980 %}
9981 
9982 // XXX do mem variants
9983 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
9984 %{
9985   match(Set dst (ConvF2I src));
9986   effect(KILL cr);
9987 
9988   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
9989             "cmpl    $dst, #0x80000000\n\t"
9990             "jne,s   done\n\t"
9991             "subq    rsp, #8\n\t"
9992             "movss   [rsp], $src\n\t"
9993             "call    f2i_fixup\n\t"
9994             "popq    $dst\n"
9995     "done:   "%}
9996   ins_encode %{
9997     Label done;
9998     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
9999     __ cmpl($dst$$Register, 0x80000000);
10000     __ jccb(Assembler::notEqual, done);
10001     __ subptr(rsp, 8);
10002     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10003     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10004     __ pop($dst$$Register);
10005     __ bind(done);
10006   %}
10007   ins_pipe(pipe_slow);
10008 %}
10009 
10010 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10011 %{
10012   match(Set dst (ConvF2L src));
10013   effect(KILL cr);
10014 
10015   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10016             "cmpq    $dst, [0x8000000000000000]\n\t"
10017             "jne,s   done\n\t"
10018             "subq    rsp, #8\n\t"
10019             "movss   [rsp], $src\n\t"
10020             "call    f2l_fixup\n\t"
10021             "popq    $dst\n"
10022     "done:   "%}
10023   ins_encode %{
10024     Label done;
10025     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10026     __ cmp64($dst$$Register,
10027              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10028     __ jccb(Assembler::notEqual, done);
10029     __ subptr(rsp, 8);
10030     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10031     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10032     __ pop($dst$$Register);
10033     __ bind(done);
10034   %}
10035   ins_pipe(pipe_slow);
10036 %}
10037 
10038 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10039 %{
10040   match(Set dst (ConvD2I src));
10041   effect(KILL cr);
10042 
10043   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10044             "cmpl    $dst, #0x80000000\n\t"
10045             "jne,s   done\n\t"
10046             "subq    rsp, #8\n\t"
10047             "movsd   [rsp], $src\n\t"
10048             "call    d2i_fixup\n\t"
10049             "popq    $dst\n"
10050     "done:   "%}
10051   ins_encode %{
10052     Label done;
10053     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10054     __ cmpl($dst$$Register, 0x80000000);
10055     __ jccb(Assembler::notEqual, done);
10056     __ subptr(rsp, 8);
10057     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10058     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10059     __ pop($dst$$Register);
10060     __ bind(done);
10061   %}
10062   ins_pipe(pipe_slow);
10063 %}
10064 
10065 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10066 %{
10067   match(Set dst (ConvD2L src));
10068   effect(KILL cr);
10069 
10070   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10071             "cmpq    $dst, [0x8000000000000000]\n\t"
10072             "jne,s   done\n\t"
10073             "subq    rsp, #8\n\t"
10074             "movsd   [rsp], $src\n\t"
10075             "call    d2l_fixup\n\t"
10076             "popq    $dst\n"
10077     "done:   "%}
10078   ins_encode %{
10079     Label done;
10080     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10081     __ cmp64($dst$$Register,
10082              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10083     __ jccb(Assembler::notEqual, done);
10084     __ subptr(rsp, 8);
10085     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10086     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10087     __ pop($dst$$Register);
10088     __ bind(done);
10089   %}
10090   ins_pipe(pipe_slow);
10091 %}
10092 
10093 instruct convI2F_reg_reg(regF dst, rRegI src)
10094 %{
10095   predicate(!UseXmmI2F);
10096   match(Set dst (ConvI2F src));
10097 
10098   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10099   ins_encode %{
10100     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10101   %}
10102   ins_pipe(pipe_slow); // XXX
10103 %}
10104 
10105 instruct convI2F_reg_mem(regF dst, memory src)
10106 %{
10107   match(Set dst (ConvI2F (LoadI src)));
10108 
10109   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10110   ins_encode %{
10111     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10112   %}
10113   ins_pipe(pipe_slow); // XXX
10114 %}
10115 
10116 instruct convI2D_reg_reg(regD dst, rRegI src)
10117 %{
10118   predicate(!UseXmmI2D);
10119   match(Set dst (ConvI2D src));
10120 
10121   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10122   ins_encode %{
10123     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10124   %}
10125   ins_pipe(pipe_slow); // XXX
10126 %}
10127 
10128 instruct convI2D_reg_mem(regD dst, memory src)
10129 %{
10130   match(Set dst (ConvI2D (LoadI src)));
10131 
10132   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10133   ins_encode %{
10134     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10135   %}
10136   ins_pipe(pipe_slow); // XXX
10137 %}
10138 
10139 instruct convXI2F_reg(regF dst, rRegI src)
10140 %{
10141   predicate(UseXmmI2F);
10142   match(Set dst (ConvI2F src));
10143 
10144   format %{ "movdl $dst, $src\n\t"
10145             "cvtdq2psl $dst, $dst\t# i2f" %}
10146   ins_encode %{
10147     __ movdl($dst$$XMMRegister, $src$$Register);
10148     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10149   %}
10150   ins_pipe(pipe_slow); // XXX
10151 %}
10152 
10153 instruct convXI2D_reg(regD dst, rRegI src)
10154 %{
10155   predicate(UseXmmI2D);
10156   match(Set dst (ConvI2D src));
10157 
10158   format %{ "movdl $dst, $src\n\t"
10159             "cvtdq2pdl $dst, $dst\t# i2d" %}
10160   ins_encode %{
10161     __ movdl($dst$$XMMRegister, $src$$Register);
10162     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10163   %}
10164   ins_pipe(pipe_slow); // XXX
10165 %}
10166 
10167 instruct convL2F_reg_reg(regF dst, rRegL src)
10168 %{
10169   match(Set dst (ConvL2F src));
10170 
10171   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10172   ins_encode %{
10173     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10174   %}
10175   ins_pipe(pipe_slow); // XXX
10176 %}
10177 
10178 instruct convL2F_reg_mem(regF dst, memory src)
10179 %{
10180   match(Set dst (ConvL2F (LoadL src)));
10181 
10182   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10183   ins_encode %{
10184     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10185   %}
10186   ins_pipe(pipe_slow); // XXX
10187 %}
10188 
10189 instruct convL2D_reg_reg(regD dst, rRegL src)
10190 %{
10191   match(Set dst (ConvL2D src));
10192 
10193   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10194   ins_encode %{
10195     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10196   %}
10197   ins_pipe(pipe_slow); // XXX
10198 %}
10199 
10200 instruct convL2D_reg_mem(regD dst, memory src)
10201 %{
10202   match(Set dst (ConvL2D (LoadL src)));
10203 
10204   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10205   ins_encode %{
10206     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10207   %}
10208   ins_pipe(pipe_slow); // XXX
10209 %}
10210 
10211 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10212 %{
10213   match(Set dst (ConvI2L src));
10214 
10215   ins_cost(125);
10216   format %{ "movslq  $dst, $src\t# i2l" %}
10217   ins_encode %{
10218     __ movslq($dst$$Register, $src$$Register);
10219   %}
10220   ins_pipe(ialu_reg_reg);
10221 %}
10222 
10223 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10224 // %{
10225 //   match(Set dst (ConvI2L src));
10226 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10227 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10228 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10229 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10230 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10231 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10232 
10233 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10234 //   ins_encode(enc_copy(dst, src));
10235 // //   opcode(0x63); // needs REX.W
10236 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10237 //   ins_pipe(ialu_reg_reg);
10238 // %}
10239 
10240 // Zero-extend convert int to long
10241 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10242 %{
10243   match(Set dst (AndL (ConvI2L src) mask));
10244 
10245   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10246   ins_encode %{
10247     if ($dst$$reg != $src$$reg) {
10248       __ movl($dst$$Register, $src$$Register);
10249     }
10250   %}
10251   ins_pipe(ialu_reg_reg);
10252 %}
10253 
10254 // Zero-extend convert int to long
10255 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10256 %{
10257   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10258 
10259   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10260   ins_encode %{
10261     __ movl($dst$$Register, $src$$Address);
10262   %}
10263   ins_pipe(ialu_reg_mem);
10264 %}
10265 
10266 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10267 %{
10268   match(Set dst (AndL src mask));
10269 
10270   format %{ "movl    $dst, $src\t# zero-extend long" %}
10271   ins_encode %{
10272     __ movl($dst$$Register, $src$$Register);
10273   %}
10274   ins_pipe(ialu_reg_reg);
10275 %}
10276 
10277 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10278 %{
10279   match(Set dst (ConvL2I src));
10280 
10281   format %{ "movl    $dst, $src\t# l2i" %}
10282   ins_encode %{
10283     __ movl($dst$$Register, $src$$Register);
10284   %}
10285   ins_pipe(ialu_reg_reg);
10286 %}
10287 
10288 
10289 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10290   match(Set dst (MoveF2I src));
10291   effect(DEF dst, USE src);
10292 
10293   ins_cost(125);
10294   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10295   ins_encode %{
10296     __ movl($dst$$Register, Address(rsp, $src$$disp));
10297   %}
10298   ins_pipe(ialu_reg_mem);
10299 %}
10300 
10301 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10302   match(Set dst (MoveI2F src));
10303   effect(DEF dst, USE src);
10304 
10305   ins_cost(125);
10306   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10307   ins_encode %{
10308     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10309   %}
10310   ins_pipe(pipe_slow);
10311 %}
10312 
10313 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10314   match(Set dst (MoveD2L src));
10315   effect(DEF dst, USE src);
10316 
10317   ins_cost(125);
10318   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10319   ins_encode %{
10320     __ movq($dst$$Register, Address(rsp, $src$$disp));
10321   %}
10322   ins_pipe(ialu_reg_mem);
10323 %}
10324 
10325 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10326   predicate(!UseXmmLoadAndClearUpper);
10327   match(Set dst (MoveL2D src));
10328   effect(DEF dst, USE src);
10329 
10330   ins_cost(125);
10331   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10332   ins_encode %{
10333     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10334   %}
10335   ins_pipe(pipe_slow);
10336 %}
10337 
10338 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10339   predicate(UseXmmLoadAndClearUpper);
10340   match(Set dst (MoveL2D src));
10341   effect(DEF dst, USE src);
10342 
10343   ins_cost(125);
10344   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10345   ins_encode %{
10346     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10347   %}
10348   ins_pipe(pipe_slow);
10349 %}
10350 
10351 
10352 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10353   match(Set dst (MoveF2I src));
10354   effect(DEF dst, USE src);
10355 
10356   ins_cost(95); // XXX
10357   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10358   ins_encode %{
10359     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10360   %}
10361   ins_pipe(pipe_slow);
10362 %}
10363 
10364 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10365   match(Set dst (MoveI2F src));
10366   effect(DEF dst, USE src);
10367 
10368   ins_cost(100);
10369   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10370   ins_encode %{
10371     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10372   %}
10373   ins_pipe( ialu_mem_reg );
10374 %}
10375 
10376 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10377   match(Set dst (MoveD2L src));
10378   effect(DEF dst, USE src);
10379 
10380   ins_cost(95); // XXX
10381   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10382   ins_encode %{
10383     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10384   %}
10385   ins_pipe(pipe_slow);
10386 %}
10387 
10388 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10389   match(Set dst (MoveL2D src));
10390   effect(DEF dst, USE src);
10391 
10392   ins_cost(100);
10393   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10394   ins_encode %{
10395     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10396   %}
10397   ins_pipe(ialu_mem_reg);
10398 %}
10399 
10400 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10401   match(Set dst (MoveF2I src));
10402   effect(DEF dst, USE src);
10403   ins_cost(85);
10404   format %{ "movd    $dst,$src\t# MoveF2I" %}
10405   ins_encode %{
10406     __ movdl($dst$$Register, $src$$XMMRegister);
10407   %}
10408   ins_pipe( pipe_slow );
10409 %}
10410 
10411 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10412   match(Set dst (MoveD2L src));
10413   effect(DEF dst, USE src);
10414   ins_cost(85);
10415   format %{ "movd    $dst,$src\t# MoveD2L" %}
10416   ins_encode %{
10417     __ movdq($dst$$Register, $src$$XMMRegister);
10418   %}
10419   ins_pipe( pipe_slow );
10420 %}
10421 
10422 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10423   match(Set dst (MoveI2F src));
10424   effect(DEF dst, USE src);
10425   ins_cost(100);
10426   format %{ "movd    $dst,$src\t# MoveI2F" %}
10427   ins_encode %{
10428     __ movdl($dst$$XMMRegister, $src$$Register);
10429   %}
10430   ins_pipe( pipe_slow );
10431 %}
10432 
10433 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10434   match(Set dst (MoveL2D src));
10435   effect(DEF dst, USE src);
10436   ins_cost(100);
10437   format %{ "movd    $dst,$src\t# MoveL2D" %}
10438   ins_encode %{
10439      __ movdq($dst$$XMMRegister, $src$$Register);
10440   %}
10441   ins_pipe( pipe_slow );
10442 %}
10443 
10444 
10445 // =======================================================================
10446 // fast clearing of an array
10447 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10448                   rFlagsReg cr)
10449 %{
10450   predicate(!UseFastStosb);
10451   match(Set dummy (ClearArray cnt base));
10452   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10453 
10454   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10455             "rep     stosq\t# Store rax to *rdi++ while rcx--" %}
10456   ins_encode %{
10457     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10458   %}
10459   ins_pipe(pipe_slow);
10460 %}
10461 
10462 instruct rep_fast_stosb(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10463                         rFlagsReg cr)
10464 %{
10465   predicate(UseFastStosb);
10466   match(Set dummy (ClearArray cnt base));
10467   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10468   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10469             "shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10470             "rep     stosb\t# Store rax to *rdi++ while rcx--" %}
10471   ins_encode %{
10472     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10473   %}
10474   ins_pipe( pipe_slow );
10475 %}
10476 
10477 instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10478                         rax_RegI result, regD tmp1, rFlagsReg cr)
10479 %{
10480   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10481   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10482 
10483   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10484   ins_encode %{
10485     __ string_compare($str1$$Register, $str2$$Register,
10486                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10487                       $tmp1$$XMMRegister);
10488   %}
10489   ins_pipe( pipe_slow );
10490 %}
10491 
10492 // fast search of substring with known size.
10493 instruct string_indexof_con(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10494                             rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10495 %{
10496   predicate(UseSSE42Intrinsics);
10497   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10498   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10499 
10500   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10501   ins_encode %{
10502     int icnt2 = (int)$int_cnt2$$constant;
10503     if (icnt2 >= 8) {
10504       // IndexOf for constant substrings with size >= 8 elements
10505       // which don't need to be loaded through stack.
10506       __ string_indexofC8($str1$$Register, $str2$$Register,
10507                           $cnt1$$Register, $cnt2$$Register,
10508                           icnt2, $result$$Register,
10509                           $vec$$XMMRegister, $tmp$$Register);
10510     } else {
10511       // Small strings are loaded through stack if they cross page boundary.
10512       __ string_indexof($str1$$Register, $str2$$Register,
10513                         $cnt1$$Register, $cnt2$$Register,
10514                         icnt2, $result$$Register,
10515                         $vec$$XMMRegister, $tmp$$Register);
10516     }
10517   %}
10518   ins_pipe( pipe_slow );
10519 %}
10520 
10521 instruct string_indexof(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10522                         rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10523 %{
10524   predicate(UseSSE42Intrinsics);
10525   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10526   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10527 
10528   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10529   ins_encode %{
10530     __ string_indexof($str1$$Register, $str2$$Register,
10531                       $cnt1$$Register, $cnt2$$Register,
10532                       (-1), $result$$Register,
10533                       $vec$$XMMRegister, $tmp$$Register);
10534   %}
10535   ins_pipe( pipe_slow );
10536 %}
10537 
10538 // fast string equals
10539 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10540                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10541 %{
10542   match(Set result (StrEquals (Binary str1 str2) cnt));
10543   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10544 
10545   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10546   ins_encode %{
10547     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
10548                           $cnt$$Register, $result$$Register, $tmp3$$Register,
10549                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10550   %}
10551   ins_pipe( pipe_slow );
10552 %}
10553 
10554 // fast array equals
10555 instruct array_equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10556                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10557 %{
10558   match(Set result (AryEq ary1 ary2));
10559   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10560   //ins_cost(300);
10561 
10562   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10563   ins_encode %{
10564     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
10565                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
10566                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10567   %}
10568   ins_pipe( pipe_slow );
10569 %}
10570 
10571 // encode char[] to byte[] in ISO_8859_1
10572 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10573                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10574                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10575   match(Set result (EncodeISOArray src (Binary dst len)));
10576   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10577 
10578   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10579   ins_encode %{
10580     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10581                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10582                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10583   %}
10584   ins_pipe( pipe_slow );
10585 %}
10586 
10587 //----------Overflow Math Instructions-----------------------------------------
10588 
10589 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10590 %{
10591   match(Set cr (OverflowAddI op1 op2));
10592   effect(DEF cr, USE_KILL op1, USE op2);
10593 
10594   format %{ "addl    $op1, $op2\t# overflow check int" %}
10595 
10596   ins_encode %{
10597     __ addl($op1$$Register, $op2$$Register);
10598   %}
10599   ins_pipe(ialu_reg_reg);
10600 %}
10601 
10602 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
10603 %{
10604   match(Set cr (OverflowAddI op1 op2));
10605   effect(DEF cr, USE_KILL op1, USE op2);
10606 
10607   format %{ "addl    $op1, $op2\t# overflow check int" %}
10608 
10609   ins_encode %{
10610     __ addl($op1$$Register, $op2$$constant);
10611   %}
10612   ins_pipe(ialu_reg_reg);
10613 %}
10614 
10615 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10616 %{
10617   match(Set cr (OverflowAddL op1 op2));
10618   effect(DEF cr, USE_KILL op1, USE op2);
10619 
10620   format %{ "addq    $op1, $op2\t# overflow check long" %}
10621   ins_encode %{
10622     __ addq($op1$$Register, $op2$$Register);
10623   %}
10624   ins_pipe(ialu_reg_reg);
10625 %}
10626 
10627 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
10628 %{
10629   match(Set cr (OverflowAddL op1 op2));
10630   effect(DEF cr, USE_KILL op1, USE op2);
10631 
10632   format %{ "addq    $op1, $op2\t# overflow check long" %}
10633   ins_encode %{
10634     __ addq($op1$$Register, $op2$$constant);
10635   %}
10636   ins_pipe(ialu_reg_reg);
10637 %}
10638 
10639 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10640 %{
10641   match(Set cr (OverflowSubI op1 op2));
10642 
10643   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10644   ins_encode %{
10645     __ cmpl($op1$$Register, $op2$$Register);
10646   %}
10647   ins_pipe(ialu_reg_reg);
10648 %}
10649 
10650 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10651 %{
10652   match(Set cr (OverflowSubI op1 op2));
10653 
10654   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10655   ins_encode %{
10656     __ cmpl($op1$$Register, $op2$$constant);
10657   %}
10658   ins_pipe(ialu_reg_reg);
10659 %}
10660 
10661 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10662 %{
10663   match(Set cr (OverflowSubL op1 op2));
10664 
10665   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10666   ins_encode %{
10667     __ cmpq($op1$$Register, $op2$$Register);
10668   %}
10669   ins_pipe(ialu_reg_reg);
10670 %}
10671 
10672 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10673 %{
10674   match(Set cr (OverflowSubL op1 op2));
10675 
10676   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10677   ins_encode %{
10678     __ cmpq($op1$$Register, $op2$$constant);
10679   %}
10680   ins_pipe(ialu_reg_reg);
10681 %}
10682 
10683 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
10684 %{
10685   match(Set cr (OverflowSubI zero op2));
10686   effect(DEF cr, USE_KILL op2);
10687 
10688   format %{ "negl    $op2\t# overflow check int" %}
10689   ins_encode %{
10690     __ negl($op2$$Register);
10691   %}
10692   ins_pipe(ialu_reg_reg);
10693 %}
10694 
10695 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
10696 %{
10697   match(Set cr (OverflowSubL zero op2));
10698   effect(DEF cr, USE_KILL op2);
10699 
10700   format %{ "negq    $op2\t# overflow check long" %}
10701   ins_encode %{
10702     __ negq($op2$$Register);
10703   %}
10704   ins_pipe(ialu_reg_reg);
10705 %}
10706 
10707 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10708 %{
10709   match(Set cr (OverflowMulI op1 op2));
10710   effect(DEF cr, USE_KILL op1, USE op2);
10711 
10712   format %{ "imull    $op1, $op2\t# overflow check int" %}
10713   ins_encode %{
10714     __ imull($op1$$Register, $op2$$Register);
10715   %}
10716   ins_pipe(ialu_reg_reg_alu0);
10717 %}
10718 
10719 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
10720 %{
10721   match(Set cr (OverflowMulI op1 op2));
10722   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10723 
10724   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
10725   ins_encode %{
10726     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
10727   %}
10728   ins_pipe(ialu_reg_reg_alu0);
10729 %}
10730 
10731 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10732 %{
10733   match(Set cr (OverflowMulL op1 op2));
10734   effect(DEF cr, USE_KILL op1, USE op2);
10735 
10736   format %{ "imulq    $op1, $op2\t# overflow check long" %}
10737   ins_encode %{
10738     __ imulq($op1$$Register, $op2$$Register);
10739   %}
10740   ins_pipe(ialu_reg_reg_alu0);
10741 %}
10742 
10743 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
10744 %{
10745   match(Set cr (OverflowMulL op1 op2));
10746   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10747 
10748   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
10749   ins_encode %{
10750     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
10751   %}
10752   ins_pipe(ialu_reg_reg_alu0);
10753 %}
10754 
10755 
10756 //----------Control Flow Instructions------------------------------------------
10757 // Signed compare Instructions
10758 
10759 // XXX more variants!!
10760 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10761 %{
10762   match(Set cr (CmpI op1 op2));
10763   effect(DEF cr, USE op1, USE op2);
10764 
10765   format %{ "cmpl    $op1, $op2" %}
10766   opcode(0x3B);  /* Opcode 3B /r */
10767   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10768   ins_pipe(ialu_cr_reg_reg);
10769 %}
10770 
10771 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10772 %{
10773   match(Set cr (CmpI op1 op2));
10774 
10775   format %{ "cmpl    $op1, $op2" %}
10776   opcode(0x81, 0x07); /* Opcode 81 /7 */
10777   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10778   ins_pipe(ialu_cr_reg_imm);
10779 %}
10780 
10781 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
10782 %{
10783   match(Set cr (CmpI op1 (LoadI op2)));
10784 
10785   ins_cost(500); // XXX
10786   format %{ "cmpl    $op1, $op2" %}
10787   opcode(0x3B); /* Opcode 3B /r */
10788   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10789   ins_pipe(ialu_cr_reg_mem);
10790 %}
10791 
10792 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
10793 %{
10794   match(Set cr (CmpI src zero));
10795 
10796   format %{ "testl   $src, $src" %}
10797   opcode(0x85);
10798   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10799   ins_pipe(ialu_cr_reg_imm);
10800 %}
10801 
10802 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
10803 %{
10804   match(Set cr (CmpI (AndI src con) zero));
10805 
10806   format %{ "testl   $src, $con" %}
10807   opcode(0xF7, 0x00);
10808   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
10809   ins_pipe(ialu_cr_reg_imm);
10810 %}
10811 
10812 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
10813 %{
10814   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
10815 
10816   format %{ "testl   $src, $mem" %}
10817   opcode(0x85);
10818   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
10819   ins_pipe(ialu_cr_reg_mem);
10820 %}
10821 
10822 // Unsigned compare Instructions; really, same as signed except they
10823 // produce an rFlagsRegU instead of rFlagsReg.
10824 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
10825 %{
10826   match(Set cr (CmpU op1 op2));
10827 
10828   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10829   opcode(0x3B); /* Opcode 3B /r */
10830   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10831   ins_pipe(ialu_cr_reg_reg);
10832 %}
10833 
10834 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
10835 %{
10836   match(Set cr (CmpU op1 op2));
10837 
10838   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10839   opcode(0x81,0x07); /* Opcode 81 /7 */
10840   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10841   ins_pipe(ialu_cr_reg_imm);
10842 %}
10843 
10844 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
10845 %{
10846   match(Set cr (CmpU op1 (LoadI op2)));
10847 
10848   ins_cost(500); // XXX
10849   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10850   opcode(0x3B); /* Opcode 3B /r */
10851   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10852   ins_pipe(ialu_cr_reg_mem);
10853 %}
10854 
10855 // // // Cisc-spilled version of cmpU_rReg
10856 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
10857 // //%{
10858 // //  match(Set cr (CmpU (LoadI op1) op2));
10859 // //
10860 // //  format %{ "CMPu   $op1,$op2" %}
10861 // //  ins_cost(500);
10862 // //  opcode(0x39);  /* Opcode 39 /r */
10863 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10864 // //%}
10865 
10866 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
10867 %{
10868   match(Set cr (CmpU src zero));
10869 
10870   format %{ "testl  $src, $src\t# unsigned" %}
10871   opcode(0x85);
10872   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10873   ins_pipe(ialu_cr_reg_imm);
10874 %}
10875 
10876 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
10877 %{
10878   match(Set cr (CmpP op1 op2));
10879 
10880   format %{ "cmpq    $op1, $op2\t# ptr" %}
10881   opcode(0x3B); /* Opcode 3B /r */
10882   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10883   ins_pipe(ialu_cr_reg_reg);
10884 %}
10885 
10886 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
10887 %{
10888   match(Set cr (CmpP op1 (LoadP op2)));
10889 
10890   ins_cost(500); // XXX
10891   format %{ "cmpq    $op1, $op2\t# ptr" %}
10892   opcode(0x3B); /* Opcode 3B /r */
10893   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10894   ins_pipe(ialu_cr_reg_mem);
10895 %}
10896 
10897 // // // Cisc-spilled version of cmpP_rReg
10898 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
10899 // //%{
10900 // //  match(Set cr (CmpP (LoadP op1) op2));
10901 // //
10902 // //  format %{ "CMPu   $op1,$op2" %}
10903 // //  ins_cost(500);
10904 // //  opcode(0x39);  /* Opcode 39 /r */
10905 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10906 // //%}
10907 
10908 // XXX this is generalized by compP_rReg_mem???
10909 // Compare raw pointer (used in out-of-heap check).
10910 // Only works because non-oop pointers must be raw pointers
10911 // and raw pointers have no anti-dependencies.
10912 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
10913 %{
10914   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
10915   match(Set cr (CmpP op1 (LoadP op2)));
10916 
10917   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
10918   opcode(0x3B); /* Opcode 3B /r */
10919   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10920   ins_pipe(ialu_cr_reg_mem);
10921 %}
10922 
10923 // This will generate a signed flags result. This should be OK since
10924 // any compare to a zero should be eq/neq.
10925 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
10926 %{
10927   match(Set cr (CmpP src zero));
10928 
10929   format %{ "testq   $src, $src\t# ptr" %}
10930   opcode(0x85);
10931   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
10932   ins_pipe(ialu_cr_reg_imm);
10933 %}
10934 
10935 // This will generate a signed flags result. This should be OK since
10936 // any compare to a zero should be eq/neq.
10937 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
10938 %{
10939   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
10940   match(Set cr (CmpP (LoadP op) zero));
10941 
10942   ins_cost(500); // XXX
10943   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
10944   opcode(0xF7); /* Opcode F7 /0 */
10945   ins_encode(REX_mem_wide(op),
10946              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
10947   ins_pipe(ialu_cr_reg_imm);
10948 %}
10949 
10950 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
10951 %{
10952   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
10953   match(Set cr (CmpP (LoadP mem) zero));
10954 
10955   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
10956   ins_encode %{
10957     __ cmpq(r12, $mem$$Address);
10958   %}
10959   ins_pipe(ialu_cr_reg_mem);
10960 %}
10961 
10962 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
10963 %{
10964   match(Set cr (CmpN op1 op2));
10965 
10966   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10967   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
10968   ins_pipe(ialu_cr_reg_reg);
10969 %}
10970 
10971 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
10972 %{
10973   match(Set cr (CmpN src (LoadN mem)));
10974 
10975   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
10976   ins_encode %{
10977     __ cmpl($src$$Register, $mem$$Address);
10978   %}
10979   ins_pipe(ialu_cr_reg_mem);
10980 %}
10981 
10982 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
10983   match(Set cr (CmpN op1 op2));
10984 
10985   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10986   ins_encode %{
10987     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
10988   %}
10989   ins_pipe(ialu_cr_reg_imm);
10990 %}
10991 
10992 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
10993 %{
10994   match(Set cr (CmpN src (LoadN mem)));
10995 
10996   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
10997   ins_encode %{
10998     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
10999   %}
11000   ins_pipe(ialu_cr_reg_mem);
11001 %}
11002 
11003 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11004   match(Set cr (CmpN op1 op2));
11005 
11006   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11007   ins_encode %{
11008     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11009   %}
11010   ins_pipe(ialu_cr_reg_imm);
11011 %}
11012 
11013 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11014 %{
11015   match(Set cr (CmpN src (LoadNKlass mem)));
11016 
11017   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11018   ins_encode %{
11019     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11020   %}
11021   ins_pipe(ialu_cr_reg_mem);
11022 %}
11023 
11024 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11025   match(Set cr (CmpN src zero));
11026 
11027   format %{ "testl   $src, $src\t# compressed ptr" %}
11028   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11029   ins_pipe(ialu_cr_reg_imm);
11030 %}
11031 
11032 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11033 %{
11034   predicate(Universe::narrow_oop_base() != NULL);
11035   match(Set cr (CmpN (LoadN mem) zero));
11036 
11037   ins_cost(500); // XXX
11038   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11039   ins_encode %{
11040     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11041   %}
11042   ins_pipe(ialu_cr_reg_mem);
11043 %}
11044 
11045 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11046 %{
11047   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11048   match(Set cr (CmpN (LoadN mem) zero));
11049 
11050   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11051   ins_encode %{
11052     __ cmpl(r12, $mem$$Address);
11053   %}
11054   ins_pipe(ialu_cr_reg_mem);
11055 %}
11056 
11057 // Yanked all unsigned pointer compare operations.
11058 // Pointer compares are done with CmpP which is already unsigned.
11059 
11060 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11061 %{
11062   match(Set cr (CmpL op1 op2));
11063 
11064   format %{ "cmpq    $op1, $op2" %}
11065   opcode(0x3B);  /* Opcode 3B /r */
11066   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11067   ins_pipe(ialu_cr_reg_reg);
11068 %}
11069 
11070 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11071 %{
11072   match(Set cr (CmpL op1 op2));
11073 
11074   format %{ "cmpq    $op1, $op2" %}
11075   opcode(0x81, 0x07); /* Opcode 81 /7 */
11076   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11077   ins_pipe(ialu_cr_reg_imm);
11078 %}
11079 
11080 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11081 %{
11082   match(Set cr (CmpL op1 (LoadL op2)));
11083 
11084   format %{ "cmpq    $op1, $op2" %}
11085   opcode(0x3B); /* Opcode 3B /r */
11086   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11087   ins_pipe(ialu_cr_reg_mem);
11088 %}
11089 
11090 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11091 %{
11092   match(Set cr (CmpL src zero));
11093 
11094   format %{ "testq   $src, $src" %}
11095   opcode(0x85);
11096   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11097   ins_pipe(ialu_cr_reg_imm);
11098 %}
11099 
11100 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11101 %{
11102   match(Set cr (CmpL (AndL src con) zero));
11103 
11104   format %{ "testq   $src, $con\t# long" %}
11105   opcode(0xF7, 0x00);
11106   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11107   ins_pipe(ialu_cr_reg_imm);
11108 %}
11109 
11110 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11111 %{
11112   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11113 
11114   format %{ "testq   $src, $mem" %}
11115   opcode(0x85);
11116   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11117   ins_pipe(ialu_cr_reg_mem);
11118 %}
11119 
11120 // Manifest a CmpL result in an integer register.  Very painful.
11121 // This is the test to avoid.
11122 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11123 %{
11124   match(Set dst (CmpL3 src1 src2));
11125   effect(KILL flags);
11126 
11127   ins_cost(275); // XXX
11128   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11129             "movl    $dst, -1\n\t"
11130             "jl,s    done\n\t"
11131             "setne   $dst\n\t"
11132             "movzbl  $dst, $dst\n\t"
11133     "done:" %}
11134   ins_encode(cmpl3_flag(src1, src2, dst));
11135   ins_pipe(pipe_slow);
11136 %}
11137 
11138 //----------Max and Min--------------------------------------------------------
11139 // Min Instructions
11140 
11141 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11142 %{
11143   effect(USE_DEF dst, USE src, USE cr);
11144 
11145   format %{ "cmovlgt $dst, $src\t# min" %}
11146   opcode(0x0F, 0x4F);
11147   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11148   ins_pipe(pipe_cmov_reg);
11149 %}
11150 
11151 
11152 instruct minI_rReg(rRegI dst, rRegI src)
11153 %{
11154   match(Set dst (MinI dst src));
11155 
11156   ins_cost(200);
11157   expand %{
11158     rFlagsReg cr;
11159     compI_rReg(cr, dst, src);
11160     cmovI_reg_g(dst, src, cr);
11161   %}
11162 %}
11163 
11164 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11165 %{
11166   effect(USE_DEF dst, USE src, USE cr);
11167 
11168   format %{ "cmovllt $dst, $src\t# max" %}
11169   opcode(0x0F, 0x4C);
11170   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11171   ins_pipe(pipe_cmov_reg);
11172 %}
11173 
11174 
11175 instruct maxI_rReg(rRegI dst, rRegI src)
11176 %{
11177   match(Set dst (MaxI dst src));
11178 
11179   ins_cost(200);
11180   expand %{
11181     rFlagsReg cr;
11182     compI_rReg(cr, dst, src);
11183     cmovI_reg_l(dst, src, cr);
11184   %}
11185 %}
11186 
11187 // ============================================================================
11188 // Branch Instructions
11189 
11190 // Jump Direct - Label defines a relative address from JMP+1
11191 instruct jmpDir(label labl)
11192 %{
11193   match(Goto);
11194   effect(USE labl);
11195 
11196   ins_cost(300);
11197   format %{ "jmp     $labl" %}
11198   size(5);
11199   ins_encode %{
11200     Label* L = $labl$$label;
11201     __ jmp(*L, false); // Always long jump
11202   %}
11203   ins_pipe(pipe_jmp);
11204 %}
11205 
11206 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11207 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11208 %{
11209   match(If cop cr);
11210   effect(USE labl);
11211 
11212   ins_cost(300);
11213   format %{ "j$cop     $labl" %}
11214   size(6);
11215   ins_encode %{
11216     Label* L = $labl$$label;
11217     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11218   %}
11219   ins_pipe(pipe_jcc);
11220 %}
11221 
11222 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11223 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11224 %{
11225   match(CountedLoopEnd cop cr);
11226   effect(USE labl);
11227 
11228   ins_cost(300);
11229   format %{ "j$cop     $labl\t# loop end" %}
11230   size(6);
11231   ins_encode %{
11232     Label* L = $labl$$label;
11233     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11234   %}
11235   ins_pipe(pipe_jcc);
11236 %}
11237 
11238 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11239 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11240   match(CountedLoopEnd cop cmp);
11241   effect(USE labl);
11242 
11243   ins_cost(300);
11244   format %{ "j$cop,u   $labl\t# loop end" %}
11245   size(6);
11246   ins_encode %{
11247     Label* L = $labl$$label;
11248     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11249   %}
11250   ins_pipe(pipe_jcc);
11251 %}
11252 
11253 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11254   match(CountedLoopEnd cop cmp);
11255   effect(USE labl);
11256 
11257   ins_cost(200);
11258   format %{ "j$cop,u   $labl\t# loop end" %}
11259   size(6);
11260   ins_encode %{
11261     Label* L = $labl$$label;
11262     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11263   %}
11264   ins_pipe(pipe_jcc);
11265 %}
11266 
11267 // Jump Direct Conditional - using unsigned comparison
11268 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11269   match(If cop cmp);
11270   effect(USE labl);
11271 
11272   ins_cost(300);
11273   format %{ "j$cop,u  $labl" %}
11274   size(6);
11275   ins_encode %{
11276     Label* L = $labl$$label;
11277     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11278   %}
11279   ins_pipe(pipe_jcc);
11280 %}
11281 
11282 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11283   match(If cop cmp);
11284   effect(USE labl);
11285 
11286   ins_cost(200);
11287   format %{ "j$cop,u  $labl" %}
11288   size(6);
11289   ins_encode %{
11290     Label* L = $labl$$label;
11291     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11292   %}
11293   ins_pipe(pipe_jcc);
11294 %}
11295 
11296 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11297   match(If cop cmp);
11298   effect(USE labl);
11299 
11300   ins_cost(200);
11301   format %{ $$template
11302     if ($cop$$cmpcode == Assembler::notEqual) {
11303       $$emit$$"jp,u   $labl\n\t"
11304       $$emit$$"j$cop,u   $labl"
11305     } else {
11306       $$emit$$"jp,u   done\n\t"
11307       $$emit$$"j$cop,u   $labl\n\t"
11308       $$emit$$"done:"
11309     }
11310   %}
11311   ins_encode %{
11312     Label* l = $labl$$label;
11313     if ($cop$$cmpcode == Assembler::notEqual) {
11314       __ jcc(Assembler::parity, *l, false);
11315       __ jcc(Assembler::notEqual, *l, false);
11316     } else if ($cop$$cmpcode == Assembler::equal) {
11317       Label done;
11318       __ jccb(Assembler::parity, done);
11319       __ jcc(Assembler::equal, *l, false);
11320       __ bind(done);
11321     } else {
11322        ShouldNotReachHere();
11323     }
11324   %}
11325   ins_pipe(pipe_jcc);
11326 %}
11327 
11328 // ============================================================================
11329 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11330 // superklass array for an instance of the superklass.  Set a hidden
11331 // internal cache on a hit (cache is checked with exposed code in
11332 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11333 // encoding ALSO sets flags.
11334 
11335 instruct partialSubtypeCheck(rdi_RegP result,
11336                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11337                              rFlagsReg cr)
11338 %{
11339   match(Set result (PartialSubtypeCheck sub super));
11340   effect(KILL rcx, KILL cr);
11341 
11342   ins_cost(1100);  // slightly larger than the next version
11343   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11344             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11345             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11346             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11347             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11348             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11349             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11350     "miss:\t" %}
11351 
11352   opcode(0x1); // Force a XOR of RDI
11353   ins_encode(enc_PartialSubtypeCheck());
11354   ins_pipe(pipe_slow);
11355 %}
11356 
11357 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11358                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11359                                      immP0 zero,
11360                                      rdi_RegP result)
11361 %{
11362   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11363   effect(KILL rcx, KILL result);
11364 
11365   ins_cost(1000);
11366   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11367             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11368             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11369             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11370             "jne,s   miss\t\t# Missed: flags nz\n\t"
11371             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11372     "miss:\t" %}
11373 
11374   opcode(0x0); // No need to XOR RDI
11375   ins_encode(enc_PartialSubtypeCheck());
11376   ins_pipe(pipe_slow);
11377 %}
11378 
11379 // ============================================================================
11380 // Branch Instructions -- short offset versions
11381 //
11382 // These instructions are used to replace jumps of a long offset (the default
11383 // match) with jumps of a shorter offset.  These instructions are all tagged
11384 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11385 // match rules in general matching.  Instead, the ADLC generates a conversion
11386 // method in the MachNode which can be used to do in-place replacement of the
11387 // long variant with the shorter variant.  The compiler will determine if a
11388 // branch can be taken by the is_short_branch_offset() predicate in the machine
11389 // specific code section of the file.
11390 
11391 // Jump Direct - Label defines a relative address from JMP+1
11392 instruct jmpDir_short(label labl) %{
11393   match(Goto);
11394   effect(USE labl);
11395 
11396   ins_cost(300);
11397   format %{ "jmp,s   $labl" %}
11398   size(2);
11399   ins_encode %{
11400     Label* L = $labl$$label;
11401     __ jmpb(*L);
11402   %}
11403   ins_pipe(pipe_jmp);
11404   ins_short_branch(1);
11405 %}
11406 
11407 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11408 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11409   match(If cop cr);
11410   effect(USE labl);
11411 
11412   ins_cost(300);
11413   format %{ "j$cop,s   $labl" %}
11414   size(2);
11415   ins_encode %{
11416     Label* L = $labl$$label;
11417     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11418   %}
11419   ins_pipe(pipe_jcc);
11420   ins_short_branch(1);
11421 %}
11422 
11423 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11424 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11425   match(CountedLoopEnd cop cr);
11426   effect(USE labl);
11427 
11428   ins_cost(300);
11429   format %{ "j$cop,s   $labl\t# loop end" %}
11430   size(2);
11431   ins_encode %{
11432     Label* L = $labl$$label;
11433     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11434   %}
11435   ins_pipe(pipe_jcc);
11436   ins_short_branch(1);
11437 %}
11438 
11439 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11440 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11441   match(CountedLoopEnd cop cmp);
11442   effect(USE labl);
11443 
11444   ins_cost(300);
11445   format %{ "j$cop,us  $labl\t# loop end" %}
11446   size(2);
11447   ins_encode %{
11448     Label* L = $labl$$label;
11449     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11450   %}
11451   ins_pipe(pipe_jcc);
11452   ins_short_branch(1);
11453 %}
11454 
11455 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11456   match(CountedLoopEnd cop cmp);
11457   effect(USE labl);
11458 
11459   ins_cost(300);
11460   format %{ "j$cop,us  $labl\t# loop end" %}
11461   size(2);
11462   ins_encode %{
11463     Label* L = $labl$$label;
11464     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11465   %}
11466   ins_pipe(pipe_jcc);
11467   ins_short_branch(1);
11468 %}
11469 
11470 // Jump Direct Conditional - using unsigned comparison
11471 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11472   match(If cop cmp);
11473   effect(USE labl);
11474 
11475   ins_cost(300);
11476   format %{ "j$cop,us  $labl" %}
11477   size(2);
11478   ins_encode %{
11479     Label* L = $labl$$label;
11480     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11481   %}
11482   ins_pipe(pipe_jcc);
11483   ins_short_branch(1);
11484 %}
11485 
11486 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11487   match(If cop cmp);
11488   effect(USE labl);
11489 
11490   ins_cost(300);
11491   format %{ "j$cop,us  $labl" %}
11492   size(2);
11493   ins_encode %{
11494     Label* L = $labl$$label;
11495     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11496   %}
11497   ins_pipe(pipe_jcc);
11498   ins_short_branch(1);
11499 %}
11500 
11501 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11502   match(If cop cmp);
11503   effect(USE labl);
11504 
11505   ins_cost(300);
11506   format %{ $$template
11507     if ($cop$$cmpcode == Assembler::notEqual) {
11508       $$emit$$"jp,u,s   $labl\n\t"
11509       $$emit$$"j$cop,u,s   $labl"
11510     } else {
11511       $$emit$$"jp,u,s   done\n\t"
11512       $$emit$$"j$cop,u,s  $labl\n\t"
11513       $$emit$$"done:"
11514     }
11515   %}
11516   size(4);
11517   ins_encode %{
11518     Label* l = $labl$$label;
11519     if ($cop$$cmpcode == Assembler::notEqual) {
11520       __ jccb(Assembler::parity, *l);
11521       __ jccb(Assembler::notEqual, *l);
11522     } else if ($cop$$cmpcode == Assembler::equal) {
11523       Label done;
11524       __ jccb(Assembler::parity, done);
11525       __ jccb(Assembler::equal, *l);
11526       __ bind(done);
11527     } else {
11528        ShouldNotReachHere();
11529     }
11530   %}
11531   ins_pipe(pipe_jcc);
11532   ins_short_branch(1);
11533 %}
11534 
11535 // ============================================================================
11536 // inlined locking and unlocking
11537 
11538 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
11539   predicate(Compile::current()->use_rtm());
11540   match(Set cr (FastLock object box));
11541   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
11542   ins_cost(300);
11543   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
11544   ins_encode %{
11545     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11546                  $scr$$Register, $cx1$$Register, $cx2$$Register,
11547                  _counters, _rtm_counters, _stack_rtm_counters,
11548                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
11549                  true, ra_->C->profile_rtm());
11550   %}
11551   ins_pipe(pipe_slow);
11552 %}
11553 
11554 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
11555   predicate(!Compile::current()->use_rtm());
11556   match(Set cr (FastLock object box));
11557   effect(TEMP tmp, TEMP scr, USE_KILL box);
11558   ins_cost(300);
11559   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11560   ins_encode %{
11561     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11562                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
11563   %}
11564   ins_pipe(pipe_slow);
11565 %}
11566 
11567 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
11568   match(Set cr (FastUnlock object box));
11569   effect(TEMP tmp, USE_KILL box);
11570   ins_cost(300);
11571   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11572   ins_encode %{
11573     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
11574   %}
11575   ins_pipe(pipe_slow);
11576 %}
11577 
11578 
11579 // ============================================================================
11580 // Safepoint Instructions
11581 instruct safePoint_poll(rFlagsReg cr)
11582 %{
11583   predicate(!Assembler::is_polling_page_far());
11584   match(SafePoint);
11585   effect(KILL cr);
11586 
11587   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
11588             "# Safepoint: poll for GC" %}
11589   ins_cost(125);
11590   ins_encode %{
11591     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11592     __ testl(rax, addr);
11593   %}
11594   ins_pipe(ialu_reg_mem);
11595 %}
11596 
11597 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
11598 %{
11599   predicate(Assembler::is_polling_page_far());
11600   match(SafePoint poll);
11601   effect(KILL cr, USE poll);
11602 
11603   format %{ "testl  rax, [$poll]\t"
11604             "# Safepoint: poll for GC" %}
11605   ins_cost(125);
11606   ins_encode %{
11607     __ relocate(relocInfo::poll_type);
11608     __ testl(rax, Address($poll$$Register, 0));
11609   %}
11610   ins_pipe(ialu_reg_mem);
11611 %}
11612 
11613 // ============================================================================
11614 // Procedure Call/Return Instructions
11615 // Call Java Static Instruction
11616 // Note: If this code changes, the corresponding ret_addr_offset() and
11617 //       compute_padding() functions will have to be adjusted.
11618 instruct CallStaticJavaDirect(method meth) %{
11619   match(CallStaticJava);
11620   effect(USE meth);
11621 
11622   ins_cost(300);
11623   format %{ "call,static " %}
11624   opcode(0xE8); /* E8 cd */
11625   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
11626   ins_pipe(pipe_slow);
11627   ins_alignment(4);
11628 %}
11629 
11630 // Call Java Dynamic Instruction
11631 // Note: If this code changes, the corresponding ret_addr_offset() and
11632 //       compute_padding() functions will have to be adjusted.
11633 instruct CallDynamicJavaDirect(method meth)
11634 %{
11635   match(CallDynamicJava);
11636   effect(USE meth);
11637 
11638   ins_cost(300);
11639   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
11640             "call,dynamic " %}
11641   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
11642   ins_pipe(pipe_slow);
11643   ins_alignment(4);
11644 %}
11645 
11646 // Call Runtime Instruction
11647 instruct CallRuntimeDirect(method meth)
11648 %{
11649   match(CallRuntime);
11650   effect(USE meth);
11651 
11652   ins_cost(300);
11653   format %{ "call,runtime " %}
11654   ins_encode(clear_avx, Java_To_Runtime(meth));
11655   ins_pipe(pipe_slow);
11656 %}
11657 
11658 // Call runtime without safepoint
11659 instruct CallLeafDirect(method meth)
11660 %{
11661   match(CallLeaf);
11662   effect(USE meth);
11663 
11664   ins_cost(300);
11665   format %{ "call_leaf,runtime " %}
11666   ins_encode(clear_avx, Java_To_Runtime(meth));
11667   ins_pipe(pipe_slow);
11668 %}
11669 
11670 // Call runtime without safepoint
11671 instruct CallLeafNoFPDirect(method meth)
11672 %{
11673   match(CallLeafNoFP);
11674   effect(USE meth);
11675 
11676   ins_cost(300);
11677   format %{ "call_leaf_nofp,runtime " %}
11678   ins_encode(Java_To_Runtime(meth));
11679   ins_pipe(pipe_slow);
11680 %}
11681 
11682 // Return Instruction
11683 // Remove the return address & jump to it.
11684 // Notice: We always emit a nop after a ret to make sure there is room
11685 // for safepoint patching
11686 instruct Ret()
11687 %{
11688   match(Return);
11689 
11690   format %{ "ret" %}
11691   opcode(0xC3);
11692   ins_encode(OpcP);
11693   ins_pipe(pipe_jmp);
11694 %}
11695 
11696 // Tail Call; Jump from runtime stub to Java code.
11697 // Also known as an 'interprocedural jump'.
11698 // Target of jump will eventually return to caller.
11699 // TailJump below removes the return address.
11700 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
11701 %{
11702   match(TailCall jump_target method_oop);
11703 
11704   ins_cost(300);
11705   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
11706   opcode(0xFF, 0x4); /* Opcode FF /4 */
11707   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
11708   ins_pipe(pipe_jmp);
11709 %}
11710 
11711 // Tail Jump; remove the return address; jump to target.
11712 // TailCall above leaves the return address around.
11713 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
11714 %{
11715   match(TailJump jump_target ex_oop);
11716 
11717   ins_cost(300);
11718   format %{ "popq    rdx\t# pop return address\n\t"
11719             "jmp     $jump_target" %}
11720   opcode(0xFF, 0x4); /* Opcode FF /4 */
11721   ins_encode(Opcode(0x5a), // popq rdx
11722              REX_reg(jump_target), OpcP, reg_opc(jump_target));
11723   ins_pipe(pipe_jmp);
11724 %}
11725 
11726 // Create exception oop: created by stack-crawling runtime code.
11727 // Created exception is now available to this handler, and is setup
11728 // just prior to jumping to this handler.  No code emitted.
11729 instruct CreateException(rax_RegP ex_oop)
11730 %{
11731   match(Set ex_oop (CreateEx));
11732 
11733   size(0);
11734   // use the following format syntax
11735   format %{ "# exception oop is in rax; no code emitted" %}
11736   ins_encode();
11737   ins_pipe(empty);
11738 %}
11739 
11740 // Rethrow exception:
11741 // The exception oop will come in the first argument position.
11742 // Then JUMP (not call) to the rethrow stub code.
11743 instruct RethrowException()
11744 %{
11745   match(Rethrow);
11746 
11747   // use the following format syntax
11748   format %{ "jmp     rethrow_stub" %}
11749   ins_encode(enc_rethrow);
11750   ins_pipe(pipe_jmp);
11751 %}
11752 
11753 
11754 // ============================================================================
11755 // This name is KNOWN by the ADLC and cannot be changed.
11756 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
11757 // for this guy.
11758 instruct tlsLoadP(r15_RegP dst) %{
11759   match(Set dst (ThreadLocal));
11760   effect(DEF dst);
11761 
11762   size(0);
11763   format %{ "# TLS is in R15" %}
11764   ins_encode( /*empty encoding*/ );
11765   ins_pipe(ialu_reg_reg);
11766 %}
11767 
11768 
11769 //----------PEEPHOLE RULES-----------------------------------------------------
11770 // These must follow all instruction definitions as they use the names
11771 // defined in the instructions definitions.
11772 //
11773 // peepmatch ( root_instr_name [preceding_instruction]* );
11774 //
11775 // peepconstraint %{
11776 // (instruction_number.operand_name relational_op instruction_number.operand_name
11777 //  [, ...] );
11778 // // instruction numbers are zero-based using left to right order in peepmatch
11779 //
11780 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
11781 // // provide an instruction_number.operand_name for each operand that appears
11782 // // in the replacement instruction's match rule
11783 //
11784 // ---------VM FLAGS---------------------------------------------------------
11785 //
11786 // All peephole optimizations can be turned off using -XX:-OptoPeephole
11787 //
11788 // Each peephole rule is given an identifying number starting with zero and
11789 // increasing by one in the order seen by the parser.  An individual peephole
11790 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
11791 // on the command-line.
11792 //
11793 // ---------CURRENT LIMITATIONS----------------------------------------------
11794 //
11795 // Only match adjacent instructions in same basic block
11796 // Only equality constraints
11797 // Only constraints between operands, not (0.dest_reg == RAX_enc)
11798 // Only one replacement instruction
11799 //
11800 // ---------EXAMPLE----------------------------------------------------------
11801 //
11802 // // pertinent parts of existing instructions in architecture description
11803 // instruct movI(rRegI dst, rRegI src)
11804 // %{
11805 //   match(Set dst (CopyI src));
11806 // %}
11807 //
11808 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
11809 // %{
11810 //   match(Set dst (AddI dst src));
11811 //   effect(KILL cr);
11812 // %}
11813 //
11814 // // Change (inc mov) to lea
11815 // peephole %{
11816 //   // increment preceeded by register-register move
11817 //   peepmatch ( incI_rReg movI );
11818 //   // require that the destination register of the increment
11819 //   // match the destination register of the move
11820 //   peepconstraint ( 0.dst == 1.dst );
11821 //   // construct a replacement instruction that sets
11822 //   // the destination to ( move's source register + one )
11823 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
11824 // %}
11825 //
11826 
11827 // Implementation no longer uses movX instructions since
11828 // machine-independent system no longer uses CopyX nodes.
11829 //
11830 // peephole
11831 // %{
11832 //   peepmatch (incI_rReg movI);
11833 //   peepconstraint (0.dst == 1.dst);
11834 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11835 // %}
11836 
11837 // peephole
11838 // %{
11839 //   peepmatch (decI_rReg movI);
11840 //   peepconstraint (0.dst == 1.dst);
11841 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11842 // %}
11843 
11844 // peephole
11845 // %{
11846 //   peepmatch (addI_rReg_imm movI);
11847 //   peepconstraint (0.dst == 1.dst);
11848 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11849 // %}
11850 
11851 // peephole
11852 // %{
11853 //   peepmatch (incL_rReg movL);
11854 //   peepconstraint (0.dst == 1.dst);
11855 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11856 // %}
11857 
11858 // peephole
11859 // %{
11860 //   peepmatch (decL_rReg movL);
11861 //   peepconstraint (0.dst == 1.dst);
11862 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11863 // %}
11864 
11865 // peephole
11866 // %{
11867 //   peepmatch (addL_rReg_imm movL);
11868 //   peepconstraint (0.dst == 1.dst);
11869 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11870 // %}
11871 
11872 // peephole
11873 // %{
11874 //   peepmatch (addP_rReg_imm movP);
11875 //   peepconstraint (0.dst == 1.dst);
11876 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
11877 // %}
11878 
11879 // // Change load of spilled value to only a spill
11880 // instruct storeI(memory mem, rRegI src)
11881 // %{
11882 //   match(Set mem (StoreI mem src));
11883 // %}
11884 //
11885 // instruct loadI(rRegI dst, memory mem)
11886 // %{
11887 //   match(Set dst (LoadI mem));
11888 // %}
11889 //
11890 
11891 peephole
11892 %{
11893   peepmatch (loadI storeI);
11894   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11895   peepreplace (storeI(1.mem 1.mem 1.src));
11896 %}
11897 
11898 peephole
11899 %{
11900   peepmatch (loadL storeL);
11901   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11902   peepreplace (storeL(1.mem 1.mem 1.src));
11903 %}
11904 
11905 //----------SMARTSPILL RULES---------------------------------------------------
11906 // These must follow all instruction definitions as they use the names
11907 // defined in the instructions definitions.
--- EOF ---