1 //
   2 // Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211 
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // Class for all long registers (excluding RSP)
 321 reg_class long_reg_with_rbp(RAX, RAX_H,
 322                             RDX, RDX_H,
 323                             RBP, RBP_H,
 324                             RDI, RDI_H,
 325                             RSI, RSI_H,
 326                             RCX, RCX_H,
 327                             RBX, RBX_H,
 328                             R8,  R8_H,
 329                             R9,  R9_H,
 330                             R10, R10_H,
 331                             R11, R11_H,
 332                             R13, R13_H,
 333                             R14, R14_H);
 334 
 335 // Class for all long registers (excluding RSP and RBP)
 336 reg_class long_reg_no_rbp(RAX, RAX_H,
 337                           RDX, RDX_H,
 338                           RDI, RDI_H,
 339                           RSI, RSI_H,
 340                           RCX, RCX_H,
 341                           RBX, RBX_H,
 342                           R8,  R8_H,
 343                           R9,  R9_H,
 344                           R10, R10_H,
 345                           R11, R11_H,
 346                           R13, R13_H,
 347                           R14, R14_H);
 348 
 349 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 350 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 351 
 352 // Class for all long registers (excluding RAX, RDX and RSP)
 353 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 354                                        RDI, RDI_H,
 355                                        RSI, RSI_H,
 356                                        RCX, RCX_H,
 357                                        RBX, RBX_H,
 358                                        R8,  R8_H,
 359                                        R9,  R9_H,
 360                                        R10, R10_H,
 361                                        R11, R11_H,
 362                                        R13, R13_H,
 363                                        R14, R14_H);
 364 
 365 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 366 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 367                                      RSI, RSI_H,
 368                                      RCX, RCX_H,
 369                                      RBX, RBX_H,
 370                                      R8,  R8_H,
 371                                      R9,  R9_H,
 372                                      R10, R10_H,
 373                                      R11, R11_H,
 374                                      R13, R13_H,
 375                                      R14, R14_H);
 376 
 377 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 378 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 379 
 380 // Class for all long registers (excluding RCX and RSP)
 381 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 382                                    RDI, RDI_H,
 383                                    RSI, RSI_H,
 384                                    RAX, RAX_H,
 385                                    RDX, RDX_H,
 386                                    RBX, RBX_H,
 387                                    R8,  R8_H,
 388                                    R9,  R9_H,
 389                                    R10, R10_H,
 390                                    R11, R11_H,
 391                                    R13, R13_H,
 392                                    R14, R14_H);
 393 
 394 // Class for all long registers (excluding RCX, RSP, and RBP)
 395 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 396                                  RSI, RSI_H,
 397                                  RAX, RAX_H,
 398                                  RDX, RDX_H,
 399                                  RBX, RBX_H,
 400                                  R8,  R8_H,
 401                                  R9,  R9_H,
 402                                  R10, R10_H,
 403                                  R11, R11_H,
 404                                  R13, R13_H,
 405                                  R14, R14_H);
 406 
 407 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 408 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 409 
 410 // Singleton class for RAX long register
 411 reg_class long_rax_reg(RAX, RAX_H);
 412 
 413 // Singleton class for RCX long register
 414 reg_class long_rcx_reg(RCX, RCX_H);
 415 
 416 // Singleton class for RDX long register
 417 reg_class long_rdx_reg(RDX, RDX_H);
 418 
 419 // Class for all int registers (excluding RSP)
 420 reg_class int_reg_with_rbp(RAX,
 421                            RDX,
 422                            RBP,
 423                            RDI,
 424                            RSI,
 425                            RCX,
 426                            RBX,
 427                            R8,
 428                            R9,
 429                            R10,
 430                            R11,
 431                            R13,
 432                            R14);
 433 
 434 // Class for all int registers (excluding RSP and RBP)
 435 reg_class int_reg_no_rbp(RAX,
 436                          RDX,
 437                          RDI,
 438                          RSI,
 439                          RCX,
 440                          RBX,
 441                          R8,
 442                          R9,
 443                          R10,
 444                          R11,
 445                          R13,
 446                          R14);
 447 
 448 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 449 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 450 
 451 // Class for all int registers (excluding RCX and RSP)
 452 reg_class int_no_rcx_reg_with_rbp(RAX,
 453                                   RDX,
 454                                   RBP,
 455                                   RDI,
 456                                   RSI,
 457                                   RBX,
 458                                   R8,
 459                                   R9,
 460                                   R10,
 461                                   R11,
 462                                   R13,
 463                                   R14);
 464 
 465 // Class for all int registers (excluding RCX, RSP, and RBP)
 466 reg_class int_no_rcx_reg_no_rbp(RAX,
 467                                 RDX,
 468                                 RDI,
 469                                 RSI,
 470                                 RBX,
 471                                 R8,
 472                                 R9,
 473                                 R10,
 474                                 R11,
 475                                 R13,
 476                                 R14);
 477 
 478 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 479 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 480 
 481 // Class for all int registers (excluding RAX, RDX, and RSP)
 482 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 483                                       RDI,
 484                                       RSI,
 485                                       RCX,
 486                                       RBX,
 487                                       R8,
 488                                       R9,
 489                                       R10,
 490                                       R11,
 491                                       R13,
 492                                       R14);
 493 
 494 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 495 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 496                                     RSI,
 497                                     RCX,
 498                                     RBX,
 499                                     R8,
 500                                     R9,
 501                                     R10,
 502                                     R11,
 503                                     R13,
 504                                     R14);
 505 
 506 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 507 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 508 
 509 // Singleton class for RAX int register
 510 reg_class int_rax_reg(RAX);
 511 
 512 // Singleton class for RBX int register
 513 reg_class int_rbx_reg(RBX);
 514 
 515 // Singleton class for RCX int register
 516 reg_class int_rcx_reg(RCX);
 517 
 518 // Singleton class for RCX int register
 519 reg_class int_rdx_reg(RDX);
 520 
 521 // Singleton class for RCX int register
 522 reg_class int_rdi_reg(RDI);
 523 
 524 // Singleton class for instruction pointer
 525 // reg_class ip_reg(RIP);
 526 
 527 %}
 528 
 529 source_hpp %{
 530 #if INCLUDE_ZGC
 531 #include "gc/z/zBarrierSetAssembler.hpp"
 532 #endif
 533 %}
 534 
 535 //----------SOURCE BLOCK-------------------------------------------------------
 536 // This is a block of C++ code which provides values, functions, and
 537 // definitions necessary in the rest of the architecture description
 538 source %{
 539 #define   RELOC_IMM64    Assembler::imm_operand
 540 #define   RELOC_DISP32   Assembler::disp32_operand
 541 
 542 #define __ _masm.
 543 
 544 static bool generate_vzeroupper(Compile* C) {
 545   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
 546 }
 547 
 548 static int clear_avx_size() {
 549   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
 550 }
 551 
 552 // !!!!! Special hack to get all types of calls to specify the byte offset
 553 //       from the start of the call to the point where the return address
 554 //       will point.
 555 int MachCallStaticJavaNode::ret_addr_offset()
 556 {
 557   int offset = 5; // 5 bytes from start of call to where return address points
 558   offset += clear_avx_size();
 559   return offset;
 560 }
 561 
 562 int MachCallDynamicJavaNode::ret_addr_offset()
 563 {
 564   int offset = 15; // 15 bytes from start of call to where return address points
 565   offset += clear_avx_size();
 566   return offset;
 567 }
 568 
 569 int MachCallRuntimeNode::ret_addr_offset() {
 570   int offset = 13; // movq r10,#addr; callq (r10)
 571   offset += clear_avx_size();
 572   return offset;
 573 }
 574 
 575 // Indicate if the safepoint node needs the polling page as an input,
 576 // it does if the polling page is more than disp32 away.
 577 bool SafePointNode::needs_polling_address_input()
 578 {
 579   return SafepointMechanism::uses_thread_local_poll() || Assembler::is_polling_page_far();
 580 }
 581 
 582 //
 583 // Compute padding required for nodes which need alignment
 584 //
 585 
 586 // The address of the call instruction needs to be 4-byte aligned to
 587 // ensure that it does not span a cache line so that it can be patched.
 588 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 589 {
 590   current_offset += clear_avx_size(); // skip vzeroupper
 591   current_offset += 1; // skip call opcode byte
 592   return align_up(current_offset, alignment_required()) - current_offset;
 593 }
 594 
 595 // The address of the call instruction needs to be 4-byte aligned to
 596 // ensure that it does not span a cache line so that it can be patched.
 597 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 598 {
 599   current_offset += clear_avx_size(); // skip vzeroupper
 600   current_offset += 11; // skip movq instruction + call opcode byte
 601   return align_up(current_offset, alignment_required()) - current_offset;
 602 }
 603 
 604 // EMIT_RM()
 605 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 606   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 607   cbuf.insts()->emit_int8(c);
 608 }
 609 
 610 // EMIT_CC()
 611 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 612   unsigned char c = (unsigned char) (f1 | f2);
 613   cbuf.insts()->emit_int8(c);
 614 }
 615 
 616 // EMIT_OPCODE()
 617 void emit_opcode(CodeBuffer &cbuf, int code) {
 618   cbuf.insts()->emit_int8((unsigned char) code);
 619 }
 620 
 621 // EMIT_OPCODE() w/ relocation information
 622 void emit_opcode(CodeBuffer &cbuf,
 623                  int code, relocInfo::relocType reloc, int offset, int format)
 624 {
 625   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 626   emit_opcode(cbuf, code);
 627 }
 628 
 629 // EMIT_D8()
 630 void emit_d8(CodeBuffer &cbuf, int d8) {
 631   cbuf.insts()->emit_int8((unsigned char) d8);
 632 }
 633 
 634 // EMIT_D16()
 635 void emit_d16(CodeBuffer &cbuf, int d16) {
 636   cbuf.insts()->emit_int16(d16);
 637 }
 638 
 639 // EMIT_D32()
 640 void emit_d32(CodeBuffer &cbuf, int d32) {
 641   cbuf.insts()->emit_int32(d32);
 642 }
 643 
 644 // EMIT_D64()
 645 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 646   cbuf.insts()->emit_int64(d64);
 647 }
 648 
 649 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 650 void emit_d32_reloc(CodeBuffer& cbuf,
 651                     int d32,
 652                     relocInfo::relocType reloc,
 653                     int format)
 654 {
 655   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 656   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 657   cbuf.insts()->emit_int32(d32);
 658 }
 659 
 660 // emit 32 bit value and construct relocation entry from RelocationHolder
 661 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 662 #ifdef ASSERT
 663   if (rspec.reloc()->type() == relocInfo::oop_type &&
 664       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 665     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 666     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop((intptr_t)d32))), "cannot embed scavengable oops in code");
 667   }
 668 #endif
 669   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 670   cbuf.insts()->emit_int32(d32);
 671 }
 672 
 673 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 674   address next_ip = cbuf.insts_end() + 4;
 675   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 676                  external_word_Relocation::spec(addr),
 677                  RELOC_DISP32);
 678 }
 679 
 680 
 681 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 682 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 683   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 684   cbuf.insts()->emit_int64(d64);
 685 }
 686 
 687 // emit 64 bit value and construct relocation entry from RelocationHolder
 688 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 689 #ifdef ASSERT
 690   if (rspec.reloc()->type() == relocInfo::oop_type &&
 691       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 692     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 693     assert(oopDesc::is_oop(cast_to_oop(d64)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop(d64))),
 694            "cannot embed scavengable oops in code");
 695   }
 696 #endif
 697   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 698   cbuf.insts()->emit_int64(d64);
 699 }
 700 
 701 // Access stack slot for load or store
 702 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 703 {
 704   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 705   if (-0x80 <= disp && disp < 0x80) {
 706     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 707     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 708     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 709   } else {
 710     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 711     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 712     emit_d32(cbuf, disp);     // Displacement // R/M byte
 713   }
 714 }
 715 
 716    // rRegI ereg, memory mem) %{    // emit_reg_mem
 717 void encode_RegMem(CodeBuffer &cbuf,
 718                    int reg,
 719                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 720 {
 721   assert(disp_reloc == relocInfo::none, "cannot have disp");
 722   int regenc = reg & 7;
 723   int baseenc = base & 7;
 724   int indexenc = index & 7;
 725 
 726   // There is no index & no scale, use form without SIB byte
 727   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 728     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 729     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 730       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 731     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 732       // If 8-bit displacement, mode 0x1
 733       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 734       emit_d8(cbuf, disp);
 735     } else {
 736       // If 32-bit displacement
 737       if (base == -1) { // Special flag for absolute address
 738         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 739         if (disp_reloc != relocInfo::none) {
 740           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 741         } else {
 742           emit_d32(cbuf, disp);
 743         }
 744       } else {
 745         // Normal base + offset
 746         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 747         if (disp_reloc != relocInfo::none) {
 748           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 749         } else {
 750           emit_d32(cbuf, disp);
 751         }
 752       }
 753     }
 754   } else {
 755     // Else, encode with the SIB byte
 756     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 757     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 758       // If no displacement
 759       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 760       emit_rm(cbuf, scale, indexenc, baseenc);
 761     } else {
 762       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 763         // If 8-bit displacement, mode 0x1
 764         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 765         emit_rm(cbuf, scale, indexenc, baseenc);
 766         emit_d8(cbuf, disp);
 767       } else {
 768         // If 32-bit displacement
 769         if (base == 0x04 ) {
 770           emit_rm(cbuf, 0x2, regenc, 0x4);
 771           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 772         } else {
 773           emit_rm(cbuf, 0x2, regenc, 0x4);
 774           emit_rm(cbuf, scale, indexenc, baseenc); // *
 775         }
 776         if (disp_reloc != relocInfo::none) {
 777           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 778         } else {
 779           emit_d32(cbuf, disp);
 780         }
 781       }
 782     }
 783   }
 784 }
 785 
 786 // This could be in MacroAssembler but it's fairly C2 specific
 787 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 788   Label exit;
 789   __ jccb(Assembler::noParity, exit);
 790   __ pushf();
 791   //
 792   // comiss/ucomiss instructions set ZF,PF,CF flags and
 793   // zero OF,AF,SF for NaN values.
 794   // Fixup flags by zeroing ZF,PF so that compare of NaN
 795   // values returns 'less than' result (CF is set).
 796   // Leave the rest of flags unchanged.
 797   //
 798   //    7 6 5 4 3 2 1 0
 799   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 800   //    0 0 1 0 1 0 1 1   (0x2B)
 801   //
 802   __ andq(Address(rsp, 0), 0xffffff2b);
 803   __ popf();
 804   __ bind(exit);
 805 }
 806 
 807 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 808   Label done;
 809   __ movl(dst, -1);
 810   __ jcc(Assembler::parity, done);
 811   __ jcc(Assembler::below, done);
 812   __ setb(Assembler::notEqual, dst);
 813   __ movzbl(dst, dst);
 814   __ bind(done);
 815 }
 816 
 817 
 818 //=============================================================================
 819 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 820 
 821 int Compile::ConstantTable::calculate_table_base_offset() const {
 822   return 0;  // absolute addressing, no offset
 823 }
 824 
 825 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 826 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 827   ShouldNotReachHere();
 828 }
 829 
 830 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 831   // Empty encoding
 832 }
 833 
 834 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 835   return 0;
 836 }
 837 
 838 #ifndef PRODUCT
 839 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 840   st->print("# MachConstantBaseNode (empty encoding)");
 841 }
 842 #endif
 843 
 844 
 845 //=============================================================================
 846 #ifndef PRODUCT
 847 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 848   Compile* C = ra_->C;
 849 
 850   int framesize = C->frame_size_in_bytes();
 851   int bangsize = C->bang_size_in_bytes();
 852   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 853   // Remove wordSize for return addr which is already pushed.
 854   framesize -= wordSize;
 855 
 856   if (C->need_stack_bang(bangsize)) {
 857     framesize -= wordSize;
 858     st->print("# stack bang (%d bytes)", bangsize);
 859     st->print("\n\t");
 860     st->print("pushq   rbp\t# Save rbp");
 861     if (PreserveFramePointer) {
 862         st->print("\n\t");
 863         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 864     }
 865     if (framesize) {
 866       st->print("\n\t");
 867       st->print("subq    rsp, #%d\t# Create frame",framesize);
 868     }
 869   } else {
 870     st->print("subq    rsp, #%d\t# Create frame",framesize);
 871     st->print("\n\t");
 872     framesize -= wordSize;
 873     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 874     if (PreserveFramePointer) {
 875       st->print("\n\t");
 876       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 877       if (framesize > 0) {
 878         st->print("\n\t");
 879         st->print("addq    rbp, #%d", framesize);
 880       }
 881     }
 882   }
 883 
 884   if (VerifyStackAtCalls) {
 885     st->print("\n\t");
 886     framesize -= wordSize;
 887     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 888 #ifdef ASSERT
 889     st->print("\n\t");
 890     st->print("# stack alignment check");
 891 #endif
 892   }
 893   st->cr();
 894 }
 895 #endif
 896 
 897 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 898   Compile* C = ra_->C;
 899   MacroAssembler _masm(&cbuf);
 900 
 901   int framesize = C->frame_size_in_bytes();
 902   int bangsize = C->bang_size_in_bytes();
 903 
 904   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 905 
 906   C->set_frame_complete(cbuf.insts_size());
 907 
 908   if (C->has_mach_constant_base_node()) {
 909     // NOTE: We set the table base offset here because users might be
 910     // emitted before MachConstantBaseNode.
 911     Compile::ConstantTable& constant_table = C->constant_table();
 912     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 913   }
 914 }
 915 
 916 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 917 {
 918   return MachNode::size(ra_); // too many variables; just compute it
 919                               // the hard way
 920 }
 921 
 922 int MachPrologNode::reloc() const
 923 {
 924   return 0; // a large enough number
 925 }
 926 
 927 //=============================================================================
 928 #ifndef PRODUCT
 929 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 930 {
 931   Compile* C = ra_->C;
 932   if (generate_vzeroupper(C)) {
 933     st->print("vzeroupper");
 934     st->cr(); st->print("\t");
 935   }
 936 
 937   int framesize = C->frame_size_in_bytes();
 938   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 939   // Remove word for return adr already pushed
 940   // and RBP
 941   framesize -= 2*wordSize;
 942 
 943   if (framesize) {
 944     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 945     st->print("\t");
 946   }
 947 
 948   st->print_cr("popq   rbp");
 949   if (do_polling() && C->is_method_compilation()) {
 950     st->print("\t");
 951     if (SafepointMechanism::uses_thread_local_poll()) {
 952       st->print_cr("movq   rscratch1, poll_offset[r15_thread] #polling_page_address\n\t"
 953                    "testl  rax, [rscratch1]\t"
 954                    "# Safepoint: poll for GC");
 955     } else if (Assembler::is_polling_page_far()) {
 956       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 957                    "testl  rax, [rscratch1]\t"
 958                    "# Safepoint: poll for GC");
 959     } else {
 960       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 961                    "# Safepoint: poll for GC");
 962     }
 963   }
 964 }
 965 #endif
 966 
 967 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 968 {
 969   Compile* C = ra_->C;
 970   MacroAssembler _masm(&cbuf);
 971 
 972   if (generate_vzeroupper(C)) {
 973     // Clear upper bits of YMM registers when current compiled code uses
 974     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 975     __ vzeroupper();
 976   }
 977 
 978   int framesize = C->frame_size_in_bytes();
 979   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 980   // Remove word for return adr already pushed
 981   // and RBP
 982   framesize -= 2*wordSize;
 983 
 984   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 985 
 986   if (framesize) {
 987     emit_opcode(cbuf, Assembler::REX_W);
 988     if (framesize < 0x80) {
 989       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 990       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 991       emit_d8(cbuf, framesize);
 992     } else {
 993       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 994       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 995       emit_d32(cbuf, framesize);
 996     }
 997   }
 998 
 999   // popq rbp
1000   emit_opcode(cbuf, 0x58 | RBP_enc);
1001 
1002   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1003     __ reserved_stack_check();
1004   }
1005 
1006   if (do_polling() && C->is_method_compilation()) {
1007     MacroAssembler _masm(&cbuf);
1008     if (SafepointMechanism::uses_thread_local_poll()) {
1009       __ movq(rscratch1, Address(r15_thread, Thread::polling_page_offset()));
1010       __ relocate(relocInfo::poll_return_type);
1011       __ testl(rax, Address(rscratch1, 0));
1012     } else {
1013       AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
1014       if (Assembler::is_polling_page_far()) {
1015         __ lea(rscratch1, polling_page);
1016         __ relocate(relocInfo::poll_return_type);
1017         __ testl(rax, Address(rscratch1, 0));
1018       } else {
1019         __ testl(rax, polling_page);
1020       }
1021     }
1022   }
1023 }
1024 
1025 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1026 {
1027   return MachNode::size(ra_); // too many variables; just compute it
1028                               // the hard way
1029 }
1030 
1031 int MachEpilogNode::reloc() const
1032 {
1033   return 2; // a large enough number
1034 }
1035 
1036 const Pipeline* MachEpilogNode::pipeline() const
1037 {
1038   return MachNode::pipeline_class();
1039 }
1040 
1041 int MachEpilogNode::safepoint_offset() const
1042 {
1043   return 0;
1044 }
1045 
1046 //=============================================================================
1047 
1048 enum RC {
1049   rc_bad,
1050   rc_int,
1051   rc_float,
1052   rc_stack
1053 };
1054 
1055 static enum RC rc_class(OptoReg::Name reg)
1056 {
1057   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1058 
1059   if (OptoReg::is_stack(reg)) return rc_stack;
1060 
1061   VMReg r = OptoReg::as_VMReg(reg);
1062 
1063   if (r->is_Register()) return rc_int;
1064 
1065   assert(r->is_XMMRegister(), "must be");
1066   return rc_float;
1067 }
1068 
1069 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1070 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1071                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1072 
1073 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1074                             int stack_offset, int reg, uint ireg, outputStream* st);
1075 
1076 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1077                                       int dst_offset, uint ireg, outputStream* st) {
1078   if (cbuf) {
1079     MacroAssembler _masm(cbuf);
1080     switch (ireg) {
1081     case Op_VecS:
1082       __ movq(Address(rsp, -8), rax);
1083       __ movl(rax, Address(rsp, src_offset));
1084       __ movl(Address(rsp, dst_offset), rax);
1085       __ movq(rax, Address(rsp, -8));
1086       break;
1087     case Op_VecD:
1088       __ pushq(Address(rsp, src_offset));
1089       __ popq (Address(rsp, dst_offset));
1090       break;
1091     case Op_VecX:
1092       __ pushq(Address(rsp, src_offset));
1093       __ popq (Address(rsp, dst_offset));
1094       __ pushq(Address(rsp, src_offset+8));
1095       __ popq (Address(rsp, dst_offset+8));
1096       break;
1097     case Op_VecY:
1098       __ vmovdqu(Address(rsp, -32), xmm0);
1099       __ vmovdqu(xmm0, Address(rsp, src_offset));
1100       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1101       __ vmovdqu(xmm0, Address(rsp, -32));
1102       break;
1103     case Op_VecZ:
1104       __ evmovdquq(Address(rsp, -64), xmm0, 2);
1105       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1106       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1107       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1108       break;
1109     default:
1110       ShouldNotReachHere();
1111     }
1112 #ifndef PRODUCT
1113   } else {
1114     switch (ireg) {
1115     case Op_VecS:
1116       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1117                 "movl    rax, [rsp + #%d]\n\t"
1118                 "movl    [rsp + #%d], rax\n\t"
1119                 "movq    rax, [rsp - #8]",
1120                 src_offset, dst_offset);
1121       break;
1122     case Op_VecD:
1123       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1124                 "popq    [rsp + #%d]",
1125                 src_offset, dst_offset);
1126       break;
1127      case Op_VecX:
1128       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1129                 "popq    [rsp + #%d]\n\t"
1130                 "pushq   [rsp + #%d]\n\t"
1131                 "popq    [rsp + #%d]",
1132                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1133       break;
1134     case Op_VecY:
1135       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1136                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1137                 "vmovdqu [rsp + #%d], xmm0\n\t"
1138                 "vmovdqu xmm0, [rsp - #32]",
1139                 src_offset, dst_offset);
1140       break;
1141     case Op_VecZ:
1142       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1143                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1144                 "vmovdqu [rsp + #%d], xmm0\n\t"
1145                 "vmovdqu xmm0, [rsp - #64]",
1146                 src_offset, dst_offset);
1147       break;
1148     default:
1149       ShouldNotReachHere();
1150     }
1151 #endif
1152   }
1153 }
1154 
1155 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1156                                        PhaseRegAlloc* ra_,
1157                                        bool do_size,
1158                                        outputStream* st) const {
1159   assert(cbuf != NULL || st  != NULL, "sanity");
1160   // Get registers to move
1161   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1162   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1163   OptoReg::Name dst_second = ra_->get_reg_second(this);
1164   OptoReg::Name dst_first = ra_->get_reg_first(this);
1165 
1166   enum RC src_second_rc = rc_class(src_second);
1167   enum RC src_first_rc = rc_class(src_first);
1168   enum RC dst_second_rc = rc_class(dst_second);
1169   enum RC dst_first_rc = rc_class(dst_first);
1170 
1171   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1172          "must move at least 1 register" );
1173 
1174   if (src_first == dst_first && src_second == dst_second) {
1175     // Self copy, no move
1176     return 0;
1177   }
1178   if (bottom_type()->isa_vect() != NULL) {
1179     uint ireg = ideal_reg();
1180     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1181     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1182     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1183       // mem -> mem
1184       int src_offset = ra_->reg2offset(src_first);
1185       int dst_offset = ra_->reg2offset(dst_first);
1186       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1187     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1188       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1189     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1190       int stack_offset = ra_->reg2offset(dst_first);
1191       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1192     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1193       int stack_offset = ra_->reg2offset(src_first);
1194       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1195     } else {
1196       ShouldNotReachHere();
1197     }
1198     return 0;
1199   }
1200   if (src_first_rc == rc_stack) {
1201     // mem ->
1202     if (dst_first_rc == rc_stack) {
1203       // mem -> mem
1204       assert(src_second != dst_first, "overlap");
1205       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1206           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1207         // 64-bit
1208         int src_offset = ra_->reg2offset(src_first);
1209         int dst_offset = ra_->reg2offset(dst_first);
1210         if (cbuf) {
1211           MacroAssembler _masm(cbuf);
1212           __ pushq(Address(rsp, src_offset));
1213           __ popq (Address(rsp, dst_offset));
1214 #ifndef PRODUCT
1215         } else {
1216           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1217                     "popq    [rsp + #%d]",
1218                      src_offset, dst_offset);
1219 #endif
1220         }
1221       } else {
1222         // 32-bit
1223         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1224         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1225         // No pushl/popl, so:
1226         int src_offset = ra_->reg2offset(src_first);
1227         int dst_offset = ra_->reg2offset(dst_first);
1228         if (cbuf) {
1229           MacroAssembler _masm(cbuf);
1230           __ movq(Address(rsp, -8), rax);
1231           __ movl(rax, Address(rsp, src_offset));
1232           __ movl(Address(rsp, dst_offset), rax);
1233           __ movq(rax, Address(rsp, -8));
1234 #ifndef PRODUCT
1235         } else {
1236           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1237                     "movl    rax, [rsp + #%d]\n\t"
1238                     "movl    [rsp + #%d], rax\n\t"
1239                     "movq    rax, [rsp - #8]",
1240                      src_offset, dst_offset);
1241 #endif
1242         }
1243       }
1244       return 0;
1245     } else if (dst_first_rc == rc_int) {
1246       // mem -> gpr
1247       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1248           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1249         // 64-bit
1250         int offset = ra_->reg2offset(src_first);
1251         if (cbuf) {
1252           MacroAssembler _masm(cbuf);
1253           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1254 #ifndef PRODUCT
1255         } else {
1256           st->print("movq    %s, [rsp + #%d]\t# spill",
1257                      Matcher::regName[dst_first],
1258                      offset);
1259 #endif
1260         }
1261       } else {
1262         // 32-bit
1263         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1264         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1265         int offset = ra_->reg2offset(src_first);
1266         if (cbuf) {
1267           MacroAssembler _masm(cbuf);
1268           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1269 #ifndef PRODUCT
1270         } else {
1271           st->print("movl    %s, [rsp + #%d]\t# spill",
1272                      Matcher::regName[dst_first],
1273                      offset);
1274 #endif
1275         }
1276       }
1277       return 0;
1278     } else if (dst_first_rc == rc_float) {
1279       // mem-> xmm
1280       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1281           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1282         // 64-bit
1283         int offset = ra_->reg2offset(src_first);
1284         if (cbuf) {
1285           MacroAssembler _masm(cbuf);
1286           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1287 #ifndef PRODUCT
1288         } else {
1289           st->print("%s  %s, [rsp + #%d]\t# spill",
1290                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1291                      Matcher::regName[dst_first],
1292                      offset);
1293 #endif
1294         }
1295       } else {
1296         // 32-bit
1297         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1298         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1299         int offset = ra_->reg2offset(src_first);
1300         if (cbuf) {
1301           MacroAssembler _masm(cbuf);
1302           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1303 #ifndef PRODUCT
1304         } else {
1305           st->print("movss   %s, [rsp + #%d]\t# spill",
1306                      Matcher::regName[dst_first],
1307                      offset);
1308 #endif
1309         }
1310       }
1311       return 0;
1312     }
1313   } else if (src_first_rc == rc_int) {
1314     // gpr ->
1315     if (dst_first_rc == rc_stack) {
1316       // gpr -> mem
1317       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1318           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1319         // 64-bit
1320         int offset = ra_->reg2offset(dst_first);
1321         if (cbuf) {
1322           MacroAssembler _masm(cbuf);
1323           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1324 #ifndef PRODUCT
1325         } else {
1326           st->print("movq    [rsp + #%d], %s\t# spill",
1327                      offset,
1328                      Matcher::regName[src_first]);
1329 #endif
1330         }
1331       } else {
1332         // 32-bit
1333         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1334         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1335         int offset = ra_->reg2offset(dst_first);
1336         if (cbuf) {
1337           MacroAssembler _masm(cbuf);
1338           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1339 #ifndef PRODUCT
1340         } else {
1341           st->print("movl    [rsp + #%d], %s\t# spill",
1342                      offset,
1343                      Matcher::regName[src_first]);
1344 #endif
1345         }
1346       }
1347       return 0;
1348     } else if (dst_first_rc == rc_int) {
1349       // gpr -> gpr
1350       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1351           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1352         // 64-bit
1353         if (cbuf) {
1354           MacroAssembler _masm(cbuf);
1355           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1356                   as_Register(Matcher::_regEncode[src_first]));
1357 #ifndef PRODUCT
1358         } else {
1359           st->print("movq    %s, %s\t# spill",
1360                      Matcher::regName[dst_first],
1361                      Matcher::regName[src_first]);
1362 #endif
1363         }
1364         return 0;
1365       } else {
1366         // 32-bit
1367         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1368         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1369         if (cbuf) {
1370           MacroAssembler _masm(cbuf);
1371           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1372                   as_Register(Matcher::_regEncode[src_first]));
1373 #ifndef PRODUCT
1374         } else {
1375           st->print("movl    %s, %s\t# spill",
1376                      Matcher::regName[dst_first],
1377                      Matcher::regName[src_first]);
1378 #endif
1379         }
1380         return 0;
1381       }
1382     } else if (dst_first_rc == rc_float) {
1383       // gpr -> xmm
1384       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1385           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1386         // 64-bit
1387         if (cbuf) {
1388           MacroAssembler _masm(cbuf);
1389           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1390 #ifndef PRODUCT
1391         } else {
1392           st->print("movdq   %s, %s\t# spill",
1393                      Matcher::regName[dst_first],
1394                      Matcher::regName[src_first]);
1395 #endif
1396         }
1397       } else {
1398         // 32-bit
1399         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1400         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1401         if (cbuf) {
1402           MacroAssembler _masm(cbuf);
1403           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1404 #ifndef PRODUCT
1405         } else {
1406           st->print("movdl   %s, %s\t# spill",
1407                      Matcher::regName[dst_first],
1408                      Matcher::regName[src_first]);
1409 #endif
1410         }
1411       }
1412       return 0;
1413     }
1414   } else if (src_first_rc == rc_float) {
1415     // xmm ->
1416     if (dst_first_rc == rc_stack) {
1417       // xmm -> mem
1418       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1419           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1420         // 64-bit
1421         int offset = ra_->reg2offset(dst_first);
1422         if (cbuf) {
1423           MacroAssembler _masm(cbuf);
1424           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1425 #ifndef PRODUCT
1426         } else {
1427           st->print("movsd   [rsp + #%d], %s\t# spill",
1428                      offset,
1429                      Matcher::regName[src_first]);
1430 #endif
1431         }
1432       } else {
1433         // 32-bit
1434         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1435         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1436         int offset = ra_->reg2offset(dst_first);
1437         if (cbuf) {
1438           MacroAssembler _masm(cbuf);
1439           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1440 #ifndef PRODUCT
1441         } else {
1442           st->print("movss   [rsp + #%d], %s\t# spill",
1443                      offset,
1444                      Matcher::regName[src_first]);
1445 #endif
1446         }
1447       }
1448       return 0;
1449     } else if (dst_first_rc == rc_int) {
1450       // xmm -> gpr
1451       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1452           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1453         // 64-bit
1454         if (cbuf) {
1455           MacroAssembler _masm(cbuf);
1456           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1457 #ifndef PRODUCT
1458         } else {
1459           st->print("movdq   %s, %s\t# spill",
1460                      Matcher::regName[dst_first],
1461                      Matcher::regName[src_first]);
1462 #endif
1463         }
1464       } else {
1465         // 32-bit
1466         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1467         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1468         if (cbuf) {
1469           MacroAssembler _masm(cbuf);
1470           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1471 #ifndef PRODUCT
1472         } else {
1473           st->print("movdl   %s, %s\t# spill",
1474                      Matcher::regName[dst_first],
1475                      Matcher::regName[src_first]);
1476 #endif
1477         }
1478       }
1479       return 0;
1480     } else if (dst_first_rc == rc_float) {
1481       // xmm -> xmm
1482       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1483           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1484         // 64-bit
1485         if (cbuf) {
1486           MacroAssembler _masm(cbuf);
1487           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1488 #ifndef PRODUCT
1489         } else {
1490           st->print("%s  %s, %s\t# spill",
1491                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1492                      Matcher::regName[dst_first],
1493                      Matcher::regName[src_first]);
1494 #endif
1495         }
1496       } else {
1497         // 32-bit
1498         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1499         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1500         if (cbuf) {
1501           MacroAssembler _masm(cbuf);
1502           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1503 #ifndef PRODUCT
1504         } else {
1505           st->print("%s  %s, %s\t# spill",
1506                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1507                      Matcher::regName[dst_first],
1508                      Matcher::regName[src_first]);
1509 #endif
1510         }
1511       }
1512       return 0;
1513     }
1514   }
1515 
1516   assert(0," foo ");
1517   Unimplemented();
1518   return 0;
1519 }
1520 
1521 #ifndef PRODUCT
1522 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1523   implementation(NULL, ra_, false, st);
1524 }
1525 #endif
1526 
1527 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1528   implementation(&cbuf, ra_, false, NULL);
1529 }
1530 
1531 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1532   return MachNode::size(ra_);
1533 }
1534 
1535 //=============================================================================
1536 #ifndef PRODUCT
1537 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1538 {
1539   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1540   int reg = ra_->get_reg_first(this);
1541   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1542             Matcher::regName[reg], offset);
1543 }
1544 #endif
1545 
1546 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1547 {
1548   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1549   int reg = ra_->get_encode(this);
1550   if (offset >= 0x80) {
1551     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1552     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1553     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1554     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1555     emit_d32(cbuf, offset);
1556   } else {
1557     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1558     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1559     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1560     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1561     emit_d8(cbuf, offset);
1562   }
1563 }
1564 
1565 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1566 {
1567   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1568   return (offset < 0x80) ? 5 : 8; // REX
1569 }
1570 
1571 //=============================================================================
1572 #ifndef PRODUCT
1573 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1574 {
1575   if (UseCompressedClassPointers) {
1576     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1577     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1578     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1579   } else {
1580     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1581                  "# Inline cache check");
1582   }
1583   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1584   st->print_cr("\tnop\t# nops to align entry point");
1585 }
1586 #endif
1587 
1588 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1589 {
1590   MacroAssembler masm(&cbuf);
1591   uint insts_size = cbuf.insts_size();
1592   if (UseCompressedClassPointers) {
1593     masm.load_klass(rscratch1, j_rarg0);
1594     masm.cmpptr(rax, rscratch1);
1595   } else {
1596     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1597   }
1598 
1599   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1600 
1601   /* WARNING these NOPs are critical so that verified entry point is properly
1602      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1603   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1604   if (OptoBreakpoint) {
1605     // Leave space for int3
1606     nops_cnt -= 1;
1607   }
1608   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1609   if (nops_cnt > 0)
1610     masm.nop(nops_cnt);
1611 }
1612 
1613 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1614 {
1615   return MachNode::size(ra_); // too many variables; just compute it
1616                               // the hard way
1617 }
1618 
1619 
1620 //=============================================================================
1621 
1622 int Matcher::regnum_to_fpu_offset(int regnum)
1623 {
1624   return regnum - 32; // The FP registers are in the second chunk
1625 }
1626 
1627 // This is UltraSparc specific, true just means we have fast l2f conversion
1628 const bool Matcher::convL2FSupported(void) {
1629   return true;
1630 }
1631 
1632 // Is this branch offset short enough that a short branch can be used?
1633 //
1634 // NOTE: If the platform does not provide any short branch variants, then
1635 //       this method should return false for offset 0.
1636 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1637   // The passed offset is relative to address of the branch.
1638   // On 86 a branch displacement is calculated relative to address
1639   // of a next instruction.
1640   offset -= br_size;
1641 
1642   // the short version of jmpConUCF2 contains multiple branches,
1643   // making the reach slightly less
1644   if (rule == jmpConUCF2_rule)
1645     return (-126 <= offset && offset <= 125);
1646   return (-128 <= offset && offset <= 127);
1647 }
1648 
1649 const bool Matcher::isSimpleConstant64(jlong value) {
1650   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1651   //return value == (int) value;  // Cf. storeImmL and immL32.
1652 
1653   // Probably always true, even if a temp register is required.
1654   return true;
1655 }
1656 
1657 // The ecx parameter to rep stosq for the ClearArray node is in words.
1658 const bool Matcher::init_array_count_is_in_bytes = false;
1659 
1660 // No additional cost for CMOVL.
1661 const int Matcher::long_cmove_cost() { return 0; }
1662 
1663 // No CMOVF/CMOVD with SSE2
1664 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1665 
1666 // Does the CPU require late expand (see block.cpp for description of late expand)?
1667 const bool Matcher::require_postalloc_expand = false;
1668 
1669 // Do we need to mask the count passed to shift instructions or does
1670 // the cpu only look at the lower 5/6 bits anyway?
1671 const bool Matcher::need_masked_shift_count = false;
1672 
1673 bool Matcher::narrow_oop_use_complex_address() {
1674   assert(UseCompressedOops, "only for compressed oops code");
1675   return (LogMinObjAlignmentInBytes <= 3);
1676 }
1677 
1678 bool Matcher::narrow_klass_use_complex_address() {
1679   assert(UseCompressedClassPointers, "only for compressed klass code");
1680   return (LogKlassAlignmentInBytes <= 3);
1681 }
1682 
1683 bool Matcher::const_oop_prefer_decode() {
1684   // Prefer ConN+DecodeN over ConP.
1685   return true;
1686 }
1687 
1688 bool Matcher::const_klass_prefer_decode() {
1689   // TODO: Either support matching DecodeNKlass (heap-based) in operand
1690   //       or condisider the following:
1691   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
1692   //return Universe::narrow_klass_base() == NULL;
1693   return true;
1694 }
1695 
1696 // Is it better to copy float constants, or load them directly from
1697 // memory?  Intel can load a float constant from a direct address,
1698 // requiring no extra registers.  Most RISCs will have to materialize
1699 // an address into a register first, so they would do better to copy
1700 // the constant from stack.
1701 const bool Matcher::rematerialize_float_constants = true; // XXX
1702 
1703 // If CPU can load and store mis-aligned doubles directly then no
1704 // fixup is needed.  Else we split the double into 2 integer pieces
1705 // and move it piece-by-piece.  Only happens when passing doubles into
1706 // C code as the Java calling convention forces doubles to be aligned.
1707 const bool Matcher::misaligned_doubles_ok = true;
1708 
1709 // No-op on amd64
1710 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1711 
1712 // Advertise here if the CPU requires explicit rounding operations to
1713 // implement the UseStrictFP mode.
1714 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1715 
1716 // Are floats conerted to double when stored to stack during deoptimization?
1717 // On x64 it is stored without convertion so we can use normal access.
1718 bool Matcher::float_in_double() { return false; }
1719 
1720 // Do ints take an entire long register or just half?
1721 const bool Matcher::int_in_long = true;
1722 
1723 // Return whether or not this register is ever used as an argument.
1724 // This function is used on startup to build the trampoline stubs in
1725 // generateOptoStub.  Registers not mentioned will be killed by the VM
1726 // call in the trampoline, and arguments in those registers not be
1727 // available to the callee.
1728 bool Matcher::can_be_java_arg(int reg)
1729 {
1730   return
1731     reg ==  RDI_num || reg == RDI_H_num ||
1732     reg ==  RSI_num || reg == RSI_H_num ||
1733     reg ==  RDX_num || reg == RDX_H_num ||
1734     reg ==  RCX_num || reg == RCX_H_num ||
1735     reg ==   R8_num || reg ==  R8_H_num ||
1736     reg ==   R9_num || reg ==  R9_H_num ||
1737     reg ==  R12_num || reg == R12_H_num ||
1738     reg == XMM0_num || reg == XMM0b_num ||
1739     reg == XMM1_num || reg == XMM1b_num ||
1740     reg == XMM2_num || reg == XMM2b_num ||
1741     reg == XMM3_num || reg == XMM3b_num ||
1742     reg == XMM4_num || reg == XMM4b_num ||
1743     reg == XMM5_num || reg == XMM5b_num ||
1744     reg == XMM6_num || reg == XMM6b_num ||
1745     reg == XMM7_num || reg == XMM7b_num;
1746 }
1747 
1748 bool Matcher::is_spillable_arg(int reg)
1749 {
1750   return can_be_java_arg(reg);
1751 }
1752 
1753 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1754   // In 64 bit mode a code which use multiply when
1755   // devisor is constant is faster than hardware
1756   // DIV instruction (it uses MulHiL).
1757   return false;
1758 }
1759 
1760 // Register for DIVI projection of divmodI
1761 RegMask Matcher::divI_proj_mask() {
1762   return INT_RAX_REG_mask();
1763 }
1764 
1765 // Register for MODI projection of divmodI
1766 RegMask Matcher::modI_proj_mask() {
1767   return INT_RDX_REG_mask();
1768 }
1769 
1770 // Register for DIVL projection of divmodL
1771 RegMask Matcher::divL_proj_mask() {
1772   return LONG_RAX_REG_mask();
1773 }
1774 
1775 // Register for MODL projection of divmodL
1776 RegMask Matcher::modL_proj_mask() {
1777   return LONG_RDX_REG_mask();
1778 }
1779 
1780 // Register for saving SP into on method handle invokes. Not used on x86_64.
1781 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1782     return NO_REG_mask();
1783 }
1784 
1785 %}
1786 
1787 //----------ENCODING BLOCK-----------------------------------------------------
1788 // This block specifies the encoding classes used by the compiler to
1789 // output byte streams.  Encoding classes are parameterized macros
1790 // used by Machine Instruction Nodes in order to generate the bit
1791 // encoding of the instruction.  Operands specify their base encoding
1792 // interface with the interface keyword.  There are currently
1793 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1794 // COND_INTER.  REG_INTER causes an operand to generate a function
1795 // which returns its register number when queried.  CONST_INTER causes
1796 // an operand to generate a function which returns the value of the
1797 // constant when queried.  MEMORY_INTER causes an operand to generate
1798 // four functions which return the Base Register, the Index Register,
1799 // the Scale Value, and the Offset Value of the operand when queried.
1800 // COND_INTER causes an operand to generate six functions which return
1801 // the encoding code (ie - encoding bits for the instruction)
1802 // associated with each basic boolean condition for a conditional
1803 // instruction.
1804 //
1805 // Instructions specify two basic values for encoding.  Again, a
1806 // function is available to check if the constant displacement is an
1807 // oop. They use the ins_encode keyword to specify their encoding
1808 // classes (which must be a sequence of enc_class names, and their
1809 // parameters, specified in the encoding block), and they use the
1810 // opcode keyword to specify, in order, their primary, secondary, and
1811 // tertiary opcode.  Only the opcode sections which a particular
1812 // instruction needs for encoding need to be specified.
1813 encode %{
1814   // Build emit functions for each basic byte or larger field in the
1815   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1816   // from C++ code in the enc_class source block.  Emit functions will
1817   // live in the main source block for now.  In future, we can
1818   // generalize this by adding a syntax that specifies the sizes of
1819   // fields in an order, so that the adlc can build the emit functions
1820   // automagically
1821 
1822   // Emit primary opcode
1823   enc_class OpcP
1824   %{
1825     emit_opcode(cbuf, $primary);
1826   %}
1827 
1828   // Emit secondary opcode
1829   enc_class OpcS
1830   %{
1831     emit_opcode(cbuf, $secondary);
1832   %}
1833 
1834   // Emit tertiary opcode
1835   enc_class OpcT
1836   %{
1837     emit_opcode(cbuf, $tertiary);
1838   %}
1839 
1840   // Emit opcode directly
1841   enc_class Opcode(immI d8)
1842   %{
1843     emit_opcode(cbuf, $d8$$constant);
1844   %}
1845 
1846   // Emit size prefix
1847   enc_class SizePrefix
1848   %{
1849     emit_opcode(cbuf, 0x66);
1850   %}
1851 
1852   enc_class reg(rRegI reg)
1853   %{
1854     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1855   %}
1856 
1857   enc_class reg_reg(rRegI dst, rRegI src)
1858   %{
1859     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1860   %}
1861 
1862   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1863   %{
1864     emit_opcode(cbuf, $opcode$$constant);
1865     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1866   %}
1867 
1868   enc_class cdql_enc(no_rax_rdx_RegI div)
1869   %{
1870     // Full implementation of Java idiv and irem; checks for
1871     // special case as described in JVM spec., p.243 & p.271.
1872     //
1873     //         normal case                           special case
1874     //
1875     // input : rax: dividend                         min_int
1876     //         reg: divisor                          -1
1877     //
1878     // output: rax: quotient  (= rax idiv reg)       min_int
1879     //         rdx: remainder (= rax irem reg)       0
1880     //
1881     //  Code sequnce:
1882     //
1883     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1884     //    5:   75 07/08                jne    e <normal>
1885     //    7:   33 d2                   xor    %edx,%edx
1886     //  [div >= 8 -> offset + 1]
1887     //  [REX_B]
1888     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1889     //    c:   74 03/04                je     11 <done>
1890     // 000000000000000e <normal>:
1891     //    e:   99                      cltd
1892     //  [div >= 8 -> offset + 1]
1893     //  [REX_B]
1894     //    f:   f7 f9                   idiv   $div
1895     // 0000000000000011 <done>:
1896 
1897     // cmp    $0x80000000,%eax
1898     emit_opcode(cbuf, 0x3d);
1899     emit_d8(cbuf, 0x00);
1900     emit_d8(cbuf, 0x00);
1901     emit_d8(cbuf, 0x00);
1902     emit_d8(cbuf, 0x80);
1903 
1904     // jne    e <normal>
1905     emit_opcode(cbuf, 0x75);
1906     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1907 
1908     // xor    %edx,%edx
1909     emit_opcode(cbuf, 0x33);
1910     emit_d8(cbuf, 0xD2);
1911 
1912     // cmp    $0xffffffffffffffff,%ecx
1913     if ($div$$reg >= 8) {
1914       emit_opcode(cbuf, Assembler::REX_B);
1915     }
1916     emit_opcode(cbuf, 0x83);
1917     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1918     emit_d8(cbuf, 0xFF);
1919 
1920     // je     11 <done>
1921     emit_opcode(cbuf, 0x74);
1922     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1923 
1924     // <normal>
1925     // cltd
1926     emit_opcode(cbuf, 0x99);
1927 
1928     // idivl (note: must be emitted by the user of this rule)
1929     // <done>
1930   %}
1931 
1932   enc_class cdqq_enc(no_rax_rdx_RegL div)
1933   %{
1934     // Full implementation of Java ldiv and lrem; checks for
1935     // special case as described in JVM spec., p.243 & p.271.
1936     //
1937     //         normal case                           special case
1938     //
1939     // input : rax: dividend                         min_long
1940     //         reg: divisor                          -1
1941     //
1942     // output: rax: quotient  (= rax idiv reg)       min_long
1943     //         rdx: remainder (= rax irem reg)       0
1944     //
1945     //  Code sequnce:
1946     //
1947     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1948     //    7:   00 00 80
1949     //    a:   48 39 d0                cmp    %rdx,%rax
1950     //    d:   75 08                   jne    17 <normal>
1951     //    f:   33 d2                   xor    %edx,%edx
1952     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1953     //   15:   74 05                   je     1c <done>
1954     // 0000000000000017 <normal>:
1955     //   17:   48 99                   cqto
1956     //   19:   48 f7 f9                idiv   $div
1957     // 000000000000001c <done>:
1958 
1959     // mov    $0x8000000000000000,%rdx
1960     emit_opcode(cbuf, Assembler::REX_W);
1961     emit_opcode(cbuf, 0xBA);
1962     emit_d8(cbuf, 0x00);
1963     emit_d8(cbuf, 0x00);
1964     emit_d8(cbuf, 0x00);
1965     emit_d8(cbuf, 0x00);
1966     emit_d8(cbuf, 0x00);
1967     emit_d8(cbuf, 0x00);
1968     emit_d8(cbuf, 0x00);
1969     emit_d8(cbuf, 0x80);
1970 
1971     // cmp    %rdx,%rax
1972     emit_opcode(cbuf, Assembler::REX_W);
1973     emit_opcode(cbuf, 0x39);
1974     emit_d8(cbuf, 0xD0);
1975 
1976     // jne    17 <normal>
1977     emit_opcode(cbuf, 0x75);
1978     emit_d8(cbuf, 0x08);
1979 
1980     // xor    %edx,%edx
1981     emit_opcode(cbuf, 0x33);
1982     emit_d8(cbuf, 0xD2);
1983 
1984     // cmp    $0xffffffffffffffff,$div
1985     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1986     emit_opcode(cbuf, 0x83);
1987     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1988     emit_d8(cbuf, 0xFF);
1989 
1990     // je     1e <done>
1991     emit_opcode(cbuf, 0x74);
1992     emit_d8(cbuf, 0x05);
1993 
1994     // <normal>
1995     // cqto
1996     emit_opcode(cbuf, Assembler::REX_W);
1997     emit_opcode(cbuf, 0x99);
1998 
1999     // idivq (note: must be emitted by the user of this rule)
2000     // <done>
2001   %}
2002 
2003   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
2004   enc_class OpcSE(immI imm)
2005   %{
2006     // Emit primary opcode and set sign-extend bit
2007     // Check for 8-bit immediate, and set sign extend bit in opcode
2008     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2009       emit_opcode(cbuf, $primary | 0x02);
2010     } else {
2011       // 32-bit immediate
2012       emit_opcode(cbuf, $primary);
2013     }
2014   %}
2015 
2016   enc_class OpcSErm(rRegI dst, immI imm)
2017   %{
2018     // OpcSEr/m
2019     int dstenc = $dst$$reg;
2020     if (dstenc >= 8) {
2021       emit_opcode(cbuf, Assembler::REX_B);
2022       dstenc -= 8;
2023     }
2024     // Emit primary opcode and set sign-extend bit
2025     // Check for 8-bit immediate, and set sign extend bit in opcode
2026     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2027       emit_opcode(cbuf, $primary | 0x02);
2028     } else {
2029       // 32-bit immediate
2030       emit_opcode(cbuf, $primary);
2031     }
2032     // Emit r/m byte with secondary opcode, after primary opcode.
2033     emit_rm(cbuf, 0x3, $secondary, dstenc);
2034   %}
2035 
2036   enc_class OpcSErm_wide(rRegL dst, immI imm)
2037   %{
2038     // OpcSEr/m
2039     int dstenc = $dst$$reg;
2040     if (dstenc < 8) {
2041       emit_opcode(cbuf, Assembler::REX_W);
2042     } else {
2043       emit_opcode(cbuf, Assembler::REX_WB);
2044       dstenc -= 8;
2045     }
2046     // Emit primary opcode and set sign-extend bit
2047     // Check for 8-bit immediate, and set sign extend bit in opcode
2048     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2049       emit_opcode(cbuf, $primary | 0x02);
2050     } else {
2051       // 32-bit immediate
2052       emit_opcode(cbuf, $primary);
2053     }
2054     // Emit r/m byte with secondary opcode, after primary opcode.
2055     emit_rm(cbuf, 0x3, $secondary, dstenc);
2056   %}
2057 
2058   enc_class Con8or32(immI imm)
2059   %{
2060     // Check for 8-bit immediate, and set sign extend bit in opcode
2061     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2062       $$$emit8$imm$$constant;
2063     } else {
2064       // 32-bit immediate
2065       $$$emit32$imm$$constant;
2066     }
2067   %}
2068 
2069   enc_class opc2_reg(rRegI dst)
2070   %{
2071     // BSWAP
2072     emit_cc(cbuf, $secondary, $dst$$reg);
2073   %}
2074 
2075   enc_class opc3_reg(rRegI dst)
2076   %{
2077     // BSWAP
2078     emit_cc(cbuf, $tertiary, $dst$$reg);
2079   %}
2080 
2081   enc_class reg_opc(rRegI div)
2082   %{
2083     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2084     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2085   %}
2086 
2087   enc_class enc_cmov(cmpOp cop)
2088   %{
2089     // CMOV
2090     $$$emit8$primary;
2091     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2092   %}
2093 
2094   enc_class enc_PartialSubtypeCheck()
2095   %{
2096     Register Rrdi = as_Register(RDI_enc); // result register
2097     Register Rrax = as_Register(RAX_enc); // super class
2098     Register Rrcx = as_Register(RCX_enc); // killed
2099     Register Rrsi = as_Register(RSI_enc); // sub class
2100     Label miss;
2101     const bool set_cond_codes = true;
2102 
2103     MacroAssembler _masm(&cbuf);
2104     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2105                                      NULL, &miss,
2106                                      /*set_cond_codes:*/ true);
2107     if ($primary) {
2108       __ xorptr(Rrdi, Rrdi);
2109     }
2110     __ bind(miss);
2111   %}
2112 
2113   enc_class clear_avx %{
2114     debug_only(int off0 = cbuf.insts_size());
2115     if (generate_vzeroupper(Compile::current())) {
2116       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
2117       // Clear upper bits of YMM registers when current compiled code uses
2118       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2119       MacroAssembler _masm(&cbuf);
2120       __ vzeroupper();
2121     }
2122     debug_only(int off1 = cbuf.insts_size());
2123     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2124   %}
2125 
2126   enc_class Java_To_Runtime(method meth) %{
2127     // No relocation needed
2128     MacroAssembler _masm(&cbuf);
2129     __ mov64(r10, (int64_t) $meth$$method);
2130     __ call(r10);
2131   %}
2132 
2133   enc_class Java_To_Interpreter(method meth)
2134   %{
2135     // CALL Java_To_Interpreter
2136     // This is the instruction starting address for relocation info.
2137     cbuf.set_insts_mark();
2138     $$$emit8$primary;
2139     // CALL directly to the runtime
2140     emit_d32_reloc(cbuf,
2141                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2142                    runtime_call_Relocation::spec(),
2143                    RELOC_DISP32);
2144   %}
2145 
2146   enc_class Java_Static_Call(method meth)
2147   %{
2148     // JAVA STATIC CALL
2149     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2150     // determine who we intended to call.
2151     cbuf.set_insts_mark();
2152     $$$emit8$primary;
2153 
2154     if (!_method) {
2155       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2156                      runtime_call_Relocation::spec(),
2157                      RELOC_DISP32);
2158     } else {
2159       int method_index = resolved_method_index(cbuf);
2160       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2161                                                   : static_call_Relocation::spec(method_index);
2162       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2163                      rspec, RELOC_DISP32);
2164       // Emit stubs for static call.
2165       address mark = cbuf.insts_mark();
2166       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2167       if (stub == NULL) {
2168         ciEnv::current()->record_failure("CodeCache is full");
2169         return;
2170       }
2171 #if INCLUDE_AOT
2172       CompiledStaticCall::emit_to_aot_stub(cbuf, mark);
2173 #endif
2174     }
2175   %}
2176 
2177   enc_class Java_Dynamic_Call(method meth) %{
2178     MacroAssembler _masm(&cbuf);
2179     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2180   %}
2181 
2182   enc_class Java_Compiled_Call(method meth)
2183   %{
2184     // JAVA COMPILED CALL
2185     int disp = in_bytes(Method:: from_compiled_offset());
2186 
2187     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2188     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2189 
2190     // callq *disp(%rax)
2191     cbuf.set_insts_mark();
2192     $$$emit8$primary;
2193     if (disp < 0x80) {
2194       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2195       emit_d8(cbuf, disp); // Displacement
2196     } else {
2197       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2198       emit_d32(cbuf, disp); // Displacement
2199     }
2200   %}
2201 
2202   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2203   %{
2204     // SAL, SAR, SHR
2205     int dstenc = $dst$$reg;
2206     if (dstenc >= 8) {
2207       emit_opcode(cbuf, Assembler::REX_B);
2208       dstenc -= 8;
2209     }
2210     $$$emit8$primary;
2211     emit_rm(cbuf, 0x3, $secondary, dstenc);
2212     $$$emit8$shift$$constant;
2213   %}
2214 
2215   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2216   %{
2217     // SAL, SAR, SHR
2218     int dstenc = $dst$$reg;
2219     if (dstenc < 8) {
2220       emit_opcode(cbuf, Assembler::REX_W);
2221     } else {
2222       emit_opcode(cbuf, Assembler::REX_WB);
2223       dstenc -= 8;
2224     }
2225     $$$emit8$primary;
2226     emit_rm(cbuf, 0x3, $secondary, dstenc);
2227     $$$emit8$shift$$constant;
2228   %}
2229 
2230   enc_class load_immI(rRegI dst, immI src)
2231   %{
2232     int dstenc = $dst$$reg;
2233     if (dstenc >= 8) {
2234       emit_opcode(cbuf, Assembler::REX_B);
2235       dstenc -= 8;
2236     }
2237     emit_opcode(cbuf, 0xB8 | dstenc);
2238     $$$emit32$src$$constant;
2239   %}
2240 
2241   enc_class load_immL(rRegL dst, immL src)
2242   %{
2243     int dstenc = $dst$$reg;
2244     if (dstenc < 8) {
2245       emit_opcode(cbuf, Assembler::REX_W);
2246     } else {
2247       emit_opcode(cbuf, Assembler::REX_WB);
2248       dstenc -= 8;
2249     }
2250     emit_opcode(cbuf, 0xB8 | dstenc);
2251     emit_d64(cbuf, $src$$constant);
2252   %}
2253 
2254   enc_class load_immUL32(rRegL dst, immUL32 src)
2255   %{
2256     // same as load_immI, but this time we care about zeroes in the high word
2257     int dstenc = $dst$$reg;
2258     if (dstenc >= 8) {
2259       emit_opcode(cbuf, Assembler::REX_B);
2260       dstenc -= 8;
2261     }
2262     emit_opcode(cbuf, 0xB8 | dstenc);
2263     $$$emit32$src$$constant;
2264   %}
2265 
2266   enc_class load_immL32(rRegL dst, immL32 src)
2267   %{
2268     int dstenc = $dst$$reg;
2269     if (dstenc < 8) {
2270       emit_opcode(cbuf, Assembler::REX_W);
2271     } else {
2272       emit_opcode(cbuf, Assembler::REX_WB);
2273       dstenc -= 8;
2274     }
2275     emit_opcode(cbuf, 0xC7);
2276     emit_rm(cbuf, 0x03, 0x00, dstenc);
2277     $$$emit32$src$$constant;
2278   %}
2279 
2280   enc_class load_immP31(rRegP dst, immP32 src)
2281   %{
2282     // same as load_immI, but this time we care about zeroes in the high word
2283     int dstenc = $dst$$reg;
2284     if (dstenc >= 8) {
2285       emit_opcode(cbuf, Assembler::REX_B);
2286       dstenc -= 8;
2287     }
2288     emit_opcode(cbuf, 0xB8 | dstenc);
2289     $$$emit32$src$$constant;
2290   %}
2291 
2292   enc_class load_immP(rRegP dst, immP src)
2293   %{
2294     int dstenc = $dst$$reg;
2295     if (dstenc < 8) {
2296       emit_opcode(cbuf, Assembler::REX_W);
2297     } else {
2298       emit_opcode(cbuf, Assembler::REX_WB);
2299       dstenc -= 8;
2300     }
2301     emit_opcode(cbuf, 0xB8 | dstenc);
2302     // This next line should be generated from ADLC
2303     if ($src->constant_reloc() != relocInfo::none) {
2304       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2305     } else {
2306       emit_d64(cbuf, $src$$constant);
2307     }
2308   %}
2309 
2310   enc_class Con32(immI src)
2311   %{
2312     // Output immediate
2313     $$$emit32$src$$constant;
2314   %}
2315 
2316   enc_class Con32F_as_bits(immF src)
2317   %{
2318     // Output Float immediate bits
2319     jfloat jf = $src$$constant;
2320     jint jf_as_bits = jint_cast(jf);
2321     emit_d32(cbuf, jf_as_bits);
2322   %}
2323 
2324   enc_class Con16(immI src)
2325   %{
2326     // Output immediate
2327     $$$emit16$src$$constant;
2328   %}
2329 
2330   // How is this different from Con32??? XXX
2331   enc_class Con_d32(immI src)
2332   %{
2333     emit_d32(cbuf,$src$$constant);
2334   %}
2335 
2336   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2337     // Output immediate memory reference
2338     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2339     emit_d32(cbuf, 0x00);
2340   %}
2341 
2342   enc_class lock_prefix()
2343   %{
2344     emit_opcode(cbuf, 0xF0); // lock
2345   %}
2346 
2347   enc_class REX_mem(memory mem)
2348   %{
2349     if ($mem$$base >= 8) {
2350       if ($mem$$index < 8) {
2351         emit_opcode(cbuf, Assembler::REX_B);
2352       } else {
2353         emit_opcode(cbuf, Assembler::REX_XB);
2354       }
2355     } else {
2356       if ($mem$$index >= 8) {
2357         emit_opcode(cbuf, Assembler::REX_X);
2358       }
2359     }
2360   %}
2361 
2362   enc_class REX_mem_wide(memory mem)
2363   %{
2364     if ($mem$$base >= 8) {
2365       if ($mem$$index < 8) {
2366         emit_opcode(cbuf, Assembler::REX_WB);
2367       } else {
2368         emit_opcode(cbuf, Assembler::REX_WXB);
2369       }
2370     } else {
2371       if ($mem$$index < 8) {
2372         emit_opcode(cbuf, Assembler::REX_W);
2373       } else {
2374         emit_opcode(cbuf, Assembler::REX_WX);
2375       }
2376     }
2377   %}
2378 
2379   // for byte regs
2380   enc_class REX_breg(rRegI reg)
2381   %{
2382     if ($reg$$reg >= 4) {
2383       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2384     }
2385   %}
2386 
2387   // for byte regs
2388   enc_class REX_reg_breg(rRegI dst, rRegI src)
2389   %{
2390     if ($dst$$reg < 8) {
2391       if ($src$$reg >= 4) {
2392         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2393       }
2394     } else {
2395       if ($src$$reg < 8) {
2396         emit_opcode(cbuf, Assembler::REX_R);
2397       } else {
2398         emit_opcode(cbuf, Assembler::REX_RB);
2399       }
2400     }
2401   %}
2402 
2403   // for byte regs
2404   enc_class REX_breg_mem(rRegI reg, memory mem)
2405   %{
2406     if ($reg$$reg < 8) {
2407       if ($mem$$base < 8) {
2408         if ($mem$$index >= 8) {
2409           emit_opcode(cbuf, Assembler::REX_X);
2410         } else if ($reg$$reg >= 4) {
2411           emit_opcode(cbuf, Assembler::REX);
2412         }
2413       } else {
2414         if ($mem$$index < 8) {
2415           emit_opcode(cbuf, Assembler::REX_B);
2416         } else {
2417           emit_opcode(cbuf, Assembler::REX_XB);
2418         }
2419       }
2420     } else {
2421       if ($mem$$base < 8) {
2422         if ($mem$$index < 8) {
2423           emit_opcode(cbuf, Assembler::REX_R);
2424         } else {
2425           emit_opcode(cbuf, Assembler::REX_RX);
2426         }
2427       } else {
2428         if ($mem$$index < 8) {
2429           emit_opcode(cbuf, Assembler::REX_RB);
2430         } else {
2431           emit_opcode(cbuf, Assembler::REX_RXB);
2432         }
2433       }
2434     }
2435   %}
2436 
2437   enc_class REX_reg(rRegI reg)
2438   %{
2439     if ($reg$$reg >= 8) {
2440       emit_opcode(cbuf, Assembler::REX_B);
2441     }
2442   %}
2443 
2444   enc_class REX_reg_wide(rRegI reg)
2445   %{
2446     if ($reg$$reg < 8) {
2447       emit_opcode(cbuf, Assembler::REX_W);
2448     } else {
2449       emit_opcode(cbuf, Assembler::REX_WB);
2450     }
2451   %}
2452 
2453   enc_class REX_reg_reg(rRegI dst, rRegI src)
2454   %{
2455     if ($dst$$reg < 8) {
2456       if ($src$$reg >= 8) {
2457         emit_opcode(cbuf, Assembler::REX_B);
2458       }
2459     } else {
2460       if ($src$$reg < 8) {
2461         emit_opcode(cbuf, Assembler::REX_R);
2462       } else {
2463         emit_opcode(cbuf, Assembler::REX_RB);
2464       }
2465     }
2466   %}
2467 
2468   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2469   %{
2470     if ($dst$$reg < 8) {
2471       if ($src$$reg < 8) {
2472         emit_opcode(cbuf, Assembler::REX_W);
2473       } else {
2474         emit_opcode(cbuf, Assembler::REX_WB);
2475       }
2476     } else {
2477       if ($src$$reg < 8) {
2478         emit_opcode(cbuf, Assembler::REX_WR);
2479       } else {
2480         emit_opcode(cbuf, Assembler::REX_WRB);
2481       }
2482     }
2483   %}
2484 
2485   enc_class REX_reg_mem(rRegI reg, memory mem)
2486   %{
2487     if ($reg$$reg < 8) {
2488       if ($mem$$base < 8) {
2489         if ($mem$$index >= 8) {
2490           emit_opcode(cbuf, Assembler::REX_X);
2491         }
2492       } else {
2493         if ($mem$$index < 8) {
2494           emit_opcode(cbuf, Assembler::REX_B);
2495         } else {
2496           emit_opcode(cbuf, Assembler::REX_XB);
2497         }
2498       }
2499     } else {
2500       if ($mem$$base < 8) {
2501         if ($mem$$index < 8) {
2502           emit_opcode(cbuf, Assembler::REX_R);
2503         } else {
2504           emit_opcode(cbuf, Assembler::REX_RX);
2505         }
2506       } else {
2507         if ($mem$$index < 8) {
2508           emit_opcode(cbuf, Assembler::REX_RB);
2509         } else {
2510           emit_opcode(cbuf, Assembler::REX_RXB);
2511         }
2512       }
2513     }
2514   %}
2515 
2516   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2517   %{
2518     if ($reg$$reg < 8) {
2519       if ($mem$$base < 8) {
2520         if ($mem$$index < 8) {
2521           emit_opcode(cbuf, Assembler::REX_W);
2522         } else {
2523           emit_opcode(cbuf, Assembler::REX_WX);
2524         }
2525       } else {
2526         if ($mem$$index < 8) {
2527           emit_opcode(cbuf, Assembler::REX_WB);
2528         } else {
2529           emit_opcode(cbuf, Assembler::REX_WXB);
2530         }
2531       }
2532     } else {
2533       if ($mem$$base < 8) {
2534         if ($mem$$index < 8) {
2535           emit_opcode(cbuf, Assembler::REX_WR);
2536         } else {
2537           emit_opcode(cbuf, Assembler::REX_WRX);
2538         }
2539       } else {
2540         if ($mem$$index < 8) {
2541           emit_opcode(cbuf, Assembler::REX_WRB);
2542         } else {
2543           emit_opcode(cbuf, Assembler::REX_WRXB);
2544         }
2545       }
2546     }
2547   %}
2548 
2549   enc_class reg_mem(rRegI ereg, memory mem)
2550   %{
2551     // High registers handle in encode_RegMem
2552     int reg = $ereg$$reg;
2553     int base = $mem$$base;
2554     int index = $mem$$index;
2555     int scale = $mem$$scale;
2556     int disp = $mem$$disp;
2557     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2558 
2559     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2560   %}
2561 
2562   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2563   %{
2564     int rm_byte_opcode = $rm_opcode$$constant;
2565 
2566     // High registers handle in encode_RegMem
2567     int base = $mem$$base;
2568     int index = $mem$$index;
2569     int scale = $mem$$scale;
2570     int displace = $mem$$disp;
2571 
2572     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2573                                             // working with static
2574                                             // globals
2575     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2576                   disp_reloc);
2577   %}
2578 
2579   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2580   %{
2581     int reg_encoding = $dst$$reg;
2582     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2583     int index        = 0x04;            // 0x04 indicates no index
2584     int scale        = 0x00;            // 0x00 indicates no scale
2585     int displace     = $src1$$constant; // 0x00 indicates no displacement
2586     relocInfo::relocType disp_reloc = relocInfo::none;
2587     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2588                   disp_reloc);
2589   %}
2590 
2591   enc_class neg_reg(rRegI dst)
2592   %{
2593     int dstenc = $dst$$reg;
2594     if (dstenc >= 8) {
2595       emit_opcode(cbuf, Assembler::REX_B);
2596       dstenc -= 8;
2597     }
2598     // NEG $dst
2599     emit_opcode(cbuf, 0xF7);
2600     emit_rm(cbuf, 0x3, 0x03, dstenc);
2601   %}
2602 
2603   enc_class neg_reg_wide(rRegI dst)
2604   %{
2605     int dstenc = $dst$$reg;
2606     if (dstenc < 8) {
2607       emit_opcode(cbuf, Assembler::REX_W);
2608     } else {
2609       emit_opcode(cbuf, Assembler::REX_WB);
2610       dstenc -= 8;
2611     }
2612     // NEG $dst
2613     emit_opcode(cbuf, 0xF7);
2614     emit_rm(cbuf, 0x3, 0x03, dstenc);
2615   %}
2616 
2617   enc_class setLT_reg(rRegI dst)
2618   %{
2619     int dstenc = $dst$$reg;
2620     if (dstenc >= 8) {
2621       emit_opcode(cbuf, Assembler::REX_B);
2622       dstenc -= 8;
2623     } else if (dstenc >= 4) {
2624       emit_opcode(cbuf, Assembler::REX);
2625     }
2626     // SETLT $dst
2627     emit_opcode(cbuf, 0x0F);
2628     emit_opcode(cbuf, 0x9C);
2629     emit_rm(cbuf, 0x3, 0x0, dstenc);
2630   %}
2631 
2632   enc_class setNZ_reg(rRegI dst)
2633   %{
2634     int dstenc = $dst$$reg;
2635     if (dstenc >= 8) {
2636       emit_opcode(cbuf, Assembler::REX_B);
2637       dstenc -= 8;
2638     } else if (dstenc >= 4) {
2639       emit_opcode(cbuf, Assembler::REX);
2640     }
2641     // SETNZ $dst
2642     emit_opcode(cbuf, 0x0F);
2643     emit_opcode(cbuf, 0x95);
2644     emit_rm(cbuf, 0x3, 0x0, dstenc);
2645   %}
2646 
2647 
2648   // Compare the lonogs and set -1, 0, or 1 into dst
2649   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2650   %{
2651     int src1enc = $src1$$reg;
2652     int src2enc = $src2$$reg;
2653     int dstenc = $dst$$reg;
2654 
2655     // cmpq $src1, $src2
2656     if (src1enc < 8) {
2657       if (src2enc < 8) {
2658         emit_opcode(cbuf, Assembler::REX_W);
2659       } else {
2660         emit_opcode(cbuf, Assembler::REX_WB);
2661       }
2662     } else {
2663       if (src2enc < 8) {
2664         emit_opcode(cbuf, Assembler::REX_WR);
2665       } else {
2666         emit_opcode(cbuf, Assembler::REX_WRB);
2667       }
2668     }
2669     emit_opcode(cbuf, 0x3B);
2670     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2671 
2672     // movl $dst, -1
2673     if (dstenc >= 8) {
2674       emit_opcode(cbuf, Assembler::REX_B);
2675     }
2676     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2677     emit_d32(cbuf, -1);
2678 
2679     // jl,s done
2680     emit_opcode(cbuf, 0x7C);
2681     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2682 
2683     // setne $dst
2684     if (dstenc >= 4) {
2685       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2686     }
2687     emit_opcode(cbuf, 0x0F);
2688     emit_opcode(cbuf, 0x95);
2689     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2690 
2691     // movzbl $dst, $dst
2692     if (dstenc >= 4) {
2693       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2694     }
2695     emit_opcode(cbuf, 0x0F);
2696     emit_opcode(cbuf, 0xB6);
2697     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2698   %}
2699 
2700   enc_class Push_ResultXD(regD dst) %{
2701     MacroAssembler _masm(&cbuf);
2702     __ fstp_d(Address(rsp, 0));
2703     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2704     __ addptr(rsp, 8);
2705   %}
2706 
2707   enc_class Push_SrcXD(regD src) %{
2708     MacroAssembler _masm(&cbuf);
2709     __ subptr(rsp, 8);
2710     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2711     __ fld_d(Address(rsp, 0));
2712   %}
2713 
2714 
2715   enc_class enc_rethrow()
2716   %{
2717     cbuf.set_insts_mark();
2718     emit_opcode(cbuf, 0xE9); // jmp entry
2719     emit_d32_reloc(cbuf,
2720                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2721                    runtime_call_Relocation::spec(),
2722                    RELOC_DISP32);
2723   %}
2724 
2725 %}
2726 
2727 
2728 
2729 //----------FRAME--------------------------------------------------------------
2730 // Definition of frame structure and management information.
2731 //
2732 //  S T A C K   L A Y O U T    Allocators stack-slot number
2733 //                             |   (to get allocators register number
2734 //  G  Owned by    |        |  v    add OptoReg::stack0())
2735 //  r   CALLER     |        |
2736 //  o     |        +--------+      pad to even-align allocators stack-slot
2737 //  w     V        |  pad0  |        numbers; owned by CALLER
2738 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2739 //  h     ^        |   in   |  5
2740 //        |        |  args  |  4   Holes in incoming args owned by SELF
2741 //  |     |        |        |  3
2742 //  |     |        +--------+
2743 //  V     |        | old out|      Empty on Intel, window on Sparc
2744 //        |    old |preserve|      Must be even aligned.
2745 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2746 //        |        |   in   |  3   area for Intel ret address
2747 //     Owned by    |preserve|      Empty on Sparc.
2748 //       SELF      +--------+
2749 //        |        |  pad2  |  2   pad to align old SP
2750 //        |        +--------+  1
2751 //        |        | locks  |  0
2752 //        |        +--------+----> OptoReg::stack0(), even aligned
2753 //        |        |  pad1  | 11   pad to align new SP
2754 //        |        +--------+
2755 //        |        |        | 10
2756 //        |        | spills |  9   spills
2757 //        V        |        |  8   (pad0 slot for callee)
2758 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2759 //        ^        |  out   |  7
2760 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2761 //     Owned by    +--------+
2762 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2763 //        |    new |preserve|      Must be even-aligned.
2764 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2765 //        |        |        |
2766 //
2767 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2768 //         known from SELF's arguments and the Java calling convention.
2769 //         Region 6-7 is determined per call site.
2770 // Note 2: If the calling convention leaves holes in the incoming argument
2771 //         area, those holes are owned by SELF.  Holes in the outgoing area
2772 //         are owned by the CALLEE.  Holes should not be nessecary in the
2773 //         incoming area, as the Java calling convention is completely under
2774 //         the control of the AD file.  Doubles can be sorted and packed to
2775 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2776 //         varargs C calling conventions.
2777 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2778 //         even aligned with pad0 as needed.
2779 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2780 //         region 6-11 is even aligned; it may be padded out more so that
2781 //         the region from SP to FP meets the minimum stack alignment.
2782 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2783 //         alignment.  Region 11, pad1, may be dynamically extended so that
2784 //         SP meets the minimum alignment.
2785 
2786 frame
2787 %{
2788   // What direction does stack grow in (assumed to be same for C & Java)
2789   stack_direction(TOWARDS_LOW);
2790 
2791   // These three registers define part of the calling convention
2792   // between compiled code and the interpreter.
2793   inline_cache_reg(RAX);                // Inline Cache Register
2794   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2795                                         // calling interpreter
2796 
2797   // Optional: name the operand used by cisc-spilling to access
2798   // [stack_pointer + offset]
2799   cisc_spilling_operand_name(indOffset32);
2800 
2801   // Number of stack slots consumed by locking an object
2802   sync_stack_slots(2);
2803 
2804   // Compiled code's Frame Pointer
2805   frame_pointer(RSP);
2806 
2807   // Interpreter stores its frame pointer in a register which is
2808   // stored to the stack by I2CAdaptors.
2809   // I2CAdaptors convert from interpreted java to compiled java.
2810   interpreter_frame_pointer(RBP);
2811 
2812   // Stack alignment requirement
2813   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2814 
2815   // Number of stack slots between incoming argument block and the start of
2816   // a new frame.  The PROLOG must add this many slots to the stack.  The
2817   // EPILOG must remove this many slots.  amd64 needs two slots for
2818   // return address.
2819   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2820 
2821   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2822   // for calls to C.  Supports the var-args backing area for register parms.
2823   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2824 
2825   // The after-PROLOG location of the return address.  Location of
2826   // return address specifies a type (REG or STACK) and a number
2827   // representing the register number (i.e. - use a register name) or
2828   // stack slot.
2829   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2830   // Otherwise, it is above the locks and verification slot and alignment word
2831   return_addr(STACK - 2 +
2832               align_up((Compile::current()->in_preserve_stack_slots() +
2833                         Compile::current()->fixed_slots()),
2834                        stack_alignment_in_slots()));
2835 
2836   // Body of function which returns an integer array locating
2837   // arguments either in registers or in stack slots.  Passed an array
2838   // of ideal registers called "sig" and a "length" count.  Stack-slot
2839   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2840   // arguments for a CALLEE.  Incoming stack arguments are
2841   // automatically biased by the preserve_stack_slots field above.
2842 
2843   calling_convention
2844   %{
2845     // No difference between ingoing/outgoing just pass false
2846     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2847   %}
2848 
2849   c_calling_convention
2850   %{
2851     // This is obviously always outgoing
2852     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2853   %}
2854 
2855   // Location of compiled Java return values.  Same as C for now.
2856   return_value
2857   %{
2858     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2859            "only return normal values");
2860 
2861     static const int lo[Op_RegL + 1] = {
2862       0,
2863       0,
2864       RAX_num,  // Op_RegN
2865       RAX_num,  // Op_RegI
2866       RAX_num,  // Op_RegP
2867       XMM0_num, // Op_RegF
2868       XMM0_num, // Op_RegD
2869       RAX_num   // Op_RegL
2870     };
2871     static const int hi[Op_RegL + 1] = {
2872       0,
2873       0,
2874       OptoReg::Bad, // Op_RegN
2875       OptoReg::Bad, // Op_RegI
2876       RAX_H_num,    // Op_RegP
2877       OptoReg::Bad, // Op_RegF
2878       XMM0b_num,    // Op_RegD
2879       RAX_H_num     // Op_RegL
2880     };
2881     // Excluded flags and vector registers.
2882     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2883     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2884   %}
2885 %}
2886 
2887 //----------ATTRIBUTES---------------------------------------------------------
2888 //----------Operand Attributes-------------------------------------------------
2889 op_attrib op_cost(0);        // Required cost attribute
2890 
2891 //----------Instruction Attributes---------------------------------------------
2892 ins_attrib ins_cost(100);       // Required cost attribute
2893 ins_attrib ins_size(8);         // Required size attribute (in bits)
2894 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2895                                 // a non-matching short branch variant
2896                                 // of some long branch?
2897 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2898                                 // be a power of 2) specifies the
2899                                 // alignment that some part of the
2900                                 // instruction (not necessarily the
2901                                 // start) requires.  If > 1, a
2902                                 // compute_padding() function must be
2903                                 // provided for the instruction
2904 
2905 //----------OPERANDS-----------------------------------------------------------
2906 // Operand definitions must precede instruction definitions for correct parsing
2907 // in the ADLC because operands constitute user defined types which are used in
2908 // instruction definitions.
2909 
2910 //----------Simple Operands----------------------------------------------------
2911 // Immediate Operands
2912 // Integer Immediate
2913 operand immI()
2914 %{
2915   match(ConI);
2916 
2917   op_cost(10);
2918   format %{ %}
2919   interface(CONST_INTER);
2920 %}
2921 
2922 // Constant for test vs zero
2923 operand immI0()
2924 %{
2925   predicate(n->get_int() == 0);
2926   match(ConI);
2927 
2928   op_cost(0);
2929   format %{ %}
2930   interface(CONST_INTER);
2931 %}
2932 
2933 // Constant for increment
2934 operand immI1()
2935 %{
2936   predicate(n->get_int() == 1);
2937   match(ConI);
2938 
2939   op_cost(0);
2940   format %{ %}
2941   interface(CONST_INTER);
2942 %}
2943 
2944 // Constant for decrement
2945 operand immI_M1()
2946 %{
2947   predicate(n->get_int() == -1);
2948   match(ConI);
2949 
2950   op_cost(0);
2951   format %{ %}
2952   interface(CONST_INTER);
2953 %}
2954 
2955 // Valid scale values for addressing modes
2956 operand immI2()
2957 %{
2958   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2959   match(ConI);
2960 
2961   format %{ %}
2962   interface(CONST_INTER);
2963 %}
2964 
2965 operand immI8()
2966 %{
2967   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2968   match(ConI);
2969 
2970   op_cost(5);
2971   format %{ %}
2972   interface(CONST_INTER);
2973 %}
2974 
2975 operand immU8()
2976 %{
2977   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
2978   match(ConI);
2979 
2980   op_cost(5);
2981   format %{ %}
2982   interface(CONST_INTER);
2983 %}
2984 
2985 operand immI16()
2986 %{
2987   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2988   match(ConI);
2989 
2990   op_cost(10);
2991   format %{ %}
2992   interface(CONST_INTER);
2993 %}
2994 
2995 // Int Immediate non-negative
2996 operand immU31()
2997 %{
2998   predicate(n->get_int() >= 0);
2999   match(ConI);
3000 
3001   op_cost(0);
3002   format %{ %}
3003   interface(CONST_INTER);
3004 %}
3005 
3006 // Constant for long shifts
3007 operand immI_32()
3008 %{
3009   predicate( n->get_int() == 32 );
3010   match(ConI);
3011 
3012   op_cost(0);
3013   format %{ %}
3014   interface(CONST_INTER);
3015 %}
3016 
3017 // Constant for long shifts
3018 operand immI_64()
3019 %{
3020   predicate( n->get_int() == 64 );
3021   match(ConI);
3022 
3023   op_cost(0);
3024   format %{ %}
3025   interface(CONST_INTER);
3026 %}
3027 
3028 // Pointer Immediate
3029 operand immP()
3030 %{
3031   match(ConP);
3032 
3033   op_cost(10);
3034   format %{ %}
3035   interface(CONST_INTER);
3036 %}
3037 
3038 // NULL Pointer Immediate
3039 operand immP0()
3040 %{
3041   predicate(n->get_ptr() == 0);
3042   match(ConP);
3043 
3044   op_cost(5);
3045   format %{ %}
3046   interface(CONST_INTER);
3047 %}
3048 
3049 // Pointer Immediate
3050 operand immN() %{
3051   match(ConN);
3052 
3053   op_cost(10);
3054   format %{ %}
3055   interface(CONST_INTER);
3056 %}
3057 
3058 operand immNKlass() %{
3059   match(ConNKlass);
3060 
3061   op_cost(10);
3062   format %{ %}
3063   interface(CONST_INTER);
3064 %}
3065 
3066 // NULL Pointer Immediate
3067 operand immN0() %{
3068   predicate(n->get_narrowcon() == 0);
3069   match(ConN);
3070 
3071   op_cost(5);
3072   format %{ %}
3073   interface(CONST_INTER);
3074 %}
3075 
3076 operand immP31()
3077 %{
3078   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3079             && (n->get_ptr() >> 31) == 0);
3080   match(ConP);
3081 
3082   op_cost(5);
3083   format %{ %}
3084   interface(CONST_INTER);
3085 %}
3086 
3087 
3088 // Long Immediate
3089 operand immL()
3090 %{
3091   match(ConL);
3092 
3093   op_cost(20);
3094   format %{ %}
3095   interface(CONST_INTER);
3096 %}
3097 
3098 // Long Immediate 8-bit
3099 operand immL8()
3100 %{
3101   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3102   match(ConL);
3103 
3104   op_cost(5);
3105   format %{ %}
3106   interface(CONST_INTER);
3107 %}
3108 
3109 // Long Immediate 32-bit unsigned
3110 operand immUL32()
3111 %{
3112   predicate(n->get_long() == (unsigned int) (n->get_long()));
3113   match(ConL);
3114 
3115   op_cost(10);
3116   format %{ %}
3117   interface(CONST_INTER);
3118 %}
3119 
3120 // Long Immediate 32-bit signed
3121 operand immL32()
3122 %{
3123   predicate(n->get_long() == (int) (n->get_long()));
3124   match(ConL);
3125 
3126   op_cost(15);
3127   format %{ %}
3128   interface(CONST_INTER);
3129 %}
3130 
3131 // Long Immediate zero
3132 operand immL0()
3133 %{
3134   predicate(n->get_long() == 0L);
3135   match(ConL);
3136 
3137   op_cost(10);
3138   format %{ %}
3139   interface(CONST_INTER);
3140 %}
3141 
3142 // Constant for increment
3143 operand immL1()
3144 %{
3145   predicate(n->get_long() == 1);
3146   match(ConL);
3147 
3148   format %{ %}
3149   interface(CONST_INTER);
3150 %}
3151 
3152 // Constant for decrement
3153 operand immL_M1()
3154 %{
3155   predicate(n->get_long() == -1);
3156   match(ConL);
3157 
3158   format %{ %}
3159   interface(CONST_INTER);
3160 %}
3161 
3162 // Long Immediate: the value 10
3163 operand immL10()
3164 %{
3165   predicate(n->get_long() == 10);
3166   match(ConL);
3167 
3168   format %{ %}
3169   interface(CONST_INTER);
3170 %}
3171 
3172 // Long immediate from 0 to 127.
3173 // Used for a shorter form of long mul by 10.
3174 operand immL_127()
3175 %{
3176   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3177   match(ConL);
3178 
3179   op_cost(10);
3180   format %{ %}
3181   interface(CONST_INTER);
3182 %}
3183 
3184 // Long Immediate: low 32-bit mask
3185 operand immL_32bits()
3186 %{
3187   predicate(n->get_long() == 0xFFFFFFFFL);
3188   match(ConL);
3189   op_cost(20);
3190 
3191   format %{ %}
3192   interface(CONST_INTER);
3193 %}
3194 
3195 // Float Immediate zero
3196 operand immF0()
3197 %{
3198   predicate(jint_cast(n->getf()) == 0);
3199   match(ConF);
3200 
3201   op_cost(5);
3202   format %{ %}
3203   interface(CONST_INTER);
3204 %}
3205 
3206 // Float Immediate
3207 operand immF()
3208 %{
3209   match(ConF);
3210 
3211   op_cost(15);
3212   format %{ %}
3213   interface(CONST_INTER);
3214 %}
3215 
3216 // Double Immediate zero
3217 operand immD0()
3218 %{
3219   predicate(jlong_cast(n->getd()) == 0);
3220   match(ConD);
3221 
3222   op_cost(5);
3223   format %{ %}
3224   interface(CONST_INTER);
3225 %}
3226 
3227 // Double Immediate
3228 operand immD()
3229 %{
3230   match(ConD);
3231 
3232   op_cost(15);
3233   format %{ %}
3234   interface(CONST_INTER);
3235 %}
3236 
3237 // Immediates for special shifts (sign extend)
3238 
3239 // Constants for increment
3240 operand immI_16()
3241 %{
3242   predicate(n->get_int() == 16);
3243   match(ConI);
3244 
3245   format %{ %}
3246   interface(CONST_INTER);
3247 %}
3248 
3249 operand immI_24()
3250 %{
3251   predicate(n->get_int() == 24);
3252   match(ConI);
3253 
3254   format %{ %}
3255   interface(CONST_INTER);
3256 %}
3257 
3258 // Constant for byte-wide masking
3259 operand immI_255()
3260 %{
3261   predicate(n->get_int() == 255);
3262   match(ConI);
3263 
3264   format %{ %}
3265   interface(CONST_INTER);
3266 %}
3267 
3268 // Constant for short-wide masking
3269 operand immI_65535()
3270 %{
3271   predicate(n->get_int() == 65535);
3272   match(ConI);
3273 
3274   format %{ %}
3275   interface(CONST_INTER);
3276 %}
3277 
3278 // Constant for byte-wide masking
3279 operand immL_255()
3280 %{
3281   predicate(n->get_long() == 255);
3282   match(ConL);
3283 
3284   format %{ %}
3285   interface(CONST_INTER);
3286 %}
3287 
3288 // Constant for short-wide masking
3289 operand immL_65535()
3290 %{
3291   predicate(n->get_long() == 65535);
3292   match(ConL);
3293 
3294   format %{ %}
3295   interface(CONST_INTER);
3296 %}
3297 
3298 // Register Operands
3299 // Integer Register
3300 operand rRegI()
3301 %{
3302   constraint(ALLOC_IN_RC(int_reg));
3303   match(RegI);
3304 
3305   match(rax_RegI);
3306   match(rbx_RegI);
3307   match(rcx_RegI);
3308   match(rdx_RegI);
3309   match(rdi_RegI);
3310 
3311   format %{ %}
3312   interface(REG_INTER);
3313 %}
3314 
3315 // Special Registers
3316 operand rax_RegI()
3317 %{
3318   constraint(ALLOC_IN_RC(int_rax_reg));
3319   match(RegI);
3320   match(rRegI);
3321 
3322   format %{ "RAX" %}
3323   interface(REG_INTER);
3324 %}
3325 
3326 // Special Registers
3327 operand rbx_RegI()
3328 %{
3329   constraint(ALLOC_IN_RC(int_rbx_reg));
3330   match(RegI);
3331   match(rRegI);
3332 
3333   format %{ "RBX" %}
3334   interface(REG_INTER);
3335 %}
3336 
3337 operand rcx_RegI()
3338 %{
3339   constraint(ALLOC_IN_RC(int_rcx_reg));
3340   match(RegI);
3341   match(rRegI);
3342 
3343   format %{ "RCX" %}
3344   interface(REG_INTER);
3345 %}
3346 
3347 operand rdx_RegI()
3348 %{
3349   constraint(ALLOC_IN_RC(int_rdx_reg));
3350   match(RegI);
3351   match(rRegI);
3352 
3353   format %{ "RDX" %}
3354   interface(REG_INTER);
3355 %}
3356 
3357 operand rdi_RegI()
3358 %{
3359   constraint(ALLOC_IN_RC(int_rdi_reg));
3360   match(RegI);
3361   match(rRegI);
3362 
3363   format %{ "RDI" %}
3364   interface(REG_INTER);
3365 %}
3366 
3367 operand no_rcx_RegI()
3368 %{
3369   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3370   match(RegI);
3371   match(rax_RegI);
3372   match(rbx_RegI);
3373   match(rdx_RegI);
3374   match(rdi_RegI);
3375 
3376   format %{ %}
3377   interface(REG_INTER);
3378 %}
3379 
3380 operand no_rax_rdx_RegI()
3381 %{
3382   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3383   match(RegI);
3384   match(rbx_RegI);
3385   match(rcx_RegI);
3386   match(rdi_RegI);
3387 
3388   format %{ %}
3389   interface(REG_INTER);
3390 %}
3391 
3392 // Pointer Register
3393 operand any_RegP()
3394 %{
3395   constraint(ALLOC_IN_RC(any_reg));
3396   match(RegP);
3397   match(rax_RegP);
3398   match(rbx_RegP);
3399   match(rdi_RegP);
3400   match(rsi_RegP);
3401   match(rbp_RegP);
3402   match(r15_RegP);
3403   match(rRegP);
3404 
3405   format %{ %}
3406   interface(REG_INTER);
3407 %}
3408 
3409 operand rRegP()
3410 %{
3411   constraint(ALLOC_IN_RC(ptr_reg));
3412   match(RegP);
3413   match(rax_RegP);
3414   match(rbx_RegP);
3415   match(rdi_RegP);
3416   match(rsi_RegP);
3417   match(rbp_RegP);  // See Q&A below about
3418   match(r15_RegP);  // r15_RegP and rbp_RegP.
3419 
3420   format %{ %}
3421   interface(REG_INTER);
3422 %}
3423 
3424 operand rRegN() %{
3425   constraint(ALLOC_IN_RC(int_reg));
3426   match(RegN);
3427 
3428   format %{ %}
3429   interface(REG_INTER);
3430 %}
3431 
3432 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3433 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3434 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3435 // The output of an instruction is controlled by the allocator, which respects
3436 // register class masks, not match rules.  Unless an instruction mentions
3437 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3438 // by the allocator as an input.
3439 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3440 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3441 // result, RBP is not included in the output of the instruction either.
3442 
3443 operand no_rax_RegP()
3444 %{
3445   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3446   match(RegP);
3447   match(rbx_RegP);
3448   match(rsi_RegP);
3449   match(rdi_RegP);
3450 
3451   format %{ %}
3452   interface(REG_INTER);
3453 %}
3454 
3455 // This operand is not allowed to use RBP even if
3456 // RBP is not used to hold the frame pointer.
3457 operand no_rbp_RegP()
3458 %{
3459   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3460   match(RegP);
3461   match(rbx_RegP);
3462   match(rsi_RegP);
3463   match(rdi_RegP);
3464 
3465   format %{ %}
3466   interface(REG_INTER);
3467 %}
3468 
3469 operand no_rax_rbx_RegP()
3470 %{
3471   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3472   match(RegP);
3473   match(rsi_RegP);
3474   match(rdi_RegP);
3475 
3476   format %{ %}
3477   interface(REG_INTER);
3478 %}
3479 
3480 // Special Registers
3481 // Return a pointer value
3482 operand rax_RegP()
3483 %{
3484   constraint(ALLOC_IN_RC(ptr_rax_reg));
3485   match(RegP);
3486   match(rRegP);
3487 
3488   format %{ %}
3489   interface(REG_INTER);
3490 %}
3491 
3492 // Special Registers
3493 // Return a compressed pointer value
3494 operand rax_RegN()
3495 %{
3496   constraint(ALLOC_IN_RC(int_rax_reg));
3497   match(RegN);
3498   match(rRegN);
3499 
3500   format %{ %}
3501   interface(REG_INTER);
3502 %}
3503 
3504 // Used in AtomicAdd
3505 operand rbx_RegP()
3506 %{
3507   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3508   match(RegP);
3509   match(rRegP);
3510 
3511   format %{ %}
3512   interface(REG_INTER);
3513 %}
3514 
3515 operand rsi_RegP()
3516 %{
3517   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3518   match(RegP);
3519   match(rRegP);
3520 
3521   format %{ %}
3522   interface(REG_INTER);
3523 %}
3524 
3525 // Used in rep stosq
3526 operand rdi_RegP()
3527 %{
3528   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3529   match(RegP);
3530   match(rRegP);
3531 
3532   format %{ %}
3533   interface(REG_INTER);
3534 %}
3535 
3536 operand r15_RegP()
3537 %{
3538   constraint(ALLOC_IN_RC(ptr_r15_reg));
3539   match(RegP);
3540   match(rRegP);
3541 
3542   format %{ %}
3543   interface(REG_INTER);
3544 %}
3545 
3546 operand rRegL()
3547 %{
3548   constraint(ALLOC_IN_RC(long_reg));
3549   match(RegL);
3550   match(rax_RegL);
3551   match(rdx_RegL);
3552 
3553   format %{ %}
3554   interface(REG_INTER);
3555 %}
3556 
3557 // Special Registers
3558 operand no_rax_rdx_RegL()
3559 %{
3560   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3561   match(RegL);
3562   match(rRegL);
3563 
3564   format %{ %}
3565   interface(REG_INTER);
3566 %}
3567 
3568 operand no_rax_RegL()
3569 %{
3570   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3571   match(RegL);
3572   match(rRegL);
3573   match(rdx_RegL);
3574 
3575   format %{ %}
3576   interface(REG_INTER);
3577 %}
3578 
3579 operand no_rcx_RegL()
3580 %{
3581   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3582   match(RegL);
3583   match(rRegL);
3584 
3585   format %{ %}
3586   interface(REG_INTER);
3587 %}
3588 
3589 operand rax_RegL()
3590 %{
3591   constraint(ALLOC_IN_RC(long_rax_reg));
3592   match(RegL);
3593   match(rRegL);
3594 
3595   format %{ "RAX" %}
3596   interface(REG_INTER);
3597 %}
3598 
3599 operand rcx_RegL()
3600 %{
3601   constraint(ALLOC_IN_RC(long_rcx_reg));
3602   match(RegL);
3603   match(rRegL);
3604 
3605   format %{ %}
3606   interface(REG_INTER);
3607 %}
3608 
3609 operand rdx_RegL()
3610 %{
3611   constraint(ALLOC_IN_RC(long_rdx_reg));
3612   match(RegL);
3613   match(rRegL);
3614 
3615   format %{ %}
3616   interface(REG_INTER);
3617 %}
3618 
3619 // Flags register, used as output of compare instructions
3620 operand rFlagsReg()
3621 %{
3622   constraint(ALLOC_IN_RC(int_flags));
3623   match(RegFlags);
3624 
3625   format %{ "RFLAGS" %}
3626   interface(REG_INTER);
3627 %}
3628 
3629 // Flags register, used as output of FLOATING POINT compare instructions
3630 operand rFlagsRegU()
3631 %{
3632   constraint(ALLOC_IN_RC(int_flags));
3633   match(RegFlags);
3634 
3635   format %{ "RFLAGS_U" %}
3636   interface(REG_INTER);
3637 %}
3638 
3639 operand rFlagsRegUCF() %{
3640   constraint(ALLOC_IN_RC(int_flags));
3641   match(RegFlags);
3642   predicate(false);
3643 
3644   format %{ "RFLAGS_U_CF" %}
3645   interface(REG_INTER);
3646 %}
3647 
3648 // Float register operands
3649 operand regF() %{
3650    constraint(ALLOC_IN_RC(float_reg));
3651    match(RegF);
3652 
3653    format %{ %}
3654    interface(REG_INTER);
3655 %}
3656 
3657 // Double register operands
3658 operand regD() %{
3659    constraint(ALLOC_IN_RC(double_reg));
3660    match(RegD);
3661 
3662    format %{ %}
3663    interface(REG_INTER);
3664 %}
3665 
3666 // Vectors
3667 operand vecS() %{
3668   constraint(ALLOC_IN_RC(vectors_reg));
3669   match(VecS);
3670 
3671   format %{ %}
3672   interface(REG_INTER);
3673 %}
3674 
3675 operand vecD() %{
3676   constraint(ALLOC_IN_RC(vectord_reg));
3677   match(VecD);
3678 
3679   format %{ %}
3680   interface(REG_INTER);
3681 %}
3682 
3683 operand vecX() %{
3684   constraint(ALLOC_IN_RC(vectorx_reg));
3685   match(VecX);
3686 
3687   format %{ %}
3688   interface(REG_INTER);
3689 %}
3690 
3691 operand vecY() %{
3692   constraint(ALLOC_IN_RC(vectory_reg));
3693   match(VecY);
3694 
3695   format %{ %}
3696   interface(REG_INTER);
3697 %}
3698 
3699 //----------Memory Operands----------------------------------------------------
3700 // Direct Memory Operand
3701 // operand direct(immP addr)
3702 // %{
3703 //   match(addr);
3704 
3705 //   format %{ "[$addr]" %}
3706 //   interface(MEMORY_INTER) %{
3707 //     base(0xFFFFFFFF);
3708 //     index(0x4);
3709 //     scale(0x0);
3710 //     disp($addr);
3711 //   %}
3712 // %}
3713 
3714 // Indirect Memory Operand
3715 operand indirect(any_RegP reg)
3716 %{
3717   constraint(ALLOC_IN_RC(ptr_reg));
3718   match(reg);
3719 
3720   format %{ "[$reg]" %}
3721   interface(MEMORY_INTER) %{
3722     base($reg);
3723     index(0x4);
3724     scale(0x0);
3725     disp(0x0);
3726   %}
3727 %}
3728 
3729 // Indirect Memory Plus Short Offset Operand
3730 operand indOffset8(any_RegP reg, immL8 off)
3731 %{
3732   constraint(ALLOC_IN_RC(ptr_reg));
3733   match(AddP reg off);
3734 
3735   format %{ "[$reg + $off (8-bit)]" %}
3736   interface(MEMORY_INTER) %{
3737     base($reg);
3738     index(0x4);
3739     scale(0x0);
3740     disp($off);
3741   %}
3742 %}
3743 
3744 // Indirect Memory Plus Long Offset Operand
3745 operand indOffset32(any_RegP reg, immL32 off)
3746 %{
3747   constraint(ALLOC_IN_RC(ptr_reg));
3748   match(AddP reg off);
3749 
3750   format %{ "[$reg + $off (32-bit)]" %}
3751   interface(MEMORY_INTER) %{
3752     base($reg);
3753     index(0x4);
3754     scale(0x0);
3755     disp($off);
3756   %}
3757 %}
3758 
3759 // Indirect Memory Plus Index Register Plus Offset Operand
3760 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3761 %{
3762   constraint(ALLOC_IN_RC(ptr_reg));
3763   match(AddP (AddP reg lreg) off);
3764 
3765   op_cost(10);
3766   format %{"[$reg + $off + $lreg]" %}
3767   interface(MEMORY_INTER) %{
3768     base($reg);
3769     index($lreg);
3770     scale(0x0);
3771     disp($off);
3772   %}
3773 %}
3774 
3775 // Indirect Memory Plus Index Register Plus Offset Operand
3776 operand indIndex(any_RegP reg, rRegL lreg)
3777 %{
3778   constraint(ALLOC_IN_RC(ptr_reg));
3779   match(AddP reg lreg);
3780 
3781   op_cost(10);
3782   format %{"[$reg + $lreg]" %}
3783   interface(MEMORY_INTER) %{
3784     base($reg);
3785     index($lreg);
3786     scale(0x0);
3787     disp(0x0);
3788   %}
3789 %}
3790 
3791 // Indirect Memory Times Scale Plus Index Register
3792 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3793 %{
3794   constraint(ALLOC_IN_RC(ptr_reg));
3795   match(AddP reg (LShiftL lreg scale));
3796 
3797   op_cost(10);
3798   format %{"[$reg + $lreg << $scale]" %}
3799   interface(MEMORY_INTER) %{
3800     base($reg);
3801     index($lreg);
3802     scale($scale);
3803     disp(0x0);
3804   %}
3805 %}
3806 
3807 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3808 %{
3809   constraint(ALLOC_IN_RC(ptr_reg));
3810   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3811   match(AddP reg (LShiftL (ConvI2L idx) scale));
3812 
3813   op_cost(10);
3814   format %{"[$reg + pos $idx << $scale]" %}
3815   interface(MEMORY_INTER) %{
3816     base($reg);
3817     index($idx);
3818     scale($scale);
3819     disp(0x0);
3820   %}
3821 %}
3822 
3823 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3824 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3825 %{
3826   constraint(ALLOC_IN_RC(ptr_reg));
3827   match(AddP (AddP reg (LShiftL lreg scale)) off);
3828 
3829   op_cost(10);
3830   format %{"[$reg + $off + $lreg << $scale]" %}
3831   interface(MEMORY_INTER) %{
3832     base($reg);
3833     index($lreg);
3834     scale($scale);
3835     disp($off);
3836   %}
3837 %}
3838 
3839 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3840 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3841 %{
3842   constraint(ALLOC_IN_RC(ptr_reg));
3843   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3844   match(AddP (AddP reg (ConvI2L idx)) off);
3845 
3846   op_cost(10);
3847   format %{"[$reg + $off + $idx]" %}
3848   interface(MEMORY_INTER) %{
3849     base($reg);
3850     index($idx);
3851     scale(0x0);
3852     disp($off);
3853   %}
3854 %}
3855 
3856 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3857 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3858 %{
3859   constraint(ALLOC_IN_RC(ptr_reg));
3860   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3861   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3862 
3863   op_cost(10);
3864   format %{"[$reg + $off + $idx << $scale]" %}
3865   interface(MEMORY_INTER) %{
3866     base($reg);
3867     index($idx);
3868     scale($scale);
3869     disp($off);
3870   %}
3871 %}
3872 
3873 // Indirect Narrow Oop Plus Offset Operand
3874 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3875 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3876 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3877   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3878   constraint(ALLOC_IN_RC(ptr_reg));
3879   match(AddP (DecodeN reg) off);
3880 
3881   op_cost(10);
3882   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3883   interface(MEMORY_INTER) %{
3884     base(0xc); // R12
3885     index($reg);
3886     scale(0x3);
3887     disp($off);
3888   %}
3889 %}
3890 
3891 // Indirect Memory Operand
3892 operand indirectNarrow(rRegN reg)
3893 %{
3894   predicate(Universe::narrow_oop_shift() == 0);
3895   constraint(ALLOC_IN_RC(ptr_reg));
3896   match(DecodeN reg);
3897 
3898   format %{ "[$reg]" %}
3899   interface(MEMORY_INTER) %{
3900     base($reg);
3901     index(0x4);
3902     scale(0x0);
3903     disp(0x0);
3904   %}
3905 %}
3906 
3907 // Indirect Memory Plus Short Offset Operand
3908 operand indOffset8Narrow(rRegN reg, immL8 off)
3909 %{
3910   predicate(Universe::narrow_oop_shift() == 0);
3911   constraint(ALLOC_IN_RC(ptr_reg));
3912   match(AddP (DecodeN reg) off);
3913 
3914   format %{ "[$reg + $off (8-bit)]" %}
3915   interface(MEMORY_INTER) %{
3916     base($reg);
3917     index(0x4);
3918     scale(0x0);
3919     disp($off);
3920   %}
3921 %}
3922 
3923 // Indirect Memory Plus Long Offset Operand
3924 operand indOffset32Narrow(rRegN reg, immL32 off)
3925 %{
3926   predicate(Universe::narrow_oop_shift() == 0);
3927   constraint(ALLOC_IN_RC(ptr_reg));
3928   match(AddP (DecodeN reg) off);
3929 
3930   format %{ "[$reg + $off (32-bit)]" %}
3931   interface(MEMORY_INTER) %{
3932     base($reg);
3933     index(0x4);
3934     scale(0x0);
3935     disp($off);
3936   %}
3937 %}
3938 
3939 // Indirect Memory Plus Index Register Plus Offset Operand
3940 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3941 %{
3942   predicate(Universe::narrow_oop_shift() == 0);
3943   constraint(ALLOC_IN_RC(ptr_reg));
3944   match(AddP (AddP (DecodeN reg) lreg) off);
3945 
3946   op_cost(10);
3947   format %{"[$reg + $off + $lreg]" %}
3948   interface(MEMORY_INTER) %{
3949     base($reg);
3950     index($lreg);
3951     scale(0x0);
3952     disp($off);
3953   %}
3954 %}
3955 
3956 // Indirect Memory Plus Index Register Plus Offset Operand
3957 operand indIndexNarrow(rRegN reg, rRegL lreg)
3958 %{
3959   predicate(Universe::narrow_oop_shift() == 0);
3960   constraint(ALLOC_IN_RC(ptr_reg));
3961   match(AddP (DecodeN reg) lreg);
3962 
3963   op_cost(10);
3964   format %{"[$reg + $lreg]" %}
3965   interface(MEMORY_INTER) %{
3966     base($reg);
3967     index($lreg);
3968     scale(0x0);
3969     disp(0x0);
3970   %}
3971 %}
3972 
3973 // Indirect Memory Times Scale Plus Index Register
3974 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3975 %{
3976   predicate(Universe::narrow_oop_shift() == 0);
3977   constraint(ALLOC_IN_RC(ptr_reg));
3978   match(AddP (DecodeN reg) (LShiftL lreg scale));
3979 
3980   op_cost(10);
3981   format %{"[$reg + $lreg << $scale]" %}
3982   interface(MEMORY_INTER) %{
3983     base($reg);
3984     index($lreg);
3985     scale($scale);
3986     disp(0x0);
3987   %}
3988 %}
3989 
3990 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3991 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3992 %{
3993   predicate(Universe::narrow_oop_shift() == 0);
3994   constraint(ALLOC_IN_RC(ptr_reg));
3995   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3996 
3997   op_cost(10);
3998   format %{"[$reg + $off + $lreg << $scale]" %}
3999   interface(MEMORY_INTER) %{
4000     base($reg);
4001     index($lreg);
4002     scale($scale);
4003     disp($off);
4004   %}
4005 %}
4006 
4007 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
4008 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
4009 %{
4010   constraint(ALLOC_IN_RC(ptr_reg));
4011   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
4012   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
4013 
4014   op_cost(10);
4015   format %{"[$reg + $off + $idx]" %}
4016   interface(MEMORY_INTER) %{
4017     base($reg);
4018     index($idx);
4019     scale(0x0);
4020     disp($off);
4021   %}
4022 %}
4023 
4024 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4025 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
4026 %{
4027   constraint(ALLOC_IN_RC(ptr_reg));
4028   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4029   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
4030 
4031   op_cost(10);
4032   format %{"[$reg + $off + $idx << $scale]" %}
4033   interface(MEMORY_INTER) %{
4034     base($reg);
4035     index($idx);
4036     scale($scale);
4037     disp($off);
4038   %}
4039 %}
4040 
4041 //----------Special Memory Operands--------------------------------------------
4042 // Stack Slot Operand - This operand is used for loading and storing temporary
4043 //                      values on the stack where a match requires a value to
4044 //                      flow through memory.
4045 operand stackSlotP(sRegP reg)
4046 %{
4047   constraint(ALLOC_IN_RC(stack_slots));
4048   // No match rule because this operand is only generated in matching
4049 
4050   format %{ "[$reg]" %}
4051   interface(MEMORY_INTER) %{
4052     base(0x4);   // RSP
4053     index(0x4);  // No Index
4054     scale(0x0);  // No Scale
4055     disp($reg);  // Stack Offset
4056   %}
4057 %}
4058 
4059 operand stackSlotI(sRegI reg)
4060 %{
4061   constraint(ALLOC_IN_RC(stack_slots));
4062   // No match rule because this operand is only generated in matching
4063 
4064   format %{ "[$reg]" %}
4065   interface(MEMORY_INTER) %{
4066     base(0x4);   // RSP
4067     index(0x4);  // No Index
4068     scale(0x0);  // No Scale
4069     disp($reg);  // Stack Offset
4070   %}
4071 %}
4072 
4073 operand stackSlotF(sRegF reg)
4074 %{
4075   constraint(ALLOC_IN_RC(stack_slots));
4076   // No match rule because this operand is only generated in matching
4077 
4078   format %{ "[$reg]" %}
4079   interface(MEMORY_INTER) %{
4080     base(0x4);   // RSP
4081     index(0x4);  // No Index
4082     scale(0x0);  // No Scale
4083     disp($reg);  // Stack Offset
4084   %}
4085 %}
4086 
4087 operand stackSlotD(sRegD reg)
4088 %{
4089   constraint(ALLOC_IN_RC(stack_slots));
4090   // No match rule because this operand is only generated in matching
4091 
4092   format %{ "[$reg]" %}
4093   interface(MEMORY_INTER) %{
4094     base(0x4);   // RSP
4095     index(0x4);  // No Index
4096     scale(0x0);  // No Scale
4097     disp($reg);  // Stack Offset
4098   %}
4099 %}
4100 operand stackSlotL(sRegL reg)
4101 %{
4102   constraint(ALLOC_IN_RC(stack_slots));
4103   // No match rule because this operand is only generated in matching
4104 
4105   format %{ "[$reg]" %}
4106   interface(MEMORY_INTER) %{
4107     base(0x4);   // RSP
4108     index(0x4);  // No Index
4109     scale(0x0);  // No Scale
4110     disp($reg);  // Stack Offset
4111   %}
4112 %}
4113 
4114 //----------Conditional Branch Operands----------------------------------------
4115 // Comparison Op  - This is the operation of the comparison, and is limited to
4116 //                  the following set of codes:
4117 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4118 //
4119 // Other attributes of the comparison, such as unsignedness, are specified
4120 // by the comparison instruction that sets a condition code flags register.
4121 // That result is represented by a flags operand whose subtype is appropriate
4122 // to the unsignedness (etc.) of the comparison.
4123 //
4124 // Later, the instruction which matches both the Comparison Op (a Bool) and
4125 // the flags (produced by the Cmp) specifies the coding of the comparison op
4126 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4127 
4128 // Comparision Code
4129 operand cmpOp()
4130 %{
4131   match(Bool);
4132 
4133   format %{ "" %}
4134   interface(COND_INTER) %{
4135     equal(0x4, "e");
4136     not_equal(0x5, "ne");
4137     less(0xC, "l");
4138     greater_equal(0xD, "ge");
4139     less_equal(0xE, "le");
4140     greater(0xF, "g");
4141     overflow(0x0, "o");
4142     no_overflow(0x1, "no");
4143   %}
4144 %}
4145 
4146 // Comparison Code, unsigned compare.  Used by FP also, with
4147 // C2 (unordered) turned into GT or LT already.  The other bits
4148 // C0 and C3 are turned into Carry & Zero flags.
4149 operand cmpOpU()
4150 %{
4151   match(Bool);
4152 
4153   format %{ "" %}
4154   interface(COND_INTER) %{
4155     equal(0x4, "e");
4156     not_equal(0x5, "ne");
4157     less(0x2, "b");
4158     greater_equal(0x3, "nb");
4159     less_equal(0x6, "be");
4160     greater(0x7, "nbe");
4161     overflow(0x0, "o");
4162     no_overflow(0x1, "no");
4163   %}
4164 %}
4165 
4166 
4167 // Floating comparisons that don't require any fixup for the unordered case
4168 operand cmpOpUCF() %{
4169   match(Bool);
4170   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4171             n->as_Bool()->_test._test == BoolTest::ge ||
4172             n->as_Bool()->_test._test == BoolTest::le ||
4173             n->as_Bool()->_test._test == BoolTest::gt);
4174   format %{ "" %}
4175   interface(COND_INTER) %{
4176     equal(0x4, "e");
4177     not_equal(0x5, "ne");
4178     less(0x2, "b");
4179     greater_equal(0x3, "nb");
4180     less_equal(0x6, "be");
4181     greater(0x7, "nbe");
4182     overflow(0x0, "o");
4183     no_overflow(0x1, "no");
4184   %}
4185 %}
4186 
4187 
4188 // Floating comparisons that can be fixed up with extra conditional jumps
4189 operand cmpOpUCF2() %{
4190   match(Bool);
4191   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4192             n->as_Bool()->_test._test == BoolTest::eq);
4193   format %{ "" %}
4194   interface(COND_INTER) %{
4195     equal(0x4, "e");
4196     not_equal(0x5, "ne");
4197     less(0x2, "b");
4198     greater_equal(0x3, "nb");
4199     less_equal(0x6, "be");
4200     greater(0x7, "nbe");
4201     overflow(0x0, "o");
4202     no_overflow(0x1, "no");
4203   %}
4204 %}
4205 
4206 // Operands for bound floating pointer register arguments
4207 operand rxmm0() %{
4208   constraint(ALLOC_IN_RC(xmm0_reg));  match(VecX);
4209   predicate((UseSSE > 0) && (UseAVX<= 2));  format%{%}  interface(REG_INTER);
4210 %}
4211 operand rxmm1() %{
4212   constraint(ALLOC_IN_RC(xmm1_reg));  match(VecX);
4213   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4214 %}
4215 operand rxmm2() %{
4216   constraint(ALLOC_IN_RC(xmm2_reg));  match(VecX);
4217   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4218 %}
4219 operand rxmm3() %{
4220   constraint(ALLOC_IN_RC(xmm3_reg));  match(VecX);
4221   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4222 %}
4223 operand rxmm4() %{
4224   constraint(ALLOC_IN_RC(xmm4_reg));  match(VecX);
4225   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4226 %}
4227 operand rxmm5() %{
4228   constraint(ALLOC_IN_RC(xmm5_reg));  match(VecX);
4229   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4230 %}
4231 operand rxmm6() %{
4232   constraint(ALLOC_IN_RC(xmm6_reg));  match(VecX);
4233   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4234 %}
4235 operand rxmm7() %{
4236   constraint(ALLOC_IN_RC(xmm7_reg));  match(VecX);
4237   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4238 %}
4239 operand rxmm8() %{
4240   constraint(ALLOC_IN_RC(xmm8_reg));  match(VecX);
4241   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4242 %}
4243 operand rxmm9() %{
4244   constraint(ALLOC_IN_RC(xmm9_reg));  match(VecX);
4245   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4246 %}
4247 operand rxmm10() %{
4248   constraint(ALLOC_IN_RC(xmm10_reg));  match(VecX);
4249   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4250 %}
4251 operand rxmm11() %{
4252   constraint(ALLOC_IN_RC(xmm11_reg));  match(VecX);
4253   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4254 %}
4255 operand rxmm12() %{
4256   constraint(ALLOC_IN_RC(xmm12_reg));  match(VecX);
4257   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4258 %}
4259 operand rxmm13() %{
4260   constraint(ALLOC_IN_RC(xmm13_reg));  match(VecX);
4261   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4262 %}
4263 operand rxmm14() %{
4264   constraint(ALLOC_IN_RC(xmm14_reg));  match(VecX);
4265   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4266 %}
4267 operand rxmm15() %{
4268   constraint(ALLOC_IN_RC(xmm15_reg));  match(VecX);
4269   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4270 %}
4271 operand rxmm16() %{
4272   constraint(ALLOC_IN_RC(xmm16_reg));  match(VecX);
4273   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4274 %}
4275 operand rxmm17() %{
4276   constraint(ALLOC_IN_RC(xmm17_reg));  match(VecX);
4277   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4278 %}
4279 operand rxmm18() %{
4280   constraint(ALLOC_IN_RC(xmm18_reg));  match(VecX);
4281   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4282 %}
4283 operand rxmm19() %{
4284   constraint(ALLOC_IN_RC(xmm19_reg));  match(VecX);
4285   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4286 %}
4287 operand rxmm20() %{
4288   constraint(ALLOC_IN_RC(xmm20_reg));  match(VecX);
4289   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4290 %}
4291 operand rxmm21() %{
4292   constraint(ALLOC_IN_RC(xmm21_reg));  match(VecX);
4293   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4294 %}
4295 operand rxmm22() %{
4296   constraint(ALLOC_IN_RC(xmm22_reg));  match(VecX);
4297   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4298 %}
4299 operand rxmm23() %{
4300   constraint(ALLOC_IN_RC(xmm23_reg));  match(VecX);
4301   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4302 %}
4303 operand rxmm24() %{
4304   constraint(ALLOC_IN_RC(xmm24_reg));  match(VecX);
4305   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4306 %}
4307 operand rxmm25() %{
4308   constraint(ALLOC_IN_RC(xmm25_reg));  match(VecX);
4309   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4310 %}
4311 operand rxmm26() %{
4312   constraint(ALLOC_IN_RC(xmm26_reg));  match(VecX);
4313   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4314 %}
4315 operand rxmm27() %{
4316   constraint(ALLOC_IN_RC(xmm27_reg));  match(VecX);
4317   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4318 %}
4319 operand rxmm28() %{
4320   constraint(ALLOC_IN_RC(xmm28_reg));  match(VecX);
4321   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4322 %}
4323 operand rxmm29() %{
4324   constraint(ALLOC_IN_RC(xmm29_reg));  match(VecX);
4325   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4326 %}
4327 operand rxmm30() %{
4328   constraint(ALLOC_IN_RC(xmm30_reg));  match(VecX);
4329   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4330 %}
4331 operand rxmm31() %{
4332   constraint(ALLOC_IN_RC(xmm31_reg));  match(VecX);
4333   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4334 %}
4335 
4336 //----------OPERAND CLASSES----------------------------------------------------
4337 // Operand Classes are groups of operands that are used as to simplify
4338 // instruction definitions by not requiring the AD writer to specify separate
4339 // instructions for every form of operand when the instruction accepts
4340 // multiple operand types with the same basic encoding and format.  The classic
4341 // case of this is memory operands.
4342 
4343 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4344                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4345                indCompressedOopOffset,
4346                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4347                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4348                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4349 
4350 //----------PIPELINE-----------------------------------------------------------
4351 // Rules which define the behavior of the target architectures pipeline.
4352 pipeline %{
4353 
4354 //----------ATTRIBUTES---------------------------------------------------------
4355 attributes %{
4356   variable_size_instructions;        // Fixed size instructions
4357   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4358   instruction_unit_size = 1;         // An instruction is 1 bytes long
4359   instruction_fetch_unit_size = 16;  // The processor fetches one line
4360   instruction_fetch_units = 1;       // of 16 bytes
4361 
4362   // List of nop instructions
4363   nops( MachNop );
4364 %}
4365 
4366 //----------RESOURCES----------------------------------------------------------
4367 // Resources are the functional units available to the machine
4368 
4369 // Generic P2/P3 pipeline
4370 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4371 // 3 instructions decoded per cycle.
4372 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4373 // 3 ALU op, only ALU0 handles mul instructions.
4374 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4375            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4376            BR, FPU,
4377            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4378 
4379 //----------PIPELINE DESCRIPTION-----------------------------------------------
4380 // Pipeline Description specifies the stages in the machine's pipeline
4381 
4382 // Generic P2/P3 pipeline
4383 pipe_desc(S0, S1, S2, S3, S4, S5);
4384 
4385 //----------PIPELINE CLASSES---------------------------------------------------
4386 // Pipeline Classes describe the stages in which input and output are
4387 // referenced by the hardware pipeline.
4388 
4389 // Naming convention: ialu or fpu
4390 // Then: _reg
4391 // Then: _reg if there is a 2nd register
4392 // Then: _long if it's a pair of instructions implementing a long
4393 // Then: _fat if it requires the big decoder
4394 //   Or: _mem if it requires the big decoder and a memory unit.
4395 
4396 // Integer ALU reg operation
4397 pipe_class ialu_reg(rRegI dst)
4398 %{
4399     single_instruction;
4400     dst    : S4(write);
4401     dst    : S3(read);
4402     DECODE : S0;        // any decoder
4403     ALU    : S3;        // any alu
4404 %}
4405 
4406 // Long ALU reg operation
4407 pipe_class ialu_reg_long(rRegL dst)
4408 %{
4409     instruction_count(2);
4410     dst    : S4(write);
4411     dst    : S3(read);
4412     DECODE : S0(2);     // any 2 decoders
4413     ALU    : S3(2);     // both alus
4414 %}
4415 
4416 // Integer ALU reg operation using big decoder
4417 pipe_class ialu_reg_fat(rRegI dst)
4418 %{
4419     single_instruction;
4420     dst    : S4(write);
4421     dst    : S3(read);
4422     D0     : S0;        // big decoder only
4423     ALU    : S3;        // any alu
4424 %}
4425 
4426 // Long ALU reg operation using big decoder
4427 pipe_class ialu_reg_long_fat(rRegL dst)
4428 %{
4429     instruction_count(2);
4430     dst    : S4(write);
4431     dst    : S3(read);
4432     D0     : S0(2);     // big decoder only; twice
4433     ALU    : S3(2);     // any 2 alus
4434 %}
4435 
4436 // Integer ALU reg-reg operation
4437 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4438 %{
4439     single_instruction;
4440     dst    : S4(write);
4441     src    : S3(read);
4442     DECODE : S0;        // any decoder
4443     ALU    : S3;        // any alu
4444 %}
4445 
4446 // Long ALU reg-reg operation
4447 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4448 %{
4449     instruction_count(2);
4450     dst    : S4(write);
4451     src    : S3(read);
4452     DECODE : S0(2);     // any 2 decoders
4453     ALU    : S3(2);     // both alus
4454 %}
4455 
4456 // Integer ALU reg-reg operation
4457 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4458 %{
4459     single_instruction;
4460     dst    : S4(write);
4461     src    : S3(read);
4462     D0     : S0;        // big decoder only
4463     ALU    : S3;        // any alu
4464 %}
4465 
4466 // Long ALU reg-reg operation
4467 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4468 %{
4469     instruction_count(2);
4470     dst    : S4(write);
4471     src    : S3(read);
4472     D0     : S0(2);     // big decoder only; twice
4473     ALU    : S3(2);     // both alus
4474 %}
4475 
4476 // Integer ALU reg-mem operation
4477 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4478 %{
4479     single_instruction;
4480     dst    : S5(write);
4481     mem    : S3(read);
4482     D0     : S0;        // big decoder only
4483     ALU    : S4;        // any alu
4484     MEM    : S3;        // any mem
4485 %}
4486 
4487 // Integer mem operation (prefetch)
4488 pipe_class ialu_mem(memory mem)
4489 %{
4490     single_instruction;
4491     mem    : S3(read);
4492     D0     : S0;        // big decoder only
4493     MEM    : S3;        // any mem
4494 %}
4495 
4496 // Integer Store to Memory
4497 pipe_class ialu_mem_reg(memory mem, rRegI src)
4498 %{
4499     single_instruction;
4500     mem    : S3(read);
4501     src    : S5(read);
4502     D0     : S0;        // big decoder only
4503     ALU    : S4;        // any alu
4504     MEM    : S3;
4505 %}
4506 
4507 // // Long Store to Memory
4508 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4509 // %{
4510 //     instruction_count(2);
4511 //     mem    : S3(read);
4512 //     src    : S5(read);
4513 //     D0     : S0(2);          // big decoder only; twice
4514 //     ALU    : S4(2);     // any 2 alus
4515 //     MEM    : S3(2);  // Both mems
4516 // %}
4517 
4518 // Integer Store to Memory
4519 pipe_class ialu_mem_imm(memory mem)
4520 %{
4521     single_instruction;
4522     mem    : S3(read);
4523     D0     : S0;        // big decoder only
4524     ALU    : S4;        // any alu
4525     MEM    : S3;
4526 %}
4527 
4528 // Integer ALU0 reg-reg operation
4529 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4530 %{
4531     single_instruction;
4532     dst    : S4(write);
4533     src    : S3(read);
4534     D0     : S0;        // Big decoder only
4535     ALU0   : S3;        // only alu0
4536 %}
4537 
4538 // Integer ALU0 reg-mem operation
4539 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4540 %{
4541     single_instruction;
4542     dst    : S5(write);
4543     mem    : S3(read);
4544     D0     : S0;        // big decoder only
4545     ALU0   : S4;        // ALU0 only
4546     MEM    : S3;        // any mem
4547 %}
4548 
4549 // Integer ALU reg-reg operation
4550 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4551 %{
4552     single_instruction;
4553     cr     : S4(write);
4554     src1   : S3(read);
4555     src2   : S3(read);
4556     DECODE : S0;        // any decoder
4557     ALU    : S3;        // any alu
4558 %}
4559 
4560 // Integer ALU reg-imm operation
4561 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4562 %{
4563     single_instruction;
4564     cr     : S4(write);
4565     src1   : S3(read);
4566     DECODE : S0;        // any decoder
4567     ALU    : S3;        // any alu
4568 %}
4569 
4570 // Integer ALU reg-mem operation
4571 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4572 %{
4573     single_instruction;
4574     cr     : S4(write);
4575     src1   : S3(read);
4576     src2   : S3(read);
4577     D0     : S0;        // big decoder only
4578     ALU    : S4;        // any alu
4579     MEM    : S3;
4580 %}
4581 
4582 // Conditional move reg-reg
4583 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4584 %{
4585     instruction_count(4);
4586     y      : S4(read);
4587     q      : S3(read);
4588     p      : S3(read);
4589     DECODE : S0(4);     // any decoder
4590 %}
4591 
4592 // Conditional move reg-reg
4593 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4594 %{
4595     single_instruction;
4596     dst    : S4(write);
4597     src    : S3(read);
4598     cr     : S3(read);
4599     DECODE : S0;        // any decoder
4600 %}
4601 
4602 // Conditional move reg-mem
4603 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4604 %{
4605     single_instruction;
4606     dst    : S4(write);
4607     src    : S3(read);
4608     cr     : S3(read);
4609     DECODE : S0;        // any decoder
4610     MEM    : S3;
4611 %}
4612 
4613 // Conditional move reg-reg long
4614 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4615 %{
4616     single_instruction;
4617     dst    : S4(write);
4618     src    : S3(read);
4619     cr     : S3(read);
4620     DECODE : S0(2);     // any 2 decoders
4621 %}
4622 
4623 // XXX
4624 // // Conditional move double reg-reg
4625 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4626 // %{
4627 //     single_instruction;
4628 //     dst    : S4(write);
4629 //     src    : S3(read);
4630 //     cr     : S3(read);
4631 //     DECODE : S0;     // any decoder
4632 // %}
4633 
4634 // Float reg-reg operation
4635 pipe_class fpu_reg(regD dst)
4636 %{
4637     instruction_count(2);
4638     dst    : S3(read);
4639     DECODE : S0(2);     // any 2 decoders
4640     FPU    : S3;
4641 %}
4642 
4643 // Float reg-reg operation
4644 pipe_class fpu_reg_reg(regD dst, regD src)
4645 %{
4646     instruction_count(2);
4647     dst    : S4(write);
4648     src    : S3(read);
4649     DECODE : S0(2);     // any 2 decoders
4650     FPU    : S3;
4651 %}
4652 
4653 // Float reg-reg operation
4654 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4655 %{
4656     instruction_count(3);
4657     dst    : S4(write);
4658     src1   : S3(read);
4659     src2   : S3(read);
4660     DECODE : S0(3);     // any 3 decoders
4661     FPU    : S3(2);
4662 %}
4663 
4664 // Float reg-reg operation
4665 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4666 %{
4667     instruction_count(4);
4668     dst    : S4(write);
4669     src1   : S3(read);
4670     src2   : S3(read);
4671     src3   : S3(read);
4672     DECODE : S0(4);     // any 3 decoders
4673     FPU    : S3(2);
4674 %}
4675 
4676 // Float reg-reg operation
4677 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4678 %{
4679     instruction_count(4);
4680     dst    : S4(write);
4681     src1   : S3(read);
4682     src2   : S3(read);
4683     src3   : S3(read);
4684     DECODE : S1(3);     // any 3 decoders
4685     D0     : S0;        // Big decoder only
4686     FPU    : S3(2);
4687     MEM    : S3;
4688 %}
4689 
4690 // Float reg-mem operation
4691 pipe_class fpu_reg_mem(regD dst, memory mem)
4692 %{
4693     instruction_count(2);
4694     dst    : S5(write);
4695     mem    : S3(read);
4696     D0     : S0;        // big decoder only
4697     DECODE : S1;        // any decoder for FPU POP
4698     FPU    : S4;
4699     MEM    : S3;        // any mem
4700 %}
4701 
4702 // Float reg-mem operation
4703 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4704 %{
4705     instruction_count(3);
4706     dst    : S5(write);
4707     src1   : S3(read);
4708     mem    : S3(read);
4709     D0     : S0;        // big decoder only
4710     DECODE : S1(2);     // any decoder for FPU POP
4711     FPU    : S4;
4712     MEM    : S3;        // any mem
4713 %}
4714 
4715 // Float mem-reg operation
4716 pipe_class fpu_mem_reg(memory mem, regD src)
4717 %{
4718     instruction_count(2);
4719     src    : S5(read);
4720     mem    : S3(read);
4721     DECODE : S0;        // any decoder for FPU PUSH
4722     D0     : S1;        // big decoder only
4723     FPU    : S4;
4724     MEM    : S3;        // any mem
4725 %}
4726 
4727 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4728 %{
4729     instruction_count(3);
4730     src1   : S3(read);
4731     src2   : S3(read);
4732     mem    : S3(read);
4733     DECODE : S0(2);     // any decoder for FPU PUSH
4734     D0     : S1;        // big decoder only
4735     FPU    : S4;
4736     MEM    : S3;        // any mem
4737 %}
4738 
4739 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4740 %{
4741     instruction_count(3);
4742     src1   : S3(read);
4743     src2   : S3(read);
4744     mem    : S4(read);
4745     DECODE : S0;        // any decoder for FPU PUSH
4746     D0     : S0(2);     // big decoder only
4747     FPU    : S4;
4748     MEM    : S3(2);     // any mem
4749 %}
4750 
4751 pipe_class fpu_mem_mem(memory dst, memory src1)
4752 %{
4753     instruction_count(2);
4754     src1   : S3(read);
4755     dst    : S4(read);
4756     D0     : S0(2);     // big decoder only
4757     MEM    : S3(2);     // any mem
4758 %}
4759 
4760 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4761 %{
4762     instruction_count(3);
4763     src1   : S3(read);
4764     src2   : S3(read);
4765     dst    : S4(read);
4766     D0     : S0(3);     // big decoder only
4767     FPU    : S4;
4768     MEM    : S3(3);     // any mem
4769 %}
4770 
4771 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4772 %{
4773     instruction_count(3);
4774     src1   : S4(read);
4775     mem    : S4(read);
4776     DECODE : S0;        // any decoder for FPU PUSH
4777     D0     : S0(2);     // big decoder only
4778     FPU    : S4;
4779     MEM    : S3(2);     // any mem
4780 %}
4781 
4782 // Float load constant
4783 pipe_class fpu_reg_con(regD dst)
4784 %{
4785     instruction_count(2);
4786     dst    : S5(write);
4787     D0     : S0;        // big decoder only for the load
4788     DECODE : S1;        // any decoder for FPU POP
4789     FPU    : S4;
4790     MEM    : S3;        // any mem
4791 %}
4792 
4793 // Float load constant
4794 pipe_class fpu_reg_reg_con(regD dst, regD src)
4795 %{
4796     instruction_count(3);
4797     dst    : S5(write);
4798     src    : S3(read);
4799     D0     : S0;        // big decoder only for the load
4800     DECODE : S1(2);     // any decoder for FPU POP
4801     FPU    : S4;
4802     MEM    : S3;        // any mem
4803 %}
4804 
4805 // UnConditional branch
4806 pipe_class pipe_jmp(label labl)
4807 %{
4808     single_instruction;
4809     BR   : S3;
4810 %}
4811 
4812 // Conditional branch
4813 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4814 %{
4815     single_instruction;
4816     cr    : S1(read);
4817     BR    : S3;
4818 %}
4819 
4820 // Allocation idiom
4821 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4822 %{
4823     instruction_count(1); force_serialization;
4824     fixed_latency(6);
4825     heap_ptr : S3(read);
4826     DECODE   : S0(3);
4827     D0       : S2;
4828     MEM      : S3;
4829     ALU      : S3(2);
4830     dst      : S5(write);
4831     BR       : S5;
4832 %}
4833 
4834 // Generic big/slow expanded idiom
4835 pipe_class pipe_slow()
4836 %{
4837     instruction_count(10); multiple_bundles; force_serialization;
4838     fixed_latency(100);
4839     D0  : S0(2);
4840     MEM : S3(2);
4841 %}
4842 
4843 // The real do-nothing guy
4844 pipe_class empty()
4845 %{
4846     instruction_count(0);
4847 %}
4848 
4849 // Define the class for the Nop node
4850 define
4851 %{
4852    MachNop = empty;
4853 %}
4854 
4855 %}
4856 
4857 //----------INSTRUCTIONS-------------------------------------------------------
4858 //
4859 // match      -- States which machine-independent subtree may be replaced
4860 //               by this instruction.
4861 // ins_cost   -- The estimated cost of this instruction is used by instruction
4862 //               selection to identify a minimum cost tree of machine
4863 //               instructions that matches a tree of machine-independent
4864 //               instructions.
4865 // format     -- A string providing the disassembly for this instruction.
4866 //               The value of an instruction's operand may be inserted
4867 //               by referring to it with a '$' prefix.
4868 // opcode     -- Three instruction opcodes may be provided.  These are referred
4869 //               to within an encode class as $primary, $secondary, and $tertiary
4870 //               rrspectively.  The primary opcode is commonly used to
4871 //               indicate the type of machine instruction, while secondary
4872 //               and tertiary are often used for prefix options or addressing
4873 //               modes.
4874 // ins_encode -- A list of encode classes with parameters. The encode class
4875 //               name must have been defined in an 'enc_class' specification
4876 //               in the encode section of the architecture description.
4877 
4878 
4879 //----------Load/Store/Move Instructions---------------------------------------
4880 //----------Load Instructions--------------------------------------------------
4881 
4882 // Load Byte (8 bit signed)
4883 instruct loadB(rRegI dst, memory mem)
4884 %{
4885   match(Set dst (LoadB mem));
4886 
4887   ins_cost(125);
4888   format %{ "movsbl  $dst, $mem\t# byte" %}
4889 
4890   ins_encode %{
4891     __ movsbl($dst$$Register, $mem$$Address);
4892   %}
4893 
4894   ins_pipe(ialu_reg_mem);
4895 %}
4896 
4897 // Load Byte (8 bit signed) into Long Register
4898 instruct loadB2L(rRegL dst, memory mem)
4899 %{
4900   match(Set dst (ConvI2L (LoadB mem)));
4901 
4902   ins_cost(125);
4903   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4904 
4905   ins_encode %{
4906     __ movsbq($dst$$Register, $mem$$Address);
4907   %}
4908 
4909   ins_pipe(ialu_reg_mem);
4910 %}
4911 
4912 // Load Unsigned Byte (8 bit UNsigned)
4913 instruct loadUB(rRegI dst, memory mem)
4914 %{
4915   match(Set dst (LoadUB mem));
4916 
4917   ins_cost(125);
4918   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4919 
4920   ins_encode %{
4921     __ movzbl($dst$$Register, $mem$$Address);
4922   %}
4923 
4924   ins_pipe(ialu_reg_mem);
4925 %}
4926 
4927 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4928 instruct loadUB2L(rRegL dst, memory mem)
4929 %{
4930   match(Set dst (ConvI2L (LoadUB mem)));
4931 
4932   ins_cost(125);
4933   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4934 
4935   ins_encode %{
4936     __ movzbq($dst$$Register, $mem$$Address);
4937   %}
4938 
4939   ins_pipe(ialu_reg_mem);
4940 %}
4941 
4942 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
4943 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4944   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4945   effect(KILL cr);
4946 
4947   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
4948             "andl    $dst, right_n_bits($mask, 8)" %}
4949   ins_encode %{
4950     Register Rdst = $dst$$Register;
4951     __ movzbq(Rdst, $mem$$Address);
4952     __ andl(Rdst, $mask$$constant & right_n_bits(8));
4953   %}
4954   ins_pipe(ialu_reg_mem);
4955 %}
4956 
4957 // Load Short (16 bit signed)
4958 instruct loadS(rRegI dst, memory mem)
4959 %{
4960   match(Set dst (LoadS mem));
4961 
4962   ins_cost(125);
4963   format %{ "movswl $dst, $mem\t# short" %}
4964 
4965   ins_encode %{
4966     __ movswl($dst$$Register, $mem$$Address);
4967   %}
4968 
4969   ins_pipe(ialu_reg_mem);
4970 %}
4971 
4972 // Load Short (16 bit signed) to Byte (8 bit signed)
4973 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4974   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4975 
4976   ins_cost(125);
4977   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4978   ins_encode %{
4979     __ movsbl($dst$$Register, $mem$$Address);
4980   %}
4981   ins_pipe(ialu_reg_mem);
4982 %}
4983 
4984 // Load Short (16 bit signed) into Long Register
4985 instruct loadS2L(rRegL dst, memory mem)
4986 %{
4987   match(Set dst (ConvI2L (LoadS mem)));
4988 
4989   ins_cost(125);
4990   format %{ "movswq $dst, $mem\t# short -> long" %}
4991 
4992   ins_encode %{
4993     __ movswq($dst$$Register, $mem$$Address);
4994   %}
4995 
4996   ins_pipe(ialu_reg_mem);
4997 %}
4998 
4999 // Load Unsigned Short/Char (16 bit UNsigned)
5000 instruct loadUS(rRegI dst, memory mem)
5001 %{
5002   match(Set dst (LoadUS mem));
5003 
5004   ins_cost(125);
5005   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
5006 
5007   ins_encode %{
5008     __ movzwl($dst$$Register, $mem$$Address);
5009   %}
5010 
5011   ins_pipe(ialu_reg_mem);
5012 %}
5013 
5014 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
5015 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5016   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
5017 
5018   ins_cost(125);
5019   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
5020   ins_encode %{
5021     __ movsbl($dst$$Register, $mem$$Address);
5022   %}
5023   ins_pipe(ialu_reg_mem);
5024 %}
5025 
5026 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
5027 instruct loadUS2L(rRegL dst, memory mem)
5028 %{
5029   match(Set dst (ConvI2L (LoadUS mem)));
5030 
5031   ins_cost(125);
5032   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
5033 
5034   ins_encode %{
5035     __ movzwq($dst$$Register, $mem$$Address);
5036   %}
5037 
5038   ins_pipe(ialu_reg_mem);
5039 %}
5040 
5041 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
5042 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5043   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5044 
5045   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
5046   ins_encode %{
5047     __ movzbq($dst$$Register, $mem$$Address);
5048   %}
5049   ins_pipe(ialu_reg_mem);
5050 %}
5051 
5052 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
5053 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5054   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5055   effect(KILL cr);
5056 
5057   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
5058             "andl    $dst, right_n_bits($mask, 16)" %}
5059   ins_encode %{
5060     Register Rdst = $dst$$Register;
5061     __ movzwq(Rdst, $mem$$Address);
5062     __ andl(Rdst, $mask$$constant & right_n_bits(16));
5063   %}
5064   ins_pipe(ialu_reg_mem);
5065 %}
5066 
5067 // Load Integer
5068 instruct loadI(rRegI dst, memory mem)
5069 %{
5070   match(Set dst (LoadI mem));
5071 
5072   ins_cost(125);
5073   format %{ "movl    $dst, $mem\t# int" %}
5074 
5075   ins_encode %{
5076     __ movl($dst$$Register, $mem$$Address);
5077   %}
5078 
5079   ins_pipe(ialu_reg_mem);
5080 %}
5081 
5082 // Load Integer (32 bit signed) to Byte (8 bit signed)
5083 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5084   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
5085 
5086   ins_cost(125);
5087   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
5088   ins_encode %{
5089     __ movsbl($dst$$Register, $mem$$Address);
5090   %}
5091   ins_pipe(ialu_reg_mem);
5092 %}
5093 
5094 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
5095 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
5096   match(Set dst (AndI (LoadI mem) mask));
5097 
5098   ins_cost(125);
5099   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
5100   ins_encode %{
5101     __ movzbl($dst$$Register, $mem$$Address);
5102   %}
5103   ins_pipe(ialu_reg_mem);
5104 %}
5105 
5106 // Load Integer (32 bit signed) to Short (16 bit signed)
5107 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5108   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5109 
5110   ins_cost(125);
5111   format %{ "movswl  $dst, $mem\t# int -> short" %}
5112   ins_encode %{
5113     __ movswl($dst$$Register, $mem$$Address);
5114   %}
5115   ins_pipe(ialu_reg_mem);
5116 %}
5117 
5118 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5119 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5120   match(Set dst (AndI (LoadI mem) mask));
5121 
5122   ins_cost(125);
5123   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
5124   ins_encode %{
5125     __ movzwl($dst$$Register, $mem$$Address);
5126   %}
5127   ins_pipe(ialu_reg_mem);
5128 %}
5129 
5130 // Load Integer into Long Register
5131 instruct loadI2L(rRegL dst, memory mem)
5132 %{
5133   match(Set dst (ConvI2L (LoadI mem)));
5134 
5135   ins_cost(125);
5136   format %{ "movslq  $dst, $mem\t# int -> long" %}
5137 
5138   ins_encode %{
5139     __ movslq($dst$$Register, $mem$$Address);
5140   %}
5141 
5142   ins_pipe(ialu_reg_mem);
5143 %}
5144 
5145 // Load Integer with mask 0xFF into Long Register
5146 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5147   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5148 
5149   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
5150   ins_encode %{
5151     __ movzbq($dst$$Register, $mem$$Address);
5152   %}
5153   ins_pipe(ialu_reg_mem);
5154 %}
5155 
5156 // Load Integer with mask 0xFFFF into Long Register
5157 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
5158   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5159 
5160   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5161   ins_encode %{
5162     __ movzwq($dst$$Register, $mem$$Address);
5163   %}
5164   ins_pipe(ialu_reg_mem);
5165 %}
5166 
5167 // Load Integer with a 31-bit mask into Long Register
5168 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5169   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5170   effect(KILL cr);
5171 
5172   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5173             "andl    $dst, $mask" %}
5174   ins_encode %{
5175     Register Rdst = $dst$$Register;
5176     __ movl(Rdst, $mem$$Address);
5177     __ andl(Rdst, $mask$$constant);
5178   %}
5179   ins_pipe(ialu_reg_mem);
5180 %}
5181 
5182 // Load Unsigned Integer into Long Register
5183 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5184 %{
5185   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5186 
5187   ins_cost(125);
5188   format %{ "movl    $dst, $mem\t# uint -> long" %}
5189 
5190   ins_encode %{
5191     __ movl($dst$$Register, $mem$$Address);
5192   %}
5193 
5194   ins_pipe(ialu_reg_mem);
5195 %}
5196 
5197 // Load Long
5198 instruct loadL(rRegL dst, memory mem)
5199 %{
5200   match(Set dst (LoadL mem));
5201 
5202   ins_cost(125);
5203   format %{ "movq    $dst, $mem\t# long" %}
5204 
5205   ins_encode %{
5206     __ movq($dst$$Register, $mem$$Address);
5207   %}
5208 
5209   ins_pipe(ialu_reg_mem); // XXX
5210 %}
5211 
5212 // Load Range
5213 instruct loadRange(rRegI dst, memory mem)
5214 %{
5215   match(Set dst (LoadRange mem));
5216 
5217   ins_cost(125); // XXX
5218   format %{ "movl    $dst, $mem\t# range" %}
5219   opcode(0x8B);
5220   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5221   ins_pipe(ialu_reg_mem);
5222 %}
5223 
5224 // Load Pointer
5225 instruct loadP(rRegP dst, memory mem)
5226 %{
5227   match(Set dst (LoadP mem));
5228 
5229   ins_cost(125); // XXX
5230   format %{ "movq    $dst, $mem\t# ptr" %}
5231   opcode(0x8B);
5232   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5233   ins_pipe(ialu_reg_mem); // XXX
5234 %}
5235 
5236 // Load Compressed Pointer
5237 instruct loadN(rRegN dst, memory mem)
5238 %{
5239    match(Set dst (LoadN mem));
5240 
5241    ins_cost(125); // XXX
5242    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5243    ins_encode %{
5244      __ movl($dst$$Register, $mem$$Address);
5245    %}
5246    ins_pipe(ialu_reg_mem); // XXX
5247 %}
5248 
5249 
5250 // Load Klass Pointer
5251 instruct loadKlass(rRegP dst, memory mem)
5252 %{
5253   match(Set dst (LoadKlass mem));
5254 
5255   ins_cost(125); // XXX
5256   format %{ "movq    $dst, $mem\t# class" %}
5257   opcode(0x8B);
5258   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5259   ins_pipe(ialu_reg_mem); // XXX
5260 %}
5261 
5262 // Load narrow Klass Pointer
5263 instruct loadNKlass(rRegN dst, memory mem)
5264 %{
5265   match(Set dst (LoadNKlass mem));
5266 
5267   ins_cost(125); // XXX
5268   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5269   ins_encode %{
5270     __ movl($dst$$Register, $mem$$Address);
5271   %}
5272   ins_pipe(ialu_reg_mem); // XXX
5273 %}
5274 
5275 // Load Float
5276 instruct loadF(regF dst, memory mem)
5277 %{
5278   match(Set dst (LoadF mem));
5279 
5280   ins_cost(145); // XXX
5281   format %{ "movss   $dst, $mem\t# float" %}
5282   ins_encode %{
5283     __ movflt($dst$$XMMRegister, $mem$$Address);
5284   %}
5285   ins_pipe(pipe_slow); // XXX
5286 %}
5287 
5288 // Load Double
5289 instruct loadD_partial(regD dst, memory mem)
5290 %{
5291   predicate(!UseXmmLoadAndClearUpper);
5292   match(Set dst (LoadD mem));
5293 
5294   ins_cost(145); // XXX
5295   format %{ "movlpd  $dst, $mem\t# double" %}
5296   ins_encode %{
5297     __ movdbl($dst$$XMMRegister, $mem$$Address);
5298   %}
5299   ins_pipe(pipe_slow); // XXX
5300 %}
5301 
5302 instruct loadD(regD dst, memory mem)
5303 %{
5304   predicate(UseXmmLoadAndClearUpper);
5305   match(Set dst (LoadD mem));
5306 
5307   ins_cost(145); // XXX
5308   format %{ "movsd   $dst, $mem\t# double" %}
5309   ins_encode %{
5310     __ movdbl($dst$$XMMRegister, $mem$$Address);
5311   %}
5312   ins_pipe(pipe_slow); // XXX
5313 %}
5314 
5315 // Load Effective Address
5316 instruct leaP8(rRegP dst, indOffset8 mem)
5317 %{
5318   match(Set dst mem);
5319 
5320   ins_cost(110); // XXX
5321   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5322   opcode(0x8D);
5323   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5324   ins_pipe(ialu_reg_reg_fat);
5325 %}
5326 
5327 instruct leaP32(rRegP dst, indOffset32 mem)
5328 %{
5329   match(Set dst mem);
5330 
5331   ins_cost(110);
5332   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5333   opcode(0x8D);
5334   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5335   ins_pipe(ialu_reg_reg_fat);
5336 %}
5337 
5338 // instruct leaPIdx(rRegP dst, indIndex mem)
5339 // %{
5340 //   match(Set dst mem);
5341 
5342 //   ins_cost(110);
5343 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5344 //   opcode(0x8D);
5345 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5346 //   ins_pipe(ialu_reg_reg_fat);
5347 // %}
5348 
5349 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5350 %{
5351   match(Set dst mem);
5352 
5353   ins_cost(110);
5354   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5355   opcode(0x8D);
5356   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5357   ins_pipe(ialu_reg_reg_fat);
5358 %}
5359 
5360 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5361 %{
5362   match(Set dst mem);
5363 
5364   ins_cost(110);
5365   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5366   opcode(0x8D);
5367   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5368   ins_pipe(ialu_reg_reg_fat);
5369 %}
5370 
5371 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5372 %{
5373   match(Set dst mem);
5374 
5375   ins_cost(110);
5376   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5377   opcode(0x8D);
5378   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5379   ins_pipe(ialu_reg_reg_fat);
5380 %}
5381 
5382 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5383 %{
5384   match(Set dst mem);
5385 
5386   ins_cost(110);
5387   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5388   opcode(0x8D);
5389   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5390   ins_pipe(ialu_reg_reg_fat);
5391 %}
5392 
5393 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5394 %{
5395   match(Set dst mem);
5396 
5397   ins_cost(110);
5398   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5399   opcode(0x8D);
5400   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5401   ins_pipe(ialu_reg_reg_fat);
5402 %}
5403 
5404 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5405 %{
5406   match(Set dst mem);
5407 
5408   ins_cost(110);
5409   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5410   opcode(0x8D);
5411   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5412   ins_pipe(ialu_reg_reg_fat);
5413 %}
5414 
5415 // Load Effective Address which uses Narrow (32-bits) oop
5416 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5417 %{
5418   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5419   match(Set dst mem);
5420 
5421   ins_cost(110);
5422   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5423   opcode(0x8D);
5424   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5425   ins_pipe(ialu_reg_reg_fat);
5426 %}
5427 
5428 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5429 %{
5430   predicate(Universe::narrow_oop_shift() == 0);
5431   match(Set dst mem);
5432 
5433   ins_cost(110); // XXX
5434   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5435   opcode(0x8D);
5436   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5437   ins_pipe(ialu_reg_reg_fat);
5438 %}
5439 
5440 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5441 %{
5442   predicate(Universe::narrow_oop_shift() == 0);
5443   match(Set dst mem);
5444 
5445   ins_cost(110);
5446   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5447   opcode(0x8D);
5448   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5449   ins_pipe(ialu_reg_reg_fat);
5450 %}
5451 
5452 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5453 %{
5454   predicate(Universe::narrow_oop_shift() == 0);
5455   match(Set dst mem);
5456 
5457   ins_cost(110);
5458   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5459   opcode(0x8D);
5460   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5461   ins_pipe(ialu_reg_reg_fat);
5462 %}
5463 
5464 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5465 %{
5466   predicate(Universe::narrow_oop_shift() == 0);
5467   match(Set dst mem);
5468 
5469   ins_cost(110);
5470   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5471   opcode(0x8D);
5472   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5473   ins_pipe(ialu_reg_reg_fat);
5474 %}
5475 
5476 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5477 %{
5478   predicate(Universe::narrow_oop_shift() == 0);
5479   match(Set dst mem);
5480 
5481   ins_cost(110);
5482   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5483   opcode(0x8D);
5484   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5485   ins_pipe(ialu_reg_reg_fat);
5486 %}
5487 
5488 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5489 %{
5490   predicate(Universe::narrow_oop_shift() == 0);
5491   match(Set dst mem);
5492 
5493   ins_cost(110);
5494   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5495   opcode(0x8D);
5496   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5497   ins_pipe(ialu_reg_reg_fat);
5498 %}
5499 
5500 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5501 %{
5502   predicate(Universe::narrow_oop_shift() == 0);
5503   match(Set dst mem);
5504 
5505   ins_cost(110);
5506   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5507   opcode(0x8D);
5508   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5509   ins_pipe(ialu_reg_reg_fat);
5510 %}
5511 
5512 instruct loadConI(rRegI dst, immI src)
5513 %{
5514   match(Set dst src);
5515 
5516   format %{ "movl    $dst, $src\t# int" %}
5517   ins_encode(load_immI(dst, src));
5518   ins_pipe(ialu_reg_fat); // XXX
5519 %}
5520 
5521 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5522 %{
5523   match(Set dst src);
5524   effect(KILL cr);
5525 
5526   ins_cost(50);
5527   format %{ "xorl    $dst, $dst\t# int" %}
5528   opcode(0x33); /* + rd */
5529   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5530   ins_pipe(ialu_reg);
5531 %}
5532 
5533 instruct loadConL(rRegL dst, immL src)
5534 %{
5535   match(Set dst src);
5536 
5537   ins_cost(150);
5538   format %{ "movq    $dst, $src\t# long" %}
5539   ins_encode(load_immL(dst, src));
5540   ins_pipe(ialu_reg);
5541 %}
5542 
5543 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5544 %{
5545   match(Set dst src);
5546   effect(KILL cr);
5547 
5548   ins_cost(50);
5549   format %{ "xorl    $dst, $dst\t# long" %}
5550   opcode(0x33); /* + rd */
5551   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5552   ins_pipe(ialu_reg); // XXX
5553 %}
5554 
5555 instruct loadConUL32(rRegL dst, immUL32 src)
5556 %{
5557   match(Set dst src);
5558 
5559   ins_cost(60);
5560   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5561   ins_encode(load_immUL32(dst, src));
5562   ins_pipe(ialu_reg);
5563 %}
5564 
5565 instruct loadConL32(rRegL dst, immL32 src)
5566 %{
5567   match(Set dst src);
5568 
5569   ins_cost(70);
5570   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5571   ins_encode(load_immL32(dst, src));
5572   ins_pipe(ialu_reg);
5573 %}
5574 
5575 instruct loadConP(rRegP dst, immP con) %{
5576   match(Set dst con);
5577 
5578   format %{ "movq    $dst, $con\t# ptr" %}
5579   ins_encode(load_immP(dst, con));
5580   ins_pipe(ialu_reg_fat); // XXX
5581 %}
5582 
5583 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5584 %{
5585   match(Set dst src);
5586   effect(KILL cr);
5587 
5588   ins_cost(50);
5589   format %{ "xorl    $dst, $dst\t# ptr" %}
5590   opcode(0x33); /* + rd */
5591   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5592   ins_pipe(ialu_reg);
5593 %}
5594 
5595 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5596 %{
5597   match(Set dst src);
5598   effect(KILL cr);
5599 
5600   ins_cost(60);
5601   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5602   ins_encode(load_immP31(dst, src));
5603   ins_pipe(ialu_reg);
5604 %}
5605 
5606 instruct loadConF(regF dst, immF con) %{
5607   match(Set dst con);
5608   ins_cost(125);
5609   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5610   ins_encode %{
5611     __ movflt($dst$$XMMRegister, $constantaddress($con));
5612   %}
5613   ins_pipe(pipe_slow);
5614 %}
5615 
5616 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5617   match(Set dst src);
5618   effect(KILL cr);
5619   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5620   ins_encode %{
5621     __ xorq($dst$$Register, $dst$$Register);
5622   %}
5623   ins_pipe(ialu_reg);
5624 %}
5625 
5626 instruct loadConN(rRegN dst, immN src) %{
5627   match(Set dst src);
5628 
5629   ins_cost(125);
5630   format %{ "movl    $dst, $src\t# compressed ptr" %}
5631   ins_encode %{
5632     address con = (address)$src$$constant;
5633     if (con == NULL) {
5634       ShouldNotReachHere();
5635     } else {
5636       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5637     }
5638   %}
5639   ins_pipe(ialu_reg_fat); // XXX
5640 %}
5641 
5642 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5643   match(Set dst src);
5644 
5645   ins_cost(125);
5646   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5647   ins_encode %{
5648     address con = (address)$src$$constant;
5649     if (con == NULL) {
5650       ShouldNotReachHere();
5651     } else {
5652       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5653     }
5654   %}
5655   ins_pipe(ialu_reg_fat); // XXX
5656 %}
5657 
5658 instruct loadConF0(regF dst, immF0 src)
5659 %{
5660   match(Set dst src);
5661   ins_cost(100);
5662 
5663   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5664   ins_encode %{
5665     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5666   %}
5667   ins_pipe(pipe_slow);
5668 %}
5669 
5670 // Use the same format since predicate() can not be used here.
5671 instruct loadConD(regD dst, immD con) %{
5672   match(Set dst con);
5673   ins_cost(125);
5674   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5675   ins_encode %{
5676     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5677   %}
5678   ins_pipe(pipe_slow);
5679 %}
5680 
5681 instruct loadConD0(regD dst, immD0 src)
5682 %{
5683   match(Set dst src);
5684   ins_cost(100);
5685 
5686   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5687   ins_encode %{
5688     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5689   %}
5690   ins_pipe(pipe_slow);
5691 %}
5692 
5693 instruct loadSSI(rRegI dst, stackSlotI src)
5694 %{
5695   match(Set dst src);
5696 
5697   ins_cost(125);
5698   format %{ "movl    $dst, $src\t# int stk" %}
5699   opcode(0x8B);
5700   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5701   ins_pipe(ialu_reg_mem);
5702 %}
5703 
5704 instruct loadSSL(rRegL dst, stackSlotL src)
5705 %{
5706   match(Set dst src);
5707 
5708   ins_cost(125);
5709   format %{ "movq    $dst, $src\t# long stk" %}
5710   opcode(0x8B);
5711   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5712   ins_pipe(ialu_reg_mem);
5713 %}
5714 
5715 instruct loadSSP(rRegP dst, stackSlotP src)
5716 %{
5717   match(Set dst src);
5718 
5719   ins_cost(125);
5720   format %{ "movq    $dst, $src\t# ptr stk" %}
5721   opcode(0x8B);
5722   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5723   ins_pipe(ialu_reg_mem);
5724 %}
5725 
5726 instruct loadSSF(regF dst, stackSlotF src)
5727 %{
5728   match(Set dst src);
5729 
5730   ins_cost(125);
5731   format %{ "movss   $dst, $src\t# float stk" %}
5732   ins_encode %{
5733     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5734   %}
5735   ins_pipe(pipe_slow); // XXX
5736 %}
5737 
5738 // Use the same format since predicate() can not be used here.
5739 instruct loadSSD(regD dst, stackSlotD src)
5740 %{
5741   match(Set dst src);
5742 
5743   ins_cost(125);
5744   format %{ "movsd   $dst, $src\t# double stk" %}
5745   ins_encode  %{
5746     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5747   %}
5748   ins_pipe(pipe_slow); // XXX
5749 %}
5750 
5751 // Prefetch instructions for allocation.
5752 // Must be safe to execute with invalid address (cannot fault).
5753 
5754 instruct prefetchAlloc( memory mem ) %{
5755   predicate(AllocatePrefetchInstr==3);
5756   match(PrefetchAllocation mem);
5757   ins_cost(125);
5758 
5759   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5760   ins_encode %{
5761     __ prefetchw($mem$$Address);
5762   %}
5763   ins_pipe(ialu_mem);
5764 %}
5765 
5766 instruct prefetchAllocNTA( memory mem ) %{
5767   predicate(AllocatePrefetchInstr==0);
5768   match(PrefetchAllocation mem);
5769   ins_cost(125);
5770 
5771   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5772   ins_encode %{
5773     __ prefetchnta($mem$$Address);
5774   %}
5775   ins_pipe(ialu_mem);
5776 %}
5777 
5778 instruct prefetchAllocT0( memory mem ) %{
5779   predicate(AllocatePrefetchInstr==1);
5780   match(PrefetchAllocation mem);
5781   ins_cost(125);
5782 
5783   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5784   ins_encode %{
5785     __ prefetcht0($mem$$Address);
5786   %}
5787   ins_pipe(ialu_mem);
5788 %}
5789 
5790 instruct prefetchAllocT2( memory mem ) %{
5791   predicate(AllocatePrefetchInstr==2);
5792   match(PrefetchAllocation mem);
5793   ins_cost(125);
5794 
5795   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5796   ins_encode %{
5797     __ prefetcht2($mem$$Address);
5798   %}
5799   ins_pipe(ialu_mem);
5800 %}
5801 
5802 //----------Store Instructions-------------------------------------------------
5803 
5804 // Store Byte
5805 instruct storeB(memory mem, rRegI src)
5806 %{
5807   match(Set mem (StoreB mem src));
5808 
5809   ins_cost(125); // XXX
5810   format %{ "movb    $mem, $src\t# byte" %}
5811   opcode(0x88);
5812   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5813   ins_pipe(ialu_mem_reg);
5814 %}
5815 
5816 // Store Char/Short
5817 instruct storeC(memory mem, rRegI src)
5818 %{
5819   match(Set mem (StoreC mem src));
5820 
5821   ins_cost(125); // XXX
5822   format %{ "movw    $mem, $src\t# char/short" %}
5823   opcode(0x89);
5824   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5825   ins_pipe(ialu_mem_reg);
5826 %}
5827 
5828 // Store Integer
5829 instruct storeI(memory mem, rRegI src)
5830 %{
5831   match(Set mem (StoreI mem src));
5832 
5833   ins_cost(125); // XXX
5834   format %{ "movl    $mem, $src\t# int" %}
5835   opcode(0x89);
5836   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5837   ins_pipe(ialu_mem_reg);
5838 %}
5839 
5840 // Store Long
5841 instruct storeL(memory mem, rRegL src)
5842 %{
5843   match(Set mem (StoreL mem src));
5844 
5845   ins_cost(125); // XXX
5846   format %{ "movq    $mem, $src\t# long" %}
5847   opcode(0x89);
5848   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5849   ins_pipe(ialu_mem_reg); // XXX
5850 %}
5851 
5852 // Store Pointer
5853 instruct storeP(memory mem, any_RegP src)
5854 %{
5855   match(Set mem (StoreP mem src));
5856 
5857   ins_cost(125); // XXX
5858   format %{ "movq    $mem, $src\t# ptr" %}
5859   opcode(0x89);
5860   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5861   ins_pipe(ialu_mem_reg);
5862 %}
5863 
5864 instruct storeImmP0(memory mem, immP0 zero)
5865 %{
5866   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5867   match(Set mem (StoreP mem zero));
5868 
5869   ins_cost(125); // XXX
5870   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5871   ins_encode %{
5872     __ movq($mem$$Address, r12);
5873   %}
5874   ins_pipe(ialu_mem_reg);
5875 %}
5876 
5877 // Store NULL Pointer, mark word, or other simple pointer constant.
5878 instruct storeImmP(memory mem, immP31 src)
5879 %{
5880   match(Set mem (StoreP mem src));
5881 
5882   ins_cost(150); // XXX
5883   format %{ "movq    $mem, $src\t# ptr" %}
5884   opcode(0xC7); /* C7 /0 */
5885   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5886   ins_pipe(ialu_mem_imm);
5887 %}
5888 
5889 // Store Compressed Pointer
5890 instruct storeN(memory mem, rRegN src)
5891 %{
5892   match(Set mem (StoreN mem src));
5893 
5894   ins_cost(125); // XXX
5895   format %{ "movl    $mem, $src\t# compressed ptr" %}
5896   ins_encode %{
5897     __ movl($mem$$Address, $src$$Register);
5898   %}
5899   ins_pipe(ialu_mem_reg);
5900 %}
5901 
5902 instruct storeNKlass(memory mem, rRegN src)
5903 %{
5904   match(Set mem (StoreNKlass mem src));
5905 
5906   ins_cost(125); // XXX
5907   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5908   ins_encode %{
5909     __ movl($mem$$Address, $src$$Register);
5910   %}
5911   ins_pipe(ialu_mem_reg);
5912 %}
5913 
5914 instruct storeImmN0(memory mem, immN0 zero)
5915 %{
5916   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5917   match(Set mem (StoreN mem zero));
5918 
5919   ins_cost(125); // XXX
5920   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5921   ins_encode %{
5922     __ movl($mem$$Address, r12);
5923   %}
5924   ins_pipe(ialu_mem_reg);
5925 %}
5926 
5927 instruct storeImmN(memory mem, immN src)
5928 %{
5929   match(Set mem (StoreN mem src));
5930 
5931   ins_cost(150); // XXX
5932   format %{ "movl    $mem, $src\t# compressed ptr" %}
5933   ins_encode %{
5934     address con = (address)$src$$constant;
5935     if (con == NULL) {
5936       __ movl($mem$$Address, (int32_t)0);
5937     } else {
5938       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5939     }
5940   %}
5941   ins_pipe(ialu_mem_imm);
5942 %}
5943 
5944 instruct storeImmNKlass(memory mem, immNKlass src)
5945 %{
5946   match(Set mem (StoreNKlass mem src));
5947 
5948   ins_cost(150); // XXX
5949   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5950   ins_encode %{
5951     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5952   %}
5953   ins_pipe(ialu_mem_imm);
5954 %}
5955 
5956 // Store Integer Immediate
5957 instruct storeImmI0(memory mem, immI0 zero)
5958 %{
5959   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5960   match(Set mem (StoreI mem zero));
5961 
5962   ins_cost(125); // XXX
5963   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5964   ins_encode %{
5965     __ movl($mem$$Address, r12);
5966   %}
5967   ins_pipe(ialu_mem_reg);
5968 %}
5969 
5970 instruct storeImmI(memory mem, immI src)
5971 %{
5972   match(Set mem (StoreI mem src));
5973 
5974   ins_cost(150);
5975   format %{ "movl    $mem, $src\t# int" %}
5976   opcode(0xC7); /* C7 /0 */
5977   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5978   ins_pipe(ialu_mem_imm);
5979 %}
5980 
5981 // Store Long Immediate
5982 instruct storeImmL0(memory mem, immL0 zero)
5983 %{
5984   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5985   match(Set mem (StoreL mem zero));
5986 
5987   ins_cost(125); // XXX
5988   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5989   ins_encode %{
5990     __ movq($mem$$Address, r12);
5991   %}
5992   ins_pipe(ialu_mem_reg);
5993 %}
5994 
5995 instruct storeImmL(memory mem, immL32 src)
5996 %{
5997   match(Set mem (StoreL mem src));
5998 
5999   ins_cost(150);
6000   format %{ "movq    $mem, $src\t# long" %}
6001   opcode(0xC7); /* C7 /0 */
6002   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6003   ins_pipe(ialu_mem_imm);
6004 %}
6005 
6006 // Store Short/Char Immediate
6007 instruct storeImmC0(memory mem, immI0 zero)
6008 %{
6009   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6010   match(Set mem (StoreC mem zero));
6011 
6012   ins_cost(125); // XXX
6013   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
6014   ins_encode %{
6015     __ movw($mem$$Address, r12);
6016   %}
6017   ins_pipe(ialu_mem_reg);
6018 %}
6019 
6020 instruct storeImmI16(memory mem, immI16 src)
6021 %{
6022   predicate(UseStoreImmI16);
6023   match(Set mem (StoreC mem src));
6024 
6025   ins_cost(150);
6026   format %{ "movw    $mem, $src\t# short/char" %}
6027   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
6028   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
6029   ins_pipe(ialu_mem_imm);
6030 %}
6031 
6032 // Store Byte Immediate
6033 instruct storeImmB0(memory mem, immI0 zero)
6034 %{
6035   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6036   match(Set mem (StoreB mem zero));
6037 
6038   ins_cost(125); // XXX
6039   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
6040   ins_encode %{
6041     __ movb($mem$$Address, r12);
6042   %}
6043   ins_pipe(ialu_mem_reg);
6044 %}
6045 
6046 instruct storeImmB(memory mem, immI8 src)
6047 %{
6048   match(Set mem (StoreB mem src));
6049 
6050   ins_cost(150); // XXX
6051   format %{ "movb    $mem, $src\t# byte" %}
6052   opcode(0xC6); /* C6 /0 */
6053   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6054   ins_pipe(ialu_mem_imm);
6055 %}
6056 
6057 // Store CMS card-mark Immediate
6058 instruct storeImmCM0_reg(memory mem, immI0 zero)
6059 %{
6060   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6061   match(Set mem (StoreCM mem zero));
6062 
6063   ins_cost(125); // XXX
6064   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
6065   ins_encode %{
6066     __ movb($mem$$Address, r12);
6067   %}
6068   ins_pipe(ialu_mem_reg);
6069 %}
6070 
6071 instruct storeImmCM0(memory mem, immI0 src)
6072 %{
6073   match(Set mem (StoreCM mem src));
6074 
6075   ins_cost(150); // XXX
6076   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
6077   opcode(0xC6); /* C6 /0 */
6078   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6079   ins_pipe(ialu_mem_imm);
6080 %}
6081 
6082 // Store Float
6083 instruct storeF(memory mem, regF src)
6084 %{
6085   match(Set mem (StoreF mem src));
6086 
6087   ins_cost(95); // XXX
6088   format %{ "movss   $mem, $src\t# float" %}
6089   ins_encode %{
6090     __ movflt($mem$$Address, $src$$XMMRegister);
6091   %}
6092   ins_pipe(pipe_slow); // XXX
6093 %}
6094 
6095 // Store immediate Float value (it is faster than store from XMM register)
6096 instruct storeF0(memory mem, immF0 zero)
6097 %{
6098   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6099   match(Set mem (StoreF mem zero));
6100 
6101   ins_cost(25); // XXX
6102   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
6103   ins_encode %{
6104     __ movl($mem$$Address, r12);
6105   %}
6106   ins_pipe(ialu_mem_reg);
6107 %}
6108 
6109 instruct storeF_imm(memory mem, immF src)
6110 %{
6111   match(Set mem (StoreF mem src));
6112 
6113   ins_cost(50);
6114   format %{ "movl    $mem, $src\t# float" %}
6115   opcode(0xC7); /* C7 /0 */
6116   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6117   ins_pipe(ialu_mem_imm);
6118 %}
6119 
6120 // Store Double
6121 instruct storeD(memory mem, regD src)
6122 %{
6123   match(Set mem (StoreD mem src));
6124 
6125   ins_cost(95); // XXX
6126   format %{ "movsd   $mem, $src\t# double" %}
6127   ins_encode %{
6128     __ movdbl($mem$$Address, $src$$XMMRegister);
6129   %}
6130   ins_pipe(pipe_slow); // XXX
6131 %}
6132 
6133 // Store immediate double 0.0 (it is faster than store from XMM register)
6134 instruct storeD0_imm(memory mem, immD0 src)
6135 %{
6136   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
6137   match(Set mem (StoreD mem src));
6138 
6139   ins_cost(50);
6140   format %{ "movq    $mem, $src\t# double 0." %}
6141   opcode(0xC7); /* C7 /0 */
6142   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6143   ins_pipe(ialu_mem_imm);
6144 %}
6145 
6146 instruct storeD0(memory mem, immD0 zero)
6147 %{
6148   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6149   match(Set mem (StoreD mem zero));
6150 
6151   ins_cost(25); // XXX
6152   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
6153   ins_encode %{
6154     __ movq($mem$$Address, r12);
6155   %}
6156   ins_pipe(ialu_mem_reg);
6157 %}
6158 
6159 instruct storeSSI(stackSlotI dst, rRegI src)
6160 %{
6161   match(Set dst src);
6162 
6163   ins_cost(100);
6164   format %{ "movl    $dst, $src\t# int stk" %}
6165   opcode(0x89);
6166   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6167   ins_pipe( ialu_mem_reg );
6168 %}
6169 
6170 instruct storeSSL(stackSlotL dst, rRegL src)
6171 %{
6172   match(Set dst src);
6173 
6174   ins_cost(100);
6175   format %{ "movq    $dst, $src\t# long stk" %}
6176   opcode(0x89);
6177   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6178   ins_pipe(ialu_mem_reg);
6179 %}
6180 
6181 instruct storeSSP(stackSlotP dst, rRegP src)
6182 %{
6183   match(Set dst src);
6184 
6185   ins_cost(100);
6186   format %{ "movq    $dst, $src\t# ptr stk" %}
6187   opcode(0x89);
6188   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6189   ins_pipe(ialu_mem_reg);
6190 %}
6191 
6192 instruct storeSSF(stackSlotF dst, regF src)
6193 %{
6194   match(Set dst src);
6195 
6196   ins_cost(95); // XXX
6197   format %{ "movss   $dst, $src\t# float stk" %}
6198   ins_encode %{
6199     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6200   %}
6201   ins_pipe(pipe_slow); // XXX
6202 %}
6203 
6204 instruct storeSSD(stackSlotD dst, regD src)
6205 %{
6206   match(Set dst src);
6207 
6208   ins_cost(95); // XXX
6209   format %{ "movsd   $dst, $src\t# double stk" %}
6210   ins_encode %{
6211     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6212   %}
6213   ins_pipe(pipe_slow); // XXX
6214 %}
6215 
6216 //----------BSWAP Instructions-------------------------------------------------
6217 instruct bytes_reverse_int(rRegI dst) %{
6218   match(Set dst (ReverseBytesI dst));
6219 
6220   format %{ "bswapl  $dst" %}
6221   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6222   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6223   ins_pipe( ialu_reg );
6224 %}
6225 
6226 instruct bytes_reverse_long(rRegL dst) %{
6227   match(Set dst (ReverseBytesL dst));
6228 
6229   format %{ "bswapq  $dst" %}
6230   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6231   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6232   ins_pipe( ialu_reg);
6233 %}
6234 
6235 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6236   match(Set dst (ReverseBytesUS dst));
6237   effect(KILL cr);
6238 
6239   format %{ "bswapl  $dst\n\t"
6240             "shrl    $dst,16\n\t" %}
6241   ins_encode %{
6242     __ bswapl($dst$$Register);
6243     __ shrl($dst$$Register, 16);
6244   %}
6245   ins_pipe( ialu_reg );
6246 %}
6247 
6248 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6249   match(Set dst (ReverseBytesS dst));
6250   effect(KILL cr);
6251 
6252   format %{ "bswapl  $dst\n\t"
6253             "sar     $dst,16\n\t" %}
6254   ins_encode %{
6255     __ bswapl($dst$$Register);
6256     __ sarl($dst$$Register, 16);
6257   %}
6258   ins_pipe( ialu_reg );
6259 %}
6260 
6261 //---------- Zeros Count Instructions ------------------------------------------
6262 
6263 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6264   predicate(UseCountLeadingZerosInstruction);
6265   match(Set dst (CountLeadingZerosI src));
6266   effect(KILL cr);
6267 
6268   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6269   ins_encode %{
6270     __ lzcntl($dst$$Register, $src$$Register);
6271   %}
6272   ins_pipe(ialu_reg);
6273 %}
6274 
6275 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6276   predicate(!UseCountLeadingZerosInstruction);
6277   match(Set dst (CountLeadingZerosI src));
6278   effect(KILL cr);
6279 
6280   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6281             "jnz     skip\n\t"
6282             "movl    $dst, -1\n"
6283       "skip:\n\t"
6284             "negl    $dst\n\t"
6285             "addl    $dst, 31" %}
6286   ins_encode %{
6287     Register Rdst = $dst$$Register;
6288     Register Rsrc = $src$$Register;
6289     Label skip;
6290     __ bsrl(Rdst, Rsrc);
6291     __ jccb(Assembler::notZero, skip);
6292     __ movl(Rdst, -1);
6293     __ bind(skip);
6294     __ negl(Rdst);
6295     __ addl(Rdst, BitsPerInt - 1);
6296   %}
6297   ins_pipe(ialu_reg);
6298 %}
6299 
6300 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6301   predicate(UseCountLeadingZerosInstruction);
6302   match(Set dst (CountLeadingZerosL src));
6303   effect(KILL cr);
6304 
6305   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6306   ins_encode %{
6307     __ lzcntq($dst$$Register, $src$$Register);
6308   %}
6309   ins_pipe(ialu_reg);
6310 %}
6311 
6312 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6313   predicate(!UseCountLeadingZerosInstruction);
6314   match(Set dst (CountLeadingZerosL src));
6315   effect(KILL cr);
6316 
6317   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6318             "jnz     skip\n\t"
6319             "movl    $dst, -1\n"
6320       "skip:\n\t"
6321             "negl    $dst\n\t"
6322             "addl    $dst, 63" %}
6323   ins_encode %{
6324     Register Rdst = $dst$$Register;
6325     Register Rsrc = $src$$Register;
6326     Label skip;
6327     __ bsrq(Rdst, Rsrc);
6328     __ jccb(Assembler::notZero, skip);
6329     __ movl(Rdst, -1);
6330     __ bind(skip);
6331     __ negl(Rdst);
6332     __ addl(Rdst, BitsPerLong - 1);
6333   %}
6334   ins_pipe(ialu_reg);
6335 %}
6336 
6337 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6338   predicate(UseCountTrailingZerosInstruction);
6339   match(Set dst (CountTrailingZerosI src));
6340   effect(KILL cr);
6341 
6342   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6343   ins_encode %{
6344     __ tzcntl($dst$$Register, $src$$Register);
6345   %}
6346   ins_pipe(ialu_reg);
6347 %}
6348 
6349 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6350   predicate(!UseCountTrailingZerosInstruction);
6351   match(Set dst (CountTrailingZerosI src));
6352   effect(KILL cr);
6353 
6354   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6355             "jnz     done\n\t"
6356             "movl    $dst, 32\n"
6357       "done:" %}
6358   ins_encode %{
6359     Register Rdst = $dst$$Register;
6360     Label done;
6361     __ bsfl(Rdst, $src$$Register);
6362     __ jccb(Assembler::notZero, done);
6363     __ movl(Rdst, BitsPerInt);
6364     __ bind(done);
6365   %}
6366   ins_pipe(ialu_reg);
6367 %}
6368 
6369 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6370   predicate(UseCountTrailingZerosInstruction);
6371   match(Set dst (CountTrailingZerosL src));
6372   effect(KILL cr);
6373 
6374   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6375   ins_encode %{
6376     __ tzcntq($dst$$Register, $src$$Register);
6377   %}
6378   ins_pipe(ialu_reg);
6379 %}
6380 
6381 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6382   predicate(!UseCountTrailingZerosInstruction);
6383   match(Set dst (CountTrailingZerosL src));
6384   effect(KILL cr);
6385 
6386   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6387             "jnz     done\n\t"
6388             "movl    $dst, 64\n"
6389       "done:" %}
6390   ins_encode %{
6391     Register Rdst = $dst$$Register;
6392     Label done;
6393     __ bsfq(Rdst, $src$$Register);
6394     __ jccb(Assembler::notZero, done);
6395     __ movl(Rdst, BitsPerLong);
6396     __ bind(done);
6397   %}
6398   ins_pipe(ialu_reg);
6399 %}
6400 
6401 
6402 //---------- Population Count Instructions -------------------------------------
6403 
6404 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6405   predicate(UsePopCountInstruction);
6406   match(Set dst (PopCountI src));
6407   effect(KILL cr);
6408 
6409   format %{ "popcnt  $dst, $src" %}
6410   ins_encode %{
6411     __ popcntl($dst$$Register, $src$$Register);
6412   %}
6413   ins_pipe(ialu_reg);
6414 %}
6415 
6416 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6417   predicate(UsePopCountInstruction);
6418   match(Set dst (PopCountI (LoadI mem)));
6419   effect(KILL cr);
6420 
6421   format %{ "popcnt  $dst, $mem" %}
6422   ins_encode %{
6423     __ popcntl($dst$$Register, $mem$$Address);
6424   %}
6425   ins_pipe(ialu_reg);
6426 %}
6427 
6428 // Note: Long.bitCount(long) returns an int.
6429 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6430   predicate(UsePopCountInstruction);
6431   match(Set dst (PopCountL src));
6432   effect(KILL cr);
6433 
6434   format %{ "popcnt  $dst, $src" %}
6435   ins_encode %{
6436     __ popcntq($dst$$Register, $src$$Register);
6437   %}
6438   ins_pipe(ialu_reg);
6439 %}
6440 
6441 // Note: Long.bitCount(long) returns an int.
6442 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6443   predicate(UsePopCountInstruction);
6444   match(Set dst (PopCountL (LoadL mem)));
6445   effect(KILL cr);
6446 
6447   format %{ "popcnt  $dst, $mem" %}
6448   ins_encode %{
6449     __ popcntq($dst$$Register, $mem$$Address);
6450   %}
6451   ins_pipe(ialu_reg);
6452 %}
6453 
6454 
6455 //----------MemBar Instructions-----------------------------------------------
6456 // Memory barrier flavors
6457 
6458 instruct membar_acquire()
6459 %{
6460   match(MemBarAcquire);
6461   match(LoadFence);
6462   ins_cost(0);
6463 
6464   size(0);
6465   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6466   ins_encode();
6467   ins_pipe(empty);
6468 %}
6469 
6470 instruct membar_acquire_lock()
6471 %{
6472   match(MemBarAcquireLock);
6473   ins_cost(0);
6474 
6475   size(0);
6476   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6477   ins_encode();
6478   ins_pipe(empty);
6479 %}
6480 
6481 instruct membar_release()
6482 %{
6483   match(MemBarRelease);
6484   match(StoreFence);
6485   ins_cost(0);
6486 
6487   size(0);
6488   format %{ "MEMBAR-release ! (empty encoding)" %}
6489   ins_encode();
6490   ins_pipe(empty);
6491 %}
6492 
6493 instruct membar_release_lock()
6494 %{
6495   match(MemBarReleaseLock);
6496   ins_cost(0);
6497 
6498   size(0);
6499   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6500   ins_encode();
6501   ins_pipe(empty);
6502 %}
6503 
6504 instruct membar_volatile(rFlagsReg cr) %{
6505   match(MemBarVolatile);
6506   effect(KILL cr);
6507   ins_cost(400);
6508 
6509   format %{
6510     $$template
6511     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6512   %}
6513   ins_encode %{
6514     __ membar(Assembler::StoreLoad);
6515   %}
6516   ins_pipe(pipe_slow);
6517 %}
6518 
6519 instruct unnecessary_membar_volatile()
6520 %{
6521   match(MemBarVolatile);
6522   predicate(Matcher::post_store_load_barrier(n));
6523   ins_cost(0);
6524 
6525   size(0);
6526   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6527   ins_encode();
6528   ins_pipe(empty);
6529 %}
6530 
6531 instruct membar_storestore() %{
6532   match(MemBarStoreStore);
6533   ins_cost(0);
6534 
6535   size(0);
6536   format %{ "MEMBAR-storestore (empty encoding)" %}
6537   ins_encode( );
6538   ins_pipe(empty);
6539 %}
6540 
6541 //----------Move Instructions--------------------------------------------------
6542 
6543 instruct castX2P(rRegP dst, rRegL src)
6544 %{
6545   match(Set dst (CastX2P src));
6546 
6547   format %{ "movq    $dst, $src\t# long->ptr" %}
6548   ins_encode %{
6549     if ($dst$$reg != $src$$reg) {
6550       __ movptr($dst$$Register, $src$$Register);
6551     }
6552   %}
6553   ins_pipe(ialu_reg_reg); // XXX
6554 %}
6555 
6556 instruct castP2X(rRegL dst, rRegP src)
6557 %{
6558   match(Set dst (CastP2X src));
6559 
6560   format %{ "movq    $dst, $src\t# ptr -> long" %}
6561   ins_encode %{
6562     if ($dst$$reg != $src$$reg) {
6563       __ movptr($dst$$Register, $src$$Register);
6564     }
6565   %}
6566   ins_pipe(ialu_reg_reg); // XXX
6567 %}
6568 
6569 // Convert oop into int for vectors alignment masking
6570 instruct convP2I(rRegI dst, rRegP src)
6571 %{
6572   match(Set dst (ConvL2I (CastP2X src)));
6573 
6574   format %{ "movl    $dst, $src\t# ptr -> int" %}
6575   ins_encode %{
6576     __ movl($dst$$Register, $src$$Register);
6577   %}
6578   ins_pipe(ialu_reg_reg); // XXX
6579 %}
6580 
6581 // Convert compressed oop into int for vectors alignment masking
6582 // in case of 32bit oops (heap < 4Gb).
6583 instruct convN2I(rRegI dst, rRegN src)
6584 %{
6585   predicate(Universe::narrow_oop_shift() == 0);
6586   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6587 
6588   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6589   ins_encode %{
6590     __ movl($dst$$Register, $src$$Register);
6591   %}
6592   ins_pipe(ialu_reg_reg); // XXX
6593 %}
6594 
6595 // Convert oop pointer into compressed form
6596 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6597   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6598   match(Set dst (EncodeP src));
6599   effect(KILL cr);
6600   format %{ "encode_heap_oop $dst,$src" %}
6601   ins_encode %{
6602     Register s = $src$$Register;
6603     Register d = $dst$$Register;
6604     if (s != d) {
6605       __ movq(d, s);
6606     }
6607     __ encode_heap_oop(d);
6608   %}
6609   ins_pipe(ialu_reg_long);
6610 %}
6611 
6612 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6613   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6614   match(Set dst (EncodeP src));
6615   effect(KILL cr);
6616   format %{ "encode_heap_oop_not_null $dst,$src" %}
6617   ins_encode %{
6618     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6619   %}
6620   ins_pipe(ialu_reg_long);
6621 %}
6622 
6623 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6624   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6625             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6626   match(Set dst (DecodeN src));
6627   effect(KILL cr);
6628   format %{ "decode_heap_oop $dst,$src" %}
6629   ins_encode %{
6630     Register s = $src$$Register;
6631     Register d = $dst$$Register;
6632     if (s != d) {
6633       __ movq(d, s);
6634     }
6635     __ decode_heap_oop(d);
6636   %}
6637   ins_pipe(ialu_reg_long);
6638 %}
6639 
6640 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6641   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6642             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6643   match(Set dst (DecodeN src));
6644   effect(KILL cr);
6645   format %{ "decode_heap_oop_not_null $dst,$src" %}
6646   ins_encode %{
6647     Register s = $src$$Register;
6648     Register d = $dst$$Register;
6649     if (s != d) {
6650       __ decode_heap_oop_not_null(d, s);
6651     } else {
6652       __ decode_heap_oop_not_null(d);
6653     }
6654   %}
6655   ins_pipe(ialu_reg_long);
6656 %}
6657 
6658 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6659   match(Set dst (EncodePKlass src));
6660   effect(KILL cr);
6661   format %{ "encode_klass_not_null $dst,$src" %}
6662   ins_encode %{
6663     __ encode_klass_not_null($dst$$Register, $src$$Register);
6664   %}
6665   ins_pipe(ialu_reg_long);
6666 %}
6667 
6668 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6669   match(Set dst (DecodeNKlass src));
6670   effect(KILL cr);
6671   format %{ "decode_klass_not_null $dst,$src" %}
6672   ins_encode %{
6673     Register s = $src$$Register;
6674     Register d = $dst$$Register;
6675     if (s != d) {
6676       __ decode_klass_not_null(d, s);
6677     } else {
6678       __ decode_klass_not_null(d);
6679     }
6680   %}
6681   ins_pipe(ialu_reg_long);
6682 %}
6683 
6684 
6685 //----------Conditional Move---------------------------------------------------
6686 // Jump
6687 // dummy instruction for generating temp registers
6688 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6689   match(Jump (LShiftL switch_val shift));
6690   ins_cost(350);
6691   predicate(false);
6692   effect(TEMP dest);
6693 
6694   format %{ "leaq    $dest, [$constantaddress]\n\t"
6695             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6696   ins_encode %{
6697     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6698     // to do that and the compiler is using that register as one it can allocate.
6699     // So we build it all by hand.
6700     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6701     // ArrayAddress dispatch(table, index);
6702     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6703     __ lea($dest$$Register, $constantaddress);
6704     __ jmp(dispatch);
6705   %}
6706   ins_pipe(pipe_jmp);
6707 %}
6708 
6709 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6710   match(Jump (AddL (LShiftL switch_val shift) offset));
6711   ins_cost(350);
6712   effect(TEMP dest);
6713 
6714   format %{ "leaq    $dest, [$constantaddress]\n\t"
6715             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6716   ins_encode %{
6717     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6718     // to do that and the compiler is using that register as one it can allocate.
6719     // So we build it all by hand.
6720     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6721     // ArrayAddress dispatch(table, index);
6722     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6723     __ lea($dest$$Register, $constantaddress);
6724     __ jmp(dispatch);
6725   %}
6726   ins_pipe(pipe_jmp);
6727 %}
6728 
6729 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6730   match(Jump switch_val);
6731   ins_cost(350);
6732   effect(TEMP dest);
6733 
6734   format %{ "leaq    $dest, [$constantaddress]\n\t"
6735             "jmp     [$dest + $switch_val]\n\t" %}
6736   ins_encode %{
6737     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6738     // to do that and the compiler is using that register as one it can allocate.
6739     // So we build it all by hand.
6740     // Address index(noreg, switch_reg, Address::times_1);
6741     // ArrayAddress dispatch(table, index);
6742     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6743     __ lea($dest$$Register, $constantaddress);
6744     __ jmp(dispatch);
6745   %}
6746   ins_pipe(pipe_jmp);
6747 %}
6748 
6749 // Conditional move
6750 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6751 %{
6752   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6753 
6754   ins_cost(200); // XXX
6755   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6756   opcode(0x0F, 0x40);
6757   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6758   ins_pipe(pipe_cmov_reg);
6759 %}
6760 
6761 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6762   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6763 
6764   ins_cost(200); // XXX
6765   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6766   opcode(0x0F, 0x40);
6767   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6768   ins_pipe(pipe_cmov_reg);
6769 %}
6770 
6771 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6772   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6773   ins_cost(200);
6774   expand %{
6775     cmovI_regU(cop, cr, dst, src);
6776   %}
6777 %}
6778 
6779 // Conditional move
6780 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6781   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6782 
6783   ins_cost(250); // XXX
6784   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6785   opcode(0x0F, 0x40);
6786   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6787   ins_pipe(pipe_cmov_mem);
6788 %}
6789 
6790 // Conditional move
6791 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6792 %{
6793   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6794 
6795   ins_cost(250); // XXX
6796   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6797   opcode(0x0F, 0x40);
6798   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6799   ins_pipe(pipe_cmov_mem);
6800 %}
6801 
6802 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6803   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6804   ins_cost(250);
6805   expand %{
6806     cmovI_memU(cop, cr, dst, src);
6807   %}
6808 %}
6809 
6810 // Conditional move
6811 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6812 %{
6813   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6814 
6815   ins_cost(200); // XXX
6816   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6817   opcode(0x0F, 0x40);
6818   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6819   ins_pipe(pipe_cmov_reg);
6820 %}
6821 
6822 // Conditional move
6823 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6824 %{
6825   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6826 
6827   ins_cost(200); // XXX
6828   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6829   opcode(0x0F, 0x40);
6830   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6831   ins_pipe(pipe_cmov_reg);
6832 %}
6833 
6834 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6835   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6836   ins_cost(200);
6837   expand %{
6838     cmovN_regU(cop, cr, dst, src);
6839   %}
6840 %}
6841 
6842 // Conditional move
6843 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6844 %{
6845   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6846 
6847   ins_cost(200); // XXX
6848   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6849   opcode(0x0F, 0x40);
6850   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6851   ins_pipe(pipe_cmov_reg);  // XXX
6852 %}
6853 
6854 // Conditional move
6855 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6856 %{
6857   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6858 
6859   ins_cost(200); // XXX
6860   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6861   opcode(0x0F, 0x40);
6862   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6863   ins_pipe(pipe_cmov_reg); // XXX
6864 %}
6865 
6866 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6867   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6868   ins_cost(200);
6869   expand %{
6870     cmovP_regU(cop, cr, dst, src);
6871   %}
6872 %}
6873 
6874 // DISABLED: Requires the ADLC to emit a bottom_type call that
6875 // correctly meets the two pointer arguments; one is an incoming
6876 // register but the other is a memory operand.  ALSO appears to
6877 // be buggy with implicit null checks.
6878 //
6879 //// Conditional move
6880 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6881 //%{
6882 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6883 //  ins_cost(250);
6884 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6885 //  opcode(0x0F,0x40);
6886 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6887 //  ins_pipe( pipe_cmov_mem );
6888 //%}
6889 //
6890 //// Conditional move
6891 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6892 //%{
6893 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6894 //  ins_cost(250);
6895 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6896 //  opcode(0x0F,0x40);
6897 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6898 //  ins_pipe( pipe_cmov_mem );
6899 //%}
6900 
6901 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6902 %{
6903   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6904 
6905   ins_cost(200); // XXX
6906   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6907   opcode(0x0F, 0x40);
6908   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6909   ins_pipe(pipe_cmov_reg);  // XXX
6910 %}
6911 
6912 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6913 %{
6914   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6915 
6916   ins_cost(200); // XXX
6917   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6918   opcode(0x0F, 0x40);
6919   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6920   ins_pipe(pipe_cmov_mem);  // XXX
6921 %}
6922 
6923 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6924 %{
6925   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6926 
6927   ins_cost(200); // XXX
6928   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6929   opcode(0x0F, 0x40);
6930   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6931   ins_pipe(pipe_cmov_reg); // XXX
6932 %}
6933 
6934 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6935   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6936   ins_cost(200);
6937   expand %{
6938     cmovL_regU(cop, cr, dst, src);
6939   %}
6940 %}
6941 
6942 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6943 %{
6944   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6945 
6946   ins_cost(200); // XXX
6947   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6948   opcode(0x0F, 0x40);
6949   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6950   ins_pipe(pipe_cmov_mem); // XXX
6951 %}
6952 
6953 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6954   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6955   ins_cost(200);
6956   expand %{
6957     cmovL_memU(cop, cr, dst, src);
6958   %}
6959 %}
6960 
6961 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6962 %{
6963   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6964 
6965   ins_cost(200); // XXX
6966   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6967             "movss     $dst, $src\n"
6968     "skip:" %}
6969   ins_encode %{
6970     Label Lskip;
6971     // Invert sense of branch from sense of CMOV
6972     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6973     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6974     __ bind(Lskip);
6975   %}
6976   ins_pipe(pipe_slow);
6977 %}
6978 
6979 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6980 // %{
6981 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6982 
6983 //   ins_cost(200); // XXX
6984 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6985 //             "movss     $dst, $src\n"
6986 //     "skip:" %}
6987 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6988 //   ins_pipe(pipe_slow);
6989 // %}
6990 
6991 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6992 %{
6993   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6994 
6995   ins_cost(200); // XXX
6996   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6997             "movss     $dst, $src\n"
6998     "skip:" %}
6999   ins_encode %{
7000     Label Lskip;
7001     // Invert sense of branch from sense of CMOV
7002     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7003     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7004     __ bind(Lskip);
7005   %}
7006   ins_pipe(pipe_slow);
7007 %}
7008 
7009 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
7010   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7011   ins_cost(200);
7012   expand %{
7013     cmovF_regU(cop, cr, dst, src);
7014   %}
7015 %}
7016 
7017 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
7018 %{
7019   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7020 
7021   ins_cost(200); // XXX
7022   format %{ "jn$cop    skip\t# signed cmove double\n\t"
7023             "movsd     $dst, $src\n"
7024     "skip:" %}
7025   ins_encode %{
7026     Label Lskip;
7027     // Invert sense of branch from sense of CMOV
7028     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7029     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7030     __ bind(Lskip);
7031   %}
7032   ins_pipe(pipe_slow);
7033 %}
7034 
7035 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
7036 %{
7037   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7038 
7039   ins_cost(200); // XXX
7040   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
7041             "movsd     $dst, $src\n"
7042     "skip:" %}
7043   ins_encode %{
7044     Label Lskip;
7045     // Invert sense of branch from sense of CMOV
7046     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7047     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7048     __ bind(Lskip);
7049   %}
7050   ins_pipe(pipe_slow);
7051 %}
7052 
7053 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
7054   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7055   ins_cost(200);
7056   expand %{
7057     cmovD_regU(cop, cr, dst, src);
7058   %}
7059 %}
7060 
7061 //----------Arithmetic Instructions--------------------------------------------
7062 //----------Addition Instructions----------------------------------------------
7063 
7064 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7065 %{
7066   match(Set dst (AddI dst src));
7067   effect(KILL cr);
7068 
7069   format %{ "addl    $dst, $src\t# int" %}
7070   opcode(0x03);
7071   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7072   ins_pipe(ialu_reg_reg);
7073 %}
7074 
7075 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7076 %{
7077   match(Set dst (AddI dst src));
7078   effect(KILL cr);
7079 
7080   format %{ "addl    $dst, $src\t# int" %}
7081   opcode(0x81, 0x00); /* /0 id */
7082   ins_encode(OpcSErm(dst, src), Con8or32(src));
7083   ins_pipe( ialu_reg );
7084 %}
7085 
7086 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7087 %{
7088   match(Set dst (AddI dst (LoadI src)));
7089   effect(KILL cr);
7090 
7091   ins_cost(125); // XXX
7092   format %{ "addl    $dst, $src\t# int" %}
7093   opcode(0x03);
7094   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7095   ins_pipe(ialu_reg_mem);
7096 %}
7097 
7098 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7099 %{
7100   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7101   effect(KILL cr);
7102 
7103   ins_cost(150); // XXX
7104   format %{ "addl    $dst, $src\t# int" %}
7105   opcode(0x01); /* Opcode 01 /r */
7106   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7107   ins_pipe(ialu_mem_reg);
7108 %}
7109 
7110 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
7111 %{
7112   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7113   effect(KILL cr);
7114 
7115   ins_cost(125); // XXX
7116   format %{ "addl    $dst, $src\t# int" %}
7117   opcode(0x81); /* Opcode 81 /0 id */
7118   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7119   ins_pipe(ialu_mem_imm);
7120 %}
7121 
7122 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7123 %{
7124   predicate(UseIncDec);
7125   match(Set dst (AddI dst src));
7126   effect(KILL cr);
7127 
7128   format %{ "incl    $dst\t# int" %}
7129   opcode(0xFF, 0x00); // FF /0
7130   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7131   ins_pipe(ialu_reg);
7132 %}
7133 
7134 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7135 %{
7136   predicate(UseIncDec);
7137   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7138   effect(KILL cr);
7139 
7140   ins_cost(125); // XXX
7141   format %{ "incl    $dst\t# int" %}
7142   opcode(0xFF); /* Opcode FF /0 */
7143   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7144   ins_pipe(ialu_mem_imm);
7145 %}
7146 
7147 // XXX why does that use AddI
7148 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7149 %{
7150   predicate(UseIncDec);
7151   match(Set dst (AddI dst src));
7152   effect(KILL cr);
7153 
7154   format %{ "decl    $dst\t# int" %}
7155   opcode(0xFF, 0x01); // FF /1
7156   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7157   ins_pipe(ialu_reg);
7158 %}
7159 
7160 // XXX why does that use AddI
7161 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7162 %{
7163   predicate(UseIncDec);
7164   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7165   effect(KILL cr);
7166 
7167   ins_cost(125); // XXX
7168   format %{ "decl    $dst\t# int" %}
7169   opcode(0xFF); /* Opcode FF /1 */
7170   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7171   ins_pipe(ialu_mem_imm);
7172 %}
7173 
7174 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7175 %{
7176   match(Set dst (AddI src0 src1));
7177 
7178   ins_cost(110);
7179   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7180   opcode(0x8D); /* 0x8D /r */
7181   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7182   ins_pipe(ialu_reg_reg);
7183 %}
7184 
7185 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7186 %{
7187   match(Set dst (AddL dst src));
7188   effect(KILL cr);
7189 
7190   format %{ "addq    $dst, $src\t# long" %}
7191   opcode(0x03);
7192   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7193   ins_pipe(ialu_reg_reg);
7194 %}
7195 
7196 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7197 %{
7198   match(Set dst (AddL dst src));
7199   effect(KILL cr);
7200 
7201   format %{ "addq    $dst, $src\t# long" %}
7202   opcode(0x81, 0x00); /* /0 id */
7203   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7204   ins_pipe( ialu_reg );
7205 %}
7206 
7207 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7208 %{
7209   match(Set dst (AddL dst (LoadL src)));
7210   effect(KILL cr);
7211 
7212   ins_cost(125); // XXX
7213   format %{ "addq    $dst, $src\t# long" %}
7214   opcode(0x03);
7215   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7216   ins_pipe(ialu_reg_mem);
7217 %}
7218 
7219 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7220 %{
7221   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7222   effect(KILL cr);
7223 
7224   ins_cost(150); // XXX
7225   format %{ "addq    $dst, $src\t# long" %}
7226   opcode(0x01); /* Opcode 01 /r */
7227   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7228   ins_pipe(ialu_mem_reg);
7229 %}
7230 
7231 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7232 %{
7233   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7234   effect(KILL cr);
7235 
7236   ins_cost(125); // XXX
7237   format %{ "addq    $dst, $src\t# long" %}
7238   opcode(0x81); /* Opcode 81 /0 id */
7239   ins_encode(REX_mem_wide(dst),
7240              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7241   ins_pipe(ialu_mem_imm);
7242 %}
7243 
7244 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7245 %{
7246   predicate(UseIncDec);
7247   match(Set dst (AddL dst src));
7248   effect(KILL cr);
7249 
7250   format %{ "incq    $dst\t# long" %}
7251   opcode(0xFF, 0x00); // FF /0
7252   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7253   ins_pipe(ialu_reg);
7254 %}
7255 
7256 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7257 %{
7258   predicate(UseIncDec);
7259   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7260   effect(KILL cr);
7261 
7262   ins_cost(125); // XXX
7263   format %{ "incq    $dst\t# long" %}
7264   opcode(0xFF); /* Opcode FF /0 */
7265   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7266   ins_pipe(ialu_mem_imm);
7267 %}
7268 
7269 // XXX why does that use AddL
7270 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7271 %{
7272   predicate(UseIncDec);
7273   match(Set dst (AddL dst src));
7274   effect(KILL cr);
7275 
7276   format %{ "decq    $dst\t# long" %}
7277   opcode(0xFF, 0x01); // FF /1
7278   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7279   ins_pipe(ialu_reg);
7280 %}
7281 
7282 // XXX why does that use AddL
7283 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7284 %{
7285   predicate(UseIncDec);
7286   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7287   effect(KILL cr);
7288 
7289   ins_cost(125); // XXX
7290   format %{ "decq    $dst\t# long" %}
7291   opcode(0xFF); /* Opcode FF /1 */
7292   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7293   ins_pipe(ialu_mem_imm);
7294 %}
7295 
7296 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7297 %{
7298   match(Set dst (AddL src0 src1));
7299 
7300   ins_cost(110);
7301   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7302   opcode(0x8D); /* 0x8D /r */
7303   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7304   ins_pipe(ialu_reg_reg);
7305 %}
7306 
7307 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7308 %{
7309   match(Set dst (AddP dst src));
7310   effect(KILL cr);
7311 
7312   format %{ "addq    $dst, $src\t# ptr" %}
7313   opcode(0x03);
7314   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7315   ins_pipe(ialu_reg_reg);
7316 %}
7317 
7318 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7319 %{
7320   match(Set dst (AddP dst src));
7321   effect(KILL cr);
7322 
7323   format %{ "addq    $dst, $src\t# ptr" %}
7324   opcode(0x81, 0x00); /* /0 id */
7325   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7326   ins_pipe( ialu_reg );
7327 %}
7328 
7329 // XXX addP mem ops ????
7330 
7331 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7332 %{
7333   match(Set dst (AddP src0 src1));
7334 
7335   ins_cost(110);
7336   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7337   opcode(0x8D); /* 0x8D /r */
7338   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7339   ins_pipe(ialu_reg_reg);
7340 %}
7341 
7342 instruct checkCastPP(rRegP dst)
7343 %{
7344   match(Set dst (CheckCastPP dst));
7345 
7346   size(0);
7347   format %{ "# checkcastPP of $dst" %}
7348   ins_encode(/* empty encoding */);
7349   ins_pipe(empty);
7350 %}
7351 
7352 instruct castPP(rRegP dst)
7353 %{
7354   match(Set dst (CastPP dst));
7355 
7356   size(0);
7357   format %{ "# castPP of $dst" %}
7358   ins_encode(/* empty encoding */);
7359   ins_pipe(empty);
7360 %}
7361 
7362 instruct castII(rRegI dst)
7363 %{
7364   match(Set dst (CastII dst));
7365 
7366   size(0);
7367   format %{ "# castII of $dst" %}
7368   ins_encode(/* empty encoding */);
7369   ins_cost(0);
7370   ins_pipe(empty);
7371 %}
7372 
7373 // LoadP-locked same as a regular LoadP when used with compare-swap
7374 instruct loadPLocked(rRegP dst, memory mem)
7375 %{
7376   match(Set dst (LoadPLocked mem));
7377 
7378   ins_cost(125); // XXX
7379   format %{ "movq    $dst, $mem\t# ptr locked" %}
7380   opcode(0x8B);
7381   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7382   ins_pipe(ialu_reg_mem); // XXX
7383 %}
7384 
7385 // Conditional-store of the updated heap-top.
7386 // Used during allocation of the shared heap.
7387 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7388 
7389 instruct storePConditional(memory heap_top_ptr,
7390                            rax_RegP oldval, rRegP newval,
7391                            rFlagsReg cr)
7392 %{
7393   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7394 
7395   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7396             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7397   opcode(0x0F, 0xB1);
7398   ins_encode(lock_prefix,
7399              REX_reg_mem_wide(newval, heap_top_ptr),
7400              OpcP, OpcS,
7401              reg_mem(newval, heap_top_ptr));
7402   ins_pipe(pipe_cmpxchg);
7403 %}
7404 
7405 // Conditional-store of an int value.
7406 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7407 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7408 %{
7409   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7410   effect(KILL oldval);
7411 
7412   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7413   opcode(0x0F, 0xB1);
7414   ins_encode(lock_prefix,
7415              REX_reg_mem(newval, mem),
7416              OpcP, OpcS,
7417              reg_mem(newval, mem));
7418   ins_pipe(pipe_cmpxchg);
7419 %}
7420 
7421 // Conditional-store of a long value.
7422 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7423 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7424 %{
7425   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7426   effect(KILL oldval);
7427 
7428   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7429   opcode(0x0F, 0xB1);
7430   ins_encode(lock_prefix,
7431              REX_reg_mem_wide(newval, mem),
7432              OpcP, OpcS,
7433              reg_mem(newval, mem));
7434   ins_pipe(pipe_cmpxchg);
7435 %}
7436 
7437 
7438 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7439 instruct compareAndSwapP(rRegI res,
7440                          memory mem_ptr,
7441                          rax_RegP oldval, rRegP newval,
7442                          rFlagsReg cr)
7443 %{
7444   predicate(VM_Version::supports_cx8());
7445   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7446   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7447   effect(KILL cr, KILL oldval);
7448 
7449   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7450             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7451             "sete    $res\n\t"
7452             "movzbl  $res, $res" %}
7453   opcode(0x0F, 0xB1);
7454   ins_encode(lock_prefix,
7455              REX_reg_mem_wide(newval, mem_ptr),
7456              OpcP, OpcS,
7457              reg_mem(newval, mem_ptr),
7458              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7459              REX_reg_breg(res, res), // movzbl
7460              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7461   ins_pipe( pipe_cmpxchg );
7462 %}
7463 
7464 instruct compareAndSwapL(rRegI res,
7465                          memory mem_ptr,
7466                          rax_RegL oldval, rRegL newval,
7467                          rFlagsReg cr)
7468 %{
7469   predicate(VM_Version::supports_cx8());
7470   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7471   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7472   effect(KILL cr, KILL oldval);
7473 
7474   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7475             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7476             "sete    $res\n\t"
7477             "movzbl  $res, $res" %}
7478   opcode(0x0F, 0xB1);
7479   ins_encode(lock_prefix,
7480              REX_reg_mem_wide(newval, mem_ptr),
7481              OpcP, OpcS,
7482              reg_mem(newval, mem_ptr),
7483              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7484              REX_reg_breg(res, res), // movzbl
7485              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7486   ins_pipe( pipe_cmpxchg );
7487 %}
7488 
7489 instruct compareAndSwapI(rRegI res,
7490                          memory mem_ptr,
7491                          rax_RegI oldval, rRegI newval,
7492                          rFlagsReg cr)
7493 %{
7494   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7495   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7496   effect(KILL cr, KILL oldval);
7497 
7498   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7499             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7500             "sete    $res\n\t"
7501             "movzbl  $res, $res" %}
7502   opcode(0x0F, 0xB1);
7503   ins_encode(lock_prefix,
7504              REX_reg_mem(newval, mem_ptr),
7505              OpcP, OpcS,
7506              reg_mem(newval, mem_ptr),
7507              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7508              REX_reg_breg(res, res), // movzbl
7509              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7510   ins_pipe( pipe_cmpxchg );
7511 %}
7512 
7513 instruct compareAndSwapB(rRegI res,
7514                          memory mem_ptr,
7515                          rax_RegI oldval, rRegI newval,
7516                          rFlagsReg cr)
7517 %{
7518   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7519   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7520   effect(KILL cr, KILL oldval);
7521 
7522   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7523             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7524             "sete    $res\n\t"
7525             "movzbl  $res, $res" %}
7526   opcode(0x0F, 0xB0);
7527   ins_encode(lock_prefix,
7528              REX_breg_mem(newval, mem_ptr),
7529              OpcP, OpcS,
7530              reg_mem(newval, mem_ptr),
7531              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7532              REX_reg_breg(res, res), // movzbl
7533              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7534   ins_pipe( pipe_cmpxchg );
7535 %}
7536 
7537 instruct compareAndSwapS(rRegI res,
7538                          memory mem_ptr,
7539                          rax_RegI oldval, rRegI newval,
7540                          rFlagsReg cr)
7541 %{
7542   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7543   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7544   effect(KILL cr, KILL oldval);
7545 
7546   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7547             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7548             "sete    $res\n\t"
7549             "movzbl  $res, $res" %}
7550   opcode(0x0F, 0xB1);
7551   ins_encode(lock_prefix,
7552              SizePrefix,
7553              REX_reg_mem(newval, mem_ptr),
7554              OpcP, OpcS,
7555              reg_mem(newval, mem_ptr),
7556              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7557              REX_reg_breg(res, res), // movzbl
7558              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7559   ins_pipe( pipe_cmpxchg );
7560 %}
7561 
7562 instruct compareAndSwapN(rRegI res,
7563                           memory mem_ptr,
7564                           rax_RegN oldval, rRegN newval,
7565                           rFlagsReg cr) %{
7566   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7567   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7568   effect(KILL cr, KILL oldval);
7569 
7570   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7571             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7572             "sete    $res\n\t"
7573             "movzbl  $res, $res" %}
7574   opcode(0x0F, 0xB1);
7575   ins_encode(lock_prefix,
7576              REX_reg_mem(newval, mem_ptr),
7577              OpcP, OpcS,
7578              reg_mem(newval, mem_ptr),
7579              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7580              REX_reg_breg(res, res), // movzbl
7581              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7582   ins_pipe( pipe_cmpxchg );
7583 %}
7584 
7585 instruct compareAndExchangeB(
7586                          memory mem_ptr,
7587                          rax_RegI oldval, rRegI newval,
7588                          rFlagsReg cr)
7589 %{
7590   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
7591   effect(KILL cr);
7592 
7593   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7594             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7595   opcode(0x0F, 0xB0);
7596   ins_encode(lock_prefix,
7597              REX_breg_mem(newval, mem_ptr),
7598              OpcP, OpcS,
7599              reg_mem(newval, mem_ptr) // lock cmpxchg
7600              );
7601   ins_pipe( pipe_cmpxchg );
7602 %}
7603 
7604 instruct compareAndExchangeS(
7605                          memory mem_ptr,
7606                          rax_RegI oldval, rRegI newval,
7607                          rFlagsReg cr)
7608 %{
7609   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
7610   effect(KILL cr);
7611 
7612   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7613             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7614   opcode(0x0F, 0xB1);
7615   ins_encode(lock_prefix,
7616              SizePrefix,
7617              REX_reg_mem(newval, mem_ptr),
7618              OpcP, OpcS,
7619              reg_mem(newval, mem_ptr) // lock cmpxchg
7620              );
7621   ins_pipe( pipe_cmpxchg );
7622 %}
7623 
7624 instruct compareAndExchangeI(
7625                          memory mem_ptr,
7626                          rax_RegI oldval, rRegI newval,
7627                          rFlagsReg cr)
7628 %{
7629   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7630   effect(KILL cr);
7631 
7632   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7633             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7634   opcode(0x0F, 0xB1);
7635   ins_encode(lock_prefix,
7636              REX_reg_mem(newval, mem_ptr),
7637              OpcP, OpcS,
7638              reg_mem(newval, mem_ptr) // lock cmpxchg
7639              );
7640   ins_pipe( pipe_cmpxchg );
7641 %}
7642 
7643 instruct compareAndExchangeL(
7644                          memory mem_ptr,
7645                          rax_RegL oldval, rRegL newval,
7646                          rFlagsReg cr)
7647 %{
7648   predicate(VM_Version::supports_cx8());
7649   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
7650   effect(KILL cr);
7651 
7652   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7653             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7654   opcode(0x0F, 0xB1);
7655   ins_encode(lock_prefix,
7656              REX_reg_mem_wide(newval, mem_ptr),
7657              OpcP, OpcS,
7658              reg_mem(newval, mem_ptr)  // lock cmpxchg
7659             );
7660   ins_pipe( pipe_cmpxchg );
7661 %}
7662 
7663 instruct compareAndExchangeN(
7664                           memory mem_ptr,
7665                           rax_RegN oldval, rRegN newval,
7666                           rFlagsReg cr) %{
7667   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
7668   effect(KILL cr);
7669 
7670   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7671             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7672   opcode(0x0F, 0xB1);
7673   ins_encode(lock_prefix,
7674              REX_reg_mem(newval, mem_ptr),
7675              OpcP, OpcS,
7676              reg_mem(newval, mem_ptr)  // lock cmpxchg
7677           );
7678   ins_pipe( pipe_cmpxchg );
7679 %}
7680 
7681 instruct compareAndExchangeP(
7682                          memory mem_ptr,
7683                          rax_RegP oldval, rRegP newval,
7684                          rFlagsReg cr)
7685 %{
7686   predicate(VM_Version::supports_cx8());
7687   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
7688   effect(KILL cr);
7689 
7690   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7691             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7692   opcode(0x0F, 0xB1);
7693   ins_encode(lock_prefix,
7694              REX_reg_mem_wide(newval, mem_ptr),
7695              OpcP, OpcS,
7696              reg_mem(newval, mem_ptr)  // lock cmpxchg
7697           );
7698   ins_pipe( pipe_cmpxchg );
7699 %}
7700 
7701 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7702   predicate(n->as_LoadStore()->result_not_used());
7703   match(Set dummy (GetAndAddB mem add));
7704   effect(KILL cr);
7705   format %{ "ADDB  [$mem],$add" %}
7706   ins_encode %{
7707     __ lock();
7708     __ addb($mem$$Address, $add$$constant);
7709   %}
7710   ins_pipe( pipe_cmpxchg );
7711 %}
7712 
7713 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
7714   match(Set newval (GetAndAddB mem newval));
7715   effect(KILL cr);
7716   format %{ "XADDB  [$mem],$newval" %}
7717   ins_encode %{
7718     __ lock();
7719     __ xaddb($mem$$Address, $newval$$Register);
7720   %}
7721   ins_pipe( pipe_cmpxchg );
7722 %}
7723 
7724 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7725   predicate(n->as_LoadStore()->result_not_used());
7726   match(Set dummy (GetAndAddS mem add));
7727   effect(KILL cr);
7728   format %{ "ADDW  [$mem],$add" %}
7729   ins_encode %{
7730     __ lock();
7731     __ addw($mem$$Address, $add$$constant);
7732   %}
7733   ins_pipe( pipe_cmpxchg );
7734 %}
7735 
7736 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
7737   match(Set newval (GetAndAddS mem newval));
7738   effect(KILL cr);
7739   format %{ "XADDW  [$mem],$newval" %}
7740   ins_encode %{
7741     __ lock();
7742     __ xaddw($mem$$Address, $newval$$Register);
7743   %}
7744   ins_pipe( pipe_cmpxchg );
7745 %}
7746 
7747 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7748   predicate(n->as_LoadStore()->result_not_used());
7749   match(Set dummy (GetAndAddI mem add));
7750   effect(KILL cr);
7751   format %{ "ADDL  [$mem],$add" %}
7752   ins_encode %{
7753     __ lock();
7754     __ addl($mem$$Address, $add$$constant);
7755   %}
7756   ins_pipe( pipe_cmpxchg );
7757 %}
7758 
7759 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7760   match(Set newval (GetAndAddI mem newval));
7761   effect(KILL cr);
7762   format %{ "XADDL  [$mem],$newval" %}
7763   ins_encode %{
7764     __ lock();
7765     __ xaddl($mem$$Address, $newval$$Register);
7766   %}
7767   ins_pipe( pipe_cmpxchg );
7768 %}
7769 
7770 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7771   predicate(n->as_LoadStore()->result_not_used());
7772   match(Set dummy (GetAndAddL mem add));
7773   effect(KILL cr);
7774   format %{ "ADDQ  [$mem],$add" %}
7775   ins_encode %{
7776     __ lock();
7777     __ addq($mem$$Address, $add$$constant);
7778   %}
7779   ins_pipe( pipe_cmpxchg );
7780 %}
7781 
7782 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7783   match(Set newval (GetAndAddL mem newval));
7784   effect(KILL cr);
7785   format %{ "XADDQ  [$mem],$newval" %}
7786   ins_encode %{
7787     __ lock();
7788     __ xaddq($mem$$Address, $newval$$Register);
7789   %}
7790   ins_pipe( pipe_cmpxchg );
7791 %}
7792 
7793 instruct xchgB( memory mem, rRegI newval) %{
7794   match(Set newval (GetAndSetB mem newval));
7795   format %{ "XCHGB  $newval,[$mem]" %}
7796   ins_encode %{
7797     __ xchgb($newval$$Register, $mem$$Address);
7798   %}
7799   ins_pipe( pipe_cmpxchg );
7800 %}
7801 
7802 instruct xchgS( memory mem, rRegI newval) %{
7803   match(Set newval (GetAndSetS mem newval));
7804   format %{ "XCHGW  $newval,[$mem]" %}
7805   ins_encode %{
7806     __ xchgw($newval$$Register, $mem$$Address);
7807   %}
7808   ins_pipe( pipe_cmpxchg );
7809 %}
7810 
7811 instruct xchgI( memory mem, rRegI newval) %{
7812   match(Set newval (GetAndSetI mem newval));
7813   format %{ "XCHGL  $newval,[$mem]" %}
7814   ins_encode %{
7815     __ xchgl($newval$$Register, $mem$$Address);
7816   %}
7817   ins_pipe( pipe_cmpxchg );
7818 %}
7819 
7820 instruct xchgL( memory mem, rRegL newval) %{
7821   match(Set newval (GetAndSetL mem newval));
7822   format %{ "XCHGL  $newval,[$mem]" %}
7823   ins_encode %{
7824     __ xchgq($newval$$Register, $mem$$Address);
7825   %}
7826   ins_pipe( pipe_cmpxchg );
7827 %}
7828 
7829 instruct xchgP( memory mem, rRegP newval) %{
7830   match(Set newval (GetAndSetP mem newval));
7831   format %{ "XCHGQ  $newval,[$mem]" %}
7832   ins_encode %{
7833     __ xchgq($newval$$Register, $mem$$Address);
7834   %}
7835   ins_pipe( pipe_cmpxchg );
7836 %}
7837 
7838 instruct xchgN( memory mem, rRegN newval) %{
7839   match(Set newval (GetAndSetN mem newval));
7840   format %{ "XCHGL  $newval,$mem]" %}
7841   ins_encode %{
7842     __ xchgl($newval$$Register, $mem$$Address);
7843   %}
7844   ins_pipe( pipe_cmpxchg );
7845 %}
7846 
7847 //----------Subtraction Instructions-------------------------------------------
7848 
7849 // Integer Subtraction Instructions
7850 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7851 %{
7852   match(Set dst (SubI dst src));
7853   effect(KILL cr);
7854 
7855   format %{ "subl    $dst, $src\t# int" %}
7856   opcode(0x2B);
7857   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7858   ins_pipe(ialu_reg_reg);
7859 %}
7860 
7861 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7862 %{
7863   match(Set dst (SubI dst src));
7864   effect(KILL cr);
7865 
7866   format %{ "subl    $dst, $src\t# int" %}
7867   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7868   ins_encode(OpcSErm(dst, src), Con8or32(src));
7869   ins_pipe(ialu_reg);
7870 %}
7871 
7872 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7873 %{
7874   match(Set dst (SubI dst (LoadI src)));
7875   effect(KILL cr);
7876 
7877   ins_cost(125);
7878   format %{ "subl    $dst, $src\t# int" %}
7879   opcode(0x2B);
7880   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7881   ins_pipe(ialu_reg_mem);
7882 %}
7883 
7884 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7885 %{
7886   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7887   effect(KILL cr);
7888 
7889   ins_cost(150);
7890   format %{ "subl    $dst, $src\t# int" %}
7891   opcode(0x29); /* Opcode 29 /r */
7892   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7893   ins_pipe(ialu_mem_reg);
7894 %}
7895 
7896 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7897 %{
7898   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7899   effect(KILL cr);
7900 
7901   ins_cost(125); // XXX
7902   format %{ "subl    $dst, $src\t# int" %}
7903   opcode(0x81); /* Opcode 81 /5 id */
7904   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7905   ins_pipe(ialu_mem_imm);
7906 %}
7907 
7908 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7909 %{
7910   match(Set dst (SubL dst src));
7911   effect(KILL cr);
7912 
7913   format %{ "subq    $dst, $src\t# long" %}
7914   opcode(0x2B);
7915   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7916   ins_pipe(ialu_reg_reg);
7917 %}
7918 
7919 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7920 %{
7921   match(Set dst (SubL dst src));
7922   effect(KILL cr);
7923 
7924   format %{ "subq    $dst, $src\t# long" %}
7925   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7926   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7927   ins_pipe(ialu_reg);
7928 %}
7929 
7930 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7931 %{
7932   match(Set dst (SubL dst (LoadL src)));
7933   effect(KILL cr);
7934 
7935   ins_cost(125);
7936   format %{ "subq    $dst, $src\t# long" %}
7937   opcode(0x2B);
7938   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7939   ins_pipe(ialu_reg_mem);
7940 %}
7941 
7942 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7943 %{
7944   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7945   effect(KILL cr);
7946 
7947   ins_cost(150);
7948   format %{ "subq    $dst, $src\t# long" %}
7949   opcode(0x29); /* Opcode 29 /r */
7950   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7951   ins_pipe(ialu_mem_reg);
7952 %}
7953 
7954 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7955 %{
7956   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7957   effect(KILL cr);
7958 
7959   ins_cost(125); // XXX
7960   format %{ "subq    $dst, $src\t# long" %}
7961   opcode(0x81); /* Opcode 81 /5 id */
7962   ins_encode(REX_mem_wide(dst),
7963              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7964   ins_pipe(ialu_mem_imm);
7965 %}
7966 
7967 // Subtract from a pointer
7968 // XXX hmpf???
7969 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7970 %{
7971   match(Set dst (AddP dst (SubI zero src)));
7972   effect(KILL cr);
7973 
7974   format %{ "subq    $dst, $src\t# ptr - int" %}
7975   opcode(0x2B);
7976   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7977   ins_pipe(ialu_reg_reg);
7978 %}
7979 
7980 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7981 %{
7982   match(Set dst (SubI zero dst));
7983   effect(KILL cr);
7984 
7985   format %{ "negl    $dst\t# int" %}
7986   opcode(0xF7, 0x03);  // Opcode F7 /3
7987   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7988   ins_pipe(ialu_reg);
7989 %}
7990 
7991 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7992 %{
7993   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7994   effect(KILL cr);
7995 
7996   format %{ "negl    $dst\t# int" %}
7997   opcode(0xF7, 0x03);  // Opcode F7 /3
7998   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7999   ins_pipe(ialu_reg);
8000 %}
8001 
8002 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
8003 %{
8004   match(Set dst (SubL zero dst));
8005   effect(KILL cr);
8006 
8007   format %{ "negq    $dst\t# long" %}
8008   opcode(0xF7, 0x03);  // Opcode F7 /3
8009   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8010   ins_pipe(ialu_reg);
8011 %}
8012 
8013 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
8014 %{
8015   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
8016   effect(KILL cr);
8017 
8018   format %{ "negq    $dst\t# long" %}
8019   opcode(0xF7, 0x03);  // Opcode F7 /3
8020   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8021   ins_pipe(ialu_reg);
8022 %}
8023 
8024 //----------Multiplication/Division Instructions-------------------------------
8025 // Integer Multiplication Instructions
8026 // Multiply Register
8027 
8028 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8029 %{
8030   match(Set dst (MulI dst src));
8031   effect(KILL cr);
8032 
8033   ins_cost(300);
8034   format %{ "imull   $dst, $src\t# int" %}
8035   opcode(0x0F, 0xAF);
8036   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8037   ins_pipe(ialu_reg_reg_alu0);
8038 %}
8039 
8040 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
8041 %{
8042   match(Set dst (MulI src imm));
8043   effect(KILL cr);
8044 
8045   ins_cost(300);
8046   format %{ "imull   $dst, $src, $imm\t# int" %}
8047   opcode(0x69); /* 69 /r id */
8048   ins_encode(REX_reg_reg(dst, src),
8049              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8050   ins_pipe(ialu_reg_reg_alu0);
8051 %}
8052 
8053 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
8054 %{
8055   match(Set dst (MulI dst (LoadI src)));
8056   effect(KILL cr);
8057 
8058   ins_cost(350);
8059   format %{ "imull   $dst, $src\t# int" %}
8060   opcode(0x0F, 0xAF);
8061   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
8062   ins_pipe(ialu_reg_mem_alu0);
8063 %}
8064 
8065 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
8066 %{
8067   match(Set dst (MulI (LoadI src) imm));
8068   effect(KILL cr);
8069 
8070   ins_cost(300);
8071   format %{ "imull   $dst, $src, $imm\t# int" %}
8072   opcode(0x69); /* 69 /r id */
8073   ins_encode(REX_reg_mem(dst, src),
8074              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8075   ins_pipe(ialu_reg_mem_alu0);
8076 %}
8077 
8078 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8079 %{
8080   match(Set dst (MulL dst src));
8081   effect(KILL cr);
8082 
8083   ins_cost(300);
8084   format %{ "imulq   $dst, $src\t# long" %}
8085   opcode(0x0F, 0xAF);
8086   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
8087   ins_pipe(ialu_reg_reg_alu0);
8088 %}
8089 
8090 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
8091 %{
8092   match(Set dst (MulL src imm));
8093   effect(KILL cr);
8094 
8095   ins_cost(300);
8096   format %{ "imulq   $dst, $src, $imm\t# long" %}
8097   opcode(0x69); /* 69 /r id */
8098   ins_encode(REX_reg_reg_wide(dst, src),
8099              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8100   ins_pipe(ialu_reg_reg_alu0);
8101 %}
8102 
8103 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
8104 %{
8105   match(Set dst (MulL dst (LoadL src)));
8106   effect(KILL cr);
8107 
8108   ins_cost(350);
8109   format %{ "imulq   $dst, $src\t# long" %}
8110   opcode(0x0F, 0xAF);
8111   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
8112   ins_pipe(ialu_reg_mem_alu0);
8113 %}
8114 
8115 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
8116 %{
8117   match(Set dst (MulL (LoadL src) imm));
8118   effect(KILL cr);
8119 
8120   ins_cost(300);
8121   format %{ "imulq   $dst, $src, $imm\t# long" %}
8122   opcode(0x69); /* 69 /r id */
8123   ins_encode(REX_reg_mem_wide(dst, src),
8124              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8125   ins_pipe(ialu_reg_mem_alu0);
8126 %}
8127 
8128 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8129 %{
8130   match(Set dst (MulHiL src rax));
8131   effect(USE_KILL rax, KILL cr);
8132 
8133   ins_cost(300);
8134   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
8135   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8136   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8137   ins_pipe(ialu_reg_reg_alu0);
8138 %}
8139 
8140 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8141                    rFlagsReg cr)
8142 %{
8143   match(Set rax (DivI rax div));
8144   effect(KILL rdx, KILL cr);
8145 
8146   ins_cost(30*100+10*100); // XXX
8147   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8148             "jne,s   normal\n\t"
8149             "xorl    rdx, rdx\n\t"
8150             "cmpl    $div, -1\n\t"
8151             "je,s    done\n"
8152     "normal: cdql\n\t"
8153             "idivl   $div\n"
8154     "done:"        %}
8155   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8156   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8157   ins_pipe(ialu_reg_reg_alu0);
8158 %}
8159 
8160 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8161                    rFlagsReg cr)
8162 %{
8163   match(Set rax (DivL rax div));
8164   effect(KILL rdx, KILL cr);
8165 
8166   ins_cost(30*100+10*100); // XXX
8167   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8168             "cmpq    rax, rdx\n\t"
8169             "jne,s   normal\n\t"
8170             "xorl    rdx, rdx\n\t"
8171             "cmpq    $div, -1\n\t"
8172             "je,s    done\n"
8173     "normal: cdqq\n\t"
8174             "idivq   $div\n"
8175     "done:"        %}
8176   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8177   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8178   ins_pipe(ialu_reg_reg_alu0);
8179 %}
8180 
8181 // Integer DIVMOD with Register, both quotient and mod results
8182 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8183                              rFlagsReg cr)
8184 %{
8185   match(DivModI rax div);
8186   effect(KILL cr);
8187 
8188   ins_cost(30*100+10*100); // XXX
8189   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8190             "jne,s   normal\n\t"
8191             "xorl    rdx, rdx\n\t"
8192             "cmpl    $div, -1\n\t"
8193             "je,s    done\n"
8194     "normal: cdql\n\t"
8195             "idivl   $div\n"
8196     "done:"        %}
8197   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8198   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8199   ins_pipe(pipe_slow);
8200 %}
8201 
8202 // Long DIVMOD with Register, both quotient and mod results
8203 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8204                              rFlagsReg cr)
8205 %{
8206   match(DivModL rax div);
8207   effect(KILL cr);
8208 
8209   ins_cost(30*100+10*100); // XXX
8210   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8211             "cmpq    rax, rdx\n\t"
8212             "jne,s   normal\n\t"
8213             "xorl    rdx, rdx\n\t"
8214             "cmpq    $div, -1\n\t"
8215             "je,s    done\n"
8216     "normal: cdqq\n\t"
8217             "idivq   $div\n"
8218     "done:"        %}
8219   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8220   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8221   ins_pipe(pipe_slow);
8222 %}
8223 
8224 //----------- DivL-By-Constant-Expansions--------------------------------------
8225 // DivI cases are handled by the compiler
8226 
8227 // Magic constant, reciprocal of 10
8228 instruct loadConL_0x6666666666666667(rRegL dst)
8229 %{
8230   effect(DEF dst);
8231 
8232   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8233   ins_encode(load_immL(dst, 0x6666666666666667));
8234   ins_pipe(ialu_reg);
8235 %}
8236 
8237 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8238 %{
8239   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8240 
8241   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8242   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8243   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8244   ins_pipe(ialu_reg_reg_alu0);
8245 %}
8246 
8247 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8248 %{
8249   effect(USE_DEF dst, KILL cr);
8250 
8251   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8252   opcode(0xC1, 0x7); /* C1 /7 ib */
8253   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8254   ins_pipe(ialu_reg);
8255 %}
8256 
8257 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8258 %{
8259   effect(USE_DEF dst, KILL cr);
8260 
8261   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8262   opcode(0xC1, 0x7); /* C1 /7 ib */
8263   ins_encode(reg_opc_imm_wide(dst, 0x2));
8264   ins_pipe(ialu_reg);
8265 %}
8266 
8267 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8268 %{
8269   match(Set dst (DivL src div));
8270 
8271   ins_cost((5+8)*100);
8272   expand %{
8273     rax_RegL rax;                     // Killed temp
8274     rFlagsReg cr;                     // Killed
8275     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8276     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8277     sarL_rReg_63(src, cr);            // sarq  src, 63
8278     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8279     subL_rReg(dst, src, cr);          // subl  rdx, src
8280   %}
8281 %}
8282 
8283 //-----------------------------------------------------------------------------
8284 
8285 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8286                    rFlagsReg cr)
8287 %{
8288   match(Set rdx (ModI rax div));
8289   effect(KILL rax, KILL cr);
8290 
8291   ins_cost(300); // XXX
8292   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8293             "jne,s   normal\n\t"
8294             "xorl    rdx, rdx\n\t"
8295             "cmpl    $div, -1\n\t"
8296             "je,s    done\n"
8297     "normal: cdql\n\t"
8298             "idivl   $div\n"
8299     "done:"        %}
8300   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8301   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8302   ins_pipe(ialu_reg_reg_alu0);
8303 %}
8304 
8305 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8306                    rFlagsReg cr)
8307 %{
8308   match(Set rdx (ModL rax div));
8309   effect(KILL rax, KILL cr);
8310 
8311   ins_cost(300); // XXX
8312   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8313             "cmpq    rax, rdx\n\t"
8314             "jne,s   normal\n\t"
8315             "xorl    rdx, rdx\n\t"
8316             "cmpq    $div, -1\n\t"
8317             "je,s    done\n"
8318     "normal: cdqq\n\t"
8319             "idivq   $div\n"
8320     "done:"        %}
8321   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8322   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8323   ins_pipe(ialu_reg_reg_alu0);
8324 %}
8325 
8326 // Integer Shift Instructions
8327 // Shift Left by one
8328 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8329 %{
8330   match(Set dst (LShiftI dst shift));
8331   effect(KILL cr);
8332 
8333   format %{ "sall    $dst, $shift" %}
8334   opcode(0xD1, 0x4); /* D1 /4 */
8335   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8336   ins_pipe(ialu_reg);
8337 %}
8338 
8339 // Shift Left by one
8340 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8341 %{
8342   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8343   effect(KILL cr);
8344 
8345   format %{ "sall    $dst, $shift\t" %}
8346   opcode(0xD1, 0x4); /* D1 /4 */
8347   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8348   ins_pipe(ialu_mem_imm);
8349 %}
8350 
8351 // Shift Left by 8-bit immediate
8352 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8353 %{
8354   match(Set dst (LShiftI dst shift));
8355   effect(KILL cr);
8356 
8357   format %{ "sall    $dst, $shift" %}
8358   opcode(0xC1, 0x4); /* C1 /4 ib */
8359   ins_encode(reg_opc_imm(dst, shift));
8360   ins_pipe(ialu_reg);
8361 %}
8362 
8363 // Shift Left by 8-bit immediate
8364 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8365 %{
8366   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8367   effect(KILL cr);
8368 
8369   format %{ "sall    $dst, $shift" %}
8370   opcode(0xC1, 0x4); /* C1 /4 ib */
8371   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8372   ins_pipe(ialu_mem_imm);
8373 %}
8374 
8375 // Shift Left by variable
8376 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8377 %{
8378   match(Set dst (LShiftI dst shift));
8379   effect(KILL cr);
8380 
8381   format %{ "sall    $dst, $shift" %}
8382   opcode(0xD3, 0x4); /* D3 /4 */
8383   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8384   ins_pipe(ialu_reg_reg);
8385 %}
8386 
8387 // Shift Left by variable
8388 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8389 %{
8390   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8391   effect(KILL cr);
8392 
8393   format %{ "sall    $dst, $shift" %}
8394   opcode(0xD3, 0x4); /* D3 /4 */
8395   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8396   ins_pipe(ialu_mem_reg);
8397 %}
8398 
8399 // Arithmetic shift right by one
8400 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8401 %{
8402   match(Set dst (RShiftI dst shift));
8403   effect(KILL cr);
8404 
8405   format %{ "sarl    $dst, $shift" %}
8406   opcode(0xD1, 0x7); /* D1 /7 */
8407   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8408   ins_pipe(ialu_reg);
8409 %}
8410 
8411 // Arithmetic shift right by one
8412 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8413 %{
8414   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8415   effect(KILL cr);
8416 
8417   format %{ "sarl    $dst, $shift" %}
8418   opcode(0xD1, 0x7); /* D1 /7 */
8419   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8420   ins_pipe(ialu_mem_imm);
8421 %}
8422 
8423 // Arithmetic Shift Right by 8-bit immediate
8424 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8425 %{
8426   match(Set dst (RShiftI dst shift));
8427   effect(KILL cr);
8428 
8429   format %{ "sarl    $dst, $shift" %}
8430   opcode(0xC1, 0x7); /* C1 /7 ib */
8431   ins_encode(reg_opc_imm(dst, shift));
8432   ins_pipe(ialu_mem_imm);
8433 %}
8434 
8435 // Arithmetic Shift Right by 8-bit immediate
8436 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8437 %{
8438   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8439   effect(KILL cr);
8440 
8441   format %{ "sarl    $dst, $shift" %}
8442   opcode(0xC1, 0x7); /* C1 /7 ib */
8443   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8444   ins_pipe(ialu_mem_imm);
8445 %}
8446 
8447 // Arithmetic Shift Right by variable
8448 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8449 %{
8450   match(Set dst (RShiftI dst shift));
8451   effect(KILL cr);
8452 
8453   format %{ "sarl    $dst, $shift" %}
8454   opcode(0xD3, 0x7); /* D3 /7 */
8455   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8456   ins_pipe(ialu_reg_reg);
8457 %}
8458 
8459 // Arithmetic Shift Right by variable
8460 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8461 %{
8462   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8463   effect(KILL cr);
8464 
8465   format %{ "sarl    $dst, $shift" %}
8466   opcode(0xD3, 0x7); /* D3 /7 */
8467   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8468   ins_pipe(ialu_mem_reg);
8469 %}
8470 
8471 // Logical shift right by one
8472 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8473 %{
8474   match(Set dst (URShiftI dst shift));
8475   effect(KILL cr);
8476 
8477   format %{ "shrl    $dst, $shift" %}
8478   opcode(0xD1, 0x5); /* D1 /5 */
8479   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8480   ins_pipe(ialu_reg);
8481 %}
8482 
8483 // Logical shift right by one
8484 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8485 %{
8486   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8487   effect(KILL cr);
8488 
8489   format %{ "shrl    $dst, $shift" %}
8490   opcode(0xD1, 0x5); /* D1 /5 */
8491   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8492   ins_pipe(ialu_mem_imm);
8493 %}
8494 
8495 // Logical Shift Right by 8-bit immediate
8496 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8497 %{
8498   match(Set dst (URShiftI dst shift));
8499   effect(KILL cr);
8500 
8501   format %{ "shrl    $dst, $shift" %}
8502   opcode(0xC1, 0x5); /* C1 /5 ib */
8503   ins_encode(reg_opc_imm(dst, shift));
8504   ins_pipe(ialu_reg);
8505 %}
8506 
8507 // Logical Shift Right by 8-bit immediate
8508 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8509 %{
8510   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8511   effect(KILL cr);
8512 
8513   format %{ "shrl    $dst, $shift" %}
8514   opcode(0xC1, 0x5); /* C1 /5 ib */
8515   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8516   ins_pipe(ialu_mem_imm);
8517 %}
8518 
8519 // Logical Shift Right by variable
8520 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8521 %{
8522   match(Set dst (URShiftI dst shift));
8523   effect(KILL cr);
8524 
8525   format %{ "shrl    $dst, $shift" %}
8526   opcode(0xD3, 0x5); /* D3 /5 */
8527   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8528   ins_pipe(ialu_reg_reg);
8529 %}
8530 
8531 // Logical Shift Right by variable
8532 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8533 %{
8534   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8535   effect(KILL cr);
8536 
8537   format %{ "shrl    $dst, $shift" %}
8538   opcode(0xD3, 0x5); /* D3 /5 */
8539   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8540   ins_pipe(ialu_mem_reg);
8541 %}
8542 
8543 // Long Shift Instructions
8544 // Shift Left by one
8545 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8546 %{
8547   match(Set dst (LShiftL dst shift));
8548   effect(KILL cr);
8549 
8550   format %{ "salq    $dst, $shift" %}
8551   opcode(0xD1, 0x4); /* D1 /4 */
8552   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8553   ins_pipe(ialu_reg);
8554 %}
8555 
8556 // Shift Left by one
8557 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8558 %{
8559   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8560   effect(KILL cr);
8561 
8562   format %{ "salq    $dst, $shift" %}
8563   opcode(0xD1, 0x4); /* D1 /4 */
8564   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8565   ins_pipe(ialu_mem_imm);
8566 %}
8567 
8568 // Shift Left by 8-bit immediate
8569 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8570 %{
8571   match(Set dst (LShiftL dst shift));
8572   effect(KILL cr);
8573 
8574   format %{ "salq    $dst, $shift" %}
8575   opcode(0xC1, 0x4); /* C1 /4 ib */
8576   ins_encode(reg_opc_imm_wide(dst, shift));
8577   ins_pipe(ialu_reg);
8578 %}
8579 
8580 // Shift Left by 8-bit immediate
8581 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8582 %{
8583   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8584   effect(KILL cr);
8585 
8586   format %{ "salq    $dst, $shift" %}
8587   opcode(0xC1, 0x4); /* C1 /4 ib */
8588   ins_encode(REX_mem_wide(dst), OpcP,
8589              RM_opc_mem(secondary, dst), Con8or32(shift));
8590   ins_pipe(ialu_mem_imm);
8591 %}
8592 
8593 // Shift Left by variable
8594 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8595 %{
8596   match(Set dst (LShiftL dst shift));
8597   effect(KILL cr);
8598 
8599   format %{ "salq    $dst, $shift" %}
8600   opcode(0xD3, 0x4); /* D3 /4 */
8601   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8602   ins_pipe(ialu_reg_reg);
8603 %}
8604 
8605 // Shift Left by variable
8606 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8607 %{
8608   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8609   effect(KILL cr);
8610 
8611   format %{ "salq    $dst, $shift" %}
8612   opcode(0xD3, 0x4); /* D3 /4 */
8613   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8614   ins_pipe(ialu_mem_reg);
8615 %}
8616 
8617 // Arithmetic shift right by one
8618 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8619 %{
8620   match(Set dst (RShiftL dst shift));
8621   effect(KILL cr);
8622 
8623   format %{ "sarq    $dst, $shift" %}
8624   opcode(0xD1, 0x7); /* D1 /7 */
8625   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8626   ins_pipe(ialu_reg);
8627 %}
8628 
8629 // Arithmetic shift right by one
8630 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8631 %{
8632   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8633   effect(KILL cr);
8634 
8635   format %{ "sarq    $dst, $shift" %}
8636   opcode(0xD1, 0x7); /* D1 /7 */
8637   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8638   ins_pipe(ialu_mem_imm);
8639 %}
8640 
8641 // Arithmetic Shift Right by 8-bit immediate
8642 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8643 %{
8644   match(Set dst (RShiftL dst shift));
8645   effect(KILL cr);
8646 
8647   format %{ "sarq    $dst, $shift" %}
8648   opcode(0xC1, 0x7); /* C1 /7 ib */
8649   ins_encode(reg_opc_imm_wide(dst, shift));
8650   ins_pipe(ialu_mem_imm);
8651 %}
8652 
8653 // Arithmetic Shift Right by 8-bit immediate
8654 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8655 %{
8656   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8657   effect(KILL cr);
8658 
8659   format %{ "sarq    $dst, $shift" %}
8660   opcode(0xC1, 0x7); /* C1 /7 ib */
8661   ins_encode(REX_mem_wide(dst), OpcP,
8662              RM_opc_mem(secondary, dst), Con8or32(shift));
8663   ins_pipe(ialu_mem_imm);
8664 %}
8665 
8666 // Arithmetic Shift Right by variable
8667 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8668 %{
8669   match(Set dst (RShiftL dst shift));
8670   effect(KILL cr);
8671 
8672   format %{ "sarq    $dst, $shift" %}
8673   opcode(0xD3, 0x7); /* D3 /7 */
8674   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8675   ins_pipe(ialu_reg_reg);
8676 %}
8677 
8678 // Arithmetic Shift Right by variable
8679 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8680 %{
8681   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8682   effect(KILL cr);
8683 
8684   format %{ "sarq    $dst, $shift" %}
8685   opcode(0xD3, 0x7); /* D3 /7 */
8686   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8687   ins_pipe(ialu_mem_reg);
8688 %}
8689 
8690 // Logical shift right by one
8691 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8692 %{
8693   match(Set dst (URShiftL dst shift));
8694   effect(KILL cr);
8695 
8696   format %{ "shrq    $dst, $shift" %}
8697   opcode(0xD1, 0x5); /* D1 /5 */
8698   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8699   ins_pipe(ialu_reg);
8700 %}
8701 
8702 // Logical shift right by one
8703 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8704 %{
8705   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8706   effect(KILL cr);
8707 
8708   format %{ "shrq    $dst, $shift" %}
8709   opcode(0xD1, 0x5); /* D1 /5 */
8710   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8711   ins_pipe(ialu_mem_imm);
8712 %}
8713 
8714 // Logical Shift Right by 8-bit immediate
8715 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8716 %{
8717   match(Set dst (URShiftL dst shift));
8718   effect(KILL cr);
8719 
8720   format %{ "shrq    $dst, $shift" %}
8721   opcode(0xC1, 0x5); /* C1 /5 ib */
8722   ins_encode(reg_opc_imm_wide(dst, shift));
8723   ins_pipe(ialu_reg);
8724 %}
8725 
8726 
8727 // Logical Shift Right by 8-bit immediate
8728 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8729 %{
8730   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8731   effect(KILL cr);
8732 
8733   format %{ "shrq    $dst, $shift" %}
8734   opcode(0xC1, 0x5); /* C1 /5 ib */
8735   ins_encode(REX_mem_wide(dst), OpcP,
8736              RM_opc_mem(secondary, dst), Con8or32(shift));
8737   ins_pipe(ialu_mem_imm);
8738 %}
8739 
8740 // Logical Shift Right by variable
8741 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8742 %{
8743   match(Set dst (URShiftL dst shift));
8744   effect(KILL cr);
8745 
8746   format %{ "shrq    $dst, $shift" %}
8747   opcode(0xD3, 0x5); /* D3 /5 */
8748   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8749   ins_pipe(ialu_reg_reg);
8750 %}
8751 
8752 // Logical Shift Right by variable
8753 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8754 %{
8755   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8756   effect(KILL cr);
8757 
8758   format %{ "shrq    $dst, $shift" %}
8759   opcode(0xD3, 0x5); /* D3 /5 */
8760   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8761   ins_pipe(ialu_mem_reg);
8762 %}
8763 
8764 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8765 // This idiom is used by the compiler for the i2b bytecode.
8766 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8767 %{
8768   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8769 
8770   format %{ "movsbl  $dst, $src\t# i2b" %}
8771   opcode(0x0F, 0xBE);
8772   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8773   ins_pipe(ialu_reg_reg);
8774 %}
8775 
8776 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8777 // This idiom is used by the compiler the i2s bytecode.
8778 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8779 %{
8780   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8781 
8782   format %{ "movswl  $dst, $src\t# i2s" %}
8783   opcode(0x0F, 0xBF);
8784   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8785   ins_pipe(ialu_reg_reg);
8786 %}
8787 
8788 // ROL/ROR instructions
8789 
8790 // ROL expand
8791 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8792   effect(KILL cr, USE_DEF dst);
8793 
8794   format %{ "roll    $dst" %}
8795   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8796   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8797   ins_pipe(ialu_reg);
8798 %}
8799 
8800 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8801   effect(USE_DEF dst, USE shift, KILL cr);
8802 
8803   format %{ "roll    $dst, $shift" %}
8804   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8805   ins_encode( reg_opc_imm(dst, shift) );
8806   ins_pipe(ialu_reg);
8807 %}
8808 
8809 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8810 %{
8811   effect(USE_DEF dst, USE shift, KILL cr);
8812 
8813   format %{ "roll    $dst, $shift" %}
8814   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8815   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8816   ins_pipe(ialu_reg_reg);
8817 %}
8818 // end of ROL expand
8819 
8820 // Rotate Left by one
8821 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8822 %{
8823   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8824 
8825   expand %{
8826     rolI_rReg_imm1(dst, cr);
8827   %}
8828 %}
8829 
8830 // Rotate Left by 8-bit immediate
8831 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8832 %{
8833   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8834   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8835 
8836   expand %{
8837     rolI_rReg_imm8(dst, lshift, cr);
8838   %}
8839 %}
8840 
8841 // Rotate Left by variable
8842 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8843 %{
8844   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8845 
8846   expand %{
8847     rolI_rReg_CL(dst, shift, cr);
8848   %}
8849 %}
8850 
8851 // Rotate Left by variable
8852 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8853 %{
8854   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8855 
8856   expand %{
8857     rolI_rReg_CL(dst, shift, cr);
8858   %}
8859 %}
8860 
8861 // ROR expand
8862 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8863 %{
8864   effect(USE_DEF dst, KILL cr);
8865 
8866   format %{ "rorl    $dst" %}
8867   opcode(0xD1, 0x1); /* D1 /1 */
8868   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8869   ins_pipe(ialu_reg);
8870 %}
8871 
8872 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8873 %{
8874   effect(USE_DEF dst, USE shift, KILL cr);
8875 
8876   format %{ "rorl    $dst, $shift" %}
8877   opcode(0xC1, 0x1); /* C1 /1 ib */
8878   ins_encode(reg_opc_imm(dst, shift));
8879   ins_pipe(ialu_reg);
8880 %}
8881 
8882 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8883 %{
8884   effect(USE_DEF dst, USE shift, KILL cr);
8885 
8886   format %{ "rorl    $dst, $shift" %}
8887   opcode(0xD3, 0x1); /* D3 /1 */
8888   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8889   ins_pipe(ialu_reg_reg);
8890 %}
8891 // end of ROR expand
8892 
8893 // Rotate Right by one
8894 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8895 %{
8896   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8897 
8898   expand %{
8899     rorI_rReg_imm1(dst, cr);
8900   %}
8901 %}
8902 
8903 // Rotate Right by 8-bit immediate
8904 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8905 %{
8906   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8907   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8908 
8909   expand %{
8910     rorI_rReg_imm8(dst, rshift, cr);
8911   %}
8912 %}
8913 
8914 // Rotate Right by variable
8915 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8916 %{
8917   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8918 
8919   expand %{
8920     rorI_rReg_CL(dst, shift, cr);
8921   %}
8922 %}
8923 
8924 // Rotate Right by variable
8925 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8926 %{
8927   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8928 
8929   expand %{
8930     rorI_rReg_CL(dst, shift, cr);
8931   %}
8932 %}
8933 
8934 // for long rotate
8935 // ROL expand
8936 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8937   effect(USE_DEF dst, KILL cr);
8938 
8939   format %{ "rolq    $dst" %}
8940   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8941   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8942   ins_pipe(ialu_reg);
8943 %}
8944 
8945 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8946   effect(USE_DEF dst, USE shift, KILL cr);
8947 
8948   format %{ "rolq    $dst, $shift" %}
8949   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8950   ins_encode( reg_opc_imm_wide(dst, shift) );
8951   ins_pipe(ialu_reg);
8952 %}
8953 
8954 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8955 %{
8956   effect(USE_DEF dst, USE shift, KILL cr);
8957 
8958   format %{ "rolq    $dst, $shift" %}
8959   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8960   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8961   ins_pipe(ialu_reg_reg);
8962 %}
8963 // end of ROL expand
8964 
8965 // Rotate Left by one
8966 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8967 %{
8968   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8969 
8970   expand %{
8971     rolL_rReg_imm1(dst, cr);
8972   %}
8973 %}
8974 
8975 // Rotate Left by 8-bit immediate
8976 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8977 %{
8978   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8979   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8980 
8981   expand %{
8982     rolL_rReg_imm8(dst, lshift, cr);
8983   %}
8984 %}
8985 
8986 // Rotate Left by variable
8987 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8988 %{
8989   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8990 
8991   expand %{
8992     rolL_rReg_CL(dst, shift, cr);
8993   %}
8994 %}
8995 
8996 // Rotate Left by variable
8997 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8998 %{
8999   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
9000 
9001   expand %{
9002     rolL_rReg_CL(dst, shift, cr);
9003   %}
9004 %}
9005 
9006 // ROR expand
9007 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
9008 %{
9009   effect(USE_DEF dst, KILL cr);
9010 
9011   format %{ "rorq    $dst" %}
9012   opcode(0xD1, 0x1); /* D1 /1 */
9013   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9014   ins_pipe(ialu_reg);
9015 %}
9016 
9017 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
9018 %{
9019   effect(USE_DEF dst, USE shift, KILL cr);
9020 
9021   format %{ "rorq    $dst, $shift" %}
9022   opcode(0xC1, 0x1); /* C1 /1 ib */
9023   ins_encode(reg_opc_imm_wide(dst, shift));
9024   ins_pipe(ialu_reg);
9025 %}
9026 
9027 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9028 %{
9029   effect(USE_DEF dst, USE shift, KILL cr);
9030 
9031   format %{ "rorq    $dst, $shift" %}
9032   opcode(0xD3, 0x1); /* D3 /1 */
9033   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9034   ins_pipe(ialu_reg_reg);
9035 %}
9036 // end of ROR expand
9037 
9038 // Rotate Right by one
9039 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9040 %{
9041   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9042 
9043   expand %{
9044     rorL_rReg_imm1(dst, cr);
9045   %}
9046 %}
9047 
9048 // Rotate Right by 8-bit immediate
9049 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9050 %{
9051   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9052   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9053 
9054   expand %{
9055     rorL_rReg_imm8(dst, rshift, cr);
9056   %}
9057 %}
9058 
9059 // Rotate Right by variable
9060 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9061 %{
9062   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
9063 
9064   expand %{
9065     rorL_rReg_CL(dst, shift, cr);
9066   %}
9067 %}
9068 
9069 // Rotate Right by variable
9070 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9071 %{
9072   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
9073 
9074   expand %{
9075     rorL_rReg_CL(dst, shift, cr);
9076   %}
9077 %}
9078 
9079 // Logical Instructions
9080 
9081 // Integer Logical Instructions
9082 
9083 // And Instructions
9084 // And Register with Register
9085 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9086 %{
9087   match(Set dst (AndI dst src));
9088   effect(KILL cr);
9089 
9090   format %{ "andl    $dst, $src\t# int" %}
9091   opcode(0x23);
9092   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9093   ins_pipe(ialu_reg_reg);
9094 %}
9095 
9096 // And Register with Immediate 255
9097 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
9098 %{
9099   match(Set dst (AndI dst src));
9100 
9101   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
9102   opcode(0x0F, 0xB6);
9103   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9104   ins_pipe(ialu_reg);
9105 %}
9106 
9107 // And Register with Immediate 255 and promote to long
9108 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
9109 %{
9110   match(Set dst (ConvI2L (AndI src mask)));
9111 
9112   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
9113   opcode(0x0F, 0xB6);
9114   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9115   ins_pipe(ialu_reg);
9116 %}
9117 
9118 // And Register with Immediate 65535
9119 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9120 %{
9121   match(Set dst (AndI dst src));
9122 
9123   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9124   opcode(0x0F, 0xB7);
9125   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9126   ins_pipe(ialu_reg);
9127 %}
9128 
9129 // And Register with Immediate 65535 and promote to long
9130 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9131 %{
9132   match(Set dst (ConvI2L (AndI src mask)));
9133 
9134   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9135   opcode(0x0F, 0xB7);
9136   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9137   ins_pipe(ialu_reg);
9138 %}
9139 
9140 // And Register with Immediate
9141 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9142 %{
9143   match(Set dst (AndI dst src));
9144   effect(KILL cr);
9145 
9146   format %{ "andl    $dst, $src\t# int" %}
9147   opcode(0x81, 0x04); /* Opcode 81 /4 */
9148   ins_encode(OpcSErm(dst, src), Con8or32(src));
9149   ins_pipe(ialu_reg);
9150 %}
9151 
9152 // And Register with Memory
9153 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9154 %{
9155   match(Set dst (AndI dst (LoadI src)));
9156   effect(KILL cr);
9157 
9158   ins_cost(125);
9159   format %{ "andl    $dst, $src\t# int" %}
9160   opcode(0x23);
9161   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9162   ins_pipe(ialu_reg_mem);
9163 %}
9164 
9165 // And Memory with Register
9166 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9167 %{
9168   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9169   effect(KILL cr);
9170 
9171   ins_cost(150);
9172   format %{ "andl    $dst, $src\t# int" %}
9173   opcode(0x21); /* Opcode 21 /r */
9174   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9175   ins_pipe(ialu_mem_reg);
9176 %}
9177 
9178 // And Memory with Immediate
9179 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9180 %{
9181   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9182   effect(KILL cr);
9183 
9184   ins_cost(125);
9185   format %{ "andl    $dst, $src\t# int" %}
9186   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9187   ins_encode(REX_mem(dst), OpcSE(src),
9188              RM_opc_mem(secondary, dst), Con8or32(src));
9189   ins_pipe(ialu_mem_imm);
9190 %}
9191 
9192 // BMI1 instructions
9193 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
9194   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
9195   predicate(UseBMI1Instructions);
9196   effect(KILL cr);
9197 
9198   ins_cost(125);
9199   format %{ "andnl  $dst, $src1, $src2" %}
9200 
9201   ins_encode %{
9202     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
9203   %}
9204   ins_pipe(ialu_reg_mem);
9205 %}
9206 
9207 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
9208   match(Set dst (AndI (XorI src1 minus_1) src2));
9209   predicate(UseBMI1Instructions);
9210   effect(KILL cr);
9211 
9212   format %{ "andnl  $dst, $src1, $src2" %}
9213 
9214   ins_encode %{
9215     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9216   %}
9217   ins_pipe(ialu_reg);
9218 %}
9219 
9220 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
9221   match(Set dst (AndI (SubI imm_zero src) src));
9222   predicate(UseBMI1Instructions);
9223   effect(KILL cr);
9224 
9225   format %{ "blsil  $dst, $src" %}
9226 
9227   ins_encode %{
9228     __ blsil($dst$$Register, $src$$Register);
9229   %}
9230   ins_pipe(ialu_reg);
9231 %}
9232 
9233 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
9234   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9235   predicate(UseBMI1Instructions);
9236   effect(KILL cr);
9237 
9238   ins_cost(125);
9239   format %{ "blsil  $dst, $src" %}
9240 
9241   ins_encode %{
9242     __ blsil($dst$$Register, $src$$Address);
9243   %}
9244   ins_pipe(ialu_reg_mem);
9245 %}
9246 
9247 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9248 %{
9249   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9250   predicate(UseBMI1Instructions);
9251   effect(KILL cr);
9252 
9253   ins_cost(125);
9254   format %{ "blsmskl $dst, $src" %}
9255 
9256   ins_encode %{
9257     __ blsmskl($dst$$Register, $src$$Address);
9258   %}
9259   ins_pipe(ialu_reg_mem);
9260 %}
9261 
9262 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9263 %{
9264   match(Set dst (XorI (AddI src minus_1) src));
9265   predicate(UseBMI1Instructions);
9266   effect(KILL cr);
9267 
9268   format %{ "blsmskl $dst, $src" %}
9269 
9270   ins_encode %{
9271     __ blsmskl($dst$$Register, $src$$Register);
9272   %}
9273 
9274   ins_pipe(ialu_reg);
9275 %}
9276 
9277 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9278 %{
9279   match(Set dst (AndI (AddI src minus_1) src) );
9280   predicate(UseBMI1Instructions);
9281   effect(KILL cr);
9282 
9283   format %{ "blsrl  $dst, $src" %}
9284 
9285   ins_encode %{
9286     __ blsrl($dst$$Register, $src$$Register);
9287   %}
9288 
9289   ins_pipe(ialu_reg_mem);
9290 %}
9291 
9292 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9293 %{
9294   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9295   predicate(UseBMI1Instructions);
9296   effect(KILL cr);
9297 
9298   ins_cost(125);
9299   format %{ "blsrl  $dst, $src" %}
9300 
9301   ins_encode %{
9302     __ blsrl($dst$$Register, $src$$Address);
9303   %}
9304 
9305   ins_pipe(ialu_reg);
9306 %}
9307 
9308 // Or Instructions
9309 // Or Register with Register
9310 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9311 %{
9312   match(Set dst (OrI dst src));
9313   effect(KILL cr);
9314 
9315   format %{ "orl     $dst, $src\t# int" %}
9316   opcode(0x0B);
9317   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9318   ins_pipe(ialu_reg_reg);
9319 %}
9320 
9321 // Or Register with Immediate
9322 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9323 %{
9324   match(Set dst (OrI dst src));
9325   effect(KILL cr);
9326 
9327   format %{ "orl     $dst, $src\t# int" %}
9328   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9329   ins_encode(OpcSErm(dst, src), Con8or32(src));
9330   ins_pipe(ialu_reg);
9331 %}
9332 
9333 // Or Register with Memory
9334 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9335 %{
9336   match(Set dst (OrI dst (LoadI src)));
9337   effect(KILL cr);
9338 
9339   ins_cost(125);
9340   format %{ "orl     $dst, $src\t# int" %}
9341   opcode(0x0B);
9342   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9343   ins_pipe(ialu_reg_mem);
9344 %}
9345 
9346 // Or Memory with Register
9347 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9348 %{
9349   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9350   effect(KILL cr);
9351 
9352   ins_cost(150);
9353   format %{ "orl     $dst, $src\t# int" %}
9354   opcode(0x09); /* Opcode 09 /r */
9355   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9356   ins_pipe(ialu_mem_reg);
9357 %}
9358 
9359 // Or Memory with Immediate
9360 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9361 %{
9362   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9363   effect(KILL cr);
9364 
9365   ins_cost(125);
9366   format %{ "orl     $dst, $src\t# int" %}
9367   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9368   ins_encode(REX_mem(dst), OpcSE(src),
9369              RM_opc_mem(secondary, dst), Con8or32(src));
9370   ins_pipe(ialu_mem_imm);
9371 %}
9372 
9373 // Xor Instructions
9374 // Xor Register with Register
9375 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9376 %{
9377   match(Set dst (XorI dst src));
9378   effect(KILL cr);
9379 
9380   format %{ "xorl    $dst, $src\t# int" %}
9381   opcode(0x33);
9382   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9383   ins_pipe(ialu_reg_reg);
9384 %}
9385 
9386 // Xor Register with Immediate -1
9387 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9388   match(Set dst (XorI dst imm));
9389 
9390   format %{ "not    $dst" %}
9391   ins_encode %{
9392      __ notl($dst$$Register);
9393   %}
9394   ins_pipe(ialu_reg);
9395 %}
9396 
9397 // Xor Register with Immediate
9398 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9399 %{
9400   match(Set dst (XorI dst src));
9401   effect(KILL cr);
9402 
9403   format %{ "xorl    $dst, $src\t# int" %}
9404   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9405   ins_encode(OpcSErm(dst, src), Con8or32(src));
9406   ins_pipe(ialu_reg);
9407 %}
9408 
9409 // Xor Register with Memory
9410 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9411 %{
9412   match(Set dst (XorI dst (LoadI src)));
9413   effect(KILL cr);
9414 
9415   ins_cost(125);
9416   format %{ "xorl    $dst, $src\t# int" %}
9417   opcode(0x33);
9418   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9419   ins_pipe(ialu_reg_mem);
9420 %}
9421 
9422 // Xor Memory with Register
9423 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9424 %{
9425   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9426   effect(KILL cr);
9427 
9428   ins_cost(150);
9429   format %{ "xorl    $dst, $src\t# int" %}
9430   opcode(0x31); /* Opcode 31 /r */
9431   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9432   ins_pipe(ialu_mem_reg);
9433 %}
9434 
9435 // Xor Memory with Immediate
9436 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9437 %{
9438   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9439   effect(KILL cr);
9440 
9441   ins_cost(125);
9442   format %{ "xorl    $dst, $src\t# int" %}
9443   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9444   ins_encode(REX_mem(dst), OpcSE(src),
9445              RM_opc_mem(secondary, dst), Con8or32(src));
9446   ins_pipe(ialu_mem_imm);
9447 %}
9448 
9449 
9450 // Long Logical Instructions
9451 
9452 // And Instructions
9453 // And Register with Register
9454 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9455 %{
9456   match(Set dst (AndL dst src));
9457   effect(KILL cr);
9458 
9459   format %{ "andq    $dst, $src\t# long" %}
9460   opcode(0x23);
9461   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9462   ins_pipe(ialu_reg_reg);
9463 %}
9464 
9465 // And Register with Immediate 255
9466 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9467 %{
9468   match(Set dst (AndL dst src));
9469 
9470   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9471   opcode(0x0F, 0xB6);
9472   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9473   ins_pipe(ialu_reg);
9474 %}
9475 
9476 // And Register with Immediate 65535
9477 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9478 %{
9479   match(Set dst (AndL dst src));
9480 
9481   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9482   opcode(0x0F, 0xB7);
9483   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9484   ins_pipe(ialu_reg);
9485 %}
9486 
9487 // And Register with Immediate
9488 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9489 %{
9490   match(Set dst (AndL dst src));
9491   effect(KILL cr);
9492 
9493   format %{ "andq    $dst, $src\t# long" %}
9494   opcode(0x81, 0x04); /* Opcode 81 /4 */
9495   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9496   ins_pipe(ialu_reg);
9497 %}
9498 
9499 // And Register with Memory
9500 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9501 %{
9502   match(Set dst (AndL dst (LoadL src)));
9503   effect(KILL cr);
9504 
9505   ins_cost(125);
9506   format %{ "andq    $dst, $src\t# long" %}
9507   opcode(0x23);
9508   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9509   ins_pipe(ialu_reg_mem);
9510 %}
9511 
9512 // And Memory with Register
9513 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9514 %{
9515   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9516   effect(KILL cr);
9517 
9518   ins_cost(150);
9519   format %{ "andq    $dst, $src\t# long" %}
9520   opcode(0x21); /* Opcode 21 /r */
9521   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9522   ins_pipe(ialu_mem_reg);
9523 %}
9524 
9525 // And Memory with Immediate
9526 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9527 %{
9528   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9529   effect(KILL cr);
9530 
9531   ins_cost(125);
9532   format %{ "andq    $dst, $src\t# long" %}
9533   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9534   ins_encode(REX_mem_wide(dst), OpcSE(src),
9535              RM_opc_mem(secondary, dst), Con8or32(src));
9536   ins_pipe(ialu_mem_imm);
9537 %}
9538 
9539 // BMI1 instructions
9540 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9541   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9542   predicate(UseBMI1Instructions);
9543   effect(KILL cr);
9544 
9545   ins_cost(125);
9546   format %{ "andnq  $dst, $src1, $src2" %}
9547 
9548   ins_encode %{
9549     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9550   %}
9551   ins_pipe(ialu_reg_mem);
9552 %}
9553 
9554 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9555   match(Set dst (AndL (XorL src1 minus_1) src2));
9556   predicate(UseBMI1Instructions);
9557   effect(KILL cr);
9558 
9559   format %{ "andnq  $dst, $src1, $src2" %}
9560 
9561   ins_encode %{
9562   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9563   %}
9564   ins_pipe(ialu_reg_mem);
9565 %}
9566 
9567 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9568   match(Set dst (AndL (SubL imm_zero src) src));
9569   predicate(UseBMI1Instructions);
9570   effect(KILL cr);
9571 
9572   format %{ "blsiq  $dst, $src" %}
9573 
9574   ins_encode %{
9575     __ blsiq($dst$$Register, $src$$Register);
9576   %}
9577   ins_pipe(ialu_reg);
9578 %}
9579 
9580 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9581   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9582   predicate(UseBMI1Instructions);
9583   effect(KILL cr);
9584 
9585   ins_cost(125);
9586   format %{ "blsiq  $dst, $src" %}
9587 
9588   ins_encode %{
9589     __ blsiq($dst$$Register, $src$$Address);
9590   %}
9591   ins_pipe(ialu_reg_mem);
9592 %}
9593 
9594 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9595 %{
9596   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9597   predicate(UseBMI1Instructions);
9598   effect(KILL cr);
9599 
9600   ins_cost(125);
9601   format %{ "blsmskq $dst, $src" %}
9602 
9603   ins_encode %{
9604     __ blsmskq($dst$$Register, $src$$Address);
9605   %}
9606   ins_pipe(ialu_reg_mem);
9607 %}
9608 
9609 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9610 %{
9611   match(Set dst (XorL (AddL src minus_1) src));
9612   predicate(UseBMI1Instructions);
9613   effect(KILL cr);
9614 
9615   format %{ "blsmskq $dst, $src" %}
9616 
9617   ins_encode %{
9618     __ blsmskq($dst$$Register, $src$$Register);
9619   %}
9620 
9621   ins_pipe(ialu_reg);
9622 %}
9623 
9624 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9625 %{
9626   match(Set dst (AndL (AddL src minus_1) src) );
9627   predicate(UseBMI1Instructions);
9628   effect(KILL cr);
9629 
9630   format %{ "blsrq  $dst, $src" %}
9631 
9632   ins_encode %{
9633     __ blsrq($dst$$Register, $src$$Register);
9634   %}
9635 
9636   ins_pipe(ialu_reg);
9637 %}
9638 
9639 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9640 %{
9641   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9642   predicate(UseBMI1Instructions);
9643   effect(KILL cr);
9644 
9645   ins_cost(125);
9646   format %{ "blsrq  $dst, $src" %}
9647 
9648   ins_encode %{
9649     __ blsrq($dst$$Register, $src$$Address);
9650   %}
9651 
9652   ins_pipe(ialu_reg);
9653 %}
9654 
9655 // Or Instructions
9656 // Or Register with Register
9657 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9658 %{
9659   match(Set dst (OrL dst src));
9660   effect(KILL cr);
9661 
9662   format %{ "orq     $dst, $src\t# long" %}
9663   opcode(0x0B);
9664   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9665   ins_pipe(ialu_reg_reg);
9666 %}
9667 
9668 // Use any_RegP to match R15 (TLS register) without spilling.
9669 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9670   match(Set dst (OrL dst (CastP2X src)));
9671   effect(KILL cr);
9672 
9673   format %{ "orq     $dst, $src\t# long" %}
9674   opcode(0x0B);
9675   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9676   ins_pipe(ialu_reg_reg);
9677 %}
9678 
9679 
9680 // Or Register with Immediate
9681 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9682 %{
9683   match(Set dst (OrL dst src));
9684   effect(KILL cr);
9685 
9686   format %{ "orq     $dst, $src\t# long" %}
9687   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9688   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9689   ins_pipe(ialu_reg);
9690 %}
9691 
9692 // Or Register with Memory
9693 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9694 %{
9695   match(Set dst (OrL dst (LoadL src)));
9696   effect(KILL cr);
9697 
9698   ins_cost(125);
9699   format %{ "orq     $dst, $src\t# long" %}
9700   opcode(0x0B);
9701   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9702   ins_pipe(ialu_reg_mem);
9703 %}
9704 
9705 // Or Memory with Register
9706 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9707 %{
9708   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9709   effect(KILL cr);
9710 
9711   ins_cost(150);
9712   format %{ "orq     $dst, $src\t# long" %}
9713   opcode(0x09); /* Opcode 09 /r */
9714   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9715   ins_pipe(ialu_mem_reg);
9716 %}
9717 
9718 // Or Memory with Immediate
9719 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9720 %{
9721   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9722   effect(KILL cr);
9723 
9724   ins_cost(125);
9725   format %{ "orq     $dst, $src\t# long" %}
9726   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9727   ins_encode(REX_mem_wide(dst), OpcSE(src),
9728              RM_opc_mem(secondary, dst), Con8or32(src));
9729   ins_pipe(ialu_mem_imm);
9730 %}
9731 
9732 // Xor Instructions
9733 // Xor Register with Register
9734 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9735 %{
9736   match(Set dst (XorL dst src));
9737   effect(KILL cr);
9738 
9739   format %{ "xorq    $dst, $src\t# long" %}
9740   opcode(0x33);
9741   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9742   ins_pipe(ialu_reg_reg);
9743 %}
9744 
9745 // Xor Register with Immediate -1
9746 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9747   match(Set dst (XorL dst imm));
9748 
9749   format %{ "notq   $dst" %}
9750   ins_encode %{
9751      __ notq($dst$$Register);
9752   %}
9753   ins_pipe(ialu_reg);
9754 %}
9755 
9756 // Xor Register with Immediate
9757 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9758 %{
9759   match(Set dst (XorL dst src));
9760   effect(KILL cr);
9761 
9762   format %{ "xorq    $dst, $src\t# long" %}
9763   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9764   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9765   ins_pipe(ialu_reg);
9766 %}
9767 
9768 // Xor Register with Memory
9769 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9770 %{
9771   match(Set dst (XorL dst (LoadL src)));
9772   effect(KILL cr);
9773 
9774   ins_cost(125);
9775   format %{ "xorq    $dst, $src\t# long" %}
9776   opcode(0x33);
9777   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9778   ins_pipe(ialu_reg_mem);
9779 %}
9780 
9781 // Xor Memory with Register
9782 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9783 %{
9784   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9785   effect(KILL cr);
9786 
9787   ins_cost(150);
9788   format %{ "xorq    $dst, $src\t# long" %}
9789   opcode(0x31); /* Opcode 31 /r */
9790   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9791   ins_pipe(ialu_mem_reg);
9792 %}
9793 
9794 // Xor Memory with Immediate
9795 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9796 %{
9797   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9798   effect(KILL cr);
9799 
9800   ins_cost(125);
9801   format %{ "xorq    $dst, $src\t# long" %}
9802   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9803   ins_encode(REX_mem_wide(dst), OpcSE(src),
9804              RM_opc_mem(secondary, dst), Con8or32(src));
9805   ins_pipe(ialu_mem_imm);
9806 %}
9807 
9808 // Convert Int to Boolean
9809 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9810 %{
9811   match(Set dst (Conv2B src));
9812   effect(KILL cr);
9813 
9814   format %{ "testl   $src, $src\t# ci2b\n\t"
9815             "setnz   $dst\n\t"
9816             "movzbl  $dst, $dst" %}
9817   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9818              setNZ_reg(dst),
9819              REX_reg_breg(dst, dst), // movzbl
9820              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9821   ins_pipe(pipe_slow); // XXX
9822 %}
9823 
9824 // Convert Pointer to Boolean
9825 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9826 %{
9827   match(Set dst (Conv2B src));
9828   effect(KILL cr);
9829 
9830   format %{ "testq   $src, $src\t# cp2b\n\t"
9831             "setnz   $dst\n\t"
9832             "movzbl  $dst, $dst" %}
9833   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9834              setNZ_reg(dst),
9835              REX_reg_breg(dst, dst), // movzbl
9836              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9837   ins_pipe(pipe_slow); // XXX
9838 %}
9839 
9840 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9841 %{
9842   match(Set dst (CmpLTMask p q));
9843   effect(KILL cr);
9844 
9845   ins_cost(400);
9846   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9847             "setlt   $dst\n\t"
9848             "movzbl  $dst, $dst\n\t"
9849             "negl    $dst" %}
9850   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9851              setLT_reg(dst),
9852              REX_reg_breg(dst, dst), // movzbl
9853              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9854              neg_reg(dst));
9855   ins_pipe(pipe_slow);
9856 %}
9857 
9858 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9859 %{
9860   match(Set dst (CmpLTMask dst zero));
9861   effect(KILL cr);
9862 
9863   ins_cost(100);
9864   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9865   ins_encode %{
9866   __ sarl($dst$$Register, 31);
9867   %}
9868   ins_pipe(ialu_reg);
9869 %}
9870 
9871 /* Better to save a register than avoid a branch */
9872 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9873 %{
9874   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9875   effect(KILL cr);
9876   ins_cost(300);
9877   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9878             "jge    done\n\t"
9879             "addl   $p,$y\n"
9880             "done:  " %}
9881   ins_encode %{
9882     Register Rp = $p$$Register;
9883     Register Rq = $q$$Register;
9884     Register Ry = $y$$Register;
9885     Label done;
9886     __ subl(Rp, Rq);
9887     __ jccb(Assembler::greaterEqual, done);
9888     __ addl(Rp, Ry);
9889     __ bind(done);
9890   %}
9891   ins_pipe(pipe_cmplt);
9892 %}
9893 
9894 /* Better to save a register than avoid a branch */
9895 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9896 %{
9897   match(Set y (AndI (CmpLTMask p q) y));
9898   effect(KILL cr);
9899 
9900   ins_cost(300);
9901 
9902   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9903             "jlt      done\n\t"
9904             "xorl     $y, $y\n"
9905             "done:  " %}
9906   ins_encode %{
9907     Register Rp = $p$$Register;
9908     Register Rq = $q$$Register;
9909     Register Ry = $y$$Register;
9910     Label done;
9911     __ cmpl(Rp, Rq);
9912     __ jccb(Assembler::less, done);
9913     __ xorl(Ry, Ry);
9914     __ bind(done);
9915   %}
9916   ins_pipe(pipe_cmplt);
9917 %}
9918 
9919 
9920 //---------- FP Instructions------------------------------------------------
9921 
9922 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9923 %{
9924   match(Set cr (CmpF src1 src2));
9925 
9926   ins_cost(145);
9927   format %{ "ucomiss $src1, $src2\n\t"
9928             "jnp,s   exit\n\t"
9929             "pushfq\t# saw NaN, set CF\n\t"
9930             "andq    [rsp], #0xffffff2b\n\t"
9931             "popfq\n"
9932     "exit:" %}
9933   ins_encode %{
9934     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9935     emit_cmpfp_fixup(_masm);
9936   %}
9937   ins_pipe(pipe_slow);
9938 %}
9939 
9940 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9941   match(Set cr (CmpF src1 src2));
9942 
9943   ins_cost(100);
9944   format %{ "ucomiss $src1, $src2" %}
9945   ins_encode %{
9946     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9947   %}
9948   ins_pipe(pipe_slow);
9949 %}
9950 
9951 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9952 %{
9953   match(Set cr (CmpF src1 (LoadF src2)));
9954 
9955   ins_cost(145);
9956   format %{ "ucomiss $src1, $src2\n\t"
9957             "jnp,s   exit\n\t"
9958             "pushfq\t# saw NaN, set CF\n\t"
9959             "andq    [rsp], #0xffffff2b\n\t"
9960             "popfq\n"
9961     "exit:" %}
9962   ins_encode %{
9963     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9964     emit_cmpfp_fixup(_masm);
9965   %}
9966   ins_pipe(pipe_slow);
9967 %}
9968 
9969 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9970   match(Set cr (CmpF src1 (LoadF src2)));
9971 
9972   ins_cost(100);
9973   format %{ "ucomiss $src1, $src2" %}
9974   ins_encode %{
9975     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9976   %}
9977   ins_pipe(pipe_slow);
9978 %}
9979 
9980 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9981   match(Set cr (CmpF src con));
9982 
9983   ins_cost(145);
9984   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9985             "jnp,s   exit\n\t"
9986             "pushfq\t# saw NaN, set CF\n\t"
9987             "andq    [rsp], #0xffffff2b\n\t"
9988             "popfq\n"
9989     "exit:" %}
9990   ins_encode %{
9991     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9992     emit_cmpfp_fixup(_masm);
9993   %}
9994   ins_pipe(pipe_slow);
9995 %}
9996 
9997 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9998   match(Set cr (CmpF src con));
9999   ins_cost(100);
10000   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10001   ins_encode %{
10002     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10003   %}
10004   ins_pipe(pipe_slow);
10005 %}
10006 
10007 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10008 %{
10009   match(Set cr (CmpD src1 src2));
10010 
10011   ins_cost(145);
10012   format %{ "ucomisd $src1, $src2\n\t"
10013             "jnp,s   exit\n\t"
10014             "pushfq\t# saw NaN, set CF\n\t"
10015             "andq    [rsp], #0xffffff2b\n\t"
10016             "popfq\n"
10017     "exit:" %}
10018   ins_encode %{
10019     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10020     emit_cmpfp_fixup(_masm);
10021   %}
10022   ins_pipe(pipe_slow);
10023 %}
10024 
10025 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10026   match(Set cr (CmpD src1 src2));
10027 
10028   ins_cost(100);
10029   format %{ "ucomisd $src1, $src2 test" %}
10030   ins_encode %{
10031     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10032   %}
10033   ins_pipe(pipe_slow);
10034 %}
10035 
10036 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10037 %{
10038   match(Set cr (CmpD src1 (LoadD src2)));
10039 
10040   ins_cost(145);
10041   format %{ "ucomisd $src1, $src2\n\t"
10042             "jnp,s   exit\n\t"
10043             "pushfq\t# saw NaN, set CF\n\t"
10044             "andq    [rsp], #0xffffff2b\n\t"
10045             "popfq\n"
10046     "exit:" %}
10047   ins_encode %{
10048     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10049     emit_cmpfp_fixup(_masm);
10050   %}
10051   ins_pipe(pipe_slow);
10052 %}
10053 
10054 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10055   match(Set cr (CmpD src1 (LoadD src2)));
10056 
10057   ins_cost(100);
10058   format %{ "ucomisd $src1, $src2" %}
10059   ins_encode %{
10060     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10061   %}
10062   ins_pipe(pipe_slow);
10063 %}
10064 
10065 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10066   match(Set cr (CmpD src con));
10067 
10068   ins_cost(145);
10069   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10070             "jnp,s   exit\n\t"
10071             "pushfq\t# saw NaN, set CF\n\t"
10072             "andq    [rsp], #0xffffff2b\n\t"
10073             "popfq\n"
10074     "exit:" %}
10075   ins_encode %{
10076     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10077     emit_cmpfp_fixup(_masm);
10078   %}
10079   ins_pipe(pipe_slow);
10080 %}
10081 
10082 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10083   match(Set cr (CmpD src con));
10084   ins_cost(100);
10085   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10086   ins_encode %{
10087     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10088   %}
10089   ins_pipe(pipe_slow);
10090 %}
10091 
10092 // Compare into -1,0,1
10093 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10094 %{
10095   match(Set dst (CmpF3 src1 src2));
10096   effect(KILL cr);
10097 
10098   ins_cost(275);
10099   format %{ "ucomiss $src1, $src2\n\t"
10100             "movl    $dst, #-1\n\t"
10101             "jp,s    done\n\t"
10102             "jb,s    done\n\t"
10103             "setne   $dst\n\t"
10104             "movzbl  $dst, $dst\n"
10105     "done:" %}
10106   ins_encode %{
10107     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10108     emit_cmpfp3(_masm, $dst$$Register);
10109   %}
10110   ins_pipe(pipe_slow);
10111 %}
10112 
10113 // Compare into -1,0,1
10114 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10115 %{
10116   match(Set dst (CmpF3 src1 (LoadF src2)));
10117   effect(KILL cr);
10118 
10119   ins_cost(275);
10120   format %{ "ucomiss $src1, $src2\n\t"
10121             "movl    $dst, #-1\n\t"
10122             "jp,s    done\n\t"
10123             "jb,s    done\n\t"
10124             "setne   $dst\n\t"
10125             "movzbl  $dst, $dst\n"
10126     "done:" %}
10127   ins_encode %{
10128     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10129     emit_cmpfp3(_masm, $dst$$Register);
10130   %}
10131   ins_pipe(pipe_slow);
10132 %}
10133 
10134 // Compare into -1,0,1
10135 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10136   match(Set dst (CmpF3 src con));
10137   effect(KILL cr);
10138 
10139   ins_cost(275);
10140   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10141             "movl    $dst, #-1\n\t"
10142             "jp,s    done\n\t"
10143             "jb,s    done\n\t"
10144             "setne   $dst\n\t"
10145             "movzbl  $dst, $dst\n"
10146     "done:" %}
10147   ins_encode %{
10148     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10149     emit_cmpfp3(_masm, $dst$$Register);
10150   %}
10151   ins_pipe(pipe_slow);
10152 %}
10153 
10154 // Compare into -1,0,1
10155 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10156 %{
10157   match(Set dst (CmpD3 src1 src2));
10158   effect(KILL cr);
10159 
10160   ins_cost(275);
10161   format %{ "ucomisd $src1, $src2\n\t"
10162             "movl    $dst, #-1\n\t"
10163             "jp,s    done\n\t"
10164             "jb,s    done\n\t"
10165             "setne   $dst\n\t"
10166             "movzbl  $dst, $dst\n"
10167     "done:" %}
10168   ins_encode %{
10169     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10170     emit_cmpfp3(_masm, $dst$$Register);
10171   %}
10172   ins_pipe(pipe_slow);
10173 %}
10174 
10175 // Compare into -1,0,1
10176 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10177 %{
10178   match(Set dst (CmpD3 src1 (LoadD src2)));
10179   effect(KILL cr);
10180 
10181   ins_cost(275);
10182   format %{ "ucomisd $src1, $src2\n\t"
10183             "movl    $dst, #-1\n\t"
10184             "jp,s    done\n\t"
10185             "jb,s    done\n\t"
10186             "setne   $dst\n\t"
10187             "movzbl  $dst, $dst\n"
10188     "done:" %}
10189   ins_encode %{
10190     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10191     emit_cmpfp3(_masm, $dst$$Register);
10192   %}
10193   ins_pipe(pipe_slow);
10194 %}
10195 
10196 // Compare into -1,0,1
10197 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10198   match(Set dst (CmpD3 src con));
10199   effect(KILL cr);
10200 
10201   ins_cost(275);
10202   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10203             "movl    $dst, #-1\n\t"
10204             "jp,s    done\n\t"
10205             "jb,s    done\n\t"
10206             "setne   $dst\n\t"
10207             "movzbl  $dst, $dst\n"
10208     "done:" %}
10209   ins_encode %{
10210     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10211     emit_cmpfp3(_masm, $dst$$Register);
10212   %}
10213   ins_pipe(pipe_slow);
10214 %}
10215 
10216 //----------Arithmetic Conversion Instructions---------------------------------
10217 
10218 instruct roundFloat_nop(regF dst)
10219 %{
10220   match(Set dst (RoundFloat dst));
10221 
10222   ins_cost(0);
10223   ins_encode();
10224   ins_pipe(empty);
10225 %}
10226 
10227 instruct roundDouble_nop(regD dst)
10228 %{
10229   match(Set dst (RoundDouble dst));
10230 
10231   ins_cost(0);
10232   ins_encode();
10233   ins_pipe(empty);
10234 %}
10235 
10236 instruct convF2D_reg_reg(regD dst, regF src)
10237 %{
10238   match(Set dst (ConvF2D src));
10239 
10240   format %{ "cvtss2sd $dst, $src" %}
10241   ins_encode %{
10242     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10243   %}
10244   ins_pipe(pipe_slow); // XXX
10245 %}
10246 
10247 instruct convF2D_reg_mem(regD dst, memory src)
10248 %{
10249   match(Set dst (ConvF2D (LoadF src)));
10250 
10251   format %{ "cvtss2sd $dst, $src" %}
10252   ins_encode %{
10253     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10254   %}
10255   ins_pipe(pipe_slow); // XXX
10256 %}
10257 
10258 instruct convD2F_reg_reg(regF dst, regD src)
10259 %{
10260   match(Set dst (ConvD2F src));
10261 
10262   format %{ "cvtsd2ss $dst, $src" %}
10263   ins_encode %{
10264     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10265   %}
10266   ins_pipe(pipe_slow); // XXX
10267 %}
10268 
10269 instruct convD2F_reg_mem(regF dst, memory src)
10270 %{
10271   match(Set dst (ConvD2F (LoadD src)));
10272 
10273   format %{ "cvtsd2ss $dst, $src" %}
10274   ins_encode %{
10275     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10276   %}
10277   ins_pipe(pipe_slow); // XXX
10278 %}
10279 
10280 // XXX do mem variants
10281 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10282 %{
10283   match(Set dst (ConvF2I src));
10284   effect(KILL cr);
10285 
10286   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10287             "cmpl    $dst, #0x80000000\n\t"
10288             "jne,s   done\n\t"
10289             "subq    rsp, #8\n\t"
10290             "movss   [rsp], $src\n\t"
10291             "call    f2i_fixup\n\t"
10292             "popq    $dst\n"
10293     "done:   "%}
10294   ins_encode %{
10295     Label done;
10296     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10297     __ cmpl($dst$$Register, 0x80000000);
10298     __ jccb(Assembler::notEqual, done);
10299     __ subptr(rsp, 8);
10300     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10301     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10302     __ pop($dst$$Register);
10303     __ bind(done);
10304   %}
10305   ins_pipe(pipe_slow);
10306 %}
10307 
10308 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10309 %{
10310   match(Set dst (ConvF2L src));
10311   effect(KILL cr);
10312 
10313   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10314             "cmpq    $dst, [0x8000000000000000]\n\t"
10315             "jne,s   done\n\t"
10316             "subq    rsp, #8\n\t"
10317             "movss   [rsp], $src\n\t"
10318             "call    f2l_fixup\n\t"
10319             "popq    $dst\n"
10320     "done:   "%}
10321   ins_encode %{
10322     Label done;
10323     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10324     __ cmp64($dst$$Register,
10325              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10326     __ jccb(Assembler::notEqual, done);
10327     __ subptr(rsp, 8);
10328     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10329     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10330     __ pop($dst$$Register);
10331     __ bind(done);
10332   %}
10333   ins_pipe(pipe_slow);
10334 %}
10335 
10336 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10337 %{
10338   match(Set dst (ConvD2I src));
10339   effect(KILL cr);
10340 
10341   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10342             "cmpl    $dst, #0x80000000\n\t"
10343             "jne,s   done\n\t"
10344             "subq    rsp, #8\n\t"
10345             "movsd   [rsp], $src\n\t"
10346             "call    d2i_fixup\n\t"
10347             "popq    $dst\n"
10348     "done:   "%}
10349   ins_encode %{
10350     Label done;
10351     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10352     __ cmpl($dst$$Register, 0x80000000);
10353     __ jccb(Assembler::notEqual, done);
10354     __ subptr(rsp, 8);
10355     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10356     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10357     __ pop($dst$$Register);
10358     __ bind(done);
10359   %}
10360   ins_pipe(pipe_slow);
10361 %}
10362 
10363 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10364 %{
10365   match(Set dst (ConvD2L src));
10366   effect(KILL cr);
10367 
10368   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10369             "cmpq    $dst, [0x8000000000000000]\n\t"
10370             "jne,s   done\n\t"
10371             "subq    rsp, #8\n\t"
10372             "movsd   [rsp], $src\n\t"
10373             "call    d2l_fixup\n\t"
10374             "popq    $dst\n"
10375     "done:   "%}
10376   ins_encode %{
10377     Label done;
10378     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10379     __ cmp64($dst$$Register,
10380              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10381     __ jccb(Assembler::notEqual, done);
10382     __ subptr(rsp, 8);
10383     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10384     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10385     __ pop($dst$$Register);
10386     __ bind(done);
10387   %}
10388   ins_pipe(pipe_slow);
10389 %}
10390 
10391 instruct convI2F_reg_reg(regF dst, rRegI src)
10392 %{
10393   predicate(!UseXmmI2F);
10394   match(Set dst (ConvI2F src));
10395 
10396   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10397   ins_encode %{
10398     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10399   %}
10400   ins_pipe(pipe_slow); // XXX
10401 %}
10402 
10403 instruct convI2F_reg_mem(regF dst, memory src)
10404 %{
10405   match(Set dst (ConvI2F (LoadI src)));
10406 
10407   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10408   ins_encode %{
10409     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10410   %}
10411   ins_pipe(pipe_slow); // XXX
10412 %}
10413 
10414 instruct convI2D_reg_reg(regD dst, rRegI src)
10415 %{
10416   predicate(!UseXmmI2D);
10417   match(Set dst (ConvI2D src));
10418 
10419   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10420   ins_encode %{
10421     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10422   %}
10423   ins_pipe(pipe_slow); // XXX
10424 %}
10425 
10426 instruct convI2D_reg_mem(regD dst, memory src)
10427 %{
10428   match(Set dst (ConvI2D (LoadI src)));
10429 
10430   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10431   ins_encode %{
10432     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10433   %}
10434   ins_pipe(pipe_slow); // XXX
10435 %}
10436 
10437 instruct convXI2F_reg(regF dst, rRegI src)
10438 %{
10439   predicate(UseXmmI2F);
10440   match(Set dst (ConvI2F src));
10441 
10442   format %{ "movdl $dst, $src\n\t"
10443             "cvtdq2psl $dst, $dst\t# i2f" %}
10444   ins_encode %{
10445     __ movdl($dst$$XMMRegister, $src$$Register);
10446     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10447   %}
10448   ins_pipe(pipe_slow); // XXX
10449 %}
10450 
10451 instruct convXI2D_reg(regD dst, rRegI src)
10452 %{
10453   predicate(UseXmmI2D);
10454   match(Set dst (ConvI2D src));
10455 
10456   format %{ "movdl $dst, $src\n\t"
10457             "cvtdq2pdl $dst, $dst\t# i2d" %}
10458   ins_encode %{
10459     __ movdl($dst$$XMMRegister, $src$$Register);
10460     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10461   %}
10462   ins_pipe(pipe_slow); // XXX
10463 %}
10464 
10465 instruct convL2F_reg_reg(regF dst, rRegL src)
10466 %{
10467   match(Set dst (ConvL2F src));
10468 
10469   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10470   ins_encode %{
10471     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10472   %}
10473   ins_pipe(pipe_slow); // XXX
10474 %}
10475 
10476 instruct convL2F_reg_mem(regF dst, memory src)
10477 %{
10478   match(Set dst (ConvL2F (LoadL src)));
10479 
10480   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10481   ins_encode %{
10482     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10483   %}
10484   ins_pipe(pipe_slow); // XXX
10485 %}
10486 
10487 instruct convL2D_reg_reg(regD dst, rRegL src)
10488 %{
10489   match(Set dst (ConvL2D src));
10490 
10491   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10492   ins_encode %{
10493     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10494   %}
10495   ins_pipe(pipe_slow); // XXX
10496 %}
10497 
10498 instruct convL2D_reg_mem(regD dst, memory src)
10499 %{
10500   match(Set dst (ConvL2D (LoadL src)));
10501 
10502   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10503   ins_encode %{
10504     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10505   %}
10506   ins_pipe(pipe_slow); // XXX
10507 %}
10508 
10509 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10510 %{
10511   match(Set dst (ConvI2L src));
10512 
10513   ins_cost(125);
10514   format %{ "movslq  $dst, $src\t# i2l" %}
10515   ins_encode %{
10516     __ movslq($dst$$Register, $src$$Register);
10517   %}
10518   ins_pipe(ialu_reg_reg);
10519 %}
10520 
10521 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10522 // %{
10523 //   match(Set dst (ConvI2L src));
10524 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10525 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10526 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10527 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10528 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10529 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10530 
10531 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10532 //   ins_encode(enc_copy(dst, src));
10533 // //   opcode(0x63); // needs REX.W
10534 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10535 //   ins_pipe(ialu_reg_reg);
10536 // %}
10537 
10538 // Zero-extend convert int to long
10539 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10540 %{
10541   match(Set dst (AndL (ConvI2L src) mask));
10542 
10543   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10544   ins_encode %{
10545     if ($dst$$reg != $src$$reg) {
10546       __ movl($dst$$Register, $src$$Register);
10547     }
10548   %}
10549   ins_pipe(ialu_reg_reg);
10550 %}
10551 
10552 // Zero-extend convert int to long
10553 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10554 %{
10555   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10556 
10557   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10558   ins_encode %{
10559     __ movl($dst$$Register, $src$$Address);
10560   %}
10561   ins_pipe(ialu_reg_mem);
10562 %}
10563 
10564 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10565 %{
10566   match(Set dst (AndL src mask));
10567 
10568   format %{ "movl    $dst, $src\t# zero-extend long" %}
10569   ins_encode %{
10570     __ movl($dst$$Register, $src$$Register);
10571   %}
10572   ins_pipe(ialu_reg_reg);
10573 %}
10574 
10575 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10576 %{
10577   match(Set dst (ConvL2I src));
10578 
10579   format %{ "movl    $dst, $src\t# l2i" %}
10580   ins_encode %{
10581     __ movl($dst$$Register, $src$$Register);
10582   %}
10583   ins_pipe(ialu_reg_reg);
10584 %}
10585 
10586 
10587 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10588   match(Set dst (MoveF2I src));
10589   effect(DEF dst, USE src);
10590 
10591   ins_cost(125);
10592   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10593   ins_encode %{
10594     __ movl($dst$$Register, Address(rsp, $src$$disp));
10595   %}
10596   ins_pipe(ialu_reg_mem);
10597 %}
10598 
10599 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10600   match(Set dst (MoveI2F src));
10601   effect(DEF dst, USE src);
10602 
10603   ins_cost(125);
10604   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10605   ins_encode %{
10606     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10607   %}
10608   ins_pipe(pipe_slow);
10609 %}
10610 
10611 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10612   match(Set dst (MoveD2L src));
10613   effect(DEF dst, USE src);
10614 
10615   ins_cost(125);
10616   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10617   ins_encode %{
10618     __ movq($dst$$Register, Address(rsp, $src$$disp));
10619   %}
10620   ins_pipe(ialu_reg_mem);
10621 %}
10622 
10623 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10624   predicate(!UseXmmLoadAndClearUpper);
10625   match(Set dst (MoveL2D src));
10626   effect(DEF dst, USE src);
10627 
10628   ins_cost(125);
10629   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10630   ins_encode %{
10631     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10632   %}
10633   ins_pipe(pipe_slow);
10634 %}
10635 
10636 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10637   predicate(UseXmmLoadAndClearUpper);
10638   match(Set dst (MoveL2D src));
10639   effect(DEF dst, USE src);
10640 
10641   ins_cost(125);
10642   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10643   ins_encode %{
10644     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10645   %}
10646   ins_pipe(pipe_slow);
10647 %}
10648 
10649 
10650 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10651   match(Set dst (MoveF2I src));
10652   effect(DEF dst, USE src);
10653 
10654   ins_cost(95); // XXX
10655   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10656   ins_encode %{
10657     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10658   %}
10659   ins_pipe(pipe_slow);
10660 %}
10661 
10662 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10663   match(Set dst (MoveI2F src));
10664   effect(DEF dst, USE src);
10665 
10666   ins_cost(100);
10667   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10668   ins_encode %{
10669     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10670   %}
10671   ins_pipe( ialu_mem_reg );
10672 %}
10673 
10674 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10675   match(Set dst (MoveD2L src));
10676   effect(DEF dst, USE src);
10677 
10678   ins_cost(95); // XXX
10679   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10680   ins_encode %{
10681     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10682   %}
10683   ins_pipe(pipe_slow);
10684 %}
10685 
10686 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10687   match(Set dst (MoveL2D src));
10688   effect(DEF dst, USE src);
10689 
10690   ins_cost(100);
10691   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10692   ins_encode %{
10693     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10694   %}
10695   ins_pipe(ialu_mem_reg);
10696 %}
10697 
10698 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10699   match(Set dst (MoveF2I src));
10700   effect(DEF dst, USE src);
10701   ins_cost(85);
10702   format %{ "movd    $dst,$src\t# MoveF2I" %}
10703   ins_encode %{
10704     __ movdl($dst$$Register, $src$$XMMRegister);
10705   %}
10706   ins_pipe( pipe_slow );
10707 %}
10708 
10709 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10710   match(Set dst (MoveD2L src));
10711   effect(DEF dst, USE src);
10712   ins_cost(85);
10713   format %{ "movd    $dst,$src\t# MoveD2L" %}
10714   ins_encode %{
10715     __ movdq($dst$$Register, $src$$XMMRegister);
10716   %}
10717   ins_pipe( pipe_slow );
10718 %}
10719 
10720 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10721   match(Set dst (MoveI2F src));
10722   effect(DEF dst, USE src);
10723   ins_cost(100);
10724   format %{ "movd    $dst,$src\t# MoveI2F" %}
10725   ins_encode %{
10726     __ movdl($dst$$XMMRegister, $src$$Register);
10727   %}
10728   ins_pipe( pipe_slow );
10729 %}
10730 
10731 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10732   match(Set dst (MoveL2D src));
10733   effect(DEF dst, USE src);
10734   ins_cost(100);
10735   format %{ "movd    $dst,$src\t# MoveL2D" %}
10736   ins_encode %{
10737      __ movdq($dst$$XMMRegister, $src$$Register);
10738   %}
10739   ins_pipe( pipe_slow );
10740 %}
10741 
10742 
10743 // =======================================================================
10744 // fast clearing of an array
10745 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10746                   Universe dummy, rFlagsReg cr)
10747 %{
10748   predicate(!((ClearArrayNode*)n)->is_large());
10749   match(Set dummy (ClearArray cnt base));
10750   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10751 
10752   format %{ $$template
10753     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10754     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10755     $$emit$$"jg      LARGE\n\t"
10756     $$emit$$"dec     rcx\n\t"
10757     $$emit$$"js      DONE\t# Zero length\n\t"
10758     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10759     $$emit$$"dec     rcx\n\t"
10760     $$emit$$"jge     LOOP\n\t"
10761     $$emit$$"jmp     DONE\n\t"
10762     $$emit$$"# LARGE:\n\t"
10763     if (UseFastStosb) {
10764        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10765        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10766     } else if (UseXMMForObjInit) {
10767        $$emit$$"mov     rdi,rax\n\t"
10768        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10769        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10770        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10771        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10772        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10773        $$emit$$"add     0x40,rax\n\t"
10774        $$emit$$"# L_zero_64_bytes:\n\t"
10775        $$emit$$"sub     0x8,rcx\n\t"
10776        $$emit$$"jge     L_loop\n\t"
10777        $$emit$$"add     0x4,rcx\n\t"
10778        $$emit$$"jl      L_tail\n\t"
10779        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10780        $$emit$$"add     0x20,rax\n\t"
10781        $$emit$$"sub     0x4,rcx\n\t"
10782        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10783        $$emit$$"add     0x4,rcx\n\t"
10784        $$emit$$"jle     L_end\n\t"
10785        $$emit$$"dec     rcx\n\t"
10786        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10787        $$emit$$"vmovq   xmm0,(rax)\n\t"
10788        $$emit$$"add     0x8,rax\n\t"
10789        $$emit$$"dec     rcx\n\t"
10790        $$emit$$"jge     L_sloop\n\t"
10791        $$emit$$"# L_end:\n\t"
10792     } else {
10793        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10794     }
10795     $$emit$$"# DONE"
10796   %}
10797   ins_encode %{
10798     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10799                  $tmp$$XMMRegister, false);
10800   %}
10801   ins_pipe(pipe_slow);
10802 %}
10803 
10804 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10805                         Universe dummy, rFlagsReg cr)
10806 %{
10807   predicate(((ClearArrayNode*)n)->is_large());
10808   match(Set dummy (ClearArray cnt base));
10809   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10810 
10811   format %{ $$template
10812     if (UseFastStosb) {
10813        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10814        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10815        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10816     } else if (UseXMMForObjInit) {
10817        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10818        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10819        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10820        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10821        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10822        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10823        $$emit$$"add     0x40,rax\n\t"
10824        $$emit$$"# L_zero_64_bytes:\n\t"
10825        $$emit$$"sub     0x8,rcx\n\t"
10826        $$emit$$"jge     L_loop\n\t"
10827        $$emit$$"add     0x4,rcx\n\t"
10828        $$emit$$"jl      L_tail\n\t"
10829        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10830        $$emit$$"add     0x20,rax\n\t"
10831        $$emit$$"sub     0x4,rcx\n\t"
10832        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10833        $$emit$$"add     0x4,rcx\n\t"
10834        $$emit$$"jle     L_end\n\t"
10835        $$emit$$"dec     rcx\n\t"
10836        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10837        $$emit$$"vmovq   xmm0,(rax)\n\t"
10838        $$emit$$"add     0x8,rax\n\t"
10839        $$emit$$"dec     rcx\n\t"
10840        $$emit$$"jge     L_sloop\n\t"
10841        $$emit$$"# L_end:\n\t"
10842     } else {
10843        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10844        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10845     }
10846   %}
10847   ins_encode %{
10848     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10849                  $tmp$$XMMRegister, true);
10850   %}
10851   ins_pipe(pipe_slow);
10852 %}
10853 
10854 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10855                          rax_RegI result, regD tmp1, rFlagsReg cr)
10856 %{
10857   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10858   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10859   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10860 
10861   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10862   ins_encode %{
10863     __ string_compare($str1$$Register, $str2$$Register,
10864                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10865                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
10866   %}
10867   ins_pipe( pipe_slow );
10868 %}
10869 
10870 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10871                          rax_RegI result, regD tmp1, rFlagsReg cr)
10872 %{
10873   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10874   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10875   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10876 
10877   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10878   ins_encode %{
10879     __ string_compare($str1$$Register, $str2$$Register,
10880                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10881                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
10882   %}
10883   ins_pipe( pipe_slow );
10884 %}
10885 
10886 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10887                           rax_RegI result, regD tmp1, rFlagsReg cr)
10888 %{
10889   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10890   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10891   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10892 
10893   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10894   ins_encode %{
10895     __ string_compare($str1$$Register, $str2$$Register,
10896                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10897                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
10898   %}
10899   ins_pipe( pipe_slow );
10900 %}
10901 
10902 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10903                           rax_RegI result, regD tmp1, rFlagsReg cr)
10904 %{
10905   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10906   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10907   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10908 
10909   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10910   ins_encode %{
10911     __ string_compare($str2$$Register, $str1$$Register,
10912                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10913                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
10914   %}
10915   ins_pipe( pipe_slow );
10916 %}
10917 
10918 // fast search of substring with known size.
10919 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10920                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10921 %{
10922   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10923   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10924   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10925 
10926   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10927   ins_encode %{
10928     int icnt2 = (int)$int_cnt2$$constant;
10929     if (icnt2 >= 16) {
10930       // IndexOf for constant substrings with size >= 16 elements
10931       // which don't need to be loaded through stack.
10932       __ string_indexofC8($str1$$Register, $str2$$Register,
10933                           $cnt1$$Register, $cnt2$$Register,
10934                           icnt2, $result$$Register,
10935                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10936     } else {
10937       // Small strings are loaded through stack if they cross page boundary.
10938       __ string_indexof($str1$$Register, $str2$$Register,
10939                         $cnt1$$Register, $cnt2$$Register,
10940                         icnt2, $result$$Register,
10941                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10942     }
10943   %}
10944   ins_pipe( pipe_slow );
10945 %}
10946 
10947 // fast search of substring with known size.
10948 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10949                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10950 %{
10951   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10952   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10953   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10954 
10955   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10956   ins_encode %{
10957     int icnt2 = (int)$int_cnt2$$constant;
10958     if (icnt2 >= 8) {
10959       // IndexOf for constant substrings with size >= 8 elements
10960       // which don't need to be loaded through stack.
10961       __ string_indexofC8($str1$$Register, $str2$$Register,
10962                           $cnt1$$Register, $cnt2$$Register,
10963                           icnt2, $result$$Register,
10964                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10965     } else {
10966       // Small strings are loaded through stack if they cross page boundary.
10967       __ string_indexof($str1$$Register, $str2$$Register,
10968                         $cnt1$$Register, $cnt2$$Register,
10969                         icnt2, $result$$Register,
10970                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10971     }
10972   %}
10973   ins_pipe( pipe_slow );
10974 %}
10975 
10976 // fast search of substring with known size.
10977 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10978                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10979 %{
10980   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10981   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10982   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10983 
10984   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10985   ins_encode %{
10986     int icnt2 = (int)$int_cnt2$$constant;
10987     if (icnt2 >= 8) {
10988       // IndexOf for constant substrings with size >= 8 elements
10989       // which don't need to be loaded through stack.
10990       __ string_indexofC8($str1$$Register, $str2$$Register,
10991                           $cnt1$$Register, $cnt2$$Register,
10992                           icnt2, $result$$Register,
10993                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10994     } else {
10995       // Small strings are loaded through stack if they cross page boundary.
10996       __ string_indexof($str1$$Register, $str2$$Register,
10997                         $cnt1$$Register, $cnt2$$Register,
10998                         icnt2, $result$$Register,
10999                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11000     }
11001   %}
11002   ins_pipe( pipe_slow );
11003 %}
11004 
11005 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11006                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
11007 %{
11008   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11009   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11010   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11011 
11012   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11013   ins_encode %{
11014     __ string_indexof($str1$$Register, $str2$$Register,
11015                       $cnt1$$Register, $cnt2$$Register,
11016                       (-1), $result$$Register,
11017                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11018   %}
11019   ins_pipe( pipe_slow );
11020 %}
11021 
11022 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11023                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
11024 %{
11025   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11026   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11027   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11028 
11029   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11030   ins_encode %{
11031     __ string_indexof($str1$$Register, $str2$$Register,
11032                       $cnt1$$Register, $cnt2$$Register,
11033                       (-1), $result$$Register,
11034                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11035   %}
11036   ins_pipe( pipe_slow );
11037 %}
11038 
11039 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11040                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
11041 %{
11042   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11043   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11044   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11045 
11046   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11047   ins_encode %{
11048     __ string_indexof($str1$$Register, $str2$$Register,
11049                       $cnt1$$Register, $cnt2$$Register,
11050                       (-1), $result$$Register,
11051                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11052   %}
11053   ins_pipe( pipe_slow );
11054 %}
11055 
11056 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11057                               rbx_RegI result, regD vec1, regD vec2, regD vec3, rcx_RegI tmp, rFlagsReg cr)
11058 %{
11059   predicate(UseSSE42Intrinsics);
11060   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11061   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11062   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11063   ins_encode %{
11064     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11065                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
11066   %}
11067   ins_pipe( pipe_slow );
11068 %}
11069 
11070 // fast string equals
11071 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11072                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11073 %{
11074   match(Set result (StrEquals (Binary str1 str2) cnt));
11075   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11076 
11077   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11078   ins_encode %{
11079     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11080                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11081                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11082   %}
11083   ins_pipe( pipe_slow );
11084 %}
11085 
11086 // fast array equals
11087 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11088                        regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11089 %{
11090   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11091   match(Set result (AryEq ary1 ary2));
11092   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11093 
11094   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11095   ins_encode %{
11096     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11097                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11098                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11099   %}
11100   ins_pipe( pipe_slow );
11101 %}
11102 
11103 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11104                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11105 %{
11106   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11107   match(Set result (AryEq ary1 ary2));
11108   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11109 
11110   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11111   ins_encode %{
11112     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11113                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11114                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
11115   %}
11116   ins_pipe( pipe_slow );
11117 %}
11118 
11119 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11120                       regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11121 %{
11122   match(Set result (HasNegatives ary1 len));
11123   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11124 
11125   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11126   ins_encode %{
11127     __ has_negatives($ary1$$Register, $len$$Register,
11128                      $result$$Register, $tmp3$$Register,
11129                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11130   %}
11131   ins_pipe( pipe_slow );
11132 %}
11133 
11134 // fast char[] to byte[] compression
11135 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, regD tmp1, regD tmp2, regD tmp3, regD tmp4,
11136                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11137   match(Set result (StrCompressedCopy src (Binary dst len)));
11138   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11139 
11140   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11141   ins_encode %{
11142     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11143                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11144                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11145   %}
11146   ins_pipe( pipe_slow );
11147 %}
11148 
11149 // fast byte[] to char[] inflation
11150 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11151                         regD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11152   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11153   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11154 
11155   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11156   ins_encode %{
11157     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11158                           $tmp1$$XMMRegister, $tmp2$$Register);
11159   %}
11160   ins_pipe( pipe_slow );
11161 %}
11162 
11163 // encode char[] to byte[] in ISO_8859_1
11164 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11165                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
11166                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11167   match(Set result (EncodeISOArray src (Binary dst len)));
11168   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11169 
11170   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11171   ins_encode %{
11172     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11173                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11174                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11175   %}
11176   ins_pipe( pipe_slow );
11177 %}
11178 
11179 //----------Overflow Math Instructions-----------------------------------------
11180 
11181 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11182 %{
11183   match(Set cr (OverflowAddI op1 op2));
11184   effect(DEF cr, USE_KILL op1, USE op2);
11185 
11186   format %{ "addl    $op1, $op2\t# overflow check int" %}
11187 
11188   ins_encode %{
11189     __ addl($op1$$Register, $op2$$Register);
11190   %}
11191   ins_pipe(ialu_reg_reg);
11192 %}
11193 
11194 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11195 %{
11196   match(Set cr (OverflowAddI op1 op2));
11197   effect(DEF cr, USE_KILL op1, USE op2);
11198 
11199   format %{ "addl    $op1, $op2\t# overflow check int" %}
11200 
11201   ins_encode %{
11202     __ addl($op1$$Register, $op2$$constant);
11203   %}
11204   ins_pipe(ialu_reg_reg);
11205 %}
11206 
11207 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11208 %{
11209   match(Set cr (OverflowAddL op1 op2));
11210   effect(DEF cr, USE_KILL op1, USE op2);
11211 
11212   format %{ "addq    $op1, $op2\t# overflow check long" %}
11213   ins_encode %{
11214     __ addq($op1$$Register, $op2$$Register);
11215   %}
11216   ins_pipe(ialu_reg_reg);
11217 %}
11218 
11219 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11220 %{
11221   match(Set cr (OverflowAddL op1 op2));
11222   effect(DEF cr, USE_KILL op1, USE op2);
11223 
11224   format %{ "addq    $op1, $op2\t# overflow check long" %}
11225   ins_encode %{
11226     __ addq($op1$$Register, $op2$$constant);
11227   %}
11228   ins_pipe(ialu_reg_reg);
11229 %}
11230 
11231 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11232 %{
11233   match(Set cr (OverflowSubI op1 op2));
11234 
11235   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11236   ins_encode %{
11237     __ cmpl($op1$$Register, $op2$$Register);
11238   %}
11239   ins_pipe(ialu_reg_reg);
11240 %}
11241 
11242 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11243 %{
11244   match(Set cr (OverflowSubI op1 op2));
11245 
11246   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11247   ins_encode %{
11248     __ cmpl($op1$$Register, $op2$$constant);
11249   %}
11250   ins_pipe(ialu_reg_reg);
11251 %}
11252 
11253 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11254 %{
11255   match(Set cr (OverflowSubL op1 op2));
11256 
11257   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11258   ins_encode %{
11259     __ cmpq($op1$$Register, $op2$$Register);
11260   %}
11261   ins_pipe(ialu_reg_reg);
11262 %}
11263 
11264 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11265 %{
11266   match(Set cr (OverflowSubL op1 op2));
11267 
11268   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11269   ins_encode %{
11270     __ cmpq($op1$$Register, $op2$$constant);
11271   %}
11272   ins_pipe(ialu_reg_reg);
11273 %}
11274 
11275 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
11276 %{
11277   match(Set cr (OverflowSubI zero op2));
11278   effect(DEF cr, USE_KILL op2);
11279 
11280   format %{ "negl    $op2\t# overflow check int" %}
11281   ins_encode %{
11282     __ negl($op2$$Register);
11283   %}
11284   ins_pipe(ialu_reg_reg);
11285 %}
11286 
11287 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11288 %{
11289   match(Set cr (OverflowSubL zero op2));
11290   effect(DEF cr, USE_KILL op2);
11291 
11292   format %{ "negq    $op2\t# overflow check long" %}
11293   ins_encode %{
11294     __ negq($op2$$Register);
11295   %}
11296   ins_pipe(ialu_reg_reg);
11297 %}
11298 
11299 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11300 %{
11301   match(Set cr (OverflowMulI op1 op2));
11302   effect(DEF cr, USE_KILL op1, USE op2);
11303 
11304   format %{ "imull    $op1, $op2\t# overflow check int" %}
11305   ins_encode %{
11306     __ imull($op1$$Register, $op2$$Register);
11307   %}
11308   ins_pipe(ialu_reg_reg_alu0);
11309 %}
11310 
11311 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11312 %{
11313   match(Set cr (OverflowMulI op1 op2));
11314   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11315 
11316   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11317   ins_encode %{
11318     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11319   %}
11320   ins_pipe(ialu_reg_reg_alu0);
11321 %}
11322 
11323 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11324 %{
11325   match(Set cr (OverflowMulL op1 op2));
11326   effect(DEF cr, USE_KILL op1, USE op2);
11327 
11328   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11329   ins_encode %{
11330     __ imulq($op1$$Register, $op2$$Register);
11331   %}
11332   ins_pipe(ialu_reg_reg_alu0);
11333 %}
11334 
11335 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11336 %{
11337   match(Set cr (OverflowMulL op1 op2));
11338   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11339 
11340   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11341   ins_encode %{
11342     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11343   %}
11344   ins_pipe(ialu_reg_reg_alu0);
11345 %}
11346 
11347 
11348 //----------Control Flow Instructions------------------------------------------
11349 // Signed compare Instructions
11350 
11351 // XXX more variants!!
11352 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11353 %{
11354   match(Set cr (CmpI op1 op2));
11355   effect(DEF cr, USE op1, USE op2);
11356 
11357   format %{ "cmpl    $op1, $op2" %}
11358   opcode(0x3B);  /* Opcode 3B /r */
11359   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11360   ins_pipe(ialu_cr_reg_reg);
11361 %}
11362 
11363 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11364 %{
11365   match(Set cr (CmpI op1 op2));
11366 
11367   format %{ "cmpl    $op1, $op2" %}
11368   opcode(0x81, 0x07); /* Opcode 81 /7 */
11369   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11370   ins_pipe(ialu_cr_reg_imm);
11371 %}
11372 
11373 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11374 %{
11375   match(Set cr (CmpI op1 (LoadI op2)));
11376 
11377   ins_cost(500); // XXX
11378   format %{ "cmpl    $op1, $op2" %}
11379   opcode(0x3B); /* Opcode 3B /r */
11380   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11381   ins_pipe(ialu_cr_reg_mem);
11382 %}
11383 
11384 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11385 %{
11386   match(Set cr (CmpI src zero));
11387 
11388   format %{ "testl   $src, $src" %}
11389   opcode(0x85);
11390   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11391   ins_pipe(ialu_cr_reg_imm);
11392 %}
11393 
11394 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11395 %{
11396   match(Set cr (CmpI (AndI src con) zero));
11397 
11398   format %{ "testl   $src, $con" %}
11399   opcode(0xF7, 0x00);
11400   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11401   ins_pipe(ialu_cr_reg_imm);
11402 %}
11403 
11404 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11405 %{
11406   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11407 
11408   format %{ "testl   $src, $mem" %}
11409   opcode(0x85);
11410   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11411   ins_pipe(ialu_cr_reg_mem);
11412 %}
11413 
11414 // Unsigned compare Instructions; really, same as signed except they
11415 // produce an rFlagsRegU instead of rFlagsReg.
11416 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11417 %{
11418   match(Set cr (CmpU op1 op2));
11419 
11420   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11421   opcode(0x3B); /* Opcode 3B /r */
11422   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11423   ins_pipe(ialu_cr_reg_reg);
11424 %}
11425 
11426 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11427 %{
11428   match(Set cr (CmpU op1 op2));
11429 
11430   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11431   opcode(0x81,0x07); /* Opcode 81 /7 */
11432   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11433   ins_pipe(ialu_cr_reg_imm);
11434 %}
11435 
11436 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11437 %{
11438   match(Set cr (CmpU op1 (LoadI op2)));
11439 
11440   ins_cost(500); // XXX
11441   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11442   opcode(0x3B); /* Opcode 3B /r */
11443   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11444   ins_pipe(ialu_cr_reg_mem);
11445 %}
11446 
11447 // // // Cisc-spilled version of cmpU_rReg
11448 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11449 // //%{
11450 // //  match(Set cr (CmpU (LoadI op1) op2));
11451 // //
11452 // //  format %{ "CMPu   $op1,$op2" %}
11453 // //  ins_cost(500);
11454 // //  opcode(0x39);  /* Opcode 39 /r */
11455 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11456 // //%}
11457 
11458 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11459 %{
11460   match(Set cr (CmpU src zero));
11461 
11462   format %{ "testl  $src, $src\t# unsigned" %}
11463   opcode(0x85);
11464   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11465   ins_pipe(ialu_cr_reg_imm);
11466 %}
11467 
11468 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11469 %{
11470   match(Set cr (CmpP op1 op2));
11471 
11472   format %{ "cmpq    $op1, $op2\t# ptr" %}
11473   opcode(0x3B); /* Opcode 3B /r */
11474   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11475   ins_pipe(ialu_cr_reg_reg);
11476 %}
11477 
11478 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11479 %{
11480   match(Set cr (CmpP op1 (LoadP op2)));
11481 
11482   ins_cost(500); // XXX
11483   format %{ "cmpq    $op1, $op2\t# ptr" %}
11484   opcode(0x3B); /* Opcode 3B /r */
11485   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11486   ins_pipe(ialu_cr_reg_mem);
11487 %}
11488 
11489 // // // Cisc-spilled version of cmpP_rReg
11490 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11491 // //%{
11492 // //  match(Set cr (CmpP (LoadP op1) op2));
11493 // //
11494 // //  format %{ "CMPu   $op1,$op2" %}
11495 // //  ins_cost(500);
11496 // //  opcode(0x39);  /* Opcode 39 /r */
11497 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11498 // //%}
11499 
11500 // XXX this is generalized by compP_rReg_mem???
11501 // Compare raw pointer (used in out-of-heap check).
11502 // Only works because non-oop pointers must be raw pointers
11503 // and raw pointers have no anti-dependencies.
11504 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11505 %{
11506   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11507   match(Set cr (CmpP op1 (LoadP op2)));
11508 
11509   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11510   opcode(0x3B); /* Opcode 3B /r */
11511   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11512   ins_pipe(ialu_cr_reg_mem);
11513 %}
11514 
11515 // This will generate a signed flags result. This should be OK since
11516 // any compare to a zero should be eq/neq.
11517 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11518 %{
11519   match(Set cr (CmpP src zero));
11520 
11521   format %{ "testq   $src, $src\t# ptr" %}
11522   opcode(0x85);
11523   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11524   ins_pipe(ialu_cr_reg_imm);
11525 %}
11526 
11527 // This will generate a signed flags result. This should be OK since
11528 // any compare to a zero should be eq/neq.
11529 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11530 %{
11531   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11532   match(Set cr (CmpP (LoadP op) zero));
11533 
11534   ins_cost(500); // XXX
11535   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11536   opcode(0xF7); /* Opcode F7 /0 */
11537   ins_encode(REX_mem_wide(op),
11538              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11539   ins_pipe(ialu_cr_reg_imm);
11540 %}
11541 
11542 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11543 %{
11544   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
11545   match(Set cr (CmpP (LoadP mem) zero));
11546 
11547   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11548   ins_encode %{
11549     __ cmpq(r12, $mem$$Address);
11550   %}
11551   ins_pipe(ialu_cr_reg_mem);
11552 %}
11553 
11554 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11555 %{
11556   match(Set cr (CmpN op1 op2));
11557 
11558   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11559   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11560   ins_pipe(ialu_cr_reg_reg);
11561 %}
11562 
11563 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11564 %{
11565   match(Set cr (CmpN src (LoadN mem)));
11566 
11567   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11568   ins_encode %{
11569     __ cmpl($src$$Register, $mem$$Address);
11570   %}
11571   ins_pipe(ialu_cr_reg_mem);
11572 %}
11573 
11574 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11575   match(Set cr (CmpN op1 op2));
11576 
11577   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11578   ins_encode %{
11579     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11580   %}
11581   ins_pipe(ialu_cr_reg_imm);
11582 %}
11583 
11584 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11585 %{
11586   match(Set cr (CmpN src (LoadN mem)));
11587 
11588   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11589   ins_encode %{
11590     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11591   %}
11592   ins_pipe(ialu_cr_reg_mem);
11593 %}
11594 
11595 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11596   match(Set cr (CmpN op1 op2));
11597 
11598   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11599   ins_encode %{
11600     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11601   %}
11602   ins_pipe(ialu_cr_reg_imm);
11603 %}
11604 
11605 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11606 %{
11607   match(Set cr (CmpN src (LoadNKlass mem)));
11608 
11609   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11610   ins_encode %{
11611     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11612   %}
11613   ins_pipe(ialu_cr_reg_mem);
11614 %}
11615 
11616 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11617   match(Set cr (CmpN src zero));
11618 
11619   format %{ "testl   $src, $src\t# compressed ptr" %}
11620   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11621   ins_pipe(ialu_cr_reg_imm);
11622 %}
11623 
11624 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11625 %{
11626   predicate(Universe::narrow_oop_base() != NULL);
11627   match(Set cr (CmpN (LoadN mem) zero));
11628 
11629   ins_cost(500); // XXX
11630   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11631   ins_encode %{
11632     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11633   %}
11634   ins_pipe(ialu_cr_reg_mem);
11635 %}
11636 
11637 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11638 %{
11639   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11640   match(Set cr (CmpN (LoadN mem) zero));
11641 
11642   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11643   ins_encode %{
11644     __ cmpl(r12, $mem$$Address);
11645   %}
11646   ins_pipe(ialu_cr_reg_mem);
11647 %}
11648 
11649 // Yanked all unsigned pointer compare operations.
11650 // Pointer compares are done with CmpP which is already unsigned.
11651 
11652 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11653 %{
11654   match(Set cr (CmpL op1 op2));
11655 
11656   format %{ "cmpq    $op1, $op2" %}
11657   opcode(0x3B);  /* Opcode 3B /r */
11658   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11659   ins_pipe(ialu_cr_reg_reg);
11660 %}
11661 
11662 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11663 %{
11664   match(Set cr (CmpL op1 op2));
11665 
11666   format %{ "cmpq    $op1, $op2" %}
11667   opcode(0x81, 0x07); /* Opcode 81 /7 */
11668   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11669   ins_pipe(ialu_cr_reg_imm);
11670 %}
11671 
11672 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11673 %{
11674   match(Set cr (CmpL op1 (LoadL op2)));
11675 
11676   format %{ "cmpq    $op1, $op2" %}
11677   opcode(0x3B); /* Opcode 3B /r */
11678   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11679   ins_pipe(ialu_cr_reg_mem);
11680 %}
11681 
11682 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11683 %{
11684   match(Set cr (CmpL src zero));
11685 
11686   format %{ "testq   $src, $src" %}
11687   opcode(0x85);
11688   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11689   ins_pipe(ialu_cr_reg_imm);
11690 %}
11691 
11692 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11693 %{
11694   match(Set cr (CmpL (AndL src con) zero));
11695 
11696   format %{ "testq   $src, $con\t# long" %}
11697   opcode(0xF7, 0x00);
11698   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11699   ins_pipe(ialu_cr_reg_imm);
11700 %}
11701 
11702 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11703 %{
11704   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11705 
11706   format %{ "testq   $src, $mem" %}
11707   opcode(0x85);
11708   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11709   ins_pipe(ialu_cr_reg_mem);
11710 %}
11711 
11712 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11713 %{
11714   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11715 
11716   format %{ "testq   $src, $mem" %}
11717   opcode(0x85);
11718   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11719   ins_pipe(ialu_cr_reg_mem);
11720 %}
11721 
11722 // Manifest a CmpL result in an integer register.  Very painful.
11723 // This is the test to avoid.
11724 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11725 %{
11726   match(Set dst (CmpL3 src1 src2));
11727   effect(KILL flags);
11728 
11729   ins_cost(275); // XXX
11730   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11731             "movl    $dst, -1\n\t"
11732             "jl,s    done\n\t"
11733             "setne   $dst\n\t"
11734             "movzbl  $dst, $dst\n\t"
11735     "done:" %}
11736   ins_encode(cmpl3_flag(src1, src2, dst));
11737   ins_pipe(pipe_slow);
11738 %}
11739 
11740 // Unsigned long compare Instructions; really, same as signed long except they
11741 // produce an rFlagsRegU instead of rFlagsReg.
11742 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11743 %{
11744   match(Set cr (CmpUL op1 op2));
11745 
11746   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11747   opcode(0x3B);  /* Opcode 3B /r */
11748   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11749   ins_pipe(ialu_cr_reg_reg);
11750 %}
11751 
11752 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11753 %{
11754   match(Set cr (CmpUL op1 op2));
11755 
11756   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11757   opcode(0x81, 0x07); /* Opcode 81 /7 */
11758   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11759   ins_pipe(ialu_cr_reg_imm);
11760 %}
11761 
11762 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11763 %{
11764   match(Set cr (CmpUL op1 (LoadL op2)));
11765 
11766   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11767   opcode(0x3B); /* Opcode 3B /r */
11768   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11769   ins_pipe(ialu_cr_reg_mem);
11770 %}
11771 
11772 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
11773 %{
11774   match(Set cr (CmpUL src zero));
11775 
11776   format %{ "testq   $src, $src\t# unsigned" %}
11777   opcode(0x85);
11778   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11779   ins_pipe(ialu_cr_reg_imm);
11780 %}
11781 
11782 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
11783 %{
11784   match(Set cr (CmpI (LoadB mem) imm));
11785 
11786   ins_cost(125);
11787   format %{ "cmpb    $mem, $imm" %}
11788   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
11789   ins_pipe(ialu_cr_reg_mem);
11790 %}
11791 
11792 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI0 zero)
11793 %{
11794   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
11795 
11796   ins_cost(125);
11797   format %{ "testb   $mem, $imm" %}
11798   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
11799   ins_pipe(ialu_cr_reg_mem);
11800 %}
11801 
11802 //----------Max and Min--------------------------------------------------------
11803 // Min Instructions
11804 
11805 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11806 %{
11807   effect(USE_DEF dst, USE src, USE cr);
11808 
11809   format %{ "cmovlgt $dst, $src\t# min" %}
11810   opcode(0x0F, 0x4F);
11811   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11812   ins_pipe(pipe_cmov_reg);
11813 %}
11814 
11815 
11816 instruct minI_rReg(rRegI dst, rRegI src)
11817 %{
11818   match(Set dst (MinI dst src));
11819 
11820   ins_cost(200);
11821   expand %{
11822     rFlagsReg cr;
11823     compI_rReg(cr, dst, src);
11824     cmovI_reg_g(dst, src, cr);
11825   %}
11826 %}
11827 
11828 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11829 %{
11830   effect(USE_DEF dst, USE src, USE cr);
11831 
11832   format %{ "cmovllt $dst, $src\t# max" %}
11833   opcode(0x0F, 0x4C);
11834   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11835   ins_pipe(pipe_cmov_reg);
11836 %}
11837 
11838 
11839 instruct maxI_rReg(rRegI dst, rRegI src)
11840 %{
11841   match(Set dst (MaxI dst src));
11842 
11843   ins_cost(200);
11844   expand %{
11845     rFlagsReg cr;
11846     compI_rReg(cr, dst, src);
11847     cmovI_reg_l(dst, src, cr);
11848   %}
11849 %}
11850 
11851 // ============================================================================
11852 // Branch Instructions
11853 
11854 // Jump Direct - Label defines a relative address from JMP+1
11855 instruct jmpDir(label labl)
11856 %{
11857   match(Goto);
11858   effect(USE labl);
11859 
11860   ins_cost(300);
11861   format %{ "jmp     $labl" %}
11862   size(5);
11863   ins_encode %{
11864     Label* L = $labl$$label;
11865     __ jmp(*L, false); // Always long jump
11866   %}
11867   ins_pipe(pipe_jmp);
11868 %}
11869 
11870 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11871 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11872 %{
11873   match(If cop cr);
11874   effect(USE labl);
11875 
11876   ins_cost(300);
11877   format %{ "j$cop     $labl" %}
11878   size(6);
11879   ins_encode %{
11880     Label* L = $labl$$label;
11881     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11882   %}
11883   ins_pipe(pipe_jcc);
11884 %}
11885 
11886 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11887 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11888 %{
11889   predicate(!n->has_vector_mask_set());
11890   match(CountedLoopEnd cop cr);
11891   effect(USE labl);
11892 
11893   ins_cost(300);
11894   format %{ "j$cop     $labl\t# loop end" %}
11895   size(6);
11896   ins_encode %{
11897     Label* L = $labl$$label;
11898     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11899   %}
11900   ins_pipe(pipe_jcc);
11901 %}
11902 
11903 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11904 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11905   predicate(!n->has_vector_mask_set());
11906   match(CountedLoopEnd cop cmp);
11907   effect(USE labl);
11908 
11909   ins_cost(300);
11910   format %{ "j$cop,u   $labl\t# loop end" %}
11911   size(6);
11912   ins_encode %{
11913     Label* L = $labl$$label;
11914     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11915   %}
11916   ins_pipe(pipe_jcc);
11917 %}
11918 
11919 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11920   predicate(!n->has_vector_mask_set());
11921   match(CountedLoopEnd cop cmp);
11922   effect(USE labl);
11923 
11924   ins_cost(200);
11925   format %{ "j$cop,u   $labl\t# loop end" %}
11926   size(6);
11927   ins_encode %{
11928     Label* L = $labl$$label;
11929     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11930   %}
11931   ins_pipe(pipe_jcc);
11932 %}
11933 
11934 // mask version
11935 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11936 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, rFlagsReg cr, label labl)
11937 %{
11938   predicate(n->has_vector_mask_set());
11939   match(CountedLoopEnd cop cr);
11940   effect(USE labl);
11941 
11942   ins_cost(400);
11943   format %{ "j$cop     $labl\t# loop end\n\t"
11944             "restorevectmask \t# vector mask restore for loops" %}
11945   size(10);
11946   ins_encode %{
11947     Label* L = $labl$$label;
11948     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11949     __ restorevectmask();
11950   %}
11951   ins_pipe(pipe_jcc);
11952 %}
11953 
11954 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11955 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11956   predicate(n->has_vector_mask_set());
11957   match(CountedLoopEnd cop cmp);
11958   effect(USE labl);
11959 
11960   ins_cost(400);
11961   format %{ "j$cop,u   $labl\t# loop end\n\t"
11962             "restorevectmask \t# vector mask restore for loops" %}
11963   size(10);
11964   ins_encode %{
11965     Label* L = $labl$$label;
11966     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11967     __ restorevectmask();
11968   %}
11969   ins_pipe(pipe_jcc);
11970 %}
11971 
11972 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11973   predicate(n->has_vector_mask_set());
11974   match(CountedLoopEnd cop cmp);
11975   effect(USE labl);
11976 
11977   ins_cost(300);
11978   format %{ "j$cop,u   $labl\t# loop end\n\t"
11979             "restorevectmask \t# vector mask restore for loops" %}
11980   size(10);
11981   ins_encode %{
11982     Label* L = $labl$$label;
11983     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11984     __ restorevectmask();
11985   %}
11986   ins_pipe(pipe_jcc);
11987 %}
11988 
11989 // Jump Direct Conditional - using unsigned comparison
11990 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11991   match(If cop cmp);
11992   effect(USE labl);
11993 
11994   ins_cost(300);
11995   format %{ "j$cop,u  $labl" %}
11996   size(6);
11997   ins_encode %{
11998     Label* L = $labl$$label;
11999     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12000   %}
12001   ins_pipe(pipe_jcc);
12002 %}
12003 
12004 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12005   match(If cop cmp);
12006   effect(USE labl);
12007 
12008   ins_cost(200);
12009   format %{ "j$cop,u  $labl" %}
12010   size(6);
12011   ins_encode %{
12012     Label* L = $labl$$label;
12013     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12014   %}
12015   ins_pipe(pipe_jcc);
12016 %}
12017 
12018 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12019   match(If cop cmp);
12020   effect(USE labl);
12021 
12022   ins_cost(200);
12023   format %{ $$template
12024     if ($cop$$cmpcode == Assembler::notEqual) {
12025       $$emit$$"jp,u   $labl\n\t"
12026       $$emit$$"j$cop,u   $labl"
12027     } else {
12028       $$emit$$"jp,u   done\n\t"
12029       $$emit$$"j$cop,u   $labl\n\t"
12030       $$emit$$"done:"
12031     }
12032   %}
12033   ins_encode %{
12034     Label* l = $labl$$label;
12035     if ($cop$$cmpcode == Assembler::notEqual) {
12036       __ jcc(Assembler::parity, *l, false);
12037       __ jcc(Assembler::notEqual, *l, false);
12038     } else if ($cop$$cmpcode == Assembler::equal) {
12039       Label done;
12040       __ jccb(Assembler::parity, done);
12041       __ jcc(Assembler::equal, *l, false);
12042       __ bind(done);
12043     } else {
12044        ShouldNotReachHere();
12045     }
12046   %}
12047   ins_pipe(pipe_jcc);
12048 %}
12049 
12050 // ============================================================================
12051 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12052 // superklass array for an instance of the superklass.  Set a hidden
12053 // internal cache on a hit (cache is checked with exposed code in
12054 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12055 // encoding ALSO sets flags.
12056 
12057 instruct partialSubtypeCheck(rdi_RegP result,
12058                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12059                              rFlagsReg cr)
12060 %{
12061   match(Set result (PartialSubtypeCheck sub super));
12062   effect(KILL rcx, KILL cr);
12063 
12064   ins_cost(1100);  // slightly larger than the next version
12065   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12066             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12067             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12068             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12069             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12070             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12071             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12072     "miss:\t" %}
12073 
12074   opcode(0x1); // Force a XOR of RDI
12075   ins_encode(enc_PartialSubtypeCheck());
12076   ins_pipe(pipe_slow);
12077 %}
12078 
12079 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12080                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12081                                      immP0 zero,
12082                                      rdi_RegP result)
12083 %{
12084   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12085   effect(KILL rcx, KILL result);
12086 
12087   ins_cost(1000);
12088   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12089             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12090             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12091             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12092             "jne,s   miss\t\t# Missed: flags nz\n\t"
12093             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12094     "miss:\t" %}
12095 
12096   opcode(0x0); // No need to XOR RDI
12097   ins_encode(enc_PartialSubtypeCheck());
12098   ins_pipe(pipe_slow);
12099 %}
12100 
12101 // ============================================================================
12102 // Branch Instructions -- short offset versions
12103 //
12104 // These instructions are used to replace jumps of a long offset (the default
12105 // match) with jumps of a shorter offset.  These instructions are all tagged
12106 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12107 // match rules in general matching.  Instead, the ADLC generates a conversion
12108 // method in the MachNode which can be used to do in-place replacement of the
12109 // long variant with the shorter variant.  The compiler will determine if a
12110 // branch can be taken by the is_short_branch_offset() predicate in the machine
12111 // specific code section of the file.
12112 
12113 // Jump Direct - Label defines a relative address from JMP+1
12114 instruct jmpDir_short(label labl) %{
12115   match(Goto);
12116   effect(USE labl);
12117 
12118   ins_cost(300);
12119   format %{ "jmp,s   $labl" %}
12120   size(2);
12121   ins_encode %{
12122     Label* L = $labl$$label;
12123     __ jmpb(*L);
12124   %}
12125   ins_pipe(pipe_jmp);
12126   ins_short_branch(1);
12127 %}
12128 
12129 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12130 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12131   match(If cop cr);
12132   effect(USE labl);
12133 
12134   ins_cost(300);
12135   format %{ "j$cop,s   $labl" %}
12136   size(2);
12137   ins_encode %{
12138     Label* L = $labl$$label;
12139     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12140   %}
12141   ins_pipe(pipe_jcc);
12142   ins_short_branch(1);
12143 %}
12144 
12145 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12146 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12147   match(CountedLoopEnd cop cr);
12148   effect(USE labl);
12149 
12150   ins_cost(300);
12151   format %{ "j$cop,s   $labl\t# loop end" %}
12152   size(2);
12153   ins_encode %{
12154     Label* L = $labl$$label;
12155     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12156   %}
12157   ins_pipe(pipe_jcc);
12158   ins_short_branch(1);
12159 %}
12160 
12161 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12162 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12163   match(CountedLoopEnd cop cmp);
12164   effect(USE labl);
12165 
12166   ins_cost(300);
12167   format %{ "j$cop,us  $labl\t# loop end" %}
12168   size(2);
12169   ins_encode %{
12170     Label* L = $labl$$label;
12171     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12172   %}
12173   ins_pipe(pipe_jcc);
12174   ins_short_branch(1);
12175 %}
12176 
12177 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12178   match(CountedLoopEnd cop cmp);
12179   effect(USE labl);
12180 
12181   ins_cost(300);
12182   format %{ "j$cop,us  $labl\t# loop end" %}
12183   size(2);
12184   ins_encode %{
12185     Label* L = $labl$$label;
12186     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12187   %}
12188   ins_pipe(pipe_jcc);
12189   ins_short_branch(1);
12190 %}
12191 
12192 // Jump Direct Conditional - using unsigned comparison
12193 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12194   match(If cop cmp);
12195   effect(USE labl);
12196 
12197   ins_cost(300);
12198   format %{ "j$cop,us  $labl" %}
12199   size(2);
12200   ins_encode %{
12201     Label* L = $labl$$label;
12202     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12203   %}
12204   ins_pipe(pipe_jcc);
12205   ins_short_branch(1);
12206 %}
12207 
12208 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12209   match(If cop cmp);
12210   effect(USE labl);
12211 
12212   ins_cost(300);
12213   format %{ "j$cop,us  $labl" %}
12214   size(2);
12215   ins_encode %{
12216     Label* L = $labl$$label;
12217     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12218   %}
12219   ins_pipe(pipe_jcc);
12220   ins_short_branch(1);
12221 %}
12222 
12223 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12224   match(If cop cmp);
12225   effect(USE labl);
12226 
12227   ins_cost(300);
12228   format %{ $$template
12229     if ($cop$$cmpcode == Assembler::notEqual) {
12230       $$emit$$"jp,u,s   $labl\n\t"
12231       $$emit$$"j$cop,u,s   $labl"
12232     } else {
12233       $$emit$$"jp,u,s   done\n\t"
12234       $$emit$$"j$cop,u,s  $labl\n\t"
12235       $$emit$$"done:"
12236     }
12237   %}
12238   size(4);
12239   ins_encode %{
12240     Label* l = $labl$$label;
12241     if ($cop$$cmpcode == Assembler::notEqual) {
12242       __ jccb(Assembler::parity, *l);
12243       __ jccb(Assembler::notEqual, *l);
12244     } else if ($cop$$cmpcode == Assembler::equal) {
12245       Label done;
12246       __ jccb(Assembler::parity, done);
12247       __ jccb(Assembler::equal, *l);
12248       __ bind(done);
12249     } else {
12250        ShouldNotReachHere();
12251     }
12252   %}
12253   ins_pipe(pipe_jcc);
12254   ins_short_branch(1);
12255 %}
12256 
12257 // ============================================================================
12258 // inlined locking and unlocking
12259 
12260 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12261   predicate(Compile::current()->use_rtm());
12262   match(Set cr (FastLock object box));
12263   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12264   ins_cost(300);
12265   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12266   ins_encode %{
12267     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12268                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12269                  _counters, _rtm_counters, _stack_rtm_counters,
12270                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12271                  true, ra_->C->profile_rtm());
12272   %}
12273   ins_pipe(pipe_slow);
12274 %}
12275 
12276 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12277   predicate(!Compile::current()->use_rtm());
12278   match(Set cr (FastLock object box));
12279   effect(TEMP tmp, TEMP scr, USE_KILL box);
12280   ins_cost(300);
12281   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12282   ins_encode %{
12283     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12284                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
12285   %}
12286   ins_pipe(pipe_slow);
12287 %}
12288 
12289 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12290   match(Set cr (FastUnlock object box));
12291   effect(TEMP tmp, USE_KILL box);
12292   ins_cost(300);
12293   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12294   ins_encode %{
12295     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12296   %}
12297   ins_pipe(pipe_slow);
12298 %}
12299 
12300 
12301 // ============================================================================
12302 // Safepoint Instructions
12303 instruct safePoint_poll(rFlagsReg cr)
12304 %{
12305   predicate(!Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12306   match(SafePoint);
12307   effect(KILL cr);
12308 
12309   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
12310             "# Safepoint: poll for GC" %}
12311   ins_cost(125);
12312   ins_encode %{
12313     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12314     __ testl(rax, addr);
12315   %}
12316   ins_pipe(ialu_reg_mem);
12317 %}
12318 
12319 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12320 %{
12321   predicate(Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12322   match(SafePoint poll);
12323   effect(KILL cr, USE poll);
12324 
12325   format %{ "testl  rax, [$poll]\t"
12326             "# Safepoint: poll for GC" %}
12327   ins_cost(125);
12328   ins_encode %{
12329     __ relocate(relocInfo::poll_type);
12330     __ testl(rax, Address($poll$$Register, 0));
12331   %}
12332   ins_pipe(ialu_reg_mem);
12333 %}
12334 
12335 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12336 %{
12337   predicate(SafepointMechanism::uses_thread_local_poll());
12338   match(SafePoint poll);
12339   effect(KILL cr, USE poll);
12340 
12341   format %{ "testl  rax, [$poll]\t"
12342             "# Safepoint: poll for GC" %}
12343   ins_cost(125);
12344   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12345   ins_encode %{
12346     __ relocate(relocInfo::poll_type);
12347     address pre_pc = __ pc();
12348     __ testl(rax, Address($poll$$Register, 0));
12349     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12350   %}
12351   ins_pipe(ialu_reg_mem);
12352 %}
12353 
12354 // ============================================================================
12355 // Procedure Call/Return Instructions
12356 // Call Java Static Instruction
12357 // Note: If this code changes, the corresponding ret_addr_offset() and
12358 //       compute_padding() functions will have to be adjusted.
12359 instruct CallStaticJavaDirect(method meth) %{
12360   match(CallStaticJava);
12361   effect(USE meth);
12362 
12363   ins_cost(300);
12364   format %{ "call,static " %}
12365   opcode(0xE8); /* E8 cd */
12366   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12367   ins_pipe(pipe_slow);
12368   ins_alignment(4);
12369 %}
12370 
12371 // Call Java Dynamic Instruction
12372 // Note: If this code changes, the corresponding ret_addr_offset() and
12373 //       compute_padding() functions will have to be adjusted.
12374 instruct CallDynamicJavaDirect(method meth)
12375 %{
12376   match(CallDynamicJava);
12377   effect(USE meth);
12378 
12379   ins_cost(300);
12380   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12381             "call,dynamic " %}
12382   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12383   ins_pipe(pipe_slow);
12384   ins_alignment(4);
12385 %}
12386 
12387 // Call Runtime Instruction
12388 instruct CallRuntimeDirect(method meth)
12389 %{
12390   match(CallRuntime);
12391   effect(USE meth);
12392 
12393   ins_cost(300);
12394   format %{ "call,runtime " %}
12395   ins_encode(clear_avx, Java_To_Runtime(meth));
12396   ins_pipe(pipe_slow);
12397 %}
12398 
12399 // Call runtime without safepoint
12400 instruct CallLeafDirect(method meth)
12401 %{
12402   match(CallLeaf);
12403   effect(USE meth);
12404 
12405   ins_cost(300);
12406   format %{ "call_leaf,runtime " %}
12407   ins_encode(clear_avx, Java_To_Runtime(meth));
12408   ins_pipe(pipe_slow);
12409 %}
12410 
12411 // Call runtime without safepoint
12412 instruct CallLeafNoFPDirect(method meth)
12413 %{
12414   match(CallLeafNoFP);
12415   effect(USE meth);
12416 
12417   ins_cost(300);
12418   format %{ "call_leaf_nofp,runtime " %}
12419   ins_encode(clear_avx, Java_To_Runtime(meth));
12420   ins_pipe(pipe_slow);
12421 %}
12422 
12423 // Return Instruction
12424 // Remove the return address & jump to it.
12425 // Notice: We always emit a nop after a ret to make sure there is room
12426 // for safepoint patching
12427 instruct Ret()
12428 %{
12429   match(Return);
12430 
12431   format %{ "ret" %}
12432   opcode(0xC3);
12433   ins_encode(OpcP);
12434   ins_pipe(pipe_jmp);
12435 %}
12436 
12437 // Tail Call; Jump from runtime stub to Java code.
12438 // Also known as an 'interprocedural jump'.
12439 // Target of jump will eventually return to caller.
12440 // TailJump below removes the return address.
12441 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
12442 %{
12443   match(TailCall jump_target method_oop);
12444 
12445   ins_cost(300);
12446   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
12447   opcode(0xFF, 0x4); /* Opcode FF /4 */
12448   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
12449   ins_pipe(pipe_jmp);
12450 %}
12451 
12452 // Tail Jump; remove the return address; jump to target.
12453 // TailCall above leaves the return address around.
12454 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12455 %{
12456   match(TailJump jump_target ex_oop);
12457 
12458   ins_cost(300);
12459   format %{ "popq    rdx\t# pop return address\n\t"
12460             "jmp     $jump_target" %}
12461   opcode(0xFF, 0x4); /* Opcode FF /4 */
12462   ins_encode(Opcode(0x5a), // popq rdx
12463              REX_reg(jump_target), OpcP, reg_opc(jump_target));
12464   ins_pipe(pipe_jmp);
12465 %}
12466 
12467 // Create exception oop: created by stack-crawling runtime code.
12468 // Created exception is now available to this handler, and is setup
12469 // just prior to jumping to this handler.  No code emitted.
12470 instruct CreateException(rax_RegP ex_oop)
12471 %{
12472   match(Set ex_oop (CreateEx));
12473 
12474   size(0);
12475   // use the following format syntax
12476   format %{ "# exception oop is in rax; no code emitted" %}
12477   ins_encode();
12478   ins_pipe(empty);
12479 %}
12480 
12481 // Rethrow exception:
12482 // The exception oop will come in the first argument position.
12483 // Then JUMP (not call) to the rethrow stub code.
12484 instruct RethrowException()
12485 %{
12486   match(Rethrow);
12487 
12488   // use the following format syntax
12489   format %{ "jmp     rethrow_stub" %}
12490   ins_encode(enc_rethrow);
12491   ins_pipe(pipe_jmp);
12492 %}
12493 
12494 //
12495 // Execute ZGC load barrier (strong) slow path
12496 //
12497 
12498 // When running without XMM regs
12499 instruct loadBarrierSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{
12500 
12501   match(Set dst (LoadBarrierSlowReg mem));
12502   predicate(MaxVectorSize < 16);
12503 
12504   effect(DEF dst, KILL cr);
12505 
12506   format %{"LoadBarrierSlowRegNoVec $dst, $mem" %}
12507   ins_encode %{
12508 #if INCLUDE_ZGC
12509     Register d = $dst$$Register;
12510     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12511 
12512     assert(d != r12, "Can't be R12!");
12513     assert(d != r15, "Can't be R15!");
12514     assert(d != rsp, "Can't be RSP!");
12515 
12516     __ lea(d, $mem$$Address);
12517     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12518 #else
12519     ShouldNotReachHere();
12520 #endif
12521   %}
12522   ins_pipe(pipe_slow);
12523 %}
12524 
12525 // For XMM and YMM enabled processors
12526 instruct loadBarrierSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr,
12527                                      rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12528                                      rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12529                                      rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12530                                      rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
12531 
12532   match(Set dst (LoadBarrierSlowReg mem));
12533   predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16));
12534 
12535   effect(DEF dst, KILL cr,
12536          KILL x0, KILL x1, KILL x2, KILL x3,
12537          KILL x4, KILL x5, KILL x6, KILL x7,
12538          KILL x8, KILL x9, KILL x10, KILL x11,
12539          KILL x12, KILL x13, KILL x14, KILL x15);
12540 
12541   format %{"LoadBarrierSlowRegXmm $dst, $mem" %}
12542   ins_encode %{
12543 #if INCLUDE_ZGC
12544     Register d = $dst$$Register;
12545     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12546 
12547     assert(d != r12, "Can't be R12!");
12548     assert(d != r15, "Can't be R15!");
12549     assert(d != rsp, "Can't be RSP!");
12550 
12551     __ lea(d, $mem$$Address);
12552     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12553 #else
12554     ShouldNotReachHere();
12555 #endif
12556   %}
12557   ins_pipe(pipe_slow);
12558 %}
12559 
12560 // For ZMM enabled processors
12561 instruct loadBarrierSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr,
12562                                rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12563                                rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12564                                rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12565                                rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15,
12566                                rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19,
12567                                rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23,
12568                                rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27,
12569                                rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
12570 
12571   match(Set dst (LoadBarrierSlowReg mem));
12572   predicate((UseAVX == 3) && (MaxVectorSize >= 16));
12573 
12574   effect(DEF dst, KILL cr,
12575          KILL x0, KILL x1, KILL x2, KILL x3,
12576          KILL x4, KILL x5, KILL x6, KILL x7,
12577          KILL x8, KILL x9, KILL x10, KILL x11,
12578          KILL x12, KILL x13, KILL x14, KILL x15,
12579          KILL x16, KILL x17, KILL x18, KILL x19,
12580          KILL x20, KILL x21, KILL x22, KILL x23,
12581          KILL x24, KILL x25, KILL x26, KILL x27,
12582          KILL x28, KILL x29, KILL x30, KILL x31);
12583 
12584   format %{"LoadBarrierSlowRegZmm $dst, $mem" %}
12585   ins_encode %{
12586 #if INCLUDE_ZGC
12587     Register d = $dst$$Register;
12588     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12589 
12590     assert(d != r12, "Can't be R12!");
12591     assert(d != r15, "Can't be R15!");
12592     assert(d != rsp, "Can't be RSP!");
12593 
12594     __ lea(d, $mem$$Address);
12595     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12596 #else
12597     ShouldNotReachHere();
12598 #endif
12599   %}
12600   ins_pipe(pipe_slow);
12601 %}
12602 
12603 //
12604 // Execute ZGC load barrier (weak) slow path
12605 //
12606 
12607 // When running without XMM regs
12608 instruct loadBarrierWeakSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{
12609 
12610   match(Set dst (LoadBarrierSlowReg mem));
12611   predicate(MaxVectorSize < 16);
12612 
12613   effect(DEF dst, KILL cr);
12614 
12615   format %{"LoadBarrierSlowRegNoVec $dst, $mem" %}
12616   ins_encode %{
12617 #if INCLUDE_ZGC
12618     Register d = $dst$$Register;
12619     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12620 
12621     assert(d != r12, "Can't be R12!");
12622     assert(d != r15, "Can't be R15!");
12623     assert(d != rsp, "Can't be RSP!");
12624 
12625     __ lea(d, $mem$$Address);
12626     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12627 #else
12628     ShouldNotReachHere();
12629 #endif
12630   %}
12631   ins_pipe(pipe_slow);
12632 %}
12633 
12634 // For XMM and YMM enabled processors
12635 instruct loadBarrierWeakSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr,
12636                                          rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12637                                          rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12638                                          rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12639                                          rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
12640 
12641   match(Set dst (LoadBarrierWeakSlowReg mem));
12642   predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16));
12643 
12644   effect(DEF dst, KILL cr,
12645          KILL x0, KILL x1, KILL x2, KILL x3,
12646          KILL x4, KILL x5, KILL x6, KILL x7,
12647          KILL x8, KILL x9, KILL x10, KILL x11,
12648          KILL x12, KILL x13, KILL x14, KILL x15);
12649 
12650   format %{"LoadBarrierWeakSlowRegXmm $dst, $mem" %}
12651   ins_encode %{
12652 #if INCLUDE_ZGC
12653     Register d = $dst$$Register;
12654     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12655 
12656     assert(d != r12, "Can't be R12!");
12657     assert(d != r15, "Can't be R15!");
12658     assert(d != rsp, "Can't be RSP!");
12659 
12660     __ lea(d,$mem$$Address);
12661     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12662 #else
12663     ShouldNotReachHere();
12664 #endif
12665   %}
12666   ins_pipe(pipe_slow);
12667 %}
12668 
12669 // For ZMM enabled processors
12670 instruct loadBarrierWeakSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr,
12671                                    rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12672                                    rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12673                                    rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12674                                    rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15,
12675                                    rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19,
12676                                    rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23,
12677                                    rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27,
12678                                    rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
12679 
12680   match(Set dst (LoadBarrierWeakSlowReg mem));
12681   predicate((UseAVX == 3) && (MaxVectorSize >= 16));
12682 
12683   effect(DEF dst, KILL cr,
12684          KILL x0, KILL x1, KILL x2, KILL x3,
12685          KILL x4, KILL x5, KILL x6, KILL x7,
12686          KILL x8, KILL x9, KILL x10, KILL x11,
12687          KILL x12, KILL x13, KILL x14, KILL x15,
12688          KILL x16, KILL x17, KILL x18, KILL x19,
12689          KILL x20, KILL x21, KILL x22, KILL x23,
12690          KILL x24, KILL x25, KILL x26, KILL x27,
12691          KILL x28, KILL x29, KILL x30, KILL x31);
12692 
12693   format %{"LoadBarrierWeakSlowRegZmm $dst, $mem" %}
12694   ins_encode %{
12695 #if INCLUDE_ZGC
12696     Register d = $dst$$Register;
12697     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12698 
12699     assert(d != r12, "Can't be R12!");
12700     assert(d != r15, "Can't be R15!");
12701     assert(d != rsp, "Can't be RSP!");
12702 
12703     __ lea(d,$mem$$Address);
12704     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12705 #else
12706     ShouldNotReachHere();
12707 #endif
12708   %}
12709   ins_pipe(pipe_slow);
12710 %}
12711 
12712 // ============================================================================
12713 // This name is KNOWN by the ADLC and cannot be changed.
12714 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12715 // for this guy.
12716 instruct tlsLoadP(r15_RegP dst) %{
12717   match(Set dst (ThreadLocal));
12718   effect(DEF dst);
12719 
12720   size(0);
12721   format %{ "# TLS is in R15" %}
12722   ins_encode( /*empty encoding*/ );
12723   ins_pipe(ialu_reg_reg);
12724 %}
12725 
12726 
12727 //----------PEEPHOLE RULES-----------------------------------------------------
12728 // These must follow all instruction definitions as they use the names
12729 // defined in the instructions definitions.
12730 //
12731 // peepmatch ( root_instr_name [preceding_instruction]* );
12732 //
12733 // peepconstraint %{
12734 // (instruction_number.operand_name relational_op instruction_number.operand_name
12735 //  [, ...] );
12736 // // instruction numbers are zero-based using left to right order in peepmatch
12737 //
12738 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12739 // // provide an instruction_number.operand_name for each operand that appears
12740 // // in the replacement instruction's match rule
12741 //
12742 // ---------VM FLAGS---------------------------------------------------------
12743 //
12744 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12745 //
12746 // Each peephole rule is given an identifying number starting with zero and
12747 // increasing by one in the order seen by the parser.  An individual peephole
12748 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12749 // on the command-line.
12750 //
12751 // ---------CURRENT LIMITATIONS----------------------------------------------
12752 //
12753 // Only match adjacent instructions in same basic block
12754 // Only equality constraints
12755 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12756 // Only one replacement instruction
12757 //
12758 // ---------EXAMPLE----------------------------------------------------------
12759 //
12760 // // pertinent parts of existing instructions in architecture description
12761 // instruct movI(rRegI dst, rRegI src)
12762 // %{
12763 //   match(Set dst (CopyI src));
12764 // %}
12765 //
12766 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
12767 // %{
12768 //   match(Set dst (AddI dst src));
12769 //   effect(KILL cr);
12770 // %}
12771 //
12772 // // Change (inc mov) to lea
12773 // peephole %{
12774 //   // increment preceeded by register-register move
12775 //   peepmatch ( incI_rReg movI );
12776 //   // require that the destination register of the increment
12777 //   // match the destination register of the move
12778 //   peepconstraint ( 0.dst == 1.dst );
12779 //   // construct a replacement instruction that sets
12780 //   // the destination to ( move's source register + one )
12781 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12782 // %}
12783 //
12784 
12785 // Implementation no longer uses movX instructions since
12786 // machine-independent system no longer uses CopyX nodes.
12787 //
12788 // peephole
12789 // %{
12790 //   peepmatch (incI_rReg movI);
12791 //   peepconstraint (0.dst == 1.dst);
12792 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12793 // %}
12794 
12795 // peephole
12796 // %{
12797 //   peepmatch (decI_rReg movI);
12798 //   peepconstraint (0.dst == 1.dst);
12799 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12800 // %}
12801 
12802 // peephole
12803 // %{
12804 //   peepmatch (addI_rReg_imm movI);
12805 //   peepconstraint (0.dst == 1.dst);
12806 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12807 // %}
12808 
12809 // peephole
12810 // %{
12811 //   peepmatch (incL_rReg movL);
12812 //   peepconstraint (0.dst == 1.dst);
12813 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12814 // %}
12815 
12816 // peephole
12817 // %{
12818 //   peepmatch (decL_rReg movL);
12819 //   peepconstraint (0.dst == 1.dst);
12820 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12821 // %}
12822 
12823 // peephole
12824 // %{
12825 //   peepmatch (addL_rReg_imm movL);
12826 //   peepconstraint (0.dst == 1.dst);
12827 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12828 // %}
12829 
12830 // peephole
12831 // %{
12832 //   peepmatch (addP_rReg_imm movP);
12833 //   peepconstraint (0.dst == 1.dst);
12834 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
12835 // %}
12836 
12837 // // Change load of spilled value to only a spill
12838 // instruct storeI(memory mem, rRegI src)
12839 // %{
12840 //   match(Set mem (StoreI mem src));
12841 // %}
12842 //
12843 // instruct loadI(rRegI dst, memory mem)
12844 // %{
12845 //   match(Set dst (LoadI mem));
12846 // %}
12847 //
12848 
12849 peephole
12850 %{
12851   peepmatch (loadI storeI);
12852   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12853   peepreplace (storeI(1.mem 1.mem 1.src));
12854 %}
12855 
12856 peephole
12857 %{
12858   peepmatch (loadL storeL);
12859   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12860   peepreplace (storeL(1.mem 1.mem 1.src));
12861 %}
12862 
12863 //----------SMARTSPILL RULES---------------------------------------------------
12864 // These must follow all instruction definitions as they use the names
12865 // defined in the instructions definitions.