1 //
   2 // Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,               
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,                
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer). 
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211                   
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,                         
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // Class for all long registers (excluding RSP)
 321 reg_class long_reg_with_rbp(RAX, RAX_H,
 322                             RDX, RDX_H,
 323                             RBP, RBP_H,
 324                             RDI, RDI_H,
 325                             RSI, RSI_H,
 326                             RCX, RCX_H,
 327                             RBX, RBX_H,
 328                             R8,  R8_H,
 329                             R9,  R9_H,
 330                             R10, R10_H,
 331                             R11, R11_H,
 332                             R13, R13_H,
 333                             R14, R14_H);
 334 
 335 // Class for all long registers (excluding RSP and RBP)
 336 reg_class long_reg_no_rbp(RAX, RAX_H,
 337                           RDX, RDX_H,
 338                           RDI, RDI_H,
 339                           RSI, RSI_H,
 340                           RCX, RCX_H,
 341                           RBX, RBX_H,
 342                           R8,  R8_H,
 343                           R9,  R9_H,
 344                           R10, R10_H,
 345                           R11, R11_H,
 346                           R13, R13_H,
 347                           R14, R14_H);
 348 
 349 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 350 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 351 
 352 // Class for all long registers (excluding RAX, RDX and RSP)
 353 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 354                                        RDI, RDI_H,
 355                                        RSI, RSI_H,
 356                                        RCX, RCX_H,
 357                                        RBX, RBX_H,
 358                                        R8,  R8_H,
 359                                        R9,  R9_H,
 360                                        R10, R10_H,
 361                                        R11, R11_H,
 362                                        R13, R13_H,
 363                                        R14, R14_H);
 364 
 365 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 366 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 367                                      RSI, RSI_H,
 368                                      RCX, RCX_H,
 369                                      RBX, RBX_H,
 370                                      R8,  R8_H,
 371                                      R9,  R9_H,
 372                                      R10, R10_H,
 373                                      R11, R11_H,
 374                                      R13, R13_H,
 375                                      R14, R14_H);
 376 
 377 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 378 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 379 
 380 // Class for all long registers (excluding RCX and RSP)
 381 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 382                                    RDI, RDI_H,
 383                                    RSI, RSI_H,
 384                                    RAX, RAX_H,
 385                                    RDX, RDX_H,
 386                                    RBX, RBX_H,
 387                                    R8,  R8_H,
 388                                    R9,  R9_H,
 389                                    R10, R10_H,
 390                                    R11, R11_H,
 391                                    R13, R13_H,
 392                                    R14, R14_H);
 393 
 394 // Class for all long registers (excluding RCX, RSP, and RBP)
 395 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 396                                  RSI, RSI_H,
 397                                  RAX, RAX_H,
 398                                  RDX, RDX_H,
 399                                  RBX, RBX_H,
 400                                  R8,  R8_H,
 401                                  R9,  R9_H,
 402                                  R10, R10_H,
 403                                  R11, R11_H,
 404                                  R13, R13_H,
 405                                  R14, R14_H);
 406 
 407 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 408 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 409 
 410 // Singleton class for RAX long register
 411 reg_class long_rax_reg(RAX, RAX_H);
 412 
 413 // Singleton class for RCX long register
 414 reg_class long_rcx_reg(RCX, RCX_H);
 415 
 416 // Singleton class for RDX long register
 417 reg_class long_rdx_reg(RDX, RDX_H);
 418 
 419 // Class for all int registers (excluding RSP)
 420 reg_class int_reg_with_rbp(RAX,
 421                            RDX,
 422                            RBP,
 423                            RDI,
 424                            RSI,
 425                            RCX,
 426                            RBX,
 427                            R8,
 428                            R9,
 429                            R10,
 430                            R11,
 431                            R13,
 432                            R14);
 433 
 434 // Class for all int registers (excluding RSP and RBP)
 435 reg_class int_reg_no_rbp(RAX,
 436                          RDX,
 437                          RDI,
 438                          RSI,
 439                          RCX,
 440                          RBX,
 441                          R8,
 442                          R9,
 443                          R10,
 444                          R11,
 445                          R13,
 446                          R14);
 447 
 448 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 449 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 450 
 451 // Class for all int registers (excluding RCX and RSP)
 452 reg_class int_no_rcx_reg_with_rbp(RAX,
 453                                   RDX,
 454                                   RBP,
 455                                   RDI,
 456                                   RSI,
 457                                   RBX,
 458                                   R8,
 459                                   R9,
 460                                   R10,
 461                                   R11,
 462                                   R13,
 463                                   R14);
 464 
 465 // Class for all int registers (excluding RCX, RSP, and RBP)
 466 reg_class int_no_rcx_reg_no_rbp(RAX,
 467                                 RDX,
 468                                 RDI,
 469                                 RSI,
 470                                 RBX,
 471                                 R8,
 472                                 R9,
 473                                 R10,
 474                                 R11,
 475                                 R13,
 476                                 R14);
 477 
 478 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 479 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 480 
 481 // Class for all int registers (excluding RAX, RDX, and RSP)
 482 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 483                                       RDI,
 484                                       RSI,
 485                                       RCX,
 486                                       RBX,
 487                                       R8,
 488                                       R9,
 489                                       R10,
 490                                       R11,
 491                                       R13,
 492                                       R14);
 493 
 494 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 495 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 496                                     RSI,
 497                                     RCX,
 498                                     RBX,
 499                                     R8,
 500                                     R9,
 501                                     R10,
 502                                     R11,
 503                                     R13,
 504                                     R14);
 505 
 506 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 507 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 508 
 509 // Singleton class for RAX int register
 510 reg_class int_rax_reg(RAX);
 511 
 512 // Singleton class for RBX int register
 513 reg_class int_rbx_reg(RBX);
 514 
 515 // Singleton class for RCX int register
 516 reg_class int_rcx_reg(RCX);
 517 
 518 // Singleton class for RCX int register
 519 reg_class int_rdx_reg(RDX);
 520 
 521 // Singleton class for RCX int register
 522 reg_class int_rdi_reg(RDI);
 523 
 524 // Singleton class for instruction pointer
 525 // reg_class ip_reg(RIP);
 526 
 527 %}
 528 
 529 //----------SOURCE BLOCK-------------------------------------------------------
 530 // This is a block of C++ code which provides values, functions, and
 531 // definitions necessary in the rest of the architecture description
 532 source %{
 533 #define   RELOC_IMM64    Assembler::imm_operand
 534 #define   RELOC_DISP32   Assembler::disp32_operand
 535 
 536 #define __ _masm.
 537 
 538 static int clear_avx_size() {
 539   return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 540 }
 541 
 542 // !!!!! Special hack to get all types of calls to specify the byte offset
 543 //       from the start of the call to the point where the return address
 544 //       will point.
 545 int MachCallStaticJavaNode::ret_addr_offset()
 546 {
 547   int offset = 5; // 5 bytes from start of call to where return address points
 548   offset += clear_avx_size();  
 549   return offset;
 550 }
 551 
 552 int MachCallDynamicJavaNode::ret_addr_offset()
 553 {
 554   int offset = 15; // 15 bytes from start of call to where return address points
 555   offset += clear_avx_size();
 556   return offset;
 557 }
 558 
 559 int MachCallRuntimeNode::ret_addr_offset() {
 560   int offset = 13; // movq r10,#addr; callq (r10)
 561   offset += clear_avx_size();
 562   return offset;
 563 }
 564 
 565 // Indicate if the safepoint node needs the polling page as an input,
 566 // it does if the polling page is more than disp32 away.
 567 bool SafePointNode::needs_polling_address_input()
 568 {
 569   return Assembler::is_polling_page_far();
 570 }
 571 
 572 //
 573 // Compute padding required for nodes which need alignment
 574 //
 575 
 576 // The address of the call instruction needs to be 4-byte aligned to
 577 // ensure that it does not span a cache line so that it can be patched.
 578 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 579 {
 580   current_offset += clear_avx_size(); // skip vzeroupper
 581   current_offset += 1; // skip call opcode byte
 582   return round_to(current_offset, alignment_required()) - current_offset;
 583 }
 584 
 585 // The address of the call instruction needs to be 4-byte aligned to
 586 // ensure that it does not span a cache line so that it can be patched.
 587 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 588 {
 589   current_offset += clear_avx_size(); // skip vzeroupper
 590   current_offset += 11; // skip movq instruction + call opcode byte
 591   return round_to(current_offset, alignment_required()) - current_offset;
 592 }
 593 
 594 // EMIT_RM()
 595 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 596   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 597   cbuf.insts()->emit_int8(c);
 598 }
 599 
 600 // EMIT_CC()
 601 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 602   unsigned char c = (unsigned char) (f1 | f2);
 603   cbuf.insts()->emit_int8(c);
 604 }
 605 
 606 // EMIT_OPCODE()
 607 void emit_opcode(CodeBuffer &cbuf, int code) {
 608   cbuf.insts()->emit_int8((unsigned char) code);
 609 }
 610 
 611 // EMIT_OPCODE() w/ relocation information
 612 void emit_opcode(CodeBuffer &cbuf,
 613                  int code, relocInfo::relocType reloc, int offset, int format)
 614 {
 615   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 616   emit_opcode(cbuf, code);
 617 }
 618 
 619 // EMIT_D8()
 620 void emit_d8(CodeBuffer &cbuf, int d8) {
 621   cbuf.insts()->emit_int8((unsigned char) d8);
 622 }
 623 
 624 // EMIT_D16()
 625 void emit_d16(CodeBuffer &cbuf, int d16) {
 626   cbuf.insts()->emit_int16(d16);
 627 }
 628 
 629 // EMIT_D32()
 630 void emit_d32(CodeBuffer &cbuf, int d32) {
 631   cbuf.insts()->emit_int32(d32);
 632 }
 633 
 634 // EMIT_D64()
 635 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 636   cbuf.insts()->emit_int64(d64);
 637 }
 638 
 639 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 640 void emit_d32_reloc(CodeBuffer& cbuf,
 641                     int d32,
 642                     relocInfo::relocType reloc,
 643                     int format)
 644 {
 645   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 646   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 647   cbuf.insts()->emit_int32(d32);
 648 }
 649 
 650 // emit 32 bit value and construct relocation entry from RelocationHolder
 651 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 652 #ifdef ASSERT
 653   if (rspec.reloc()->type() == relocInfo::oop_type &&
 654       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 655     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 656     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 657   }
 658 #endif
 659   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 660   cbuf.insts()->emit_int32(d32);
 661 }
 662 
 663 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 664   address next_ip = cbuf.insts_end() + 4;
 665   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 666                  external_word_Relocation::spec(addr),
 667                  RELOC_DISP32);
 668 }
 669 
 670 
 671 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 672 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 673   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 674   cbuf.insts()->emit_int64(d64);
 675 }
 676 
 677 // emit 64 bit value and construct relocation entry from RelocationHolder
 678 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 679 #ifdef ASSERT
 680   if (rspec.reloc()->type() == relocInfo::oop_type &&
 681       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 682     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 683     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 684            "cannot embed scavengable oops in code");
 685   }
 686 #endif
 687   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 688   cbuf.insts()->emit_int64(d64);
 689 }
 690 
 691 // Access stack slot for load or store
 692 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 693 {
 694   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 695   if (-0x80 <= disp && disp < 0x80) {
 696     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 697     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 698     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 699   } else {
 700     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 701     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 702     emit_d32(cbuf, disp);     // Displacement // R/M byte
 703   }
 704 }
 705 
 706    // rRegI ereg, memory mem) %{    // emit_reg_mem
 707 void encode_RegMem(CodeBuffer &cbuf,
 708                    int reg,
 709                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 710 {
 711   assert(disp_reloc == relocInfo::none, "cannot have disp");
 712   int regenc = reg & 7;
 713   int baseenc = base & 7;
 714   int indexenc = index & 7;
 715 
 716   // There is no index & no scale, use form without SIB byte
 717   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 718     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 719     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 720       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 721     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 722       // If 8-bit displacement, mode 0x1
 723       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 724       emit_d8(cbuf, disp);
 725     } else {
 726       // If 32-bit displacement
 727       if (base == -1) { // Special flag for absolute address
 728         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 729         if (disp_reloc != relocInfo::none) {
 730           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 731         } else {
 732           emit_d32(cbuf, disp);
 733         }
 734       } else {
 735         // Normal base + offset
 736         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 737         if (disp_reloc != relocInfo::none) {
 738           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 739         } else {
 740           emit_d32(cbuf, disp);
 741         }
 742       }
 743     }
 744   } else {
 745     // Else, encode with the SIB byte
 746     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 747     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 748       // If no displacement
 749       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 750       emit_rm(cbuf, scale, indexenc, baseenc);
 751     } else {
 752       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 753         // If 8-bit displacement, mode 0x1
 754         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 755         emit_rm(cbuf, scale, indexenc, baseenc);
 756         emit_d8(cbuf, disp);
 757       } else {
 758         // If 32-bit displacement
 759         if (base == 0x04 ) {
 760           emit_rm(cbuf, 0x2, regenc, 0x4);
 761           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 762         } else {
 763           emit_rm(cbuf, 0x2, regenc, 0x4);
 764           emit_rm(cbuf, scale, indexenc, baseenc); // *
 765         }
 766         if (disp_reloc != relocInfo::none) {
 767           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 768         } else {
 769           emit_d32(cbuf, disp);
 770         }
 771       }
 772     }
 773   }
 774 }
 775 
 776 // This could be in MacroAssembler but it's fairly C2 specific
 777 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 778   Label exit;
 779   __ jccb(Assembler::noParity, exit);
 780   __ pushf();
 781   //
 782   // comiss/ucomiss instructions set ZF,PF,CF flags and
 783   // zero OF,AF,SF for NaN values.
 784   // Fixup flags by zeroing ZF,PF so that compare of NaN
 785   // values returns 'less than' result (CF is set).
 786   // Leave the rest of flags unchanged.
 787   //
 788   //    7 6 5 4 3 2 1 0
 789   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 790   //    0 0 1 0 1 0 1 1   (0x2B)
 791   //
 792   __ andq(Address(rsp, 0), 0xffffff2b);
 793   __ popf();
 794   __ bind(exit);
 795 }
 796 
 797 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 798   Label done;
 799   __ movl(dst, -1);
 800   __ jcc(Assembler::parity, done);
 801   __ jcc(Assembler::below, done);
 802   __ setb(Assembler::notEqual, dst);
 803   __ movzbl(dst, dst);
 804   __ bind(done);
 805 }
 806 
 807 
 808 //=============================================================================
 809 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 810 
 811 int Compile::ConstantTable::calculate_table_base_offset() const {
 812   return 0;  // absolute addressing, no offset
 813 }
 814 
 815 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 816 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 817   ShouldNotReachHere();
 818 }
 819 
 820 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 821   // Empty encoding
 822 }
 823 
 824 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 825   return 0;
 826 }
 827 
 828 #ifndef PRODUCT
 829 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 830   st->print("# MachConstantBaseNode (empty encoding)");
 831 }
 832 #endif
 833 
 834 
 835 //=============================================================================
 836 #ifndef PRODUCT
 837 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 838   Compile* C = ra_->C;
 839 
 840   int framesize = C->frame_size_in_bytes();
 841   int bangsize = C->bang_size_in_bytes();
 842   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 843   // Remove wordSize for return addr which is already pushed.
 844   framesize -= wordSize;
 845 
 846   if (C->need_stack_bang(bangsize)) {
 847     framesize -= wordSize;
 848     st->print("# stack bang (%d bytes)", bangsize);
 849     st->print("\n\t");
 850     st->print("pushq   rbp\t# Save rbp");
 851     if (PreserveFramePointer) {
 852         st->print("\n\t");
 853         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 854     }
 855     if (framesize) {
 856       st->print("\n\t");
 857       st->print("subq    rsp, #%d\t# Create frame",framesize);
 858     }
 859   } else {
 860     st->print("subq    rsp, #%d\t# Create frame",framesize);
 861     st->print("\n\t");
 862     framesize -= wordSize;
 863     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);    
 864     if (PreserveFramePointer) {
 865       st->print("\n\t");
 866       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 867       if (framesize > 0) {
 868         st->print("\n\t");
 869         st->print("addq    rbp, #%d", framesize);
 870       }      
 871     }
 872   }
 873 
 874   if (VerifyStackAtCalls) {
 875     st->print("\n\t");
 876     framesize -= wordSize;
 877     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 878 #ifdef ASSERT
 879     st->print("\n\t");
 880     st->print("# stack alignment check");
 881 #endif
 882   }
 883   st->cr();
 884 }
 885 #endif
 886 
 887 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 888   Compile* C = ra_->C;
 889   MacroAssembler _masm(&cbuf);
 890 
 891   int framesize = C->frame_size_in_bytes();
 892   int bangsize = C->bang_size_in_bytes();
 893 
 894   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 895 
 896   C->set_frame_complete(cbuf.insts_size());
 897 
 898   if (C->has_mach_constant_base_node()) {
 899     // NOTE: We set the table base offset here because users might be
 900     // emitted before MachConstantBaseNode.
 901     Compile::ConstantTable& constant_table = C->constant_table();
 902     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 903   }
 904 }
 905 
 906 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 907 {
 908   return MachNode::size(ra_); // too many variables; just compute it
 909                               // the hard way
 910 }
 911 
 912 int MachPrologNode::reloc() const
 913 {
 914   return 0; // a large enough number
 915 }
 916 
 917 //=============================================================================
 918 #ifndef PRODUCT
 919 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 920 {
 921   Compile* C = ra_->C;
 922   if (C->max_vector_size() > 16) {
 923     st->print("vzeroupper");
 924     st->cr(); st->print("\t");
 925   }
 926 
 927   int framesize = C->frame_size_in_bytes();
 928   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 929   // Remove word for return adr already pushed
 930   // and RBP
 931   framesize -= 2*wordSize;
 932 
 933   if (framesize) {
 934     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 935     st->print("\t");
 936   }
 937 
 938   st->print_cr("popq   rbp");
 939   if (do_polling() && C->is_method_compilation()) {
 940     st->print("\t");
 941     if (Assembler::is_polling_page_far()) {
 942       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 943                    "testl  rax, [rscratch1]\t"
 944                    "# Safepoint: poll for GC");
 945     } else {
 946       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 947                    "# Safepoint: poll for GC");
 948     }
 949   }
 950 }
 951 #endif
 952 
 953 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 954 {
 955   Compile* C = ra_->C;
 956   if (C->max_vector_size() > 16) {
 957     // Clear upper bits of YMM registers when current compiled code uses
 958     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 959     MacroAssembler _masm(&cbuf);
 960     __ vzeroupper();
 961   }
 962 
 963   int framesize = C->frame_size_in_bytes();
 964   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 965   // Remove word for return adr already pushed
 966   // and RBP
 967   framesize -= 2*wordSize;
 968 
 969   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 970 
 971   if (framesize) {
 972     emit_opcode(cbuf, Assembler::REX_W);
 973     if (framesize < 0x80) {
 974       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 975       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 976       emit_d8(cbuf, framesize);
 977     } else {
 978       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 979       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 980       emit_d32(cbuf, framesize);
 981     }
 982   }
 983 
 984   // popq rbp
 985   emit_opcode(cbuf, 0x58 | RBP_enc);
 986 
 987   if (do_polling() && C->is_method_compilation()) {
 988     MacroAssembler _masm(&cbuf);
 989     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 990     if (Assembler::is_polling_page_far()) {
 991       __ lea(rscratch1, polling_page);
 992       __ relocate(relocInfo::poll_return_type);
 993       __ testl(rax, Address(rscratch1, 0));
 994     } else {
 995       __ testl(rax, polling_page);
 996     }
 997   }
 998 }
 999 
1000 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1001 {
1002   return MachNode::size(ra_); // too many variables; just compute it
1003                               // the hard way
1004 }
1005 
1006 int MachEpilogNode::reloc() const
1007 {
1008   return 2; // a large enough number
1009 }
1010 
1011 const Pipeline* MachEpilogNode::pipeline() const
1012 {
1013   return MachNode::pipeline_class();
1014 }
1015 
1016 int MachEpilogNode::safepoint_offset() const
1017 {
1018   return 0;
1019 }
1020 
1021 //=============================================================================
1022 
1023 enum RC {
1024   rc_bad,
1025   rc_int,
1026   rc_float,
1027   rc_stack
1028 };
1029 
1030 static enum RC rc_class(OptoReg::Name reg)
1031 {
1032   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1033 
1034   if (OptoReg::is_stack(reg)) return rc_stack;
1035 
1036   VMReg r = OptoReg::as_VMReg(reg);
1037 
1038   if (r->is_Register()) return rc_int;
1039 
1040   assert(r->is_XMMRegister(), "must be");
1041   return rc_float;
1042 }
1043 
1044 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1045 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1046                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1047 
1048 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1049                             int stack_offset, int reg, uint ireg, outputStream* st);
1050 
1051 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1052                                       int dst_offset, uint ireg, outputStream* st) {
1053   if (cbuf) {
1054     MacroAssembler _masm(cbuf);
1055     switch (ireg) {
1056     case Op_VecS:
1057       __ movq(Address(rsp, -8), rax);
1058       __ movl(rax, Address(rsp, src_offset));
1059       __ movl(Address(rsp, dst_offset), rax);
1060       __ movq(rax, Address(rsp, -8));
1061       break;
1062     case Op_VecD:
1063       __ pushq(Address(rsp, src_offset));
1064       __ popq (Address(rsp, dst_offset));
1065       break;
1066     case Op_VecX:
1067       __ pushq(Address(rsp, src_offset));
1068       __ popq (Address(rsp, dst_offset));
1069       __ pushq(Address(rsp, src_offset+8));
1070       __ popq (Address(rsp, dst_offset+8));
1071       break;
1072     case Op_VecY:
1073       __ vmovdqu(Address(rsp, -32), xmm0);
1074       __ vmovdqu(xmm0, Address(rsp, src_offset));
1075       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1076       __ vmovdqu(xmm0, Address(rsp, -32));
1077       break;
1078     default:
1079       ShouldNotReachHere();
1080     }
1081 #ifndef PRODUCT
1082   } else {
1083     switch (ireg) {
1084     case Op_VecS:
1085       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1086                 "movl    rax, [rsp + #%d]\n\t"
1087                 "movl    [rsp + #%d], rax\n\t"
1088                 "movq    rax, [rsp - #8]",
1089                 src_offset, dst_offset);
1090       break;
1091     case Op_VecD:
1092       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1093                 "popq    [rsp + #%d]",
1094                 src_offset, dst_offset);
1095       break;
1096      case Op_VecX:
1097       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1098                 "popq    [rsp + #%d]\n\t"
1099                 "pushq   [rsp + #%d]\n\t"
1100                 "popq    [rsp + #%d]",
1101                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1102       break;
1103     case Op_VecY:
1104       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1105                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1106                 "vmovdqu [rsp + #%d], xmm0\n\t"
1107                 "vmovdqu xmm0, [rsp - #32]",
1108                 src_offset, dst_offset);
1109       break;
1110     default:
1111       ShouldNotReachHere();
1112     }
1113 #endif
1114   }
1115 }
1116 
1117 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1118                                        PhaseRegAlloc* ra_,
1119                                        bool do_size,
1120                                        outputStream* st) const {
1121   assert(cbuf != NULL || st  != NULL, "sanity");
1122   // Get registers to move
1123   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1124   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1125   OptoReg::Name dst_second = ra_->get_reg_second(this);
1126   OptoReg::Name dst_first = ra_->get_reg_first(this);
1127 
1128   enum RC src_second_rc = rc_class(src_second);
1129   enum RC src_first_rc = rc_class(src_first);
1130   enum RC dst_second_rc = rc_class(dst_second);
1131   enum RC dst_first_rc = rc_class(dst_first);
1132 
1133   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1134          "must move at least 1 register" );
1135 
1136   if (src_first == dst_first && src_second == dst_second) {
1137     // Self copy, no move
1138     return 0;
1139   }
1140   if (bottom_type()->isa_vect() != NULL) {
1141     uint ireg = ideal_reg();
1142     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1143     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY), "sanity");
1144     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1145       // mem -> mem
1146       int src_offset = ra_->reg2offset(src_first);
1147       int dst_offset = ra_->reg2offset(dst_first);
1148       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1149     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1150       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1151     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1152       int stack_offset = ra_->reg2offset(dst_first);
1153       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1154     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1155       int stack_offset = ra_->reg2offset(src_first);
1156       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1157     } else {
1158       ShouldNotReachHere();
1159     }
1160     return 0;
1161   }
1162   if (src_first_rc == rc_stack) {
1163     // mem ->
1164     if (dst_first_rc == rc_stack) {
1165       // mem -> mem
1166       assert(src_second != dst_first, "overlap");
1167       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1168           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1169         // 64-bit
1170         int src_offset = ra_->reg2offset(src_first);
1171         int dst_offset = ra_->reg2offset(dst_first);
1172         if (cbuf) {
1173           MacroAssembler _masm(cbuf);
1174           __ pushq(Address(rsp, src_offset));
1175           __ popq (Address(rsp, dst_offset));
1176 #ifndef PRODUCT
1177         } else {
1178           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1179                     "popq    [rsp + #%d]",
1180                      src_offset, dst_offset);
1181 #endif
1182         }
1183       } else {
1184         // 32-bit
1185         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1186         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1187         // No pushl/popl, so:
1188         int src_offset = ra_->reg2offset(src_first);
1189         int dst_offset = ra_->reg2offset(dst_first);
1190         if (cbuf) {
1191           MacroAssembler _masm(cbuf);
1192           __ movq(Address(rsp, -8), rax);
1193           __ movl(rax, Address(rsp, src_offset));
1194           __ movl(Address(rsp, dst_offset), rax);
1195           __ movq(rax, Address(rsp, -8));
1196 #ifndef PRODUCT
1197         } else {
1198           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1199                     "movl    rax, [rsp + #%d]\n\t"
1200                     "movl    [rsp + #%d], rax\n\t"
1201                     "movq    rax, [rsp - #8]",
1202                      src_offset, dst_offset);
1203 #endif
1204         }
1205       }
1206       return 0;
1207     } else if (dst_first_rc == rc_int) {
1208       // mem -> gpr
1209       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1210           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1211         // 64-bit
1212         int offset = ra_->reg2offset(src_first);
1213         if (cbuf) {
1214           MacroAssembler _masm(cbuf);
1215           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1216 #ifndef PRODUCT
1217         } else {
1218           st->print("movq    %s, [rsp + #%d]\t# spill",
1219                      Matcher::regName[dst_first],
1220                      offset);
1221 #endif
1222         }
1223       } else {
1224         // 32-bit
1225         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1226         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1227         int offset = ra_->reg2offset(src_first);
1228         if (cbuf) {
1229           MacroAssembler _masm(cbuf);
1230           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1231 #ifndef PRODUCT
1232         } else {
1233           st->print("movl    %s, [rsp + #%d]\t# spill",
1234                      Matcher::regName[dst_first],
1235                      offset);
1236 #endif
1237         }
1238       }
1239       return 0;
1240     } else if (dst_first_rc == rc_float) {
1241       // mem-> xmm
1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1244         // 64-bit
1245         int offset = ra_->reg2offset(src_first);
1246         if (cbuf) {
1247           MacroAssembler _masm(cbuf);
1248           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1249 #ifndef PRODUCT
1250         } else {
1251           st->print("%s  %s, [rsp + #%d]\t# spill",
1252                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1253                      Matcher::regName[dst_first],
1254                      offset);
1255 #endif
1256         }
1257       } else {
1258         // 32-bit
1259         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1260         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1261         int offset = ra_->reg2offset(src_first);
1262         if (cbuf) {
1263           MacroAssembler _masm(cbuf);
1264           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1265 #ifndef PRODUCT
1266         } else {
1267           st->print("movss   %s, [rsp + #%d]\t# spill",
1268                      Matcher::regName[dst_first],
1269                      offset);
1270 #endif
1271         }
1272       }
1273       return 0;
1274     }
1275   } else if (src_first_rc == rc_int) {
1276     // gpr ->
1277     if (dst_first_rc == rc_stack) {
1278       // gpr -> mem
1279       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1280           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1281         // 64-bit
1282         int offset = ra_->reg2offset(dst_first);
1283         if (cbuf) {
1284           MacroAssembler _masm(cbuf);
1285           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1286 #ifndef PRODUCT
1287         } else {
1288           st->print("movq    [rsp + #%d], %s\t# spill",
1289                      offset,
1290                      Matcher::regName[src_first]);
1291 #endif
1292         }
1293       } else {
1294         // 32-bit
1295         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1296         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1297         int offset = ra_->reg2offset(dst_first);
1298         if (cbuf) {
1299           MacroAssembler _masm(cbuf);
1300           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1301 #ifndef PRODUCT
1302         } else {
1303           st->print("movl    [rsp + #%d], %s\t# spill",
1304                      offset,
1305                      Matcher::regName[src_first]);
1306 #endif
1307         }
1308       }
1309       return 0;
1310     } else if (dst_first_rc == rc_int) {
1311       // gpr -> gpr
1312       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1313           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1314         // 64-bit
1315         if (cbuf) {
1316           MacroAssembler _masm(cbuf);
1317           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1318                   as_Register(Matcher::_regEncode[src_first]));
1319 #ifndef PRODUCT
1320         } else {
1321           st->print("movq    %s, %s\t# spill",
1322                      Matcher::regName[dst_first],
1323                      Matcher::regName[src_first]);
1324 #endif
1325         }
1326         return 0;
1327       } else {
1328         // 32-bit
1329         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1330         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1331         if (cbuf) {
1332           MacroAssembler _masm(cbuf);
1333           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1334                   as_Register(Matcher::_regEncode[src_first]));
1335 #ifndef PRODUCT
1336         } else {
1337           st->print("movl    %s, %s\t# spill",
1338                      Matcher::regName[dst_first],
1339                      Matcher::regName[src_first]);
1340 #endif
1341         }
1342         return 0;
1343       }
1344     } else if (dst_first_rc == rc_float) {
1345       // gpr -> xmm
1346       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1347           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1348         // 64-bit
1349         if (cbuf) {
1350           MacroAssembler _masm(cbuf);
1351           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1352 #ifndef PRODUCT
1353         } else {
1354           st->print("movdq   %s, %s\t# spill",
1355                      Matcher::regName[dst_first],
1356                      Matcher::regName[src_first]);
1357 #endif
1358         }
1359       } else {
1360         // 32-bit
1361         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1362         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1363         if (cbuf) {
1364           MacroAssembler _masm(cbuf);
1365           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1366 #ifndef PRODUCT
1367         } else {
1368           st->print("movdl   %s, %s\t# spill",
1369                      Matcher::regName[dst_first],
1370                      Matcher::regName[src_first]);
1371 #endif
1372         }
1373       }
1374       return 0;
1375     }
1376   } else if (src_first_rc == rc_float) {
1377     // xmm ->
1378     if (dst_first_rc == rc_stack) {
1379       // xmm -> mem
1380       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1381           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1382         // 64-bit
1383         int offset = ra_->reg2offset(dst_first);
1384         if (cbuf) {
1385           MacroAssembler _masm(cbuf);
1386           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1387 #ifndef PRODUCT
1388         } else {
1389           st->print("movsd   [rsp + #%d], %s\t# spill",
1390                      offset,
1391                      Matcher::regName[src_first]);
1392 #endif
1393         }
1394       } else {
1395         // 32-bit
1396         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1397         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1398         int offset = ra_->reg2offset(dst_first);
1399         if (cbuf) {
1400           MacroAssembler _masm(cbuf);
1401           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1402 #ifndef PRODUCT
1403         } else {
1404           st->print("movss   [rsp + #%d], %s\t# spill",
1405                      offset,
1406                      Matcher::regName[src_first]);
1407 #endif
1408         }
1409       }
1410       return 0;
1411     } else if (dst_first_rc == rc_int) {
1412       // xmm -> gpr
1413       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1414           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1415         // 64-bit
1416         if (cbuf) {
1417           MacroAssembler _masm(cbuf);
1418           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1419 #ifndef PRODUCT
1420         } else {
1421           st->print("movdq   %s, %s\t# spill",
1422                      Matcher::regName[dst_first],
1423                      Matcher::regName[src_first]);
1424 #endif
1425         }
1426       } else {
1427         // 32-bit
1428         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1429         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1430         if (cbuf) {
1431           MacroAssembler _masm(cbuf);
1432           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1433 #ifndef PRODUCT
1434         } else {
1435           st->print("movdl   %s, %s\t# spill",
1436                      Matcher::regName[dst_first],
1437                      Matcher::regName[src_first]);
1438 #endif
1439         }
1440       }
1441       return 0;
1442     } else if (dst_first_rc == rc_float) {
1443       // xmm -> xmm
1444       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1445           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1446         // 64-bit
1447         if (cbuf) {
1448           MacroAssembler _masm(cbuf);
1449           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1450 #ifndef PRODUCT
1451         } else {
1452           st->print("%s  %s, %s\t# spill",
1453                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1454                      Matcher::regName[dst_first],
1455                      Matcher::regName[src_first]);
1456 #endif
1457         }
1458       } else {
1459         // 32-bit
1460         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1461         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1462         if (cbuf) {
1463           MacroAssembler _masm(cbuf);
1464           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1465 #ifndef PRODUCT
1466         } else {
1467           st->print("%s  %s, %s\t# spill",
1468                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1469                      Matcher::regName[dst_first],
1470                      Matcher::regName[src_first]);
1471 #endif
1472         }
1473       }
1474       return 0;
1475     }
1476   }
1477 
1478   assert(0," foo ");
1479   Unimplemented();
1480   return 0;
1481 }
1482 
1483 #ifndef PRODUCT
1484 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1485   implementation(NULL, ra_, false, st);
1486 }
1487 #endif
1488 
1489 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1490   implementation(&cbuf, ra_, false, NULL);
1491 }
1492 
1493 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1494   return MachNode::size(ra_);
1495 }
1496 
1497 //=============================================================================
1498 #ifndef PRODUCT
1499 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1500 {
1501   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1502   int reg = ra_->get_reg_first(this);
1503   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1504             Matcher::regName[reg], offset);
1505 }
1506 #endif
1507 
1508 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1509 {
1510   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1511   int reg = ra_->get_encode(this);
1512   if (offset >= 0x80) {
1513     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1514     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1515     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1516     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1517     emit_d32(cbuf, offset);
1518   } else {
1519     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1520     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1521     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1522     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1523     emit_d8(cbuf, offset);
1524   }
1525 }
1526 
1527 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1528 {
1529   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1530   return (offset < 0x80) ? 5 : 8; // REX
1531 }
1532 
1533 //=============================================================================
1534 #ifndef PRODUCT
1535 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1536 {
1537   if (UseCompressedClassPointers) {
1538     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1539     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1540     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1541   } else {
1542     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1543                  "# Inline cache check");
1544   }
1545   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1546   st->print_cr("\tnop\t# nops to align entry point");
1547 }
1548 #endif
1549 
1550 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1551 {
1552   MacroAssembler masm(&cbuf);
1553   uint insts_size = cbuf.insts_size();
1554   if (UseCompressedClassPointers) {
1555     masm.load_klass(rscratch1, j_rarg0);
1556     masm.cmpptr(rax, rscratch1);
1557   } else {
1558     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1559   }
1560 
1561   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1562 
1563   /* WARNING these NOPs are critical so that verified entry point is properly
1564      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1565   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1566   if (OptoBreakpoint) {
1567     // Leave space for int3
1568     nops_cnt -= 1;
1569   }
1570   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1571   if (nops_cnt > 0)
1572     masm.nop(nops_cnt);
1573 }
1574 
1575 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1576 {
1577   return MachNode::size(ra_); // too many variables; just compute it
1578                               // the hard way
1579 }
1580  
1581 
1582 //=============================================================================
1583 
1584 int Matcher::regnum_to_fpu_offset(int regnum)
1585 {
1586   return regnum - 32; // The FP registers are in the second chunk
1587 }
1588 
1589 // This is UltraSparc specific, true just means we have fast l2f conversion
1590 const bool Matcher::convL2FSupported(void) {
1591   return true;
1592 }
1593 
1594 // Is this branch offset short enough that a short branch can be used?
1595 //
1596 // NOTE: If the platform does not provide any short branch variants, then
1597 //       this method should return false for offset 0.
1598 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1599   // The passed offset is relative to address of the branch.
1600   // On 86 a branch displacement is calculated relative to address
1601   // of a next instruction.
1602   offset -= br_size;
1603 
1604   // the short version of jmpConUCF2 contains multiple branches,
1605   // making the reach slightly less
1606   if (rule == jmpConUCF2_rule)
1607     return (-126 <= offset && offset <= 125);
1608   return (-128 <= offset && offset <= 127);
1609 }
1610 
1611 const bool Matcher::isSimpleConstant64(jlong value) {
1612   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1613   //return value == (int) value;  // Cf. storeImmL and immL32.
1614 
1615   // Probably always true, even if a temp register is required.
1616   return true;
1617 }
1618 
1619 // The ecx parameter to rep stosq for the ClearArray node is in words.
1620 const bool Matcher::init_array_count_is_in_bytes = false;
1621 
1622 // Threshold size for cleararray.
1623 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1624 
1625 // No additional cost for CMOVL.
1626 const int Matcher::long_cmove_cost() { return 0; }
1627 
1628 // No CMOVF/CMOVD with SSE2
1629 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1630 
1631 // Does the CPU require late expand (see block.cpp for description of late expand)?
1632 const bool Matcher::require_postalloc_expand = false;
1633 
1634 // Should the Matcher clone shifts on addressing modes, expecting them
1635 // to be subsumed into complex addressing expressions or compute them
1636 // into registers?  True for Intel but false for most RISCs
1637 const bool Matcher::clone_shift_expressions = true;
1638 
1639 // Do we need to mask the count passed to shift instructions or does
1640 // the cpu only look at the lower 5/6 bits anyway?
1641 const bool Matcher::need_masked_shift_count = false;
1642 
1643 bool Matcher::narrow_oop_use_complex_address() {
1644   assert(UseCompressedOops, "only for compressed oops code");
1645   return (LogMinObjAlignmentInBytes <= 3);
1646 }
1647 
1648 bool Matcher::narrow_klass_use_complex_address() {
1649   assert(UseCompressedClassPointers, "only for compressed klass code");
1650   return (LogKlassAlignmentInBytes <= 3);
1651 }
1652 
1653 // Is it better to copy float constants, or load them directly from
1654 // memory?  Intel can load a float constant from a direct address,
1655 // requiring no extra registers.  Most RISCs will have to materialize
1656 // an address into a register first, so they would do better to copy
1657 // the constant from stack.
1658 const bool Matcher::rematerialize_float_constants = true; // XXX
1659 
1660 // If CPU can load and store mis-aligned doubles directly then no
1661 // fixup is needed.  Else we split the double into 2 integer pieces
1662 // and move it piece-by-piece.  Only happens when passing doubles into
1663 // C code as the Java calling convention forces doubles to be aligned.
1664 const bool Matcher::misaligned_doubles_ok = true;
1665 
1666 // No-op on amd64
1667 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1668 
1669 // Advertise here if the CPU requires explicit rounding operations to
1670 // implement the UseStrictFP mode.
1671 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1672 
1673 // Are floats conerted to double when stored to stack during deoptimization?
1674 // On x64 it is stored without convertion so we can use normal access.
1675 bool Matcher::float_in_double() { return false; }
1676 
1677 // Do ints take an entire long register or just half?
1678 const bool Matcher::int_in_long = true;
1679 
1680 // Return whether or not this register is ever used as an argument.
1681 // This function is used on startup to build the trampoline stubs in
1682 // generateOptoStub.  Registers not mentioned will be killed by the VM
1683 // call in the trampoline, and arguments in those registers not be
1684 // available to the callee.
1685 bool Matcher::can_be_java_arg(int reg)
1686 {
1687   return
1688     reg ==  RDI_num || reg == RDI_H_num ||
1689     reg ==  RSI_num || reg == RSI_H_num ||
1690     reg ==  RDX_num || reg == RDX_H_num ||
1691     reg ==  RCX_num || reg == RCX_H_num ||
1692     reg ==   R8_num || reg ==  R8_H_num ||
1693     reg ==   R9_num || reg ==  R9_H_num ||
1694     reg ==  R12_num || reg == R12_H_num ||
1695     reg == XMM0_num || reg == XMM0b_num ||
1696     reg == XMM1_num || reg == XMM1b_num ||
1697     reg == XMM2_num || reg == XMM2b_num ||
1698     reg == XMM3_num || reg == XMM3b_num ||
1699     reg == XMM4_num || reg == XMM4b_num ||
1700     reg == XMM5_num || reg == XMM5b_num ||
1701     reg == XMM6_num || reg == XMM6b_num ||
1702     reg == XMM7_num || reg == XMM7b_num;
1703 }
1704 
1705 bool Matcher::is_spillable_arg(int reg)
1706 {
1707   return can_be_java_arg(reg);
1708 }
1709 
1710 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1711   // In 64 bit mode a code which use multiply when
1712   // devisor is constant is faster than hardware
1713   // DIV instruction (it uses MulHiL).
1714   return false;
1715 }
1716 
1717 // Register for DIVI projection of divmodI
1718 RegMask Matcher::divI_proj_mask() {
1719   return INT_RAX_REG_mask();
1720 }
1721 
1722 // Register for MODI projection of divmodI
1723 RegMask Matcher::modI_proj_mask() {
1724   return INT_RDX_REG_mask();
1725 }
1726 
1727 // Register for DIVL projection of divmodL
1728 RegMask Matcher::divL_proj_mask() {
1729   return LONG_RAX_REG_mask();
1730 }
1731 
1732 // Register for MODL projection of divmodL
1733 RegMask Matcher::modL_proj_mask() {
1734   return LONG_RDX_REG_mask();
1735 }
1736 
1737 // Register for saving SP into on method handle invokes. Not used on x86_64.
1738 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1739     return NO_REG_mask();
1740 }
1741 
1742 %}
1743 
1744 //----------ENCODING BLOCK-----------------------------------------------------
1745 // This block specifies the encoding classes used by the compiler to
1746 // output byte streams.  Encoding classes are parameterized macros
1747 // used by Machine Instruction Nodes in order to generate the bit
1748 // encoding of the instruction.  Operands specify their base encoding
1749 // interface with the interface keyword.  There are currently
1750 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1751 // COND_INTER.  REG_INTER causes an operand to generate a function
1752 // which returns its register number when queried.  CONST_INTER causes
1753 // an operand to generate a function which returns the value of the
1754 // constant when queried.  MEMORY_INTER causes an operand to generate
1755 // four functions which return the Base Register, the Index Register,
1756 // the Scale Value, and the Offset Value of the operand when queried.
1757 // COND_INTER causes an operand to generate six functions which return
1758 // the encoding code (ie - encoding bits for the instruction)
1759 // associated with each basic boolean condition for a conditional
1760 // instruction.
1761 //
1762 // Instructions specify two basic values for encoding.  Again, a
1763 // function is available to check if the constant displacement is an
1764 // oop. They use the ins_encode keyword to specify their encoding
1765 // classes (which must be a sequence of enc_class names, and their
1766 // parameters, specified in the encoding block), and they use the
1767 // opcode keyword to specify, in order, their primary, secondary, and
1768 // tertiary opcode.  Only the opcode sections which a particular
1769 // instruction needs for encoding need to be specified.
1770 encode %{
1771   // Build emit functions for each basic byte or larger field in the
1772   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1773   // from C++ code in the enc_class source block.  Emit functions will
1774   // live in the main source block for now.  In future, we can
1775   // generalize this by adding a syntax that specifies the sizes of
1776   // fields in an order, so that the adlc can build the emit functions
1777   // automagically
1778 
1779   // Emit primary opcode
1780   enc_class OpcP
1781   %{
1782     emit_opcode(cbuf, $primary);
1783   %}
1784 
1785   // Emit secondary opcode
1786   enc_class OpcS
1787   %{
1788     emit_opcode(cbuf, $secondary);
1789   %}
1790 
1791   // Emit tertiary opcode
1792   enc_class OpcT
1793   %{
1794     emit_opcode(cbuf, $tertiary);
1795   %}
1796 
1797   // Emit opcode directly
1798   enc_class Opcode(immI d8)
1799   %{
1800     emit_opcode(cbuf, $d8$$constant);
1801   %}
1802 
1803   // Emit size prefix
1804   enc_class SizePrefix
1805   %{
1806     emit_opcode(cbuf, 0x66);
1807   %}
1808 
1809   enc_class reg(rRegI reg)
1810   %{
1811     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1812   %}
1813 
1814   enc_class reg_reg(rRegI dst, rRegI src)
1815   %{
1816     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1817   %}
1818 
1819   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1820   %{
1821     emit_opcode(cbuf, $opcode$$constant);
1822     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1823   %}
1824 
1825   enc_class cdql_enc(no_rax_rdx_RegI div)
1826   %{
1827     // Full implementation of Java idiv and irem; checks for
1828     // special case as described in JVM spec., p.243 & p.271.
1829     //
1830     //         normal case                           special case
1831     //
1832     // input : rax: dividend                         min_int
1833     //         reg: divisor                          -1
1834     //
1835     // output: rax: quotient  (= rax idiv reg)       min_int
1836     //         rdx: remainder (= rax irem reg)       0
1837     //
1838     //  Code sequnce:
1839     //
1840     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1841     //    5:   75 07/08                jne    e <normal>
1842     //    7:   33 d2                   xor    %edx,%edx
1843     //  [div >= 8 -> offset + 1]
1844     //  [REX_B]
1845     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1846     //    c:   74 03/04                je     11 <done>
1847     // 000000000000000e <normal>:
1848     //    e:   99                      cltd
1849     //  [div >= 8 -> offset + 1]
1850     //  [REX_B]
1851     //    f:   f7 f9                   idiv   $div
1852     // 0000000000000011 <done>:
1853 
1854     // cmp    $0x80000000,%eax
1855     emit_opcode(cbuf, 0x3d);
1856     emit_d8(cbuf, 0x00);
1857     emit_d8(cbuf, 0x00);
1858     emit_d8(cbuf, 0x00);
1859     emit_d8(cbuf, 0x80);
1860 
1861     // jne    e <normal>
1862     emit_opcode(cbuf, 0x75);
1863     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1864 
1865     // xor    %edx,%edx
1866     emit_opcode(cbuf, 0x33);
1867     emit_d8(cbuf, 0xD2);
1868 
1869     // cmp    $0xffffffffffffffff,%ecx
1870     if ($div$$reg >= 8) {
1871       emit_opcode(cbuf, Assembler::REX_B);
1872     }
1873     emit_opcode(cbuf, 0x83);
1874     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1875     emit_d8(cbuf, 0xFF);
1876 
1877     // je     11 <done>
1878     emit_opcode(cbuf, 0x74);
1879     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1880 
1881     // <normal>
1882     // cltd
1883     emit_opcode(cbuf, 0x99);
1884 
1885     // idivl (note: must be emitted by the user of this rule)
1886     // <done>
1887   %}
1888 
1889   enc_class cdqq_enc(no_rax_rdx_RegL div)
1890   %{
1891     // Full implementation of Java ldiv and lrem; checks for
1892     // special case as described in JVM spec., p.243 & p.271.
1893     //
1894     //         normal case                           special case
1895     //
1896     // input : rax: dividend                         min_long
1897     //         reg: divisor                          -1
1898     //
1899     // output: rax: quotient  (= rax idiv reg)       min_long
1900     //         rdx: remainder (= rax irem reg)       0
1901     //
1902     //  Code sequnce:
1903     //
1904     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1905     //    7:   00 00 80
1906     //    a:   48 39 d0                cmp    %rdx,%rax
1907     //    d:   75 08                   jne    17 <normal>
1908     //    f:   33 d2                   xor    %edx,%edx
1909     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1910     //   15:   74 05                   je     1c <done>
1911     // 0000000000000017 <normal>:
1912     //   17:   48 99                   cqto
1913     //   19:   48 f7 f9                idiv   $div
1914     // 000000000000001c <done>:
1915 
1916     // mov    $0x8000000000000000,%rdx
1917     emit_opcode(cbuf, Assembler::REX_W);
1918     emit_opcode(cbuf, 0xBA);
1919     emit_d8(cbuf, 0x00);
1920     emit_d8(cbuf, 0x00);
1921     emit_d8(cbuf, 0x00);
1922     emit_d8(cbuf, 0x00);
1923     emit_d8(cbuf, 0x00);
1924     emit_d8(cbuf, 0x00);
1925     emit_d8(cbuf, 0x00);
1926     emit_d8(cbuf, 0x80);
1927 
1928     // cmp    %rdx,%rax
1929     emit_opcode(cbuf, Assembler::REX_W);
1930     emit_opcode(cbuf, 0x39);
1931     emit_d8(cbuf, 0xD0);
1932 
1933     // jne    17 <normal>
1934     emit_opcode(cbuf, 0x75);
1935     emit_d8(cbuf, 0x08);
1936 
1937     // xor    %edx,%edx
1938     emit_opcode(cbuf, 0x33);
1939     emit_d8(cbuf, 0xD2);
1940 
1941     // cmp    $0xffffffffffffffff,$div
1942     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1943     emit_opcode(cbuf, 0x83);
1944     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1945     emit_d8(cbuf, 0xFF);
1946 
1947     // je     1e <done>
1948     emit_opcode(cbuf, 0x74);
1949     emit_d8(cbuf, 0x05);
1950 
1951     // <normal>
1952     // cqto
1953     emit_opcode(cbuf, Assembler::REX_W);
1954     emit_opcode(cbuf, 0x99);
1955 
1956     // idivq (note: must be emitted by the user of this rule)
1957     // <done>
1958   %}
1959 
1960   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1961   enc_class OpcSE(immI imm)
1962   %{
1963     // Emit primary opcode and set sign-extend bit
1964     // Check for 8-bit immediate, and set sign extend bit in opcode
1965     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1966       emit_opcode(cbuf, $primary | 0x02);
1967     } else {
1968       // 32-bit immediate
1969       emit_opcode(cbuf, $primary);
1970     }
1971   %}
1972 
1973   enc_class OpcSErm(rRegI dst, immI imm)
1974   %{
1975     // OpcSEr/m
1976     int dstenc = $dst$$reg;
1977     if (dstenc >= 8) {
1978       emit_opcode(cbuf, Assembler::REX_B);
1979       dstenc -= 8;
1980     }
1981     // Emit primary opcode and set sign-extend bit
1982     // Check for 8-bit immediate, and set sign extend bit in opcode
1983     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1984       emit_opcode(cbuf, $primary | 0x02);
1985     } else {
1986       // 32-bit immediate
1987       emit_opcode(cbuf, $primary);
1988     }
1989     // Emit r/m byte with secondary opcode, after primary opcode.
1990     emit_rm(cbuf, 0x3, $secondary, dstenc);
1991   %}
1992 
1993   enc_class OpcSErm_wide(rRegL dst, immI imm)
1994   %{
1995     // OpcSEr/m
1996     int dstenc = $dst$$reg;
1997     if (dstenc < 8) {
1998       emit_opcode(cbuf, Assembler::REX_W);
1999     } else {
2000       emit_opcode(cbuf, Assembler::REX_WB);
2001       dstenc -= 8;
2002     }
2003     // Emit primary opcode and set sign-extend bit
2004     // Check for 8-bit immediate, and set sign extend bit in opcode
2005     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2006       emit_opcode(cbuf, $primary | 0x02);
2007     } else {
2008       // 32-bit immediate
2009       emit_opcode(cbuf, $primary);
2010     }
2011     // Emit r/m byte with secondary opcode, after primary opcode.
2012     emit_rm(cbuf, 0x3, $secondary, dstenc);
2013   %}
2014 
2015   enc_class Con8or32(immI imm)
2016   %{
2017     // Check for 8-bit immediate, and set sign extend bit in opcode
2018     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2019       $$$emit8$imm$$constant;
2020     } else {
2021       // 32-bit immediate
2022       $$$emit32$imm$$constant;
2023     }
2024   %}
2025 
2026   enc_class opc2_reg(rRegI dst)
2027   %{
2028     // BSWAP
2029     emit_cc(cbuf, $secondary, $dst$$reg);
2030   %}
2031 
2032   enc_class opc3_reg(rRegI dst)
2033   %{
2034     // BSWAP
2035     emit_cc(cbuf, $tertiary, $dst$$reg);
2036   %}
2037 
2038   enc_class reg_opc(rRegI div)
2039   %{
2040     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2041     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2042   %}
2043 
2044   enc_class enc_cmov(cmpOp cop)
2045   %{
2046     // CMOV
2047     $$$emit8$primary;
2048     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2049   %}
2050 
2051   enc_class enc_PartialSubtypeCheck()
2052   %{
2053     Register Rrdi = as_Register(RDI_enc); // result register
2054     Register Rrax = as_Register(RAX_enc); // super class
2055     Register Rrcx = as_Register(RCX_enc); // killed
2056     Register Rrsi = as_Register(RSI_enc); // sub class
2057     Label miss;
2058     const bool set_cond_codes = true;
2059 
2060     MacroAssembler _masm(&cbuf);
2061     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2062                                      NULL, &miss,
2063                                      /*set_cond_codes:*/ true);
2064     if ($primary) {
2065       __ xorptr(Rrdi, Rrdi);
2066     }
2067     __ bind(miss);
2068   %}
2069 
2070   enc_class clear_avx %{
2071     debug_only(int off0 = cbuf.insts_size());
2072     if (ra_->C->max_vector_size() > 16) {
2073       // Clear upper bits of YMM registers when current compiled code uses
2074       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2075       MacroAssembler _masm(&cbuf);
2076       __ vzeroupper();
2077     }
2078     debug_only(int off1 = cbuf.insts_size());
2079     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2080   %}
2081 
2082   enc_class Java_To_Runtime(method meth) %{
2083     // No relocation needed
2084     MacroAssembler _masm(&cbuf);
2085     __ mov64(r10, (int64_t) $meth$$method);
2086     __ call(r10);
2087   %}
2088 
2089   enc_class Java_To_Interpreter(method meth)
2090   %{
2091     // CALL Java_To_Interpreter
2092     // This is the instruction starting address for relocation info.
2093     cbuf.set_insts_mark();
2094     $$$emit8$primary;
2095     // CALL directly to the runtime
2096     emit_d32_reloc(cbuf,
2097                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2098                    runtime_call_Relocation::spec(),
2099                    RELOC_DISP32);
2100   %}
2101 
2102   enc_class Java_Static_Call(method meth)
2103   %{
2104     // JAVA STATIC CALL
2105     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2106     // determine who we intended to call.
2107     cbuf.set_insts_mark();
2108     $$$emit8$primary;
2109 
2110     if (!_method) {
2111       emit_d32_reloc(cbuf,
2112                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2113                      runtime_call_Relocation::spec(),
2114                      RELOC_DISP32);
2115     } else if (_optimized_virtual) {
2116       emit_d32_reloc(cbuf,
2117                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2118                      opt_virtual_call_Relocation::spec(),
2119                      RELOC_DISP32);
2120     } else {
2121       emit_d32_reloc(cbuf,
2122                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2123                      static_call_Relocation::spec(),
2124                      RELOC_DISP32);
2125     }
2126     if (_method) {
2127       // Emit stub for static call.
2128       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
2129       if (stub == NULL) {
2130         ciEnv::current()->record_failure("CodeCache is full");
2131         return;
2132       } 
2133     }
2134   %}
2135 
2136   enc_class Java_Dynamic_Call(method meth) %{
2137     MacroAssembler _masm(&cbuf);
2138     __ ic_call((address)$meth$$method);
2139   %}
2140 
2141   enc_class Java_Compiled_Call(method meth)
2142   %{
2143     // JAVA COMPILED CALL
2144     int disp = in_bytes(Method:: from_compiled_offset());
2145 
2146     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2147     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2148 
2149     // callq *disp(%rax)
2150     cbuf.set_insts_mark();
2151     $$$emit8$primary;
2152     if (disp < 0x80) {
2153       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2154       emit_d8(cbuf, disp); // Displacement
2155     } else {
2156       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2157       emit_d32(cbuf, disp); // Displacement
2158     }
2159   %}
2160 
2161   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2162   %{
2163     // SAL, SAR, SHR
2164     int dstenc = $dst$$reg;
2165     if (dstenc >= 8) {
2166       emit_opcode(cbuf, Assembler::REX_B);
2167       dstenc -= 8;
2168     }
2169     $$$emit8$primary;
2170     emit_rm(cbuf, 0x3, $secondary, dstenc);
2171     $$$emit8$shift$$constant;
2172   %}
2173 
2174   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2175   %{
2176     // SAL, SAR, SHR
2177     int dstenc = $dst$$reg;
2178     if (dstenc < 8) {
2179       emit_opcode(cbuf, Assembler::REX_W);
2180     } else {
2181       emit_opcode(cbuf, Assembler::REX_WB);
2182       dstenc -= 8;
2183     }
2184     $$$emit8$primary;
2185     emit_rm(cbuf, 0x3, $secondary, dstenc);
2186     $$$emit8$shift$$constant;
2187   %}
2188 
2189   enc_class load_immI(rRegI dst, immI src)
2190   %{
2191     int dstenc = $dst$$reg;
2192     if (dstenc >= 8) {
2193       emit_opcode(cbuf, Assembler::REX_B);
2194       dstenc -= 8;
2195     }
2196     emit_opcode(cbuf, 0xB8 | dstenc);
2197     $$$emit32$src$$constant;
2198   %}
2199 
2200   enc_class load_immL(rRegL dst, immL src)
2201   %{
2202     int dstenc = $dst$$reg;
2203     if (dstenc < 8) {
2204       emit_opcode(cbuf, Assembler::REX_W);
2205     } else {
2206       emit_opcode(cbuf, Assembler::REX_WB);
2207       dstenc -= 8;
2208     }
2209     emit_opcode(cbuf, 0xB8 | dstenc);
2210     emit_d64(cbuf, $src$$constant);
2211   %}
2212 
2213   enc_class load_immUL32(rRegL dst, immUL32 src)
2214   %{
2215     // same as load_immI, but this time we care about zeroes in the high word
2216     int dstenc = $dst$$reg;
2217     if (dstenc >= 8) {
2218       emit_opcode(cbuf, Assembler::REX_B);
2219       dstenc -= 8;
2220     }
2221     emit_opcode(cbuf, 0xB8 | dstenc);
2222     $$$emit32$src$$constant;
2223   %}
2224 
2225   enc_class load_immL32(rRegL dst, immL32 src)
2226   %{
2227     int dstenc = $dst$$reg;
2228     if (dstenc < 8) {
2229       emit_opcode(cbuf, Assembler::REX_W);
2230     } else {
2231       emit_opcode(cbuf, Assembler::REX_WB);
2232       dstenc -= 8;
2233     }
2234     emit_opcode(cbuf, 0xC7);
2235     emit_rm(cbuf, 0x03, 0x00, dstenc);
2236     $$$emit32$src$$constant;
2237   %}
2238 
2239   enc_class load_immP31(rRegP dst, immP32 src)
2240   %{
2241     // same as load_immI, but this time we care about zeroes in the high word
2242     int dstenc = $dst$$reg;
2243     if (dstenc >= 8) {
2244       emit_opcode(cbuf, Assembler::REX_B);
2245       dstenc -= 8;
2246     }
2247     emit_opcode(cbuf, 0xB8 | dstenc);
2248     $$$emit32$src$$constant;
2249   %}
2250 
2251   enc_class load_immP(rRegP dst, immP src)
2252   %{
2253     int dstenc = $dst$$reg;
2254     if (dstenc < 8) {
2255       emit_opcode(cbuf, Assembler::REX_W);
2256     } else {
2257       emit_opcode(cbuf, Assembler::REX_WB);
2258       dstenc -= 8;
2259     }
2260     emit_opcode(cbuf, 0xB8 | dstenc);
2261     // This next line should be generated from ADLC
2262     if ($src->constant_reloc() != relocInfo::none) {
2263       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2264     } else {
2265       emit_d64(cbuf, $src$$constant);
2266     }
2267   %}
2268 
2269   enc_class Con32(immI src)
2270   %{
2271     // Output immediate
2272     $$$emit32$src$$constant;
2273   %}
2274 
2275   enc_class Con32F_as_bits(immF src)
2276   %{
2277     // Output Float immediate bits
2278     jfloat jf = $src$$constant;
2279     jint jf_as_bits = jint_cast(jf);
2280     emit_d32(cbuf, jf_as_bits);
2281   %}
2282 
2283   enc_class Con16(immI src)
2284   %{
2285     // Output immediate
2286     $$$emit16$src$$constant;
2287   %}
2288 
2289   // How is this different from Con32??? XXX
2290   enc_class Con_d32(immI src)
2291   %{
2292     emit_d32(cbuf,$src$$constant);
2293   %}
2294 
2295   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2296     // Output immediate memory reference
2297     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2298     emit_d32(cbuf, 0x00);
2299   %}
2300 
2301   enc_class lock_prefix()
2302   %{
2303     if (os::is_MP()) {
2304       emit_opcode(cbuf, 0xF0); // lock
2305     }
2306   %}
2307 
2308   enc_class REX_mem(memory mem)
2309   %{
2310     if ($mem$$base >= 8) {
2311       if ($mem$$index < 8) {
2312         emit_opcode(cbuf, Assembler::REX_B);
2313       } else {
2314         emit_opcode(cbuf, Assembler::REX_XB);
2315       }
2316     } else {
2317       if ($mem$$index >= 8) {
2318         emit_opcode(cbuf, Assembler::REX_X);
2319       }
2320     }
2321   %}
2322 
2323   enc_class REX_mem_wide(memory mem)
2324   %{
2325     if ($mem$$base >= 8) {
2326       if ($mem$$index < 8) {
2327         emit_opcode(cbuf, Assembler::REX_WB);
2328       } else {
2329         emit_opcode(cbuf, Assembler::REX_WXB);
2330       }
2331     } else {
2332       if ($mem$$index < 8) {
2333         emit_opcode(cbuf, Assembler::REX_W);
2334       } else {
2335         emit_opcode(cbuf, Assembler::REX_WX);
2336       }
2337     }
2338   %}
2339 
2340   // for byte regs
2341   enc_class REX_breg(rRegI reg)
2342   %{
2343     if ($reg$$reg >= 4) {
2344       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2345     }
2346   %}
2347 
2348   // for byte regs
2349   enc_class REX_reg_breg(rRegI dst, rRegI src)
2350   %{
2351     if ($dst$$reg < 8) {
2352       if ($src$$reg >= 4) {
2353         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2354       }
2355     } else {
2356       if ($src$$reg < 8) {
2357         emit_opcode(cbuf, Assembler::REX_R);
2358       } else {
2359         emit_opcode(cbuf, Assembler::REX_RB);
2360       }
2361     }
2362   %}
2363 
2364   // for byte regs
2365   enc_class REX_breg_mem(rRegI reg, memory mem)
2366   %{
2367     if ($reg$$reg < 8) {
2368       if ($mem$$base < 8) {
2369         if ($mem$$index >= 8) {
2370           emit_opcode(cbuf, Assembler::REX_X);
2371         } else if ($reg$$reg >= 4) {
2372           emit_opcode(cbuf, Assembler::REX);
2373         }
2374       } else {
2375         if ($mem$$index < 8) {
2376           emit_opcode(cbuf, Assembler::REX_B);
2377         } else {
2378           emit_opcode(cbuf, Assembler::REX_XB);
2379         }
2380       }
2381     } else {
2382       if ($mem$$base < 8) {
2383         if ($mem$$index < 8) {
2384           emit_opcode(cbuf, Assembler::REX_R);
2385         } else {
2386           emit_opcode(cbuf, Assembler::REX_RX);
2387         }
2388       } else {
2389         if ($mem$$index < 8) {
2390           emit_opcode(cbuf, Assembler::REX_RB);
2391         } else {
2392           emit_opcode(cbuf, Assembler::REX_RXB);
2393         }
2394       }
2395     }
2396   %}
2397 
2398   enc_class REX_reg(rRegI reg)
2399   %{
2400     if ($reg$$reg >= 8) {
2401       emit_opcode(cbuf, Assembler::REX_B);
2402     }
2403   %}
2404 
2405   enc_class REX_reg_wide(rRegI reg)
2406   %{
2407     if ($reg$$reg < 8) {
2408       emit_opcode(cbuf, Assembler::REX_W);
2409     } else {
2410       emit_opcode(cbuf, Assembler::REX_WB);
2411     }
2412   %}
2413 
2414   enc_class REX_reg_reg(rRegI dst, rRegI src)
2415   %{
2416     if ($dst$$reg < 8) {
2417       if ($src$$reg >= 8) {
2418         emit_opcode(cbuf, Assembler::REX_B);
2419       }
2420     } else {
2421       if ($src$$reg < 8) {
2422         emit_opcode(cbuf, Assembler::REX_R);
2423       } else {
2424         emit_opcode(cbuf, Assembler::REX_RB);
2425       }
2426     }
2427   %}
2428 
2429   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2430   %{
2431     if ($dst$$reg < 8) {
2432       if ($src$$reg < 8) {
2433         emit_opcode(cbuf, Assembler::REX_W);
2434       } else {
2435         emit_opcode(cbuf, Assembler::REX_WB);
2436       }
2437     } else {
2438       if ($src$$reg < 8) {
2439         emit_opcode(cbuf, Assembler::REX_WR);
2440       } else {
2441         emit_opcode(cbuf, Assembler::REX_WRB);
2442       }
2443     }
2444   %}
2445 
2446   enc_class REX_reg_mem(rRegI reg, memory mem)
2447   %{
2448     if ($reg$$reg < 8) {
2449       if ($mem$$base < 8) {
2450         if ($mem$$index >= 8) {
2451           emit_opcode(cbuf, Assembler::REX_X);
2452         }
2453       } else {
2454         if ($mem$$index < 8) {
2455           emit_opcode(cbuf, Assembler::REX_B);
2456         } else {
2457           emit_opcode(cbuf, Assembler::REX_XB);
2458         }
2459       }
2460     } else {
2461       if ($mem$$base < 8) {
2462         if ($mem$$index < 8) {
2463           emit_opcode(cbuf, Assembler::REX_R);
2464         } else {
2465           emit_opcode(cbuf, Assembler::REX_RX);
2466         }
2467       } else {
2468         if ($mem$$index < 8) {
2469           emit_opcode(cbuf, Assembler::REX_RB);
2470         } else {
2471           emit_opcode(cbuf, Assembler::REX_RXB);
2472         }
2473       }
2474     }
2475   %}
2476 
2477   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2478   %{
2479     if ($reg$$reg < 8) {
2480       if ($mem$$base < 8) {
2481         if ($mem$$index < 8) {
2482           emit_opcode(cbuf, Assembler::REX_W);
2483         } else {
2484           emit_opcode(cbuf, Assembler::REX_WX);
2485         }
2486       } else {
2487         if ($mem$$index < 8) {
2488           emit_opcode(cbuf, Assembler::REX_WB);
2489         } else {
2490           emit_opcode(cbuf, Assembler::REX_WXB);
2491         }
2492       }
2493     } else {
2494       if ($mem$$base < 8) {
2495         if ($mem$$index < 8) {
2496           emit_opcode(cbuf, Assembler::REX_WR);
2497         } else {
2498           emit_opcode(cbuf, Assembler::REX_WRX);
2499         }
2500       } else {
2501         if ($mem$$index < 8) {
2502           emit_opcode(cbuf, Assembler::REX_WRB);
2503         } else {
2504           emit_opcode(cbuf, Assembler::REX_WRXB);
2505         }
2506       }
2507     }
2508   %}
2509 
2510   enc_class reg_mem(rRegI ereg, memory mem)
2511   %{
2512     // High registers handle in encode_RegMem
2513     int reg = $ereg$$reg;
2514     int base = $mem$$base;
2515     int index = $mem$$index;
2516     int scale = $mem$$scale;
2517     int disp = $mem$$disp;
2518     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2519 
2520     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2521   %}
2522 
2523   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2524   %{
2525     int rm_byte_opcode = $rm_opcode$$constant;
2526 
2527     // High registers handle in encode_RegMem
2528     int base = $mem$$base;
2529     int index = $mem$$index;
2530     int scale = $mem$$scale;
2531     int displace = $mem$$disp;
2532 
2533     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2534                                             // working with static
2535                                             // globals
2536     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2537                   disp_reloc);
2538   %}
2539 
2540   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2541   %{
2542     int reg_encoding = $dst$$reg;
2543     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2544     int index        = 0x04;            // 0x04 indicates no index
2545     int scale        = 0x00;            // 0x00 indicates no scale
2546     int displace     = $src1$$constant; // 0x00 indicates no displacement
2547     relocInfo::relocType disp_reloc = relocInfo::none;
2548     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2549                   disp_reloc);
2550   %}
2551 
2552   enc_class neg_reg(rRegI dst)
2553   %{
2554     int dstenc = $dst$$reg;
2555     if (dstenc >= 8) {
2556       emit_opcode(cbuf, Assembler::REX_B);
2557       dstenc -= 8;
2558     }
2559     // NEG $dst
2560     emit_opcode(cbuf, 0xF7);
2561     emit_rm(cbuf, 0x3, 0x03, dstenc);
2562   %}
2563 
2564   enc_class neg_reg_wide(rRegI dst)
2565   %{
2566     int dstenc = $dst$$reg;
2567     if (dstenc < 8) {
2568       emit_opcode(cbuf, Assembler::REX_W);
2569     } else {
2570       emit_opcode(cbuf, Assembler::REX_WB);
2571       dstenc -= 8;
2572     }
2573     // NEG $dst
2574     emit_opcode(cbuf, 0xF7);
2575     emit_rm(cbuf, 0x3, 0x03, dstenc);
2576   %}
2577 
2578   enc_class setLT_reg(rRegI dst)
2579   %{
2580     int dstenc = $dst$$reg;
2581     if (dstenc >= 8) {
2582       emit_opcode(cbuf, Assembler::REX_B);
2583       dstenc -= 8;
2584     } else if (dstenc >= 4) {
2585       emit_opcode(cbuf, Assembler::REX);
2586     }
2587     // SETLT $dst
2588     emit_opcode(cbuf, 0x0F);
2589     emit_opcode(cbuf, 0x9C);
2590     emit_rm(cbuf, 0x3, 0x0, dstenc);
2591   %}
2592 
2593   enc_class setNZ_reg(rRegI dst)
2594   %{
2595     int dstenc = $dst$$reg;
2596     if (dstenc >= 8) {
2597       emit_opcode(cbuf, Assembler::REX_B);
2598       dstenc -= 8;
2599     } else if (dstenc >= 4) {
2600       emit_opcode(cbuf, Assembler::REX);
2601     }
2602     // SETNZ $dst
2603     emit_opcode(cbuf, 0x0F);
2604     emit_opcode(cbuf, 0x95);
2605     emit_rm(cbuf, 0x3, 0x0, dstenc);
2606   %}
2607 
2608 
2609   // Compare the lonogs and set -1, 0, or 1 into dst
2610   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2611   %{
2612     int src1enc = $src1$$reg;
2613     int src2enc = $src2$$reg;
2614     int dstenc = $dst$$reg;
2615 
2616     // cmpq $src1, $src2
2617     if (src1enc < 8) {
2618       if (src2enc < 8) {
2619         emit_opcode(cbuf, Assembler::REX_W);
2620       } else {
2621         emit_opcode(cbuf, Assembler::REX_WB);
2622       }
2623     } else {
2624       if (src2enc < 8) {
2625         emit_opcode(cbuf, Assembler::REX_WR);
2626       } else {
2627         emit_opcode(cbuf, Assembler::REX_WRB);
2628       }
2629     }
2630     emit_opcode(cbuf, 0x3B);
2631     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2632 
2633     // movl $dst, -1
2634     if (dstenc >= 8) {
2635       emit_opcode(cbuf, Assembler::REX_B);
2636     }
2637     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2638     emit_d32(cbuf, -1);
2639 
2640     // jl,s done
2641     emit_opcode(cbuf, 0x7C);
2642     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2643 
2644     // setne $dst
2645     if (dstenc >= 4) {
2646       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2647     }
2648     emit_opcode(cbuf, 0x0F);
2649     emit_opcode(cbuf, 0x95);
2650     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2651 
2652     // movzbl $dst, $dst
2653     if (dstenc >= 4) {
2654       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2655     }
2656     emit_opcode(cbuf, 0x0F);
2657     emit_opcode(cbuf, 0xB6);
2658     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2659   %}
2660 
2661   enc_class Push_ResultXD(regD dst) %{
2662     MacroAssembler _masm(&cbuf);
2663     __ fstp_d(Address(rsp, 0));
2664     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2665     __ addptr(rsp, 8);
2666   %}
2667 
2668   enc_class Push_SrcXD(regD src) %{
2669     MacroAssembler _masm(&cbuf);
2670     __ subptr(rsp, 8);
2671     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2672     __ fld_d(Address(rsp, 0));
2673   %}
2674 
2675 
2676   enc_class enc_rethrow()
2677   %{
2678     cbuf.set_insts_mark();
2679     emit_opcode(cbuf, 0xE9); // jmp entry
2680     emit_d32_reloc(cbuf,
2681                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2682                    runtime_call_Relocation::spec(),
2683                    RELOC_DISP32);
2684   %}
2685 
2686 %}
2687 
2688 
2689 
2690 //----------FRAME--------------------------------------------------------------
2691 // Definition of frame structure and management information.
2692 //
2693 //  S T A C K   L A Y O U T    Allocators stack-slot number
2694 //                             |   (to get allocators register number
2695 //  G  Owned by    |        |  v    add OptoReg::stack0())
2696 //  r   CALLER     |        |
2697 //  o     |        +--------+      pad to even-align allocators stack-slot
2698 //  w     V        |  pad0  |        numbers; owned by CALLER
2699 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2700 //  h     ^        |   in   |  5
2701 //        |        |  args  |  4   Holes in incoming args owned by SELF
2702 //  |     |        |        |  3
2703 //  |     |        +--------+
2704 //  V     |        | old out|      Empty on Intel, window on Sparc
2705 //        |    old |preserve|      Must be even aligned.
2706 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2707 //        |        |   in   |  3   area for Intel ret address
2708 //     Owned by    |preserve|      Empty on Sparc.
2709 //       SELF      +--------+
2710 //        |        |  pad2  |  2   pad to align old SP
2711 //        |        +--------+  1
2712 //        |        | locks  |  0
2713 //        |        +--------+----> OptoReg::stack0(), even aligned
2714 //        |        |  pad1  | 11   pad to align new SP
2715 //        |        +--------+
2716 //        |        |        | 10
2717 //        |        | spills |  9   spills
2718 //        V        |        |  8   (pad0 slot for callee)
2719 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2720 //        ^        |  out   |  7
2721 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2722 //     Owned by    +--------+
2723 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2724 //        |    new |preserve|      Must be even-aligned.
2725 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2726 //        |        |        |
2727 //
2728 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2729 //         known from SELF's arguments and the Java calling convention.
2730 //         Region 6-7 is determined per call site.
2731 // Note 2: If the calling convention leaves holes in the incoming argument
2732 //         area, those holes are owned by SELF.  Holes in the outgoing area
2733 //         are owned by the CALLEE.  Holes should not be nessecary in the
2734 //         incoming area, as the Java calling convention is completely under
2735 //         the control of the AD file.  Doubles can be sorted and packed to
2736 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2737 //         varargs C calling conventions.
2738 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2739 //         even aligned with pad0 as needed.
2740 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2741 //         region 6-11 is even aligned; it may be padded out more so that
2742 //         the region from SP to FP meets the minimum stack alignment.
2743 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2744 //         alignment.  Region 11, pad1, may be dynamically extended so that
2745 //         SP meets the minimum alignment.
2746 
2747 frame
2748 %{
2749   // What direction does stack grow in (assumed to be same for C & Java)
2750   stack_direction(TOWARDS_LOW);
2751 
2752   // These three registers define part of the calling convention
2753   // between compiled code and the interpreter.
2754   inline_cache_reg(RAX);                // Inline Cache Register
2755   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2756                                         // calling interpreter
2757 
2758   // Optional: name the operand used by cisc-spilling to access
2759   // [stack_pointer + offset]
2760   cisc_spilling_operand_name(indOffset32);
2761 
2762   // Number of stack slots consumed by locking an object
2763   sync_stack_slots(2);
2764 
2765   // Compiled code's Frame Pointer
2766   frame_pointer(RSP);
2767 
2768   // Interpreter stores its frame pointer in a register which is
2769   // stored to the stack by I2CAdaptors.
2770   // I2CAdaptors convert from interpreted java to compiled java.
2771   interpreter_frame_pointer(RBP);
2772 
2773   // Stack alignment requirement
2774   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2775 
2776   // Number of stack slots between incoming argument block and the start of
2777   // a new frame.  The PROLOG must add this many slots to the stack.  The
2778   // EPILOG must remove this many slots.  amd64 needs two slots for
2779   // return address.
2780   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2781 
2782   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2783   // for calls to C.  Supports the var-args backing area for register parms.
2784   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2785 
2786   // The after-PROLOG location of the return address.  Location of
2787   // return address specifies a type (REG or STACK) and a number
2788   // representing the register number (i.e. - use a register name) or
2789   // stack slot.
2790   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2791   // Otherwise, it is above the locks and verification slot and alignment word
2792   return_addr(STACK - 2 +
2793               round_to((Compile::current()->in_preserve_stack_slots() +
2794                         Compile::current()->fixed_slots()),
2795                        stack_alignment_in_slots()));
2796 
2797   // Body of function which returns an integer array locating
2798   // arguments either in registers or in stack slots.  Passed an array
2799   // of ideal registers called "sig" and a "length" count.  Stack-slot
2800   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2801   // arguments for a CALLEE.  Incoming stack arguments are
2802   // automatically biased by the preserve_stack_slots field above.
2803 
2804   calling_convention
2805   %{
2806     // No difference between ingoing/outgoing just pass false
2807     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2808   %}
2809 
2810   c_calling_convention
2811   %{
2812     // This is obviously always outgoing
2813     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2814   %}
2815 
2816   // Location of compiled Java return values.  Same as C for now.
2817   return_value
2818   %{
2819     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2820            "only return normal values");
2821 
2822     static const int lo[Op_RegL + 1] = {
2823       0,
2824       0,
2825       RAX_num,  // Op_RegN
2826       RAX_num,  // Op_RegI
2827       RAX_num,  // Op_RegP
2828       XMM0_num, // Op_RegF
2829       XMM0_num, // Op_RegD
2830       RAX_num   // Op_RegL
2831     };
2832     static const int hi[Op_RegL + 1] = {
2833       0,
2834       0,
2835       OptoReg::Bad, // Op_RegN
2836       OptoReg::Bad, // Op_RegI
2837       RAX_H_num,    // Op_RegP
2838       OptoReg::Bad, // Op_RegF
2839       XMM0b_num,    // Op_RegD
2840       RAX_H_num     // Op_RegL
2841     };
2842     // Excluded flags and vector registers.
2843     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 5, "missing type");
2844     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2845   %}
2846 %}
2847 
2848 //----------ATTRIBUTES---------------------------------------------------------
2849 //----------Operand Attributes-------------------------------------------------
2850 op_attrib op_cost(0);        // Required cost attribute
2851 
2852 //----------Instruction Attributes---------------------------------------------
2853 ins_attrib ins_cost(100);       // Required cost attribute
2854 ins_attrib ins_size(8);         // Required size attribute (in bits)
2855 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2856                                 // a non-matching short branch variant
2857                                 // of some long branch?
2858 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2859                                 // be a power of 2) specifies the
2860                                 // alignment that some part of the
2861                                 // instruction (not necessarily the
2862                                 // start) requires.  If > 1, a
2863                                 // compute_padding() function must be
2864                                 // provided for the instruction
2865 
2866 //----------OPERANDS-----------------------------------------------------------
2867 // Operand definitions must precede instruction definitions for correct parsing
2868 // in the ADLC because operands constitute user defined types which are used in
2869 // instruction definitions.
2870 
2871 //----------Simple Operands----------------------------------------------------
2872 // Immediate Operands
2873 // Integer Immediate
2874 operand immI()
2875 %{
2876   match(ConI);
2877 
2878   op_cost(10);
2879   format %{ %}
2880   interface(CONST_INTER);
2881 %}
2882 
2883 // Constant for test vs zero
2884 operand immI0()
2885 %{
2886   predicate(n->get_int() == 0);
2887   match(ConI);
2888 
2889   op_cost(0);
2890   format %{ %}
2891   interface(CONST_INTER);
2892 %}
2893 
2894 // Constant for increment
2895 operand immI1()
2896 %{
2897   predicate(n->get_int() == 1);
2898   match(ConI);
2899 
2900   op_cost(0);
2901   format %{ %}
2902   interface(CONST_INTER);
2903 %}
2904 
2905 // Constant for decrement
2906 operand immI_M1()
2907 %{
2908   predicate(n->get_int() == -1);
2909   match(ConI);
2910 
2911   op_cost(0);
2912   format %{ %}
2913   interface(CONST_INTER);
2914 %}
2915 
2916 // Valid scale values for addressing modes
2917 operand immI2()
2918 %{
2919   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2920   match(ConI);
2921 
2922   format %{ %}
2923   interface(CONST_INTER);
2924 %}
2925 
2926 operand immI8()
2927 %{
2928   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2929   match(ConI);
2930 
2931   op_cost(5);
2932   format %{ %}
2933   interface(CONST_INTER);
2934 %}
2935 
2936 operand immI16()
2937 %{
2938   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2939   match(ConI);
2940 
2941   op_cost(10);
2942   format %{ %}
2943   interface(CONST_INTER);
2944 %}
2945 
2946 // Int Immediate non-negative
2947 operand immU31()
2948 %{
2949   predicate(n->get_int() >= 0);
2950   match(ConI);
2951 
2952   op_cost(0);
2953   format %{ %}
2954   interface(CONST_INTER);
2955 %}
2956 
2957 // Constant for long shifts
2958 operand immI_32()
2959 %{
2960   predicate( n->get_int() == 32 );
2961   match(ConI);
2962 
2963   op_cost(0);
2964   format %{ %}
2965   interface(CONST_INTER);
2966 %}
2967 
2968 // Constant for long shifts
2969 operand immI_64()
2970 %{
2971   predicate( n->get_int() == 64 );
2972   match(ConI);
2973 
2974   op_cost(0);
2975   format %{ %}
2976   interface(CONST_INTER);
2977 %}
2978 
2979 // Pointer Immediate
2980 operand immP()
2981 %{
2982   match(ConP);
2983 
2984   op_cost(10);
2985   format %{ %}
2986   interface(CONST_INTER);
2987 %}
2988 
2989 // NULL Pointer Immediate
2990 operand immP0()
2991 %{
2992   predicate(n->get_ptr() == 0);
2993   match(ConP);
2994 
2995   op_cost(5);
2996   format %{ %}
2997   interface(CONST_INTER);
2998 %}
2999 
3000 // Pointer Immediate
3001 operand immN() %{
3002   match(ConN);
3003 
3004   op_cost(10);
3005   format %{ %}
3006   interface(CONST_INTER);
3007 %}
3008 
3009 operand immNKlass() %{
3010   match(ConNKlass);
3011 
3012   op_cost(10);
3013   format %{ %}
3014   interface(CONST_INTER);
3015 %}
3016 
3017 // NULL Pointer Immediate
3018 operand immN0() %{
3019   predicate(n->get_narrowcon() == 0);
3020   match(ConN);
3021 
3022   op_cost(5);
3023   format %{ %}
3024   interface(CONST_INTER);
3025 %}
3026 
3027 operand immP31()
3028 %{
3029   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3030             && (n->get_ptr() >> 31) == 0);
3031   match(ConP);
3032 
3033   op_cost(5);
3034   format %{ %}
3035   interface(CONST_INTER);
3036 %}
3037 
3038 
3039 // Long Immediate
3040 operand immL()
3041 %{
3042   match(ConL);
3043 
3044   op_cost(20);
3045   format %{ %}
3046   interface(CONST_INTER);
3047 %}
3048 
3049 // Long Immediate 8-bit
3050 operand immL8()
3051 %{
3052   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3053   match(ConL);
3054 
3055   op_cost(5);
3056   format %{ %}
3057   interface(CONST_INTER);
3058 %}
3059 
3060 // Long Immediate 32-bit unsigned
3061 operand immUL32()
3062 %{
3063   predicate(n->get_long() == (unsigned int) (n->get_long()));
3064   match(ConL);
3065 
3066   op_cost(10);
3067   format %{ %}
3068   interface(CONST_INTER);
3069 %}
3070 
3071 // Long Immediate 32-bit signed
3072 operand immL32()
3073 %{
3074   predicate(n->get_long() == (int) (n->get_long()));
3075   match(ConL);
3076 
3077   op_cost(15);
3078   format %{ %}
3079   interface(CONST_INTER);
3080 %}
3081 
3082 // Long Immediate zero
3083 operand immL0()
3084 %{
3085   predicate(n->get_long() == 0L);
3086   match(ConL);
3087 
3088   op_cost(10);
3089   format %{ %}
3090   interface(CONST_INTER);
3091 %}
3092 
3093 // Constant for increment
3094 operand immL1()
3095 %{
3096   predicate(n->get_long() == 1);
3097   match(ConL);
3098 
3099   format %{ %}
3100   interface(CONST_INTER);
3101 %}
3102 
3103 // Constant for decrement
3104 operand immL_M1()
3105 %{
3106   predicate(n->get_long() == -1);
3107   match(ConL);
3108 
3109   format %{ %}
3110   interface(CONST_INTER);
3111 %}
3112 
3113 // Long Immediate: the value 10
3114 operand immL10()
3115 %{
3116   predicate(n->get_long() == 10);
3117   match(ConL);
3118 
3119   format %{ %}
3120   interface(CONST_INTER);
3121 %}
3122 
3123 // Long immediate from 0 to 127.
3124 // Used for a shorter form of long mul by 10.
3125 operand immL_127()
3126 %{
3127   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3128   match(ConL);
3129 
3130   op_cost(10);
3131   format %{ %}
3132   interface(CONST_INTER);
3133 %}
3134 
3135 // Long Immediate: low 32-bit mask
3136 operand immL_32bits()
3137 %{
3138   predicate(n->get_long() == 0xFFFFFFFFL);
3139   match(ConL);
3140   op_cost(20);
3141 
3142   format %{ %}
3143   interface(CONST_INTER);
3144 %}
3145 
3146 // Float Immediate zero
3147 operand immF0()
3148 %{
3149   predicate(jint_cast(n->getf()) == 0);
3150   match(ConF);
3151 
3152   op_cost(5);
3153   format %{ %}
3154   interface(CONST_INTER);
3155 %}
3156 
3157 // Float Immediate
3158 operand immF()
3159 %{
3160   match(ConF);
3161 
3162   op_cost(15);
3163   format %{ %}
3164   interface(CONST_INTER);
3165 %}
3166 
3167 // Double Immediate zero
3168 operand immD0()
3169 %{
3170   predicate(jlong_cast(n->getd()) == 0);
3171   match(ConD);
3172 
3173   op_cost(5);
3174   format %{ %}
3175   interface(CONST_INTER);
3176 %}
3177 
3178 // Double Immediate
3179 operand immD()
3180 %{
3181   match(ConD);
3182 
3183   op_cost(15);
3184   format %{ %}
3185   interface(CONST_INTER);
3186 %}
3187 
3188 // Immediates for special shifts (sign extend)
3189 
3190 // Constants for increment
3191 operand immI_16()
3192 %{
3193   predicate(n->get_int() == 16);
3194   match(ConI);
3195 
3196   format %{ %}
3197   interface(CONST_INTER);
3198 %}
3199 
3200 operand immI_24()
3201 %{
3202   predicate(n->get_int() == 24);
3203   match(ConI);
3204 
3205   format %{ %}
3206   interface(CONST_INTER);
3207 %}
3208 
3209 // Constant for byte-wide masking
3210 operand immI_255()
3211 %{
3212   predicate(n->get_int() == 255);
3213   match(ConI);
3214 
3215   format %{ %}
3216   interface(CONST_INTER);
3217 %}
3218 
3219 // Constant for short-wide masking
3220 operand immI_65535()
3221 %{
3222   predicate(n->get_int() == 65535);
3223   match(ConI);
3224 
3225   format %{ %}
3226   interface(CONST_INTER);
3227 %}
3228 
3229 // Constant for byte-wide masking
3230 operand immL_255()
3231 %{
3232   predicate(n->get_long() == 255);
3233   match(ConL);
3234 
3235   format %{ %}
3236   interface(CONST_INTER);
3237 %}
3238 
3239 // Constant for short-wide masking
3240 operand immL_65535()
3241 %{
3242   predicate(n->get_long() == 65535);
3243   match(ConL);
3244 
3245   format %{ %}
3246   interface(CONST_INTER);
3247 %}
3248 
3249 // Register Operands
3250 // Integer Register
3251 operand rRegI()
3252 %{
3253   constraint(ALLOC_IN_RC(int_reg));
3254   match(RegI);
3255 
3256   match(rax_RegI);
3257   match(rbx_RegI);
3258   match(rcx_RegI);
3259   match(rdx_RegI);
3260   match(rdi_RegI);
3261 
3262   format %{ %}
3263   interface(REG_INTER);
3264 %}
3265 
3266 // Special Registers
3267 operand rax_RegI()
3268 %{
3269   constraint(ALLOC_IN_RC(int_rax_reg));
3270   match(RegI);
3271   match(rRegI);
3272 
3273   format %{ "RAX" %}
3274   interface(REG_INTER);
3275 %}
3276 
3277 // Special Registers
3278 operand rbx_RegI()
3279 %{
3280   constraint(ALLOC_IN_RC(int_rbx_reg));
3281   match(RegI);
3282   match(rRegI);
3283 
3284   format %{ "RBX" %}
3285   interface(REG_INTER);
3286 %}
3287 
3288 operand rcx_RegI()
3289 %{
3290   constraint(ALLOC_IN_RC(int_rcx_reg));
3291   match(RegI);
3292   match(rRegI);
3293 
3294   format %{ "RCX" %}
3295   interface(REG_INTER);
3296 %}
3297 
3298 operand rdx_RegI()
3299 %{
3300   constraint(ALLOC_IN_RC(int_rdx_reg));
3301   match(RegI);
3302   match(rRegI);
3303 
3304   format %{ "RDX" %}
3305   interface(REG_INTER);
3306 %}
3307 
3308 operand rdi_RegI()
3309 %{
3310   constraint(ALLOC_IN_RC(int_rdi_reg));
3311   match(RegI);
3312   match(rRegI);
3313 
3314   format %{ "RDI" %}
3315   interface(REG_INTER);
3316 %}
3317 
3318 operand no_rcx_RegI()
3319 %{
3320   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3321   match(RegI);
3322   match(rax_RegI);
3323   match(rbx_RegI);
3324   match(rdx_RegI);
3325   match(rdi_RegI);
3326 
3327   format %{ %}
3328   interface(REG_INTER);
3329 %}
3330 
3331 operand no_rax_rdx_RegI()
3332 %{
3333   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3334   match(RegI);
3335   match(rbx_RegI);
3336   match(rcx_RegI);
3337   match(rdi_RegI);
3338 
3339   format %{ %}
3340   interface(REG_INTER);
3341 %}
3342 
3343 // Pointer Register
3344 operand any_RegP()
3345 %{
3346   constraint(ALLOC_IN_RC(any_reg));  
3347   match(RegP);
3348   match(rax_RegP);
3349   match(rbx_RegP);
3350   match(rdi_RegP);
3351   match(rsi_RegP);
3352   match(rbp_RegP);
3353   match(r15_RegP);
3354   match(rRegP);
3355 
3356   format %{ %}
3357   interface(REG_INTER);
3358 %}
3359 
3360 operand rRegP()
3361 %{
3362   constraint(ALLOC_IN_RC(ptr_reg));
3363   match(RegP);
3364   match(rax_RegP);
3365   match(rbx_RegP);
3366   match(rdi_RegP);
3367   match(rsi_RegP);
3368   match(rbp_RegP);  // See Q&A below about
3369   match(r15_RegP);  // r15_RegP and rbp_RegP.
3370 
3371   format %{ %}
3372   interface(REG_INTER);
3373 %}
3374 
3375 operand rRegN() %{
3376   constraint(ALLOC_IN_RC(int_reg));
3377   match(RegN);
3378 
3379   format %{ %}
3380   interface(REG_INTER);
3381 %}
3382 
3383 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3384 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3385 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3386 // The output of an instruction is controlled by the allocator, which respects
3387 // register class masks, not match rules.  Unless an instruction mentions
3388 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3389 // by the allocator as an input.
3390 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3391 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3392 // result, RBP is not included in the output of the instruction either.
3393 
3394 operand no_rax_RegP()
3395 %{
3396   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3397   match(RegP);
3398   match(rbx_RegP);
3399   match(rsi_RegP);
3400   match(rdi_RegP);
3401 
3402   format %{ %}
3403   interface(REG_INTER);
3404 %}
3405 
3406 // This operand is not allowed to use RBP even if
3407 // RBP is not used to hold the frame pointer.
3408 operand no_rbp_RegP()
3409 %{
3410   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3411   match(RegP);
3412   match(rbx_RegP);
3413   match(rsi_RegP);
3414   match(rdi_RegP);
3415 
3416   format %{ %}
3417   interface(REG_INTER);
3418 %}
3419 
3420 operand no_rax_rbx_RegP()
3421 %{
3422   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3423   match(RegP);
3424   match(rsi_RegP);
3425   match(rdi_RegP);
3426 
3427   format %{ %}
3428   interface(REG_INTER);
3429 %}
3430 
3431 // Special Registers
3432 // Return a pointer value
3433 operand rax_RegP()
3434 %{
3435   constraint(ALLOC_IN_RC(ptr_rax_reg));
3436   match(RegP);
3437   match(rRegP);
3438 
3439   format %{ %}
3440   interface(REG_INTER);
3441 %}
3442 
3443 // Special Registers
3444 // Return a compressed pointer value
3445 operand rax_RegN()
3446 %{
3447   constraint(ALLOC_IN_RC(int_rax_reg));
3448   match(RegN);
3449   match(rRegN);
3450 
3451   format %{ %}
3452   interface(REG_INTER);
3453 %}
3454 
3455 // Used in AtomicAdd
3456 operand rbx_RegP()
3457 %{
3458   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3459   match(RegP);
3460   match(rRegP);
3461 
3462   format %{ %}
3463   interface(REG_INTER);
3464 %}
3465 
3466 operand rsi_RegP()
3467 %{
3468   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3469   match(RegP);
3470   match(rRegP);
3471 
3472   format %{ %}
3473   interface(REG_INTER);
3474 %}
3475 
3476 // Used in rep stosq
3477 operand rdi_RegP()
3478 %{
3479   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3480   match(RegP);
3481   match(rRegP);
3482 
3483   format %{ %}
3484   interface(REG_INTER);
3485 %}
3486 
3487 operand r15_RegP()
3488 %{
3489   constraint(ALLOC_IN_RC(ptr_r15_reg));
3490   match(RegP);
3491   match(rRegP);
3492 
3493   format %{ %}
3494   interface(REG_INTER);
3495 %}
3496 
3497 operand rRegL()
3498 %{
3499   constraint(ALLOC_IN_RC(long_reg));
3500   match(RegL);
3501   match(rax_RegL);
3502   match(rdx_RegL);
3503 
3504   format %{ %}
3505   interface(REG_INTER);
3506 %}
3507 
3508 // Special Registers
3509 operand no_rax_rdx_RegL()
3510 %{
3511   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3512   match(RegL);
3513   match(rRegL);
3514 
3515   format %{ %}
3516   interface(REG_INTER);
3517 %}
3518 
3519 operand no_rax_RegL()
3520 %{
3521   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3522   match(RegL);
3523   match(rRegL);
3524   match(rdx_RegL);
3525 
3526   format %{ %}
3527   interface(REG_INTER);
3528 %}
3529 
3530 operand no_rcx_RegL()
3531 %{
3532   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3533   match(RegL);
3534   match(rRegL);
3535 
3536   format %{ %}
3537   interface(REG_INTER);
3538 %}
3539 
3540 operand rax_RegL()
3541 %{
3542   constraint(ALLOC_IN_RC(long_rax_reg));
3543   match(RegL);
3544   match(rRegL);
3545 
3546   format %{ "RAX" %}
3547   interface(REG_INTER);
3548 %}
3549 
3550 operand rcx_RegL()
3551 %{
3552   constraint(ALLOC_IN_RC(long_rcx_reg));
3553   match(RegL);
3554   match(rRegL);
3555 
3556   format %{ %}
3557   interface(REG_INTER);
3558 %}
3559 
3560 operand rdx_RegL()
3561 %{
3562   constraint(ALLOC_IN_RC(long_rdx_reg));
3563   match(RegL);
3564   match(rRegL);
3565 
3566   format %{ %}
3567   interface(REG_INTER);
3568 %}
3569 
3570 // Flags register, used as output of compare instructions
3571 operand rFlagsReg()
3572 %{
3573   constraint(ALLOC_IN_RC(int_flags));
3574   match(RegFlags);
3575 
3576   format %{ "RFLAGS" %}
3577   interface(REG_INTER);
3578 %}
3579 
3580 // Flags register, used as output of FLOATING POINT compare instructions
3581 operand rFlagsRegU()
3582 %{
3583   constraint(ALLOC_IN_RC(int_flags));
3584   match(RegFlags);
3585 
3586   format %{ "RFLAGS_U" %}
3587   interface(REG_INTER);
3588 %}
3589 
3590 operand rFlagsRegUCF() %{
3591   constraint(ALLOC_IN_RC(int_flags));
3592   match(RegFlags);
3593   predicate(false);
3594 
3595   format %{ "RFLAGS_U_CF" %}
3596   interface(REG_INTER);
3597 %}
3598 
3599 // Float register operands
3600 operand regF()
3601 %{
3602   constraint(ALLOC_IN_RC(float_reg));
3603   match(RegF);
3604 
3605   format %{ %}
3606   interface(REG_INTER);
3607 %}
3608 
3609 // Double register operands
3610 operand regD()
3611 %{
3612   constraint(ALLOC_IN_RC(double_reg));
3613   match(RegD);
3614 
3615   format %{ %}
3616   interface(REG_INTER);
3617 %}
3618 
3619 //----------Memory Operands----------------------------------------------------
3620 // Direct Memory Operand
3621 // operand direct(immP addr)
3622 // %{
3623 //   match(addr);
3624 
3625 //   format %{ "[$addr]" %}
3626 //   interface(MEMORY_INTER) %{
3627 //     base(0xFFFFFFFF);
3628 //     index(0x4);
3629 //     scale(0x0);
3630 //     disp($addr);
3631 //   %}
3632 // %}
3633 
3634 // Indirect Memory Operand
3635 operand indirect(any_RegP reg)
3636 %{
3637   constraint(ALLOC_IN_RC(ptr_reg));
3638   match(reg);
3639 
3640   format %{ "[$reg]" %}
3641   interface(MEMORY_INTER) %{
3642     base($reg);
3643     index(0x4);
3644     scale(0x0);
3645     disp(0x0);
3646   %}
3647 %}
3648 
3649 // Indirect Memory Plus Short Offset Operand
3650 operand indOffset8(any_RegP reg, immL8 off)
3651 %{
3652   constraint(ALLOC_IN_RC(ptr_reg));
3653   match(AddP reg off);
3654 
3655   format %{ "[$reg + $off (8-bit)]" %}
3656   interface(MEMORY_INTER) %{
3657     base($reg);
3658     index(0x4);
3659     scale(0x0);
3660     disp($off);
3661   %}
3662 %}
3663 
3664 // Indirect Memory Plus Long Offset Operand
3665 operand indOffset32(any_RegP reg, immL32 off)
3666 %{
3667   constraint(ALLOC_IN_RC(ptr_reg));
3668   match(AddP reg off);
3669 
3670   format %{ "[$reg + $off (32-bit)]" %}
3671   interface(MEMORY_INTER) %{
3672     base($reg);
3673     index(0x4);
3674     scale(0x0);
3675     disp($off);
3676   %}
3677 %}
3678 
3679 // Indirect Memory Plus Index Register Plus Offset Operand
3680 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3681 %{
3682   constraint(ALLOC_IN_RC(ptr_reg));
3683   match(AddP (AddP reg lreg) off);
3684 
3685   op_cost(10);
3686   format %{"[$reg + $off + $lreg]" %}
3687   interface(MEMORY_INTER) %{
3688     base($reg);
3689     index($lreg);
3690     scale(0x0);
3691     disp($off);
3692   %}
3693 %}
3694 
3695 // Indirect Memory Plus Index Register Plus Offset Operand
3696 operand indIndex(any_RegP reg, rRegL lreg)
3697 %{
3698   constraint(ALLOC_IN_RC(ptr_reg));
3699   match(AddP reg lreg);
3700 
3701   op_cost(10);
3702   format %{"[$reg + $lreg]" %}
3703   interface(MEMORY_INTER) %{
3704     base($reg);
3705     index($lreg);
3706     scale(0x0);
3707     disp(0x0);
3708   %}
3709 %}
3710 
3711 // Indirect Memory Times Scale Plus Index Register
3712 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3713 %{
3714   constraint(ALLOC_IN_RC(ptr_reg));
3715   match(AddP reg (LShiftL lreg scale));
3716 
3717   op_cost(10);
3718   format %{"[$reg + $lreg << $scale]" %}
3719   interface(MEMORY_INTER) %{
3720     base($reg);
3721     index($lreg);
3722     scale($scale);
3723     disp(0x0);
3724   %}
3725 %}
3726 
3727 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3728 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3729 %{
3730   constraint(ALLOC_IN_RC(ptr_reg));
3731   match(AddP (AddP reg (LShiftL lreg scale)) off);
3732 
3733   op_cost(10);
3734   format %{"[$reg + $off + $lreg << $scale]" %}
3735   interface(MEMORY_INTER) %{
3736     base($reg);
3737     index($lreg);
3738     scale($scale);
3739     disp($off);
3740   %}
3741 %}
3742 
3743 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3744 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3745 %{
3746   constraint(ALLOC_IN_RC(ptr_reg));
3747   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3748   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3749 
3750   op_cost(10);
3751   format %{"[$reg + $off + $idx << $scale]" %}
3752   interface(MEMORY_INTER) %{
3753     base($reg);
3754     index($idx);
3755     scale($scale);
3756     disp($off);
3757   %}
3758 %}
3759 
3760 // Indirect Narrow Oop Plus Offset Operand
3761 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3762 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3763 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3764   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3765   constraint(ALLOC_IN_RC(ptr_reg));
3766   match(AddP (DecodeN reg) off);
3767 
3768   op_cost(10);
3769   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3770   interface(MEMORY_INTER) %{
3771     base(0xc); // R12
3772     index($reg);
3773     scale(0x3);
3774     disp($off);
3775   %}
3776 %}
3777 
3778 // Indirect Memory Operand
3779 operand indirectNarrow(rRegN reg)
3780 %{
3781   predicate(Universe::narrow_oop_shift() == 0);
3782   constraint(ALLOC_IN_RC(ptr_reg));
3783   match(DecodeN reg);
3784 
3785   format %{ "[$reg]" %}
3786   interface(MEMORY_INTER) %{
3787     base($reg);
3788     index(0x4);
3789     scale(0x0);
3790     disp(0x0);
3791   %}
3792 %}
3793 
3794 // Indirect Memory Plus Short Offset Operand
3795 operand indOffset8Narrow(rRegN reg, immL8 off)
3796 %{
3797   predicate(Universe::narrow_oop_shift() == 0);
3798   constraint(ALLOC_IN_RC(ptr_reg));
3799   match(AddP (DecodeN reg) off);
3800 
3801   format %{ "[$reg + $off (8-bit)]" %}
3802   interface(MEMORY_INTER) %{
3803     base($reg);
3804     index(0x4);
3805     scale(0x0);
3806     disp($off);
3807   %}
3808 %}
3809 
3810 // Indirect Memory Plus Long Offset Operand
3811 operand indOffset32Narrow(rRegN reg, immL32 off)
3812 %{
3813   predicate(Universe::narrow_oop_shift() == 0);
3814   constraint(ALLOC_IN_RC(ptr_reg));
3815   match(AddP (DecodeN reg) off);
3816 
3817   format %{ "[$reg + $off (32-bit)]" %}
3818   interface(MEMORY_INTER) %{
3819     base($reg);
3820     index(0x4);
3821     scale(0x0);
3822     disp($off);
3823   %}
3824 %}
3825 
3826 // Indirect Memory Plus Index Register Plus Offset Operand
3827 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3828 %{
3829   predicate(Universe::narrow_oop_shift() == 0);
3830   constraint(ALLOC_IN_RC(ptr_reg));
3831   match(AddP (AddP (DecodeN reg) lreg) off);
3832 
3833   op_cost(10);
3834   format %{"[$reg + $off + $lreg]" %}
3835   interface(MEMORY_INTER) %{
3836     base($reg);
3837     index($lreg);
3838     scale(0x0);
3839     disp($off);
3840   %}
3841 %}
3842 
3843 // Indirect Memory Plus Index Register Plus Offset Operand
3844 operand indIndexNarrow(rRegN reg, rRegL lreg)
3845 %{
3846   predicate(Universe::narrow_oop_shift() == 0);
3847   constraint(ALLOC_IN_RC(ptr_reg));
3848   match(AddP (DecodeN reg) lreg);
3849 
3850   op_cost(10);
3851   format %{"[$reg + $lreg]" %}
3852   interface(MEMORY_INTER) %{
3853     base($reg);
3854     index($lreg);
3855     scale(0x0);
3856     disp(0x0);
3857   %}
3858 %}
3859 
3860 // Indirect Memory Times Scale Plus Index Register
3861 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3862 %{
3863   predicate(Universe::narrow_oop_shift() == 0);
3864   constraint(ALLOC_IN_RC(ptr_reg));
3865   match(AddP (DecodeN reg) (LShiftL lreg scale));
3866 
3867   op_cost(10);
3868   format %{"[$reg + $lreg << $scale]" %}
3869   interface(MEMORY_INTER) %{
3870     base($reg);
3871     index($lreg);
3872     scale($scale);
3873     disp(0x0);
3874   %}
3875 %}
3876 
3877 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3878 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3879 %{
3880   predicate(Universe::narrow_oop_shift() == 0);
3881   constraint(ALLOC_IN_RC(ptr_reg));
3882   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3883 
3884   op_cost(10);
3885   format %{"[$reg + $off + $lreg << $scale]" %}
3886   interface(MEMORY_INTER) %{
3887     base($reg);
3888     index($lreg);
3889     scale($scale);
3890     disp($off);
3891   %}
3892 %}
3893 
3894 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3895 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3896 %{
3897   constraint(ALLOC_IN_RC(ptr_reg));
3898   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3899   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3900 
3901   op_cost(10);
3902   format %{"[$reg + $off + $idx << $scale]" %}
3903   interface(MEMORY_INTER) %{
3904     base($reg);
3905     index($idx);
3906     scale($scale);
3907     disp($off);
3908   %}
3909 %}
3910 
3911 //----------Special Memory Operands--------------------------------------------
3912 // Stack Slot Operand - This operand is used for loading and storing temporary
3913 //                      values on the stack where a match requires a value to
3914 //                      flow through memory.
3915 operand stackSlotP(sRegP reg)
3916 %{
3917   constraint(ALLOC_IN_RC(stack_slots));
3918   // No match rule because this operand is only generated in matching
3919 
3920   format %{ "[$reg]" %}
3921   interface(MEMORY_INTER) %{
3922     base(0x4);   // RSP
3923     index(0x4);  // No Index
3924     scale(0x0);  // No Scale
3925     disp($reg);  // Stack Offset
3926   %}
3927 %}
3928 
3929 operand stackSlotI(sRegI reg)
3930 %{
3931   constraint(ALLOC_IN_RC(stack_slots));
3932   // No match rule because this operand is only generated in matching
3933 
3934   format %{ "[$reg]" %}
3935   interface(MEMORY_INTER) %{
3936     base(0x4);   // RSP
3937     index(0x4);  // No Index
3938     scale(0x0);  // No Scale
3939     disp($reg);  // Stack Offset
3940   %}
3941 %}
3942 
3943 operand stackSlotF(sRegF reg)
3944 %{
3945   constraint(ALLOC_IN_RC(stack_slots));
3946   // No match rule because this operand is only generated in matching
3947 
3948   format %{ "[$reg]" %}
3949   interface(MEMORY_INTER) %{
3950     base(0x4);   // RSP
3951     index(0x4);  // No Index
3952     scale(0x0);  // No Scale
3953     disp($reg);  // Stack Offset
3954   %}
3955 %}
3956 
3957 operand stackSlotD(sRegD reg)
3958 %{
3959   constraint(ALLOC_IN_RC(stack_slots));
3960   // No match rule because this operand is only generated in matching
3961 
3962   format %{ "[$reg]" %}
3963   interface(MEMORY_INTER) %{
3964     base(0x4);   // RSP
3965     index(0x4);  // No Index
3966     scale(0x0);  // No Scale
3967     disp($reg);  // Stack Offset
3968   %}
3969 %}
3970 operand stackSlotL(sRegL reg)
3971 %{
3972   constraint(ALLOC_IN_RC(stack_slots));
3973   // No match rule because this operand is only generated in matching
3974 
3975   format %{ "[$reg]" %}
3976   interface(MEMORY_INTER) %{
3977     base(0x4);   // RSP
3978     index(0x4);  // No Index
3979     scale(0x0);  // No Scale
3980     disp($reg);  // Stack Offset
3981   %}
3982 %}
3983 
3984 //----------Conditional Branch Operands----------------------------------------
3985 // Comparison Op  - This is the operation of the comparison, and is limited to
3986 //                  the following set of codes:
3987 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
3988 //
3989 // Other attributes of the comparison, such as unsignedness, are specified
3990 // by the comparison instruction that sets a condition code flags register.
3991 // That result is represented by a flags operand whose subtype is appropriate
3992 // to the unsignedness (etc.) of the comparison.
3993 //
3994 // Later, the instruction which matches both the Comparison Op (a Bool) and
3995 // the flags (produced by the Cmp) specifies the coding of the comparison op
3996 // by matching a specific subtype of Bool operand below, such as cmpOpU.
3997 
3998 // Comparision Code
3999 operand cmpOp()
4000 %{
4001   match(Bool);
4002 
4003   format %{ "" %}
4004   interface(COND_INTER) %{
4005     equal(0x4, "e");
4006     not_equal(0x5, "ne");
4007     less(0xC, "l");
4008     greater_equal(0xD, "ge");
4009     less_equal(0xE, "le");
4010     greater(0xF, "g");
4011     overflow(0x0, "o");
4012     no_overflow(0x1, "no");
4013   %}
4014 %}
4015 
4016 // Comparison Code, unsigned compare.  Used by FP also, with
4017 // C2 (unordered) turned into GT or LT already.  The other bits
4018 // C0 and C3 are turned into Carry & Zero flags.
4019 operand cmpOpU()
4020 %{
4021   match(Bool);
4022 
4023   format %{ "" %}
4024   interface(COND_INTER) %{
4025     equal(0x4, "e");
4026     not_equal(0x5, "ne");
4027     less(0x2, "b");
4028     greater_equal(0x3, "nb");
4029     less_equal(0x6, "be");
4030     greater(0x7, "nbe");
4031     overflow(0x0, "o");
4032     no_overflow(0x1, "no");
4033   %}
4034 %}
4035 
4036 
4037 // Floating comparisons that don't require any fixup for the unordered case
4038 operand cmpOpUCF() %{
4039   match(Bool);
4040   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4041             n->as_Bool()->_test._test == BoolTest::ge ||
4042             n->as_Bool()->_test._test == BoolTest::le ||
4043             n->as_Bool()->_test._test == BoolTest::gt);
4044   format %{ "" %}
4045   interface(COND_INTER) %{
4046     equal(0x4, "e");
4047     not_equal(0x5, "ne");
4048     less(0x2, "b");
4049     greater_equal(0x3, "nb");
4050     less_equal(0x6, "be");
4051     greater(0x7, "nbe");
4052     overflow(0x0, "o");
4053     no_overflow(0x1, "no");
4054   %}
4055 %}
4056 
4057 
4058 // Floating comparisons that can be fixed up with extra conditional jumps
4059 operand cmpOpUCF2() %{
4060   match(Bool);
4061   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4062             n->as_Bool()->_test._test == BoolTest::eq);
4063   format %{ "" %}
4064   interface(COND_INTER) %{
4065     equal(0x4, "e");
4066     not_equal(0x5, "ne");
4067     less(0x2, "b");
4068     greater_equal(0x3, "nb");
4069     less_equal(0x6, "be");
4070     greater(0x7, "nbe");
4071     overflow(0x0, "o");
4072     no_overflow(0x1, "no");
4073   %}
4074 %}
4075 
4076 
4077 //----------OPERAND CLASSES----------------------------------------------------
4078 // Operand Classes are groups of operands that are used as to simplify
4079 // instruction definitions by not requiring the AD writer to specify separate
4080 // instructions for every form of operand when the instruction accepts
4081 // multiple operand types with the same basic encoding and format.  The classic
4082 // case of this is memory operands.
4083 
4084 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4085                indIndexScale, indIndexScaleOffset, indPosIndexScaleOffset,
4086                indCompressedOopOffset,
4087                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4088                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4089                indIndexScaleOffsetNarrow, indPosIndexScaleOffsetNarrow);
4090 
4091 //----------PIPELINE-----------------------------------------------------------
4092 // Rules which define the behavior of the target architectures pipeline.
4093 pipeline %{
4094 
4095 //----------ATTRIBUTES---------------------------------------------------------
4096 attributes %{
4097   variable_size_instructions;        // Fixed size instructions
4098   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4099   instruction_unit_size = 1;         // An instruction is 1 bytes long
4100   instruction_fetch_unit_size = 16;  // The processor fetches one line
4101   instruction_fetch_units = 1;       // of 16 bytes
4102 
4103   // List of nop instructions
4104   nops( MachNop );
4105 %}
4106 
4107 //----------RESOURCES----------------------------------------------------------
4108 // Resources are the functional units available to the machine
4109 
4110 // Generic P2/P3 pipeline
4111 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4112 // 3 instructions decoded per cycle.
4113 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4114 // 3 ALU op, only ALU0 handles mul instructions.
4115 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4116            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4117            BR, FPU,
4118            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4119 
4120 //----------PIPELINE DESCRIPTION-----------------------------------------------
4121 // Pipeline Description specifies the stages in the machine's pipeline
4122 
4123 // Generic P2/P3 pipeline
4124 pipe_desc(S0, S1, S2, S3, S4, S5);
4125 
4126 //----------PIPELINE CLASSES---------------------------------------------------
4127 // Pipeline Classes describe the stages in which input and output are
4128 // referenced by the hardware pipeline.
4129 
4130 // Naming convention: ialu or fpu
4131 // Then: _reg
4132 // Then: _reg if there is a 2nd register
4133 // Then: _long if it's a pair of instructions implementing a long
4134 // Then: _fat if it requires the big decoder
4135 //   Or: _mem if it requires the big decoder and a memory unit.
4136 
4137 // Integer ALU reg operation
4138 pipe_class ialu_reg(rRegI dst)
4139 %{
4140     single_instruction;
4141     dst    : S4(write);
4142     dst    : S3(read);
4143     DECODE : S0;        // any decoder
4144     ALU    : S3;        // any alu
4145 %}
4146 
4147 // Long ALU reg operation
4148 pipe_class ialu_reg_long(rRegL dst)
4149 %{
4150     instruction_count(2);
4151     dst    : S4(write);
4152     dst    : S3(read);
4153     DECODE : S0(2);     // any 2 decoders
4154     ALU    : S3(2);     // both alus
4155 %}
4156 
4157 // Integer ALU reg operation using big decoder
4158 pipe_class ialu_reg_fat(rRegI dst)
4159 %{
4160     single_instruction;
4161     dst    : S4(write);
4162     dst    : S3(read);
4163     D0     : S0;        // big decoder only
4164     ALU    : S3;        // any alu
4165 %}
4166 
4167 // Long ALU reg operation using big decoder
4168 pipe_class ialu_reg_long_fat(rRegL dst)
4169 %{
4170     instruction_count(2);
4171     dst    : S4(write);
4172     dst    : S3(read);
4173     D0     : S0(2);     // big decoder only; twice
4174     ALU    : S3(2);     // any 2 alus
4175 %}
4176 
4177 // Integer ALU reg-reg operation
4178 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4179 %{
4180     single_instruction;
4181     dst    : S4(write);
4182     src    : S3(read);
4183     DECODE : S0;        // any decoder
4184     ALU    : S3;        // any alu
4185 %}
4186 
4187 // Long ALU reg-reg operation
4188 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4189 %{
4190     instruction_count(2);
4191     dst    : S4(write);
4192     src    : S3(read);
4193     DECODE : S0(2);     // any 2 decoders
4194     ALU    : S3(2);     // both alus
4195 %}
4196 
4197 // Integer ALU reg-reg operation
4198 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4199 %{
4200     single_instruction;
4201     dst    : S4(write);
4202     src    : S3(read);
4203     D0     : S0;        // big decoder only
4204     ALU    : S3;        // any alu
4205 %}
4206 
4207 // Long ALU reg-reg operation
4208 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4209 %{
4210     instruction_count(2);
4211     dst    : S4(write);
4212     src    : S3(read);
4213     D0     : S0(2);     // big decoder only; twice
4214     ALU    : S3(2);     // both alus
4215 %}
4216 
4217 // Integer ALU reg-mem operation
4218 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4219 %{
4220     single_instruction;
4221     dst    : S5(write);
4222     mem    : S3(read);
4223     D0     : S0;        // big decoder only
4224     ALU    : S4;        // any alu
4225     MEM    : S3;        // any mem
4226 %}
4227 
4228 // Integer mem operation (prefetch)
4229 pipe_class ialu_mem(memory mem)
4230 %{
4231     single_instruction;
4232     mem    : S3(read);
4233     D0     : S0;        // big decoder only
4234     MEM    : S3;        // any mem
4235 %}
4236 
4237 // Integer Store to Memory
4238 pipe_class ialu_mem_reg(memory mem, rRegI src)
4239 %{
4240     single_instruction;
4241     mem    : S3(read);
4242     src    : S5(read);
4243     D0     : S0;        // big decoder only
4244     ALU    : S4;        // any alu
4245     MEM    : S3;
4246 %}
4247 
4248 // // Long Store to Memory
4249 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4250 // %{
4251 //     instruction_count(2);
4252 //     mem    : S3(read);
4253 //     src    : S5(read);
4254 //     D0     : S0(2);          // big decoder only; twice
4255 //     ALU    : S4(2);     // any 2 alus
4256 //     MEM    : S3(2);  // Both mems
4257 // %}
4258 
4259 // Integer Store to Memory
4260 pipe_class ialu_mem_imm(memory mem)
4261 %{
4262     single_instruction;
4263     mem    : S3(read);
4264     D0     : S0;        // big decoder only
4265     ALU    : S4;        // any alu
4266     MEM    : S3;
4267 %}
4268 
4269 // Integer ALU0 reg-reg operation
4270 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4271 %{
4272     single_instruction;
4273     dst    : S4(write);
4274     src    : S3(read);
4275     D0     : S0;        // Big decoder only
4276     ALU0   : S3;        // only alu0
4277 %}
4278 
4279 // Integer ALU0 reg-mem operation
4280 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4281 %{
4282     single_instruction;
4283     dst    : S5(write);
4284     mem    : S3(read);
4285     D0     : S0;        // big decoder only
4286     ALU0   : S4;        // ALU0 only
4287     MEM    : S3;        // any mem
4288 %}
4289 
4290 // Integer ALU reg-reg operation
4291 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4292 %{
4293     single_instruction;
4294     cr     : S4(write);
4295     src1   : S3(read);
4296     src2   : S3(read);
4297     DECODE : S0;        // any decoder
4298     ALU    : S3;        // any alu
4299 %}
4300 
4301 // Integer ALU reg-imm operation
4302 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4303 %{
4304     single_instruction;
4305     cr     : S4(write);
4306     src1   : S3(read);
4307     DECODE : S0;        // any decoder
4308     ALU    : S3;        // any alu
4309 %}
4310 
4311 // Integer ALU reg-mem operation
4312 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4313 %{
4314     single_instruction;
4315     cr     : S4(write);
4316     src1   : S3(read);
4317     src2   : S3(read);
4318     D0     : S0;        // big decoder only
4319     ALU    : S4;        // any alu
4320     MEM    : S3;
4321 %}
4322 
4323 // Conditional move reg-reg
4324 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4325 %{
4326     instruction_count(4);
4327     y      : S4(read);
4328     q      : S3(read);
4329     p      : S3(read);
4330     DECODE : S0(4);     // any decoder
4331 %}
4332 
4333 // Conditional move reg-reg
4334 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4335 %{
4336     single_instruction;
4337     dst    : S4(write);
4338     src    : S3(read);
4339     cr     : S3(read);
4340     DECODE : S0;        // any decoder
4341 %}
4342 
4343 // Conditional move reg-mem
4344 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4345 %{
4346     single_instruction;
4347     dst    : S4(write);
4348     src    : S3(read);
4349     cr     : S3(read);
4350     DECODE : S0;        // any decoder
4351     MEM    : S3;
4352 %}
4353 
4354 // Conditional move reg-reg long
4355 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4356 %{
4357     single_instruction;
4358     dst    : S4(write);
4359     src    : S3(read);
4360     cr     : S3(read);
4361     DECODE : S0(2);     // any 2 decoders
4362 %}
4363 
4364 // XXX
4365 // // Conditional move double reg-reg
4366 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4367 // %{
4368 //     single_instruction;
4369 //     dst    : S4(write);
4370 //     src    : S3(read);
4371 //     cr     : S3(read);
4372 //     DECODE : S0;     // any decoder
4373 // %}
4374 
4375 // Float reg-reg operation
4376 pipe_class fpu_reg(regD dst)
4377 %{
4378     instruction_count(2);
4379     dst    : S3(read);
4380     DECODE : S0(2);     // any 2 decoders
4381     FPU    : S3;
4382 %}
4383 
4384 // Float reg-reg operation
4385 pipe_class fpu_reg_reg(regD dst, regD src)
4386 %{
4387     instruction_count(2);
4388     dst    : S4(write);
4389     src    : S3(read);
4390     DECODE : S0(2);     // any 2 decoders
4391     FPU    : S3;
4392 %}
4393 
4394 // Float reg-reg operation
4395 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4396 %{
4397     instruction_count(3);
4398     dst    : S4(write);
4399     src1   : S3(read);
4400     src2   : S3(read);
4401     DECODE : S0(3);     // any 3 decoders
4402     FPU    : S3(2);
4403 %}
4404 
4405 // Float reg-reg operation
4406 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4407 %{
4408     instruction_count(4);
4409     dst    : S4(write);
4410     src1   : S3(read);
4411     src2   : S3(read);
4412     src3   : S3(read);
4413     DECODE : S0(4);     // any 3 decoders
4414     FPU    : S3(2);
4415 %}
4416 
4417 // Float reg-reg operation
4418 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4419 %{
4420     instruction_count(4);
4421     dst    : S4(write);
4422     src1   : S3(read);
4423     src2   : S3(read);
4424     src3   : S3(read);
4425     DECODE : S1(3);     // any 3 decoders
4426     D0     : S0;        // Big decoder only
4427     FPU    : S3(2);
4428     MEM    : S3;
4429 %}
4430 
4431 // Float reg-mem operation
4432 pipe_class fpu_reg_mem(regD dst, memory mem)
4433 %{
4434     instruction_count(2);
4435     dst    : S5(write);
4436     mem    : S3(read);
4437     D0     : S0;        // big decoder only
4438     DECODE : S1;        // any decoder for FPU POP
4439     FPU    : S4;
4440     MEM    : S3;        // any mem
4441 %}
4442 
4443 // Float reg-mem operation
4444 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4445 %{
4446     instruction_count(3);
4447     dst    : S5(write);
4448     src1   : S3(read);
4449     mem    : S3(read);
4450     D0     : S0;        // big decoder only
4451     DECODE : S1(2);     // any decoder for FPU POP
4452     FPU    : S4;
4453     MEM    : S3;        // any mem
4454 %}
4455 
4456 // Float mem-reg operation
4457 pipe_class fpu_mem_reg(memory mem, regD src)
4458 %{
4459     instruction_count(2);
4460     src    : S5(read);
4461     mem    : S3(read);
4462     DECODE : S0;        // any decoder for FPU PUSH
4463     D0     : S1;        // big decoder only
4464     FPU    : S4;
4465     MEM    : S3;        // any mem
4466 %}
4467 
4468 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4469 %{
4470     instruction_count(3);
4471     src1   : S3(read);
4472     src2   : S3(read);
4473     mem    : S3(read);
4474     DECODE : S0(2);     // any decoder for FPU PUSH
4475     D0     : S1;        // big decoder only
4476     FPU    : S4;
4477     MEM    : S3;        // any mem
4478 %}
4479 
4480 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4481 %{
4482     instruction_count(3);
4483     src1   : S3(read);
4484     src2   : S3(read);
4485     mem    : S4(read);
4486     DECODE : S0;        // any decoder for FPU PUSH
4487     D0     : S0(2);     // big decoder only
4488     FPU    : S4;
4489     MEM    : S3(2);     // any mem
4490 %}
4491 
4492 pipe_class fpu_mem_mem(memory dst, memory src1)
4493 %{
4494     instruction_count(2);
4495     src1   : S3(read);
4496     dst    : S4(read);
4497     D0     : S0(2);     // big decoder only
4498     MEM    : S3(2);     // any mem
4499 %}
4500 
4501 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4502 %{
4503     instruction_count(3);
4504     src1   : S3(read);
4505     src2   : S3(read);
4506     dst    : S4(read);
4507     D0     : S0(3);     // big decoder only
4508     FPU    : S4;
4509     MEM    : S3(3);     // any mem
4510 %}
4511 
4512 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4513 %{
4514     instruction_count(3);
4515     src1   : S4(read);
4516     mem    : S4(read);
4517     DECODE : S0;        // any decoder for FPU PUSH
4518     D0     : S0(2);     // big decoder only
4519     FPU    : S4;
4520     MEM    : S3(2);     // any mem
4521 %}
4522 
4523 // Float load constant
4524 pipe_class fpu_reg_con(regD dst)
4525 %{
4526     instruction_count(2);
4527     dst    : S5(write);
4528     D0     : S0;        // big decoder only for the load
4529     DECODE : S1;        // any decoder for FPU POP
4530     FPU    : S4;
4531     MEM    : S3;        // any mem
4532 %}
4533 
4534 // Float load constant
4535 pipe_class fpu_reg_reg_con(regD dst, regD src)
4536 %{
4537     instruction_count(3);
4538     dst    : S5(write);
4539     src    : S3(read);
4540     D0     : S0;        // big decoder only for the load
4541     DECODE : S1(2);     // any decoder for FPU POP
4542     FPU    : S4;
4543     MEM    : S3;        // any mem
4544 %}
4545 
4546 // UnConditional branch
4547 pipe_class pipe_jmp(label labl)
4548 %{
4549     single_instruction;
4550     BR   : S3;
4551 %}
4552 
4553 // Conditional branch
4554 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4555 %{
4556     single_instruction;
4557     cr    : S1(read);
4558     BR    : S3;
4559 %}
4560 
4561 // Allocation idiom
4562 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4563 %{
4564     instruction_count(1); force_serialization;
4565     fixed_latency(6);
4566     heap_ptr : S3(read);
4567     DECODE   : S0(3);
4568     D0       : S2;
4569     MEM      : S3;
4570     ALU      : S3(2);
4571     dst      : S5(write);
4572     BR       : S5;
4573 %}
4574 
4575 // Generic big/slow expanded idiom
4576 pipe_class pipe_slow()
4577 %{
4578     instruction_count(10); multiple_bundles; force_serialization;
4579     fixed_latency(100);
4580     D0  : S0(2);
4581     MEM : S3(2);
4582 %}
4583 
4584 // The real do-nothing guy
4585 pipe_class empty()
4586 %{
4587     instruction_count(0);
4588 %}
4589 
4590 // Define the class for the Nop node
4591 define
4592 %{
4593    MachNop = empty;
4594 %}
4595 
4596 %}
4597 
4598 //----------INSTRUCTIONS-------------------------------------------------------
4599 //
4600 // match      -- States which machine-independent subtree may be replaced
4601 //               by this instruction.
4602 // ins_cost   -- The estimated cost of this instruction is used by instruction
4603 //               selection to identify a minimum cost tree of machine
4604 //               instructions that matches a tree of machine-independent
4605 //               instructions.
4606 // format     -- A string providing the disassembly for this instruction.
4607 //               The value of an instruction's operand may be inserted
4608 //               by referring to it with a '$' prefix.
4609 // opcode     -- Three instruction opcodes may be provided.  These are referred
4610 //               to within an encode class as $primary, $secondary, and $tertiary
4611 //               rrspectively.  The primary opcode is commonly used to
4612 //               indicate the type of machine instruction, while secondary
4613 //               and tertiary are often used for prefix options or addressing
4614 //               modes.
4615 // ins_encode -- A list of encode classes with parameters. The encode class
4616 //               name must have been defined in an 'enc_class' specification
4617 //               in the encode section of the architecture description.
4618 
4619 
4620 //----------Load/Store/Move Instructions---------------------------------------
4621 //----------Load Instructions--------------------------------------------------
4622 
4623 // Load Byte (8 bit signed)
4624 instruct loadB(rRegI dst, memory mem)
4625 %{
4626   match(Set dst (LoadB mem));
4627 
4628   ins_cost(125);
4629   format %{ "movsbl  $dst, $mem\t# byte" %}
4630 
4631   ins_encode %{
4632     __ movsbl($dst$$Register, $mem$$Address);
4633   %}
4634 
4635   ins_pipe(ialu_reg_mem);
4636 %}
4637 
4638 // Load Byte (8 bit signed) into Long Register
4639 instruct loadB2L(rRegL dst, memory mem)
4640 %{
4641   match(Set dst (ConvI2L (LoadB mem)));
4642 
4643   ins_cost(125);
4644   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4645 
4646   ins_encode %{
4647     __ movsbq($dst$$Register, $mem$$Address);
4648   %}
4649 
4650   ins_pipe(ialu_reg_mem);
4651 %}
4652 
4653 // Load Unsigned Byte (8 bit UNsigned)
4654 instruct loadUB(rRegI dst, memory mem)
4655 %{
4656   match(Set dst (LoadUB mem));
4657 
4658   ins_cost(125);
4659   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4660 
4661   ins_encode %{
4662     __ movzbl($dst$$Register, $mem$$Address);
4663   %}
4664 
4665   ins_pipe(ialu_reg_mem);
4666 %}
4667 
4668 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4669 instruct loadUB2L(rRegL dst, memory mem)
4670 %{
4671   match(Set dst (ConvI2L (LoadUB mem)));
4672 
4673   ins_cost(125);
4674   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4675 
4676   ins_encode %{
4677     __ movzbq($dst$$Register, $mem$$Address);
4678   %}
4679 
4680   ins_pipe(ialu_reg_mem);
4681 %}
4682 
4683 // Load Unsigned Byte (8 bit UNsigned) with a 8-bit mask into Long Register
4684 instruct loadUB2L_immI8(rRegL dst, memory mem, immI8 mask, rFlagsReg cr) %{
4685   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4686   effect(KILL cr);
4687 
4688   format %{ "movzbq  $dst, $mem\t# ubyte & 8-bit mask -> long\n\t"
4689             "andl    $dst, $mask" %}
4690   ins_encode %{
4691     Register Rdst = $dst$$Register;
4692     __ movzbq(Rdst, $mem$$Address);
4693     __ andl(Rdst, $mask$$constant);
4694   %}
4695   ins_pipe(ialu_reg_mem);
4696 %}
4697 
4698 // Load Short (16 bit signed)
4699 instruct loadS(rRegI dst, memory mem)
4700 %{
4701   match(Set dst (LoadS mem));
4702 
4703   ins_cost(125);
4704   format %{ "movswl $dst, $mem\t# short" %}
4705 
4706   ins_encode %{
4707     __ movswl($dst$$Register, $mem$$Address);
4708   %}
4709 
4710   ins_pipe(ialu_reg_mem);
4711 %}
4712 
4713 // Load Short (16 bit signed) to Byte (8 bit signed)
4714 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4715   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4716 
4717   ins_cost(125);
4718   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4719   ins_encode %{
4720     __ movsbl($dst$$Register, $mem$$Address);
4721   %}
4722   ins_pipe(ialu_reg_mem);
4723 %}
4724 
4725 // Load Short (16 bit signed) into Long Register
4726 instruct loadS2L(rRegL dst, memory mem)
4727 %{
4728   match(Set dst (ConvI2L (LoadS mem)));
4729 
4730   ins_cost(125);
4731   format %{ "movswq $dst, $mem\t# short -> long" %}
4732 
4733   ins_encode %{
4734     __ movswq($dst$$Register, $mem$$Address);
4735   %}
4736 
4737   ins_pipe(ialu_reg_mem);
4738 %}
4739 
4740 // Load Unsigned Short/Char (16 bit UNsigned)
4741 instruct loadUS(rRegI dst, memory mem)
4742 %{
4743   match(Set dst (LoadUS mem));
4744 
4745   ins_cost(125);
4746   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4747 
4748   ins_encode %{
4749     __ movzwl($dst$$Register, $mem$$Address);
4750   %}
4751 
4752   ins_pipe(ialu_reg_mem);
4753 %}
4754 
4755 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4756 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4757   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4758 
4759   ins_cost(125);
4760   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4761   ins_encode %{
4762     __ movsbl($dst$$Register, $mem$$Address);
4763   %}
4764   ins_pipe(ialu_reg_mem);
4765 %}
4766 
4767 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4768 instruct loadUS2L(rRegL dst, memory mem)
4769 %{
4770   match(Set dst (ConvI2L (LoadUS mem)));
4771 
4772   ins_cost(125);
4773   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4774 
4775   ins_encode %{
4776     __ movzwq($dst$$Register, $mem$$Address);
4777   %}
4778 
4779   ins_pipe(ialu_reg_mem);
4780 %}
4781 
4782 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4783 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4784   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4785 
4786   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4787   ins_encode %{
4788     __ movzbq($dst$$Register, $mem$$Address);
4789   %}
4790   ins_pipe(ialu_reg_mem);
4791 %}
4792 
4793 // Load Unsigned Short/Char (16 bit UNsigned) with mask into Long Register
4794 instruct loadUS2L_immI16(rRegL dst, memory mem, immI16 mask, rFlagsReg cr) %{
4795   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4796   effect(KILL cr);
4797 
4798   format %{ "movzwq  $dst, $mem\t# ushort/char & 16-bit mask -> long\n\t"
4799             "andl    $dst, $mask" %}
4800   ins_encode %{
4801     Register Rdst = $dst$$Register;
4802     __ movzwq(Rdst, $mem$$Address);
4803     __ andl(Rdst, $mask$$constant);
4804   %}
4805   ins_pipe(ialu_reg_mem);
4806 %}
4807 
4808 // Load Integer
4809 instruct loadI(rRegI dst, memory mem)
4810 %{
4811   match(Set dst (LoadI mem));
4812 
4813   ins_cost(125);
4814   format %{ "movl    $dst, $mem\t# int" %}
4815 
4816   ins_encode %{
4817     __ movl($dst$$Register, $mem$$Address);
4818   %}
4819 
4820   ins_pipe(ialu_reg_mem);
4821 %}
4822 
4823 // Load Integer (32 bit signed) to Byte (8 bit signed)
4824 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4825   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4826 
4827   ins_cost(125);
4828   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4829   ins_encode %{
4830     __ movsbl($dst$$Register, $mem$$Address);
4831   %}
4832   ins_pipe(ialu_reg_mem);
4833 %}
4834 
4835 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4836 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4837   match(Set dst (AndI (LoadI mem) mask));
4838 
4839   ins_cost(125);
4840   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4841   ins_encode %{
4842     __ movzbl($dst$$Register, $mem$$Address);
4843   %}
4844   ins_pipe(ialu_reg_mem);
4845 %}
4846 
4847 // Load Integer (32 bit signed) to Short (16 bit signed)
4848 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4849   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4850 
4851   ins_cost(125);
4852   format %{ "movswl  $dst, $mem\t# int -> short" %}
4853   ins_encode %{
4854     __ movswl($dst$$Register, $mem$$Address);
4855   %}
4856   ins_pipe(ialu_reg_mem);
4857 %}
4858 
4859 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4860 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4861   match(Set dst (AndI (LoadI mem) mask));
4862 
4863   ins_cost(125);
4864   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
4865   ins_encode %{
4866     __ movzwl($dst$$Register, $mem$$Address);
4867   %}
4868   ins_pipe(ialu_reg_mem);
4869 %}
4870 
4871 // Load Integer into Long Register
4872 instruct loadI2L(rRegL dst, memory mem)
4873 %{
4874   match(Set dst (ConvI2L (LoadI mem)));
4875 
4876   ins_cost(125);
4877   format %{ "movslq  $dst, $mem\t# int -> long" %}
4878 
4879   ins_encode %{
4880     __ movslq($dst$$Register, $mem$$Address);
4881   %}
4882 
4883   ins_pipe(ialu_reg_mem);
4884 %}
4885 
4886 // Load Integer with mask 0xFF into Long Register
4887 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4888   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4889 
4890   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
4891   ins_encode %{
4892     __ movzbq($dst$$Register, $mem$$Address);
4893   %}
4894   ins_pipe(ialu_reg_mem);
4895 %}
4896 
4897 // Load Integer with mask 0xFFFF into Long Register
4898 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4899   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4900 
4901   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
4902   ins_encode %{
4903     __ movzwq($dst$$Register, $mem$$Address);
4904   %}
4905   ins_pipe(ialu_reg_mem);
4906 %}
4907 
4908 // Load Integer with a 31-bit mask into Long Register
4909 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
4910   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4911   effect(KILL cr);
4912 
4913   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
4914             "andl    $dst, $mask" %}
4915   ins_encode %{
4916     Register Rdst = $dst$$Register;
4917     __ movl(Rdst, $mem$$Address);
4918     __ andl(Rdst, $mask$$constant);
4919   %}
4920   ins_pipe(ialu_reg_mem);
4921 %}
4922 
4923 // Load Unsigned Integer into Long Register
4924 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask) 
4925 %{
4926   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
4927 
4928   ins_cost(125);
4929   format %{ "movl    $dst, $mem\t# uint -> long" %}
4930 
4931   ins_encode %{
4932     __ movl($dst$$Register, $mem$$Address);
4933   %}
4934 
4935   ins_pipe(ialu_reg_mem);
4936 %}
4937 
4938 // Load Long
4939 instruct loadL(rRegL dst, memory mem)
4940 %{
4941   match(Set dst (LoadL mem));
4942 
4943   ins_cost(125);
4944   format %{ "movq    $dst, $mem\t# long" %}
4945 
4946   ins_encode %{
4947     __ movq($dst$$Register, $mem$$Address);
4948   %}
4949 
4950   ins_pipe(ialu_reg_mem); // XXX
4951 %}
4952 
4953 // Load Range
4954 instruct loadRange(rRegI dst, memory mem)
4955 %{
4956   match(Set dst (LoadRange mem));
4957 
4958   ins_cost(125); // XXX
4959   format %{ "movl    $dst, $mem\t# range" %}
4960   opcode(0x8B);
4961   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
4962   ins_pipe(ialu_reg_mem);
4963 %}
4964 
4965 // Load Pointer
4966 instruct loadP(rRegP dst, memory mem)
4967 %{
4968   match(Set dst (LoadP mem));
4969 
4970   ins_cost(125); // XXX
4971   format %{ "movq    $dst, $mem\t# ptr" %}
4972   opcode(0x8B);
4973   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4974   ins_pipe(ialu_reg_mem); // XXX
4975 %}
4976 
4977 // Load Compressed Pointer
4978 instruct loadN(rRegN dst, memory mem)
4979 %{
4980    match(Set dst (LoadN mem));
4981 
4982    ins_cost(125); // XXX
4983    format %{ "movl    $dst, $mem\t# compressed ptr" %}
4984    ins_encode %{
4985      __ movl($dst$$Register, $mem$$Address);
4986    %}
4987    ins_pipe(ialu_reg_mem); // XXX
4988 %}
4989 
4990 
4991 // Load Klass Pointer
4992 instruct loadKlass(rRegP dst, memory mem)
4993 %{
4994   match(Set dst (LoadKlass mem));
4995 
4996   ins_cost(125); // XXX
4997   format %{ "movq    $dst, $mem\t# class" %}
4998   opcode(0x8B);
4999   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5000   ins_pipe(ialu_reg_mem); // XXX
5001 %}
5002 
5003 // Load narrow Klass Pointer
5004 instruct loadNKlass(rRegN dst, memory mem)
5005 %{
5006   match(Set dst (LoadNKlass mem));
5007 
5008   ins_cost(125); // XXX
5009   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5010   ins_encode %{
5011     __ movl($dst$$Register, $mem$$Address);
5012   %}
5013   ins_pipe(ialu_reg_mem); // XXX
5014 %}
5015 
5016 // Load Float
5017 instruct loadF(regF dst, memory mem)
5018 %{
5019   match(Set dst (LoadF mem));
5020 
5021   ins_cost(145); // XXX
5022   format %{ "movss   $dst, $mem\t# float" %}
5023   ins_encode %{
5024     __ movflt($dst$$XMMRegister, $mem$$Address);
5025   %}
5026   ins_pipe(pipe_slow); // XXX
5027 %}
5028 
5029 // Load Double
5030 instruct loadD_partial(regD dst, memory mem)
5031 %{
5032   predicate(!UseXmmLoadAndClearUpper);
5033   match(Set dst (LoadD mem));
5034 
5035   ins_cost(145); // XXX
5036   format %{ "movlpd  $dst, $mem\t# double" %}
5037   ins_encode %{
5038     __ movdbl($dst$$XMMRegister, $mem$$Address);
5039   %}
5040   ins_pipe(pipe_slow); // XXX
5041 %}
5042 
5043 instruct loadD(regD dst, memory mem)
5044 %{
5045   predicate(UseXmmLoadAndClearUpper);
5046   match(Set dst (LoadD mem));
5047 
5048   ins_cost(145); // XXX
5049   format %{ "movsd   $dst, $mem\t# double" %}
5050   ins_encode %{
5051     __ movdbl($dst$$XMMRegister, $mem$$Address);
5052   %}
5053   ins_pipe(pipe_slow); // XXX
5054 %}
5055 
5056 // Load Effective Address
5057 instruct leaP8(rRegP dst, indOffset8 mem)
5058 %{
5059   match(Set dst mem);
5060 
5061   ins_cost(110); // XXX
5062   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5063   opcode(0x8D);
5064   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5065   ins_pipe(ialu_reg_reg_fat);
5066 %}
5067 
5068 instruct leaP32(rRegP dst, indOffset32 mem)
5069 %{
5070   match(Set dst mem);
5071 
5072   ins_cost(110);
5073   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5074   opcode(0x8D);
5075   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5076   ins_pipe(ialu_reg_reg_fat);
5077 %}
5078 
5079 // instruct leaPIdx(rRegP dst, indIndex mem)
5080 // %{
5081 //   match(Set dst mem);
5082 
5083 //   ins_cost(110);
5084 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5085 //   opcode(0x8D);
5086 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5087 //   ins_pipe(ialu_reg_reg_fat);
5088 // %}
5089 
5090 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5091 %{
5092   match(Set dst mem);
5093 
5094   ins_cost(110);
5095   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5096   opcode(0x8D);
5097   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5098   ins_pipe(ialu_reg_reg_fat);
5099 %}
5100 
5101 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5102 %{
5103   match(Set dst mem);
5104 
5105   ins_cost(110);
5106   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5107   opcode(0x8D);
5108   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5109   ins_pipe(ialu_reg_reg_fat);
5110 %}
5111 
5112 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5113 %{
5114   match(Set dst mem);
5115 
5116   ins_cost(110);
5117   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5118   opcode(0x8D);
5119   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5120   ins_pipe(ialu_reg_reg_fat);
5121 %}
5122 
5123 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5124 %{
5125   match(Set dst mem);
5126 
5127   ins_cost(110);
5128   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5129   opcode(0x8D);
5130   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5131   ins_pipe(ialu_reg_reg_fat);
5132 %}
5133 
5134 // Load Effective Address which uses Narrow (32-bits) oop
5135 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5136 %{
5137   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5138   match(Set dst mem);
5139 
5140   ins_cost(110);
5141   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5142   opcode(0x8D);
5143   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5144   ins_pipe(ialu_reg_reg_fat);
5145 %}
5146 
5147 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5148 %{
5149   predicate(Universe::narrow_oop_shift() == 0);
5150   match(Set dst mem);
5151 
5152   ins_cost(110); // XXX
5153   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5154   opcode(0x8D);
5155   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5156   ins_pipe(ialu_reg_reg_fat);
5157 %}
5158 
5159 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5160 %{
5161   predicate(Universe::narrow_oop_shift() == 0);
5162   match(Set dst mem);
5163 
5164   ins_cost(110);
5165   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5166   opcode(0x8D);
5167   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5168   ins_pipe(ialu_reg_reg_fat);
5169 %}
5170 
5171 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5172 %{
5173   predicate(Universe::narrow_oop_shift() == 0);
5174   match(Set dst mem);
5175 
5176   ins_cost(110);
5177   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5178   opcode(0x8D);
5179   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5180   ins_pipe(ialu_reg_reg_fat);
5181 %}
5182 
5183 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5184 %{
5185   predicate(Universe::narrow_oop_shift() == 0);
5186   match(Set dst mem);
5187 
5188   ins_cost(110);
5189   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5190   opcode(0x8D);
5191   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5192   ins_pipe(ialu_reg_reg_fat);
5193 %}
5194 
5195 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5196 %{
5197   predicate(Universe::narrow_oop_shift() == 0);
5198   match(Set dst mem);
5199 
5200   ins_cost(110);
5201   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5202   opcode(0x8D);
5203   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5204   ins_pipe(ialu_reg_reg_fat);
5205 %}
5206 
5207 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5208 %{
5209   predicate(Universe::narrow_oop_shift() == 0);
5210   match(Set dst mem);
5211 
5212   ins_cost(110);
5213   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5214   opcode(0x8D);
5215   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5216   ins_pipe(ialu_reg_reg_fat);
5217 %}
5218 
5219 instruct loadConI(rRegI dst, immI src)
5220 %{
5221   match(Set dst src);
5222 
5223   format %{ "movl    $dst, $src\t# int" %}
5224   ins_encode(load_immI(dst, src));
5225   ins_pipe(ialu_reg_fat); // XXX
5226 %}
5227 
5228 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5229 %{
5230   match(Set dst src);
5231   effect(KILL cr);
5232 
5233   ins_cost(50);
5234   format %{ "xorl    $dst, $dst\t# int" %}
5235   opcode(0x33); /* + rd */
5236   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5237   ins_pipe(ialu_reg);
5238 %}
5239 
5240 instruct loadConL(rRegL dst, immL src)
5241 %{
5242   match(Set dst src);
5243 
5244   ins_cost(150);
5245   format %{ "movq    $dst, $src\t# long" %}
5246   ins_encode(load_immL(dst, src));
5247   ins_pipe(ialu_reg);
5248 %}
5249 
5250 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5251 %{
5252   match(Set dst src);
5253   effect(KILL cr);
5254 
5255   ins_cost(50);
5256   format %{ "xorl    $dst, $dst\t# long" %}
5257   opcode(0x33); /* + rd */
5258   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5259   ins_pipe(ialu_reg); // XXX
5260 %}
5261 
5262 instruct loadConUL32(rRegL dst, immUL32 src)
5263 %{
5264   match(Set dst src);
5265 
5266   ins_cost(60);
5267   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5268   ins_encode(load_immUL32(dst, src));
5269   ins_pipe(ialu_reg);
5270 %}
5271 
5272 instruct loadConL32(rRegL dst, immL32 src)
5273 %{
5274   match(Set dst src);
5275 
5276   ins_cost(70);
5277   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5278   ins_encode(load_immL32(dst, src));
5279   ins_pipe(ialu_reg);
5280 %}
5281 
5282 instruct loadConP(rRegP dst, immP con) %{
5283   match(Set dst con);
5284 
5285   format %{ "movq    $dst, $con\t# ptr" %}
5286   ins_encode(load_immP(dst, con));
5287   ins_pipe(ialu_reg_fat); // XXX
5288 %}
5289 
5290 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5291 %{
5292   match(Set dst src);
5293   effect(KILL cr);
5294 
5295   ins_cost(50);
5296   format %{ "xorl    $dst, $dst\t# ptr" %}
5297   opcode(0x33); /* + rd */
5298   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5299   ins_pipe(ialu_reg);
5300 %}
5301 
5302 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5303 %{
5304   match(Set dst src);
5305   effect(KILL cr);
5306 
5307   ins_cost(60);
5308   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5309   ins_encode(load_immP31(dst, src));
5310   ins_pipe(ialu_reg);
5311 %}
5312 
5313 instruct loadConF(regF dst, immF con) %{
5314   match(Set dst con);
5315   ins_cost(125);
5316   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5317   ins_encode %{
5318     __ movflt($dst$$XMMRegister, $constantaddress($con));
5319   %}
5320   ins_pipe(pipe_slow);
5321 %}
5322 
5323 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5324   match(Set dst src);
5325   effect(KILL cr);
5326   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5327   ins_encode %{
5328     __ xorq($dst$$Register, $dst$$Register);
5329   %}
5330   ins_pipe(ialu_reg);
5331 %}
5332 
5333 instruct loadConN(rRegN dst, immN src) %{
5334   match(Set dst src);
5335 
5336   ins_cost(125);
5337   format %{ "movl    $dst, $src\t# compressed ptr" %}
5338   ins_encode %{
5339     address con = (address)$src$$constant;
5340     if (con == NULL) {
5341       ShouldNotReachHere();
5342     } else {
5343       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5344     }
5345   %}
5346   ins_pipe(ialu_reg_fat); // XXX
5347 %}
5348 
5349 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5350   match(Set dst src);
5351 
5352   ins_cost(125);
5353   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5354   ins_encode %{
5355     address con = (address)$src$$constant;
5356     if (con == NULL) {
5357       ShouldNotReachHere();
5358     } else {
5359       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5360     }
5361   %}
5362   ins_pipe(ialu_reg_fat); // XXX
5363 %}
5364 
5365 instruct loadConF0(regF dst, immF0 src)
5366 %{
5367   match(Set dst src);
5368   ins_cost(100);
5369 
5370   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5371   ins_encode %{
5372     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5373   %}
5374   ins_pipe(pipe_slow);
5375 %}
5376 
5377 // Use the same format since predicate() can not be used here.
5378 instruct loadConD(regD dst, immD con) %{
5379   match(Set dst con);
5380   ins_cost(125);
5381   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5382   ins_encode %{
5383     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5384   %}
5385   ins_pipe(pipe_slow);
5386 %}
5387 
5388 instruct loadConD0(regD dst, immD0 src)
5389 %{
5390   match(Set dst src);
5391   ins_cost(100);
5392 
5393   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5394   ins_encode %{
5395     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5396   %}
5397   ins_pipe(pipe_slow);
5398 %}
5399 
5400 instruct loadSSI(rRegI dst, stackSlotI src)
5401 %{
5402   match(Set dst src);
5403 
5404   ins_cost(125);
5405   format %{ "movl    $dst, $src\t# int stk" %}
5406   opcode(0x8B);
5407   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5408   ins_pipe(ialu_reg_mem);
5409 %}
5410 
5411 instruct loadSSL(rRegL dst, stackSlotL src)
5412 %{
5413   match(Set dst src);
5414 
5415   ins_cost(125);
5416   format %{ "movq    $dst, $src\t# long stk" %}
5417   opcode(0x8B);
5418   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5419   ins_pipe(ialu_reg_mem);
5420 %}
5421 
5422 instruct loadSSP(rRegP dst, stackSlotP src)
5423 %{
5424   match(Set dst src);
5425 
5426   ins_cost(125);
5427   format %{ "movq    $dst, $src\t# ptr stk" %}
5428   opcode(0x8B);
5429   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5430   ins_pipe(ialu_reg_mem);
5431 %}
5432 
5433 instruct loadSSF(regF dst, stackSlotF src)
5434 %{
5435   match(Set dst src);
5436 
5437   ins_cost(125);
5438   format %{ "movss   $dst, $src\t# float stk" %}
5439   ins_encode %{
5440     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5441   %}
5442   ins_pipe(pipe_slow); // XXX
5443 %}
5444 
5445 // Use the same format since predicate() can not be used here.
5446 instruct loadSSD(regD dst, stackSlotD src)
5447 %{
5448   match(Set dst src);
5449 
5450   ins_cost(125);
5451   format %{ "movsd   $dst, $src\t# double stk" %}
5452   ins_encode  %{
5453     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5454   %}
5455   ins_pipe(pipe_slow); // XXX
5456 %}
5457 
5458 // Prefetch instructions.
5459 // Must be safe to execute with invalid address (cannot fault).
5460 
5461 instruct prefetchr( memory mem ) %{
5462   predicate(ReadPrefetchInstr==3);
5463   match(PrefetchRead mem);
5464   ins_cost(125);
5465 
5466   format %{ "PREFETCHR $mem\t# Prefetch into level 1 cache" %}
5467   ins_encode %{
5468     __ prefetchr($mem$$Address);
5469   %}
5470   ins_pipe(ialu_mem);
5471 %}
5472 
5473 instruct prefetchrNTA( memory mem ) %{
5474   predicate(ReadPrefetchInstr==0);
5475   match(PrefetchRead mem);
5476   ins_cost(125);
5477 
5478   format %{ "PREFETCHNTA $mem\t# Prefetch into non-temporal cache for read" %}
5479   ins_encode %{
5480     __ prefetchnta($mem$$Address);
5481   %}
5482   ins_pipe(ialu_mem);
5483 %}
5484 
5485 instruct prefetchrT0( memory mem ) %{
5486   predicate(ReadPrefetchInstr==1);
5487   match(PrefetchRead mem);
5488   ins_cost(125);
5489 
5490   format %{ "PREFETCHT0 $mem\t# prefetch into L1 and L2 caches for read" %}
5491   ins_encode %{
5492     __ prefetcht0($mem$$Address);
5493   %}
5494   ins_pipe(ialu_mem);
5495 %}
5496 
5497 instruct prefetchrT2( memory mem ) %{
5498   predicate(ReadPrefetchInstr==2);
5499   match(PrefetchRead mem);
5500   ins_cost(125);
5501 
5502   format %{ "PREFETCHT2 $mem\t# prefetch into L2 caches for read" %}
5503   ins_encode %{
5504     __ prefetcht2($mem$$Address);
5505   %}
5506   ins_pipe(ialu_mem);
5507 %}
5508 
5509 instruct prefetchwNTA( memory mem ) %{
5510   match(PrefetchWrite mem);
5511   ins_cost(125);
5512 
5513   format %{ "PREFETCHNTA $mem\t# Prefetch to non-temporal cache for write" %}
5514   ins_encode %{
5515     __ prefetchnta($mem$$Address);
5516   %}
5517   ins_pipe(ialu_mem);
5518 %}
5519 
5520 // Prefetch instructions for allocation.
5521 
5522 instruct prefetchAlloc( memory mem ) %{
5523   predicate(AllocatePrefetchInstr==3);
5524   match(PrefetchAllocation mem);
5525   ins_cost(125);
5526 
5527   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5528   ins_encode %{
5529     __ prefetchw($mem$$Address);
5530   %}
5531   ins_pipe(ialu_mem);
5532 %}
5533 
5534 instruct prefetchAllocNTA( memory mem ) %{
5535   predicate(AllocatePrefetchInstr==0);
5536   match(PrefetchAllocation mem);
5537   ins_cost(125);
5538 
5539   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5540   ins_encode %{
5541     __ prefetchnta($mem$$Address);
5542   %}
5543   ins_pipe(ialu_mem);
5544 %}
5545 
5546 instruct prefetchAllocT0( memory mem ) %{
5547   predicate(AllocatePrefetchInstr==1);
5548   match(PrefetchAllocation mem);
5549   ins_cost(125);
5550 
5551   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5552   ins_encode %{
5553     __ prefetcht0($mem$$Address);
5554   %}
5555   ins_pipe(ialu_mem);
5556 %}
5557 
5558 instruct prefetchAllocT2( memory mem ) %{
5559   predicate(AllocatePrefetchInstr==2);
5560   match(PrefetchAllocation mem);
5561   ins_cost(125);
5562 
5563   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5564   ins_encode %{
5565     __ prefetcht2($mem$$Address);
5566   %}
5567   ins_pipe(ialu_mem);
5568 %}
5569 
5570 //----------Store Instructions-------------------------------------------------
5571 
5572 // Store Byte
5573 instruct storeB(memory mem, rRegI src)
5574 %{
5575   match(Set mem (StoreB mem src));
5576 
5577   ins_cost(125); // XXX
5578   format %{ "movb    $mem, $src\t# byte" %}
5579   opcode(0x88);
5580   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5581   ins_pipe(ialu_mem_reg);
5582 %}
5583 
5584 // Store Char/Short
5585 instruct storeC(memory mem, rRegI src)
5586 %{
5587   match(Set mem (StoreC mem src));
5588 
5589   ins_cost(125); // XXX
5590   format %{ "movw    $mem, $src\t# char/short" %}
5591   opcode(0x89);
5592   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5593   ins_pipe(ialu_mem_reg);
5594 %}
5595 
5596 // Store Integer
5597 instruct storeI(memory mem, rRegI src)
5598 %{
5599   match(Set mem (StoreI mem src));
5600 
5601   ins_cost(125); // XXX
5602   format %{ "movl    $mem, $src\t# int" %}
5603   opcode(0x89);
5604   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5605   ins_pipe(ialu_mem_reg);
5606 %}
5607 
5608 // Store Long
5609 instruct storeL(memory mem, rRegL src)
5610 %{
5611   match(Set mem (StoreL mem src));
5612 
5613   ins_cost(125); // XXX
5614   format %{ "movq    $mem, $src\t# long" %}
5615   opcode(0x89);
5616   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5617   ins_pipe(ialu_mem_reg); // XXX
5618 %}
5619 
5620 // Store Pointer
5621 instruct storeP(memory mem, any_RegP src)
5622 %{
5623   match(Set mem (StoreP mem src));
5624 
5625   ins_cost(125); // XXX
5626   format %{ "movq    $mem, $src\t# ptr" %}
5627   opcode(0x89);
5628   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5629   ins_pipe(ialu_mem_reg);
5630 %}
5631 
5632 instruct storeImmP0(memory mem, immP0 zero)
5633 %{
5634   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5635   match(Set mem (StoreP mem zero));
5636 
5637   ins_cost(125); // XXX
5638   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5639   ins_encode %{
5640     __ movq($mem$$Address, r12);
5641   %}
5642   ins_pipe(ialu_mem_reg);
5643 %}
5644 
5645 // Store NULL Pointer, mark word, or other simple pointer constant.
5646 instruct storeImmP(memory mem, immP31 src)
5647 %{
5648   match(Set mem (StoreP mem src));
5649 
5650   ins_cost(150); // XXX
5651   format %{ "movq    $mem, $src\t# ptr" %}
5652   opcode(0xC7); /* C7 /0 */
5653   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5654   ins_pipe(ialu_mem_imm);
5655 %}
5656 
5657 // Store Compressed Pointer
5658 instruct storeN(memory mem, rRegN src)
5659 %{
5660   match(Set mem (StoreN mem src));
5661 
5662   ins_cost(125); // XXX
5663   format %{ "movl    $mem, $src\t# compressed ptr" %}
5664   ins_encode %{
5665     __ movl($mem$$Address, $src$$Register);
5666   %}
5667   ins_pipe(ialu_mem_reg);
5668 %}
5669 
5670 instruct storeNKlass(memory mem, rRegN src)
5671 %{
5672   match(Set mem (StoreNKlass mem src));
5673 
5674   ins_cost(125); // XXX
5675   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5676   ins_encode %{
5677     __ movl($mem$$Address, $src$$Register);
5678   %}
5679   ins_pipe(ialu_mem_reg);
5680 %}
5681 
5682 instruct storeImmN0(memory mem, immN0 zero)
5683 %{
5684   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5685   match(Set mem (StoreN mem zero));
5686 
5687   ins_cost(125); // XXX
5688   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5689   ins_encode %{
5690     __ movl($mem$$Address, r12);
5691   %}
5692   ins_pipe(ialu_mem_reg);
5693 %}
5694 
5695 instruct storeImmN(memory mem, immN src)
5696 %{
5697   match(Set mem (StoreN mem src));
5698 
5699   ins_cost(150); // XXX
5700   format %{ "movl    $mem, $src\t# compressed ptr" %}
5701   ins_encode %{
5702     address con = (address)$src$$constant;
5703     if (con == NULL) {
5704       __ movl($mem$$Address, (int32_t)0);
5705     } else {
5706       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5707     }
5708   %}
5709   ins_pipe(ialu_mem_imm);
5710 %}
5711 
5712 instruct storeImmNKlass(memory mem, immNKlass src)
5713 %{
5714   match(Set mem (StoreNKlass mem src));
5715 
5716   ins_cost(150); // XXX
5717   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5718   ins_encode %{
5719     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5720   %}
5721   ins_pipe(ialu_mem_imm);
5722 %}
5723 
5724 // Store Integer Immediate
5725 instruct storeImmI0(memory mem, immI0 zero)
5726 %{
5727   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5728   match(Set mem (StoreI mem zero));
5729 
5730   ins_cost(125); // XXX
5731   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5732   ins_encode %{
5733     __ movl($mem$$Address, r12);
5734   %}
5735   ins_pipe(ialu_mem_reg);
5736 %}
5737 
5738 instruct storeImmI(memory mem, immI src)
5739 %{
5740   match(Set mem (StoreI mem src));
5741 
5742   ins_cost(150);
5743   format %{ "movl    $mem, $src\t# int" %}
5744   opcode(0xC7); /* C7 /0 */
5745   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5746   ins_pipe(ialu_mem_imm);
5747 %}
5748 
5749 // Store Long Immediate
5750 instruct storeImmL0(memory mem, immL0 zero)
5751 %{
5752   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5753   match(Set mem (StoreL mem zero));
5754 
5755   ins_cost(125); // XXX
5756   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5757   ins_encode %{
5758     __ movq($mem$$Address, r12);
5759   %}
5760   ins_pipe(ialu_mem_reg);
5761 %}
5762 
5763 instruct storeImmL(memory mem, immL32 src)
5764 %{
5765   match(Set mem (StoreL mem src));
5766 
5767   ins_cost(150);
5768   format %{ "movq    $mem, $src\t# long" %}
5769   opcode(0xC7); /* C7 /0 */
5770   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5771   ins_pipe(ialu_mem_imm);
5772 %}
5773 
5774 // Store Short/Char Immediate
5775 instruct storeImmC0(memory mem, immI0 zero)
5776 %{
5777   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5778   match(Set mem (StoreC mem zero));
5779 
5780   ins_cost(125); // XXX
5781   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5782   ins_encode %{
5783     __ movw($mem$$Address, r12);
5784   %}
5785   ins_pipe(ialu_mem_reg);
5786 %}
5787 
5788 instruct storeImmI16(memory mem, immI16 src)
5789 %{
5790   predicate(UseStoreImmI16);
5791   match(Set mem (StoreC mem src));
5792 
5793   ins_cost(150);
5794   format %{ "movw    $mem, $src\t# short/char" %}
5795   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5796   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5797   ins_pipe(ialu_mem_imm);
5798 %}
5799 
5800 // Store Byte Immediate
5801 instruct storeImmB0(memory mem, immI0 zero)
5802 %{
5803   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5804   match(Set mem (StoreB mem zero));
5805 
5806   ins_cost(125); // XXX
5807   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5808   ins_encode %{
5809     __ movb($mem$$Address, r12);
5810   %}
5811   ins_pipe(ialu_mem_reg);
5812 %}
5813 
5814 instruct storeImmB(memory mem, immI8 src)
5815 %{
5816   match(Set mem (StoreB mem src));
5817 
5818   ins_cost(150); // XXX
5819   format %{ "movb    $mem, $src\t# byte" %}
5820   opcode(0xC6); /* C6 /0 */
5821   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5822   ins_pipe(ialu_mem_imm);
5823 %}
5824 
5825 // Store CMS card-mark Immediate
5826 instruct storeImmCM0_reg(memory mem, immI0 zero)
5827 %{
5828   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5829   match(Set mem (StoreCM mem zero));
5830 
5831   ins_cost(125); // XXX
5832   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5833   ins_encode %{
5834     __ movb($mem$$Address, r12);
5835   %}
5836   ins_pipe(ialu_mem_reg);
5837 %}
5838 
5839 instruct storeImmCM0(memory mem, immI0 src)
5840 %{
5841   match(Set mem (StoreCM mem src));
5842 
5843   ins_cost(150); // XXX
5844   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5845   opcode(0xC6); /* C6 /0 */
5846   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5847   ins_pipe(ialu_mem_imm);
5848 %}
5849 
5850 // Store Float
5851 instruct storeF(memory mem, regF src)
5852 %{
5853   match(Set mem (StoreF mem src));
5854 
5855   ins_cost(95); // XXX
5856   format %{ "movss   $mem, $src\t# float" %}
5857   ins_encode %{
5858     __ movflt($mem$$Address, $src$$XMMRegister);
5859   %}
5860   ins_pipe(pipe_slow); // XXX
5861 %}
5862 
5863 // Store immediate Float value (it is faster than store from XMM register)
5864 instruct storeF0(memory mem, immF0 zero)
5865 %{
5866   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5867   match(Set mem (StoreF mem zero));
5868 
5869   ins_cost(25); // XXX
5870   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5871   ins_encode %{
5872     __ movl($mem$$Address, r12);
5873   %}
5874   ins_pipe(ialu_mem_reg);
5875 %}
5876 
5877 instruct storeF_imm(memory mem, immF src)
5878 %{
5879   match(Set mem (StoreF mem src));
5880 
5881   ins_cost(50);
5882   format %{ "movl    $mem, $src\t# float" %}
5883   opcode(0xC7); /* C7 /0 */
5884   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5885   ins_pipe(ialu_mem_imm);
5886 %}
5887 
5888 // Store Double
5889 instruct storeD(memory mem, regD src)
5890 %{
5891   match(Set mem (StoreD mem src));
5892 
5893   ins_cost(95); // XXX
5894   format %{ "movsd   $mem, $src\t# double" %}
5895   ins_encode %{
5896     __ movdbl($mem$$Address, $src$$XMMRegister);
5897   %}
5898   ins_pipe(pipe_slow); // XXX
5899 %}
5900 
5901 // Store immediate double 0.0 (it is faster than store from XMM register)
5902 instruct storeD0_imm(memory mem, immD0 src)
5903 %{
5904   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
5905   match(Set mem (StoreD mem src));
5906 
5907   ins_cost(50);
5908   format %{ "movq    $mem, $src\t# double 0." %}
5909   opcode(0xC7); /* C7 /0 */
5910   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5911   ins_pipe(ialu_mem_imm);
5912 %}
5913 
5914 instruct storeD0(memory mem, immD0 zero)
5915 %{
5916   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5917   match(Set mem (StoreD mem zero));
5918 
5919   ins_cost(25); // XXX
5920   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
5921   ins_encode %{
5922     __ movq($mem$$Address, r12);
5923   %}
5924   ins_pipe(ialu_mem_reg);
5925 %}
5926 
5927 instruct storeSSI(stackSlotI dst, rRegI src)
5928 %{
5929   match(Set dst src);
5930 
5931   ins_cost(100);
5932   format %{ "movl    $dst, $src\t# int stk" %}
5933   opcode(0x89);
5934   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
5935   ins_pipe( ialu_mem_reg );
5936 %}
5937 
5938 instruct storeSSL(stackSlotL dst, rRegL src)
5939 %{
5940   match(Set dst src);
5941 
5942   ins_cost(100);
5943   format %{ "movq    $dst, $src\t# long stk" %}
5944   opcode(0x89);
5945   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
5946   ins_pipe(ialu_mem_reg);
5947 %}
5948 
5949 instruct storeSSP(stackSlotP dst, rRegP src)
5950 %{
5951   match(Set dst src);
5952 
5953   ins_cost(100);
5954   format %{ "movq    $dst, $src\t# ptr stk" %}
5955   opcode(0x89);
5956   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
5957   ins_pipe(ialu_mem_reg);
5958 %}
5959 
5960 instruct storeSSF(stackSlotF dst, regF src)
5961 %{
5962   match(Set dst src);
5963 
5964   ins_cost(95); // XXX
5965   format %{ "movss   $dst, $src\t# float stk" %}
5966   ins_encode %{
5967     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
5968   %}
5969   ins_pipe(pipe_slow); // XXX
5970 %}
5971 
5972 instruct storeSSD(stackSlotD dst, regD src)
5973 %{
5974   match(Set dst src);
5975 
5976   ins_cost(95); // XXX
5977   format %{ "movsd   $dst, $src\t# double stk" %}
5978   ins_encode %{
5979     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
5980   %}
5981   ins_pipe(pipe_slow); // XXX
5982 %}
5983 
5984 //----------BSWAP Instructions-------------------------------------------------
5985 instruct bytes_reverse_int(rRegI dst) %{
5986   match(Set dst (ReverseBytesI dst));
5987 
5988   format %{ "bswapl  $dst" %}
5989   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
5990   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
5991   ins_pipe( ialu_reg );
5992 %}
5993 
5994 instruct bytes_reverse_long(rRegL dst) %{
5995   match(Set dst (ReverseBytesL dst));
5996 
5997   format %{ "bswapq  $dst" %}
5998   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
5999   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6000   ins_pipe( ialu_reg);
6001 %}
6002 
6003 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6004   match(Set dst (ReverseBytesUS dst));
6005   effect(KILL cr);
6006 
6007   format %{ "bswapl  $dst\n\t"
6008             "shrl    $dst,16\n\t" %}
6009   ins_encode %{
6010     __ bswapl($dst$$Register);
6011     __ shrl($dst$$Register, 16);
6012   %}
6013   ins_pipe( ialu_reg );
6014 %}
6015 
6016 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6017   match(Set dst (ReverseBytesS dst));
6018   effect(KILL cr);
6019 
6020   format %{ "bswapl  $dst\n\t"
6021             "sar     $dst,16\n\t" %}
6022   ins_encode %{
6023     __ bswapl($dst$$Register);
6024     __ sarl($dst$$Register, 16);
6025   %}
6026   ins_pipe( ialu_reg );
6027 %}
6028 
6029 //---------- Zeros Count Instructions ------------------------------------------
6030 
6031 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6032   predicate(UseCountLeadingZerosInstruction);
6033   match(Set dst (CountLeadingZerosI src));
6034   effect(KILL cr);
6035 
6036   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6037   ins_encode %{
6038     __ lzcntl($dst$$Register, $src$$Register);
6039   %}
6040   ins_pipe(ialu_reg);
6041 %}
6042 
6043 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6044   predicate(!UseCountLeadingZerosInstruction);
6045   match(Set dst (CountLeadingZerosI src));
6046   effect(KILL cr);
6047 
6048   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6049             "jnz     skip\n\t"
6050             "movl    $dst, -1\n"
6051       "skip:\n\t"
6052             "negl    $dst\n\t"
6053             "addl    $dst, 31" %}
6054   ins_encode %{
6055     Register Rdst = $dst$$Register;
6056     Register Rsrc = $src$$Register;
6057     Label skip;
6058     __ bsrl(Rdst, Rsrc);
6059     __ jccb(Assembler::notZero, skip);
6060     __ movl(Rdst, -1);
6061     __ bind(skip);
6062     __ negl(Rdst);
6063     __ addl(Rdst, BitsPerInt - 1);
6064   %}
6065   ins_pipe(ialu_reg);
6066 %}
6067 
6068 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6069   predicate(UseCountLeadingZerosInstruction);
6070   match(Set dst (CountLeadingZerosL src));
6071   effect(KILL cr);
6072 
6073   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6074   ins_encode %{
6075     __ lzcntq($dst$$Register, $src$$Register);
6076   %}
6077   ins_pipe(ialu_reg);
6078 %}
6079 
6080 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6081   predicate(!UseCountLeadingZerosInstruction);
6082   match(Set dst (CountLeadingZerosL src));
6083   effect(KILL cr);
6084 
6085   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6086             "jnz     skip\n\t"
6087             "movl    $dst, -1\n"
6088       "skip:\n\t"
6089             "negl    $dst\n\t"
6090             "addl    $dst, 63" %}
6091   ins_encode %{
6092     Register Rdst = $dst$$Register;
6093     Register Rsrc = $src$$Register;
6094     Label skip;
6095     __ bsrq(Rdst, Rsrc);
6096     __ jccb(Assembler::notZero, skip);
6097     __ movl(Rdst, -1);
6098     __ bind(skip);
6099     __ negl(Rdst);
6100     __ addl(Rdst, BitsPerLong - 1);
6101   %}
6102   ins_pipe(ialu_reg);
6103 %}
6104 
6105 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6106   predicate(UseCountTrailingZerosInstruction);
6107   match(Set dst (CountTrailingZerosI src));
6108   effect(KILL cr);
6109 
6110   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6111   ins_encode %{
6112     __ tzcntl($dst$$Register, $src$$Register);
6113   %}
6114   ins_pipe(ialu_reg);
6115 %}
6116 
6117 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6118   predicate(!UseCountTrailingZerosInstruction);
6119   match(Set dst (CountTrailingZerosI src));
6120   effect(KILL cr);
6121 
6122   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6123             "jnz     done\n\t"
6124             "movl    $dst, 32\n"
6125       "done:" %}
6126   ins_encode %{
6127     Register Rdst = $dst$$Register;
6128     Label done;
6129     __ bsfl(Rdst, $src$$Register);
6130     __ jccb(Assembler::notZero, done);
6131     __ movl(Rdst, BitsPerInt);
6132     __ bind(done);
6133   %}
6134   ins_pipe(ialu_reg);
6135 %}
6136 
6137 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6138   predicate(UseCountTrailingZerosInstruction);
6139   match(Set dst (CountTrailingZerosL src));
6140   effect(KILL cr);
6141 
6142   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6143   ins_encode %{
6144     __ tzcntq($dst$$Register, $src$$Register);
6145   %}
6146   ins_pipe(ialu_reg);
6147 %}
6148 
6149 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6150   predicate(!UseCountTrailingZerosInstruction);
6151   match(Set dst (CountTrailingZerosL src));
6152   effect(KILL cr);
6153 
6154   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6155             "jnz     done\n\t"
6156             "movl    $dst, 64\n"
6157       "done:" %}
6158   ins_encode %{
6159     Register Rdst = $dst$$Register;
6160     Label done;
6161     __ bsfq(Rdst, $src$$Register);
6162     __ jccb(Assembler::notZero, done);
6163     __ movl(Rdst, BitsPerLong);
6164     __ bind(done);
6165   %}
6166   ins_pipe(ialu_reg);
6167 %}
6168 
6169 
6170 //---------- Population Count Instructions -------------------------------------
6171 
6172 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6173   predicate(UsePopCountInstruction);
6174   match(Set dst (PopCountI src));
6175   effect(KILL cr);
6176 
6177   format %{ "popcnt  $dst, $src" %}
6178   ins_encode %{
6179     __ popcntl($dst$$Register, $src$$Register);
6180   %}
6181   ins_pipe(ialu_reg);
6182 %}
6183 
6184 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6185   predicate(UsePopCountInstruction);
6186   match(Set dst (PopCountI (LoadI mem)));
6187   effect(KILL cr);
6188 
6189   format %{ "popcnt  $dst, $mem" %}
6190   ins_encode %{
6191     __ popcntl($dst$$Register, $mem$$Address);
6192   %}
6193   ins_pipe(ialu_reg);
6194 %}
6195 
6196 // Note: Long.bitCount(long) returns an int.
6197 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6198   predicate(UsePopCountInstruction);
6199   match(Set dst (PopCountL src));
6200   effect(KILL cr);
6201 
6202   format %{ "popcnt  $dst, $src" %}
6203   ins_encode %{
6204     __ popcntq($dst$$Register, $src$$Register);
6205   %}
6206   ins_pipe(ialu_reg);
6207 %}
6208 
6209 // Note: Long.bitCount(long) returns an int.
6210 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6211   predicate(UsePopCountInstruction);
6212   match(Set dst (PopCountL (LoadL mem)));
6213   effect(KILL cr);
6214 
6215   format %{ "popcnt  $dst, $mem" %}
6216   ins_encode %{
6217     __ popcntq($dst$$Register, $mem$$Address);
6218   %}
6219   ins_pipe(ialu_reg);
6220 %}
6221 
6222 
6223 //----------MemBar Instructions-----------------------------------------------
6224 // Memory barrier flavors
6225 
6226 instruct membar_acquire()
6227 %{
6228   match(MemBarAcquire);
6229   match(LoadFence);
6230   ins_cost(0);
6231 
6232   size(0);
6233   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6234   ins_encode();
6235   ins_pipe(empty);
6236 %}
6237 
6238 instruct membar_acquire_lock()
6239 %{
6240   match(MemBarAcquireLock);
6241   ins_cost(0);
6242 
6243   size(0);
6244   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6245   ins_encode();
6246   ins_pipe(empty);
6247 %}
6248 
6249 instruct membar_release()
6250 %{
6251   match(MemBarRelease);
6252   match(StoreFence);
6253   ins_cost(0);
6254 
6255   size(0);
6256   format %{ "MEMBAR-release ! (empty encoding)" %}
6257   ins_encode();
6258   ins_pipe(empty);
6259 %}
6260 
6261 instruct membar_release_lock()
6262 %{
6263   match(MemBarReleaseLock);
6264   ins_cost(0);
6265 
6266   size(0);
6267   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6268   ins_encode();
6269   ins_pipe(empty);
6270 %}
6271 
6272 instruct membar_volatile(rFlagsReg cr) %{
6273   match(MemBarVolatile);
6274   effect(KILL cr);
6275   ins_cost(400);
6276 
6277   format %{
6278     $$template
6279     if (os::is_MP()) {
6280       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6281     } else {
6282       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6283     }
6284   %}
6285   ins_encode %{
6286     __ membar(Assembler::StoreLoad);
6287   %}
6288   ins_pipe(pipe_slow);
6289 %}
6290 
6291 instruct unnecessary_membar_volatile()
6292 %{
6293   match(MemBarVolatile);
6294   predicate(Matcher::post_store_load_barrier(n));
6295   ins_cost(0);
6296 
6297   size(0);
6298   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6299   ins_encode();
6300   ins_pipe(empty);
6301 %}
6302 
6303 instruct membar_storestore() %{
6304   match(MemBarStoreStore);
6305   ins_cost(0);
6306 
6307   size(0);
6308   format %{ "MEMBAR-storestore (empty encoding)" %}
6309   ins_encode( );
6310   ins_pipe(empty);
6311 %}
6312 
6313 //----------Move Instructions--------------------------------------------------
6314 
6315 instruct castX2P(rRegP dst, rRegL src)
6316 %{
6317   match(Set dst (CastX2P src));
6318 
6319   format %{ "movq    $dst, $src\t# long->ptr" %}
6320   ins_encode %{
6321     if ($dst$$reg != $src$$reg) {
6322       __ movptr($dst$$Register, $src$$Register);
6323     }
6324   %}
6325   ins_pipe(ialu_reg_reg); // XXX
6326 %}
6327 
6328 instruct castP2X(rRegL dst, rRegP src)
6329 %{
6330   match(Set dst (CastP2X src));
6331 
6332   format %{ "movq    $dst, $src\t# ptr -> long" %}
6333   ins_encode %{
6334     if ($dst$$reg != $src$$reg) {
6335       __ movptr($dst$$Register, $src$$Register);
6336     }
6337   %}
6338   ins_pipe(ialu_reg_reg); // XXX
6339 %}
6340 
6341 // Convert oop into int for vectors alignment masking
6342 instruct convP2I(rRegI dst, rRegP src)
6343 %{
6344   match(Set dst (ConvL2I (CastP2X src)));
6345 
6346   format %{ "movl    $dst, $src\t# ptr -> int" %}
6347   ins_encode %{
6348     __ movl($dst$$Register, $src$$Register);
6349   %}
6350   ins_pipe(ialu_reg_reg); // XXX
6351 %}
6352 
6353 // Convert compressed oop into int for vectors alignment masking
6354 // in case of 32bit oops (heap < 4Gb).
6355 instruct convN2I(rRegI dst, rRegN src)
6356 %{
6357   predicate(Universe::narrow_oop_shift() == 0);
6358   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6359 
6360   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6361   ins_encode %{
6362     __ movl($dst$$Register, $src$$Register);
6363   %}
6364   ins_pipe(ialu_reg_reg); // XXX
6365 %}
6366 
6367 // Convert oop pointer into compressed form
6368 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6369   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6370   match(Set dst (EncodeP src));
6371   effect(KILL cr);
6372   format %{ "encode_heap_oop $dst,$src" %}
6373   ins_encode %{
6374     Register s = $src$$Register;
6375     Register d = $dst$$Register;
6376     if (s != d) {
6377       __ movq(d, s);
6378     }
6379     __ encode_heap_oop(d);
6380   %}
6381   ins_pipe(ialu_reg_long);
6382 %}
6383 
6384 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6385   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6386   match(Set dst (EncodeP src));
6387   effect(KILL cr);
6388   format %{ "encode_heap_oop_not_null $dst,$src" %}
6389   ins_encode %{
6390     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6391   %}
6392   ins_pipe(ialu_reg_long);
6393 %}
6394 
6395 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6396   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6397             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6398   match(Set dst (DecodeN src));
6399   effect(KILL cr);
6400   format %{ "decode_heap_oop $dst,$src" %}
6401   ins_encode %{
6402     Register s = $src$$Register;
6403     Register d = $dst$$Register;
6404     if (s != d) {
6405       __ movq(d, s);
6406     }
6407     __ decode_heap_oop(d);
6408   %}
6409   ins_pipe(ialu_reg_long);
6410 %}
6411 
6412 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6413   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6414             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6415   match(Set dst (DecodeN src));
6416   effect(KILL cr);
6417   format %{ "decode_heap_oop_not_null $dst,$src" %}
6418   ins_encode %{
6419     Register s = $src$$Register;
6420     Register d = $dst$$Register;
6421     if (s != d) {
6422       __ decode_heap_oop_not_null(d, s);
6423     } else {
6424       __ decode_heap_oop_not_null(d);
6425     }
6426   %}
6427   ins_pipe(ialu_reg_long);
6428 %}
6429 
6430 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6431   match(Set dst (EncodePKlass src));
6432   effect(KILL cr);
6433   format %{ "encode_klass_not_null $dst,$src" %}
6434   ins_encode %{
6435     __ encode_klass_not_null($dst$$Register, $src$$Register);
6436   %}
6437   ins_pipe(ialu_reg_long);
6438 %}
6439 
6440 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6441   match(Set dst (DecodeNKlass src));
6442   effect(KILL cr);
6443   format %{ "decode_klass_not_null $dst,$src" %}
6444   ins_encode %{
6445     Register s = $src$$Register;
6446     Register d = $dst$$Register;
6447     if (s != d) {
6448       __ decode_klass_not_null(d, s);
6449     } else {
6450       __ decode_klass_not_null(d);
6451     }
6452   %}
6453   ins_pipe(ialu_reg_long);
6454 %}
6455 
6456 
6457 //----------Conditional Move---------------------------------------------------
6458 // Jump
6459 // dummy instruction for generating temp registers
6460 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6461   match(Jump (LShiftL switch_val shift));
6462   ins_cost(350);
6463   predicate(false);
6464   effect(TEMP dest);
6465 
6466   format %{ "leaq    $dest, [$constantaddress]\n\t"
6467             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6468   ins_encode %{
6469     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6470     // to do that and the compiler is using that register as one it can allocate.
6471     // So we build it all by hand.
6472     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6473     // ArrayAddress dispatch(table, index);
6474     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6475     __ lea($dest$$Register, $constantaddress);
6476     __ jmp(dispatch);
6477   %}
6478   ins_pipe(pipe_jmp);
6479 %}
6480 
6481 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6482   match(Jump (AddL (LShiftL switch_val shift) offset));
6483   ins_cost(350);
6484   effect(TEMP dest);
6485 
6486   format %{ "leaq    $dest, [$constantaddress]\n\t"
6487             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6488   ins_encode %{
6489     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6490     // to do that and the compiler is using that register as one it can allocate.
6491     // So we build it all by hand.
6492     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6493     // ArrayAddress dispatch(table, index);
6494     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6495     __ lea($dest$$Register, $constantaddress);
6496     __ jmp(dispatch);
6497   %}
6498   ins_pipe(pipe_jmp);
6499 %}
6500 
6501 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6502   match(Jump switch_val);
6503   ins_cost(350);
6504   effect(TEMP dest);
6505 
6506   format %{ "leaq    $dest, [$constantaddress]\n\t"
6507             "jmp     [$dest + $switch_val]\n\t" %}
6508   ins_encode %{
6509     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6510     // to do that and the compiler is using that register as one it can allocate.
6511     // So we build it all by hand.
6512     // Address index(noreg, switch_reg, Address::times_1);
6513     // ArrayAddress dispatch(table, index);
6514     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6515     __ lea($dest$$Register, $constantaddress);
6516     __ jmp(dispatch);
6517   %}
6518   ins_pipe(pipe_jmp);
6519 %}
6520 
6521 // Conditional move
6522 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6523 %{
6524   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6525 
6526   ins_cost(200); // XXX
6527   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6528   opcode(0x0F, 0x40);
6529   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6530   ins_pipe(pipe_cmov_reg);
6531 %}
6532 
6533 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6534   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6535 
6536   ins_cost(200); // XXX
6537   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6538   opcode(0x0F, 0x40);
6539   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6540   ins_pipe(pipe_cmov_reg);
6541 %}
6542 
6543 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6544   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6545   ins_cost(200);
6546   expand %{
6547     cmovI_regU(cop, cr, dst, src);
6548   %}
6549 %}
6550 
6551 // Conditional move
6552 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6553   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6554 
6555   ins_cost(250); // XXX
6556   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6557   opcode(0x0F, 0x40);
6558   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6559   ins_pipe(pipe_cmov_mem);
6560 %}
6561 
6562 // Conditional move
6563 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6564 %{
6565   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6566 
6567   ins_cost(250); // XXX
6568   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6569   opcode(0x0F, 0x40);
6570   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6571   ins_pipe(pipe_cmov_mem);
6572 %}
6573 
6574 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6575   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6576   ins_cost(250);
6577   expand %{
6578     cmovI_memU(cop, cr, dst, src);
6579   %}
6580 %}
6581 
6582 // Conditional move
6583 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6584 %{
6585   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6586 
6587   ins_cost(200); // XXX
6588   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6589   opcode(0x0F, 0x40);
6590   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6591   ins_pipe(pipe_cmov_reg);
6592 %}
6593 
6594 // Conditional move
6595 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6596 %{
6597   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6598 
6599   ins_cost(200); // XXX
6600   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6601   opcode(0x0F, 0x40);
6602   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6603   ins_pipe(pipe_cmov_reg);
6604 %}
6605 
6606 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6607   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6608   ins_cost(200);
6609   expand %{
6610     cmovN_regU(cop, cr, dst, src);
6611   %}
6612 %}
6613 
6614 // Conditional move
6615 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6616 %{
6617   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6618 
6619   ins_cost(200); // XXX
6620   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6621   opcode(0x0F, 0x40);
6622   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6623   ins_pipe(pipe_cmov_reg);  // XXX
6624 %}
6625 
6626 // Conditional move
6627 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6628 %{
6629   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6630 
6631   ins_cost(200); // XXX
6632   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6633   opcode(0x0F, 0x40);
6634   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6635   ins_pipe(pipe_cmov_reg); // XXX
6636 %}
6637 
6638 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6639   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6640   ins_cost(200);
6641   expand %{
6642     cmovP_regU(cop, cr, dst, src);
6643   %}
6644 %}
6645 
6646 // DISABLED: Requires the ADLC to emit a bottom_type call that
6647 // correctly meets the two pointer arguments; one is an incoming
6648 // register but the other is a memory operand.  ALSO appears to
6649 // be buggy with implicit null checks.
6650 //
6651 //// Conditional move
6652 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6653 //%{
6654 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6655 //  ins_cost(250);
6656 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6657 //  opcode(0x0F,0x40);
6658 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6659 //  ins_pipe( pipe_cmov_mem );
6660 //%}
6661 //
6662 //// Conditional move
6663 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6664 //%{
6665 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6666 //  ins_cost(250);
6667 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6668 //  opcode(0x0F,0x40);
6669 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6670 //  ins_pipe( pipe_cmov_mem );
6671 //%}
6672 
6673 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6674 %{
6675   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6676 
6677   ins_cost(200); // XXX
6678   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6679   opcode(0x0F, 0x40);
6680   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6681   ins_pipe(pipe_cmov_reg);  // XXX
6682 %}
6683 
6684 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6685 %{
6686   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6687 
6688   ins_cost(200); // XXX
6689   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6690   opcode(0x0F, 0x40);
6691   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6692   ins_pipe(pipe_cmov_mem);  // XXX
6693 %}
6694 
6695 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6696 %{
6697   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6698 
6699   ins_cost(200); // XXX
6700   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6701   opcode(0x0F, 0x40);
6702   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6703   ins_pipe(pipe_cmov_reg); // XXX
6704 %}
6705 
6706 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6707   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6708   ins_cost(200);
6709   expand %{
6710     cmovL_regU(cop, cr, dst, src);
6711   %}
6712 %}
6713 
6714 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6715 %{
6716   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6717 
6718   ins_cost(200); // XXX
6719   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6720   opcode(0x0F, 0x40);
6721   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6722   ins_pipe(pipe_cmov_mem); // XXX
6723 %}
6724 
6725 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6726   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6727   ins_cost(200);
6728   expand %{
6729     cmovL_memU(cop, cr, dst, src);
6730   %}
6731 %}
6732 
6733 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6734 %{
6735   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6736 
6737   ins_cost(200); // XXX
6738   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6739             "movss     $dst, $src\n"
6740     "skip:" %}
6741   ins_encode %{
6742     Label Lskip;
6743     // Invert sense of branch from sense of CMOV
6744     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6745     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6746     __ bind(Lskip);
6747   %}
6748   ins_pipe(pipe_slow);
6749 %}
6750 
6751 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6752 // %{
6753 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6754 
6755 //   ins_cost(200); // XXX
6756 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6757 //             "movss     $dst, $src\n"
6758 //     "skip:" %}
6759 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6760 //   ins_pipe(pipe_slow);
6761 // %}
6762 
6763 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6764 %{
6765   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6766 
6767   ins_cost(200); // XXX
6768   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6769             "movss     $dst, $src\n"
6770     "skip:" %}
6771   ins_encode %{
6772     Label Lskip;
6773     // Invert sense of branch from sense of CMOV
6774     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6775     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6776     __ bind(Lskip);
6777   %}
6778   ins_pipe(pipe_slow);
6779 %}
6780 
6781 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6782   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6783   ins_cost(200);
6784   expand %{
6785     cmovF_regU(cop, cr, dst, src);
6786   %}
6787 %}
6788 
6789 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6790 %{
6791   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6792 
6793   ins_cost(200); // XXX
6794   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6795             "movsd     $dst, $src\n"
6796     "skip:" %}
6797   ins_encode %{
6798     Label Lskip;
6799     // Invert sense of branch from sense of CMOV
6800     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6801     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6802     __ bind(Lskip);
6803   %}
6804   ins_pipe(pipe_slow);
6805 %}
6806 
6807 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6808 %{
6809   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6810 
6811   ins_cost(200); // XXX
6812   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6813             "movsd     $dst, $src\n"
6814     "skip:" %}
6815   ins_encode %{
6816     Label Lskip;
6817     // Invert sense of branch from sense of CMOV
6818     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6819     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6820     __ bind(Lskip);
6821   %}
6822   ins_pipe(pipe_slow);
6823 %}
6824 
6825 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6826   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6827   ins_cost(200);
6828   expand %{
6829     cmovD_regU(cop, cr, dst, src);
6830   %}
6831 %}
6832 
6833 //----------Arithmetic Instructions--------------------------------------------
6834 //----------Addition Instructions----------------------------------------------
6835 
6836 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6837 %{
6838   match(Set dst (AddI dst src));
6839   effect(KILL cr);
6840 
6841   format %{ "addl    $dst, $src\t# int" %}
6842   opcode(0x03);
6843   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6844   ins_pipe(ialu_reg_reg);
6845 %}
6846 
6847 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6848 %{
6849   match(Set dst (AddI dst src));
6850   effect(KILL cr);
6851 
6852   format %{ "addl    $dst, $src\t# int" %}
6853   opcode(0x81, 0x00); /* /0 id */
6854   ins_encode(OpcSErm(dst, src), Con8or32(src));
6855   ins_pipe( ialu_reg );
6856 %}
6857 
6858 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6859 %{
6860   match(Set dst (AddI dst (LoadI src)));
6861   effect(KILL cr);
6862 
6863   ins_cost(125); // XXX
6864   format %{ "addl    $dst, $src\t# int" %}
6865   opcode(0x03);
6866   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6867   ins_pipe(ialu_reg_mem);
6868 %}
6869 
6870 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6871 %{
6872   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6873   effect(KILL cr);
6874 
6875   ins_cost(150); // XXX
6876   format %{ "addl    $dst, $src\t# int" %}
6877   opcode(0x01); /* Opcode 01 /r */
6878   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6879   ins_pipe(ialu_mem_reg);
6880 %}
6881 
6882 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6883 %{
6884   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6885   effect(KILL cr);
6886 
6887   ins_cost(125); // XXX
6888   format %{ "addl    $dst, $src\t# int" %}
6889   opcode(0x81); /* Opcode 81 /0 id */
6890   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6891   ins_pipe(ialu_mem_imm);
6892 %}
6893 
6894 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
6895 %{
6896   predicate(UseIncDec);
6897   match(Set dst (AddI dst src));
6898   effect(KILL cr);
6899 
6900   format %{ "incl    $dst\t# int" %}
6901   opcode(0xFF, 0x00); // FF /0
6902   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6903   ins_pipe(ialu_reg);
6904 %}
6905 
6906 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
6907 %{
6908   predicate(UseIncDec);
6909   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6910   effect(KILL cr);
6911 
6912   ins_cost(125); // XXX
6913   format %{ "incl    $dst\t# int" %}
6914   opcode(0xFF); /* Opcode FF /0 */
6915   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
6916   ins_pipe(ialu_mem_imm);
6917 %}
6918 
6919 // XXX why does that use AddI
6920 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
6921 %{
6922   predicate(UseIncDec);
6923   match(Set dst (AddI dst src));
6924   effect(KILL cr);
6925 
6926   format %{ "decl    $dst\t# int" %}
6927   opcode(0xFF, 0x01); // FF /1
6928   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6929   ins_pipe(ialu_reg);
6930 %}
6931 
6932 // XXX why does that use AddI
6933 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
6934 %{
6935   predicate(UseIncDec);
6936   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6937   effect(KILL cr);
6938 
6939   ins_cost(125); // XXX
6940   format %{ "decl    $dst\t# int" %}
6941   opcode(0xFF); /* Opcode FF /1 */
6942   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
6943   ins_pipe(ialu_mem_imm);
6944 %}
6945 
6946 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
6947 %{
6948   match(Set dst (AddI src0 src1));
6949 
6950   ins_cost(110);
6951   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
6952   opcode(0x8D); /* 0x8D /r */
6953   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
6954   ins_pipe(ialu_reg_reg);
6955 %}
6956 
6957 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
6958 %{
6959   match(Set dst (AddL dst src));
6960   effect(KILL cr);
6961 
6962   format %{ "addq    $dst, $src\t# long" %}
6963   opcode(0x03);
6964   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
6965   ins_pipe(ialu_reg_reg);
6966 %}
6967 
6968 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
6969 %{
6970   match(Set dst (AddL dst src));
6971   effect(KILL cr);
6972 
6973   format %{ "addq    $dst, $src\t# long" %}
6974   opcode(0x81, 0x00); /* /0 id */
6975   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
6976   ins_pipe( ialu_reg );
6977 %}
6978 
6979 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
6980 %{
6981   match(Set dst (AddL dst (LoadL src)));
6982   effect(KILL cr);
6983 
6984   ins_cost(125); // XXX
6985   format %{ "addq    $dst, $src\t# long" %}
6986   opcode(0x03);
6987   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6988   ins_pipe(ialu_reg_mem);
6989 %}
6990 
6991 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
6992 %{
6993   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6994   effect(KILL cr);
6995 
6996   ins_cost(150); // XXX
6997   format %{ "addq    $dst, $src\t# long" %}
6998   opcode(0x01); /* Opcode 01 /r */
6999   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7000   ins_pipe(ialu_mem_reg);
7001 %}
7002 
7003 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7004 %{
7005   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7006   effect(KILL cr);
7007 
7008   ins_cost(125); // XXX
7009   format %{ "addq    $dst, $src\t# long" %}
7010   opcode(0x81); /* Opcode 81 /0 id */
7011   ins_encode(REX_mem_wide(dst),
7012              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7013   ins_pipe(ialu_mem_imm);
7014 %}
7015 
7016 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7017 %{
7018   predicate(UseIncDec);
7019   match(Set dst (AddL dst src));
7020   effect(KILL cr);
7021 
7022   format %{ "incq    $dst\t# long" %}
7023   opcode(0xFF, 0x00); // FF /0
7024   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7025   ins_pipe(ialu_reg);
7026 %}
7027 
7028 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7029 %{
7030   predicate(UseIncDec);
7031   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7032   effect(KILL cr);
7033 
7034   ins_cost(125); // XXX
7035   format %{ "incq    $dst\t# long" %}
7036   opcode(0xFF); /* Opcode FF /0 */
7037   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7038   ins_pipe(ialu_mem_imm);
7039 %}
7040 
7041 // XXX why does that use AddL
7042 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7043 %{
7044   predicate(UseIncDec);
7045   match(Set dst (AddL dst src));
7046   effect(KILL cr);
7047 
7048   format %{ "decq    $dst\t# long" %}
7049   opcode(0xFF, 0x01); // FF /1
7050   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7051   ins_pipe(ialu_reg);
7052 %}
7053 
7054 // XXX why does that use AddL
7055 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7056 %{
7057   predicate(UseIncDec);
7058   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7059   effect(KILL cr);
7060 
7061   ins_cost(125); // XXX
7062   format %{ "decq    $dst\t# long" %}
7063   opcode(0xFF); /* Opcode FF /1 */
7064   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7065   ins_pipe(ialu_mem_imm);
7066 %}
7067 
7068 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7069 %{
7070   match(Set dst (AddL src0 src1));
7071 
7072   ins_cost(110);
7073   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7074   opcode(0x8D); /* 0x8D /r */
7075   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7076   ins_pipe(ialu_reg_reg);
7077 %}
7078 
7079 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7080 %{
7081   match(Set dst (AddP dst src));
7082   effect(KILL cr);
7083 
7084   format %{ "addq    $dst, $src\t# ptr" %}
7085   opcode(0x03);
7086   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7087   ins_pipe(ialu_reg_reg);
7088 %}
7089 
7090 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7091 %{
7092   match(Set dst (AddP dst src));
7093   effect(KILL cr);
7094 
7095   format %{ "addq    $dst, $src\t# ptr" %}
7096   opcode(0x81, 0x00); /* /0 id */
7097   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7098   ins_pipe( ialu_reg );
7099 %}
7100 
7101 // XXX addP mem ops ????
7102 
7103 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7104 %{
7105   match(Set dst (AddP src0 src1));
7106 
7107   ins_cost(110);
7108   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7109   opcode(0x8D); /* 0x8D /r */
7110   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7111   ins_pipe(ialu_reg_reg);
7112 %}
7113 
7114 instruct checkCastPP(rRegP dst)
7115 %{
7116   match(Set dst (CheckCastPP dst));
7117 
7118   size(0);
7119   format %{ "# checkcastPP of $dst" %}
7120   ins_encode(/* empty encoding */);
7121   ins_pipe(empty);
7122 %}
7123 
7124 instruct castPP(rRegP dst)
7125 %{
7126   match(Set dst (CastPP dst));
7127 
7128   size(0);
7129   format %{ "# castPP of $dst" %}
7130   ins_encode(/* empty encoding */);
7131   ins_pipe(empty);
7132 %}
7133 
7134 instruct castII(rRegI dst)
7135 %{
7136   match(Set dst (CastII dst));
7137 
7138   size(0);
7139   format %{ "# castII of $dst" %}
7140   ins_encode(/* empty encoding */);
7141   ins_cost(0);
7142   ins_pipe(empty);
7143 %}
7144 
7145 // LoadP-locked same as a regular LoadP when used with compare-swap
7146 instruct loadPLocked(rRegP dst, memory mem)
7147 %{
7148   match(Set dst (LoadPLocked mem));
7149 
7150   ins_cost(125); // XXX
7151   format %{ "movq    $dst, $mem\t# ptr locked" %}
7152   opcode(0x8B);
7153   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7154   ins_pipe(ialu_reg_mem); // XXX
7155 %}
7156 
7157 // Conditional-store of the updated heap-top.
7158 // Used during allocation of the shared heap.
7159 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7160 
7161 instruct storePConditional(memory heap_top_ptr,
7162                            rax_RegP oldval, rRegP newval,
7163                            rFlagsReg cr)
7164 %{
7165   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7166 
7167   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7168             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7169   opcode(0x0F, 0xB1);
7170   ins_encode(lock_prefix,
7171              REX_reg_mem_wide(newval, heap_top_ptr),
7172              OpcP, OpcS,
7173              reg_mem(newval, heap_top_ptr));
7174   ins_pipe(pipe_cmpxchg);
7175 %}
7176 
7177 // Conditional-store of an int value.
7178 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7179 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7180 %{
7181   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7182   effect(KILL oldval);
7183 
7184   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7185   opcode(0x0F, 0xB1);
7186   ins_encode(lock_prefix,
7187              REX_reg_mem(newval, mem),
7188              OpcP, OpcS,
7189              reg_mem(newval, mem));
7190   ins_pipe(pipe_cmpxchg);
7191 %}
7192 
7193 // Conditional-store of a long value.
7194 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7195 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7196 %{
7197   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7198   effect(KILL oldval);
7199 
7200   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7201   opcode(0x0F, 0xB1);
7202   ins_encode(lock_prefix,
7203              REX_reg_mem_wide(newval, mem),
7204              OpcP, OpcS,
7205              reg_mem(newval, mem));
7206   ins_pipe(pipe_cmpxchg);
7207 %}
7208 
7209 
7210 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7211 instruct compareAndSwapP(rRegI res,
7212                          memory mem_ptr,
7213                          rax_RegP oldval, rRegP newval,
7214                          rFlagsReg cr)
7215 %{
7216   predicate(VM_Version::supports_cx8());
7217   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7218   effect(KILL cr, KILL oldval);
7219 
7220   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7221             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7222             "sete    $res\n\t"
7223             "movzbl  $res, $res" %}
7224   opcode(0x0F, 0xB1);
7225   ins_encode(lock_prefix,
7226              REX_reg_mem_wide(newval, mem_ptr),
7227              OpcP, OpcS,
7228              reg_mem(newval, mem_ptr),
7229              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7230              REX_reg_breg(res, res), // movzbl
7231              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7232   ins_pipe( pipe_cmpxchg );
7233 %}
7234 
7235 instruct compareAndSwapL(rRegI res,
7236                          memory mem_ptr,
7237                          rax_RegL oldval, rRegL newval,
7238                          rFlagsReg cr)
7239 %{
7240   predicate(VM_Version::supports_cx8());
7241   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7242   effect(KILL cr, KILL oldval);
7243 
7244   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7245             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7246             "sete    $res\n\t"
7247             "movzbl  $res, $res" %}
7248   opcode(0x0F, 0xB1);
7249   ins_encode(lock_prefix,
7250              REX_reg_mem_wide(newval, mem_ptr),
7251              OpcP, OpcS,
7252              reg_mem(newval, mem_ptr),
7253              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7254              REX_reg_breg(res, res), // movzbl
7255              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7256   ins_pipe( pipe_cmpxchg );
7257 %}
7258 
7259 instruct compareAndSwapI(rRegI res,
7260                          memory mem_ptr,
7261                          rax_RegI oldval, rRegI newval,
7262                          rFlagsReg cr)
7263 %{
7264   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7265   effect(KILL cr, KILL oldval);
7266 
7267   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7268             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7269             "sete    $res\n\t"
7270             "movzbl  $res, $res" %}
7271   opcode(0x0F, 0xB1);
7272   ins_encode(lock_prefix,
7273              REX_reg_mem(newval, mem_ptr),
7274              OpcP, OpcS,
7275              reg_mem(newval, mem_ptr),
7276              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7277              REX_reg_breg(res, res), // movzbl
7278              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7279   ins_pipe( pipe_cmpxchg );
7280 %}
7281 
7282 
7283 instruct compareAndSwapN(rRegI res,
7284                           memory mem_ptr,
7285                           rax_RegN oldval, rRegN newval,
7286                           rFlagsReg cr) %{
7287   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7288   effect(KILL cr, KILL oldval);
7289 
7290   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7291             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7292             "sete    $res\n\t"
7293             "movzbl  $res, $res" %}
7294   opcode(0x0F, 0xB1);
7295   ins_encode(lock_prefix,
7296              REX_reg_mem(newval, mem_ptr),
7297              OpcP, OpcS,
7298              reg_mem(newval, mem_ptr),
7299              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7300              REX_reg_breg(res, res), // movzbl
7301              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7302   ins_pipe( pipe_cmpxchg );
7303 %}
7304 
7305 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7306   predicate(n->as_LoadStore()->result_not_used());
7307   match(Set dummy (GetAndAddI mem add));
7308   effect(KILL cr);
7309   format %{ "ADDL  [$mem],$add" %}
7310   ins_encode %{
7311     if (os::is_MP()) { __ lock(); }
7312     __ addl($mem$$Address, $add$$constant);
7313   %}
7314   ins_pipe( pipe_cmpxchg );
7315 %}
7316 
7317 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7318   match(Set newval (GetAndAddI mem newval));
7319   effect(KILL cr);
7320   format %{ "XADDL  [$mem],$newval" %}
7321   ins_encode %{
7322     if (os::is_MP()) { __ lock(); }
7323     __ xaddl($mem$$Address, $newval$$Register);
7324   %}
7325   ins_pipe( pipe_cmpxchg );
7326 %}
7327 
7328 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7329   predicate(n->as_LoadStore()->result_not_used());
7330   match(Set dummy (GetAndAddL mem add));
7331   effect(KILL cr);
7332   format %{ "ADDQ  [$mem],$add" %}
7333   ins_encode %{
7334     if (os::is_MP()) { __ lock(); }
7335     __ addq($mem$$Address, $add$$constant);
7336   %}
7337   ins_pipe( pipe_cmpxchg );
7338 %}
7339 
7340 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7341   match(Set newval (GetAndAddL mem newval));
7342   effect(KILL cr);
7343   format %{ "XADDQ  [$mem],$newval" %}
7344   ins_encode %{
7345     if (os::is_MP()) { __ lock(); }
7346     __ xaddq($mem$$Address, $newval$$Register);
7347   %}
7348   ins_pipe( pipe_cmpxchg );
7349 %}
7350 
7351 instruct xchgI( memory mem, rRegI newval) %{
7352   match(Set newval (GetAndSetI mem newval));
7353   format %{ "XCHGL  $newval,[$mem]" %}
7354   ins_encode %{
7355     __ xchgl($newval$$Register, $mem$$Address);
7356   %}
7357   ins_pipe( pipe_cmpxchg );
7358 %}
7359 
7360 instruct xchgL( memory mem, rRegL newval) %{
7361   match(Set newval (GetAndSetL mem newval));
7362   format %{ "XCHGL  $newval,[$mem]" %}
7363   ins_encode %{
7364     __ xchgq($newval$$Register, $mem$$Address);
7365   %}
7366   ins_pipe( pipe_cmpxchg );
7367 %}
7368 
7369 instruct xchgP( memory mem, rRegP newval) %{
7370   match(Set newval (GetAndSetP mem newval));
7371   format %{ "XCHGQ  $newval,[$mem]" %}
7372   ins_encode %{
7373     __ xchgq($newval$$Register, $mem$$Address);
7374   %}
7375   ins_pipe( pipe_cmpxchg );
7376 %}
7377 
7378 instruct xchgN( memory mem, rRegN newval) %{
7379   match(Set newval (GetAndSetN mem newval));
7380   format %{ "XCHGL  $newval,$mem]" %}
7381   ins_encode %{
7382     __ xchgl($newval$$Register, $mem$$Address);
7383   %}
7384   ins_pipe( pipe_cmpxchg );
7385 %}
7386 
7387 //----------Subtraction Instructions-------------------------------------------
7388 
7389 // Integer Subtraction Instructions
7390 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7391 %{
7392   match(Set dst (SubI dst src));
7393   effect(KILL cr);
7394 
7395   format %{ "subl    $dst, $src\t# int" %}
7396   opcode(0x2B);
7397   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7398   ins_pipe(ialu_reg_reg);
7399 %}
7400 
7401 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7402 %{
7403   match(Set dst (SubI dst src));
7404   effect(KILL cr);
7405 
7406   format %{ "subl    $dst, $src\t# int" %}
7407   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7408   ins_encode(OpcSErm(dst, src), Con8or32(src));
7409   ins_pipe(ialu_reg);
7410 %}
7411 
7412 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7413 %{
7414   match(Set dst (SubI dst (LoadI src)));
7415   effect(KILL cr);
7416 
7417   ins_cost(125);
7418   format %{ "subl    $dst, $src\t# int" %}
7419   opcode(0x2B);
7420   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7421   ins_pipe(ialu_reg_mem);
7422 %}
7423 
7424 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7425 %{
7426   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7427   effect(KILL cr);
7428 
7429   ins_cost(150);
7430   format %{ "subl    $dst, $src\t# int" %}
7431   opcode(0x29); /* Opcode 29 /r */
7432   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7433   ins_pipe(ialu_mem_reg);
7434 %}
7435 
7436 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7437 %{
7438   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7439   effect(KILL cr);
7440 
7441   ins_cost(125); // XXX
7442   format %{ "subl    $dst, $src\t# int" %}
7443   opcode(0x81); /* Opcode 81 /5 id */
7444   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7445   ins_pipe(ialu_mem_imm);
7446 %}
7447 
7448 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7449 %{
7450   match(Set dst (SubL dst src));
7451   effect(KILL cr);
7452 
7453   format %{ "subq    $dst, $src\t# long" %}
7454   opcode(0x2B);
7455   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7456   ins_pipe(ialu_reg_reg);
7457 %}
7458 
7459 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7460 %{
7461   match(Set dst (SubL dst src));
7462   effect(KILL cr);
7463 
7464   format %{ "subq    $dst, $src\t# long" %}
7465   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7466   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7467   ins_pipe(ialu_reg);
7468 %}
7469 
7470 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7471 %{
7472   match(Set dst (SubL dst (LoadL src)));
7473   effect(KILL cr);
7474 
7475   ins_cost(125);
7476   format %{ "subq    $dst, $src\t# long" %}
7477   opcode(0x2B);
7478   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7479   ins_pipe(ialu_reg_mem);
7480 %}
7481 
7482 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7483 %{
7484   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7485   effect(KILL cr);
7486 
7487   ins_cost(150);
7488   format %{ "subq    $dst, $src\t# long" %}
7489   opcode(0x29); /* Opcode 29 /r */
7490   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7491   ins_pipe(ialu_mem_reg);
7492 %}
7493 
7494 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7495 %{
7496   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7497   effect(KILL cr);
7498 
7499   ins_cost(125); // XXX
7500   format %{ "subq    $dst, $src\t# long" %}
7501   opcode(0x81); /* Opcode 81 /5 id */
7502   ins_encode(REX_mem_wide(dst),
7503              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7504   ins_pipe(ialu_mem_imm);
7505 %}
7506 
7507 // Subtract from a pointer
7508 // XXX hmpf???
7509 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7510 %{
7511   match(Set dst (AddP dst (SubI zero src)));
7512   effect(KILL cr);
7513 
7514   format %{ "subq    $dst, $src\t# ptr - int" %}
7515   opcode(0x2B);
7516   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7517   ins_pipe(ialu_reg_reg);
7518 %}
7519 
7520 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7521 %{
7522   match(Set dst (SubI zero dst));
7523   effect(KILL cr);
7524 
7525   format %{ "negl    $dst\t# int" %}
7526   opcode(0xF7, 0x03);  // Opcode F7 /3
7527   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7528   ins_pipe(ialu_reg);
7529 %}
7530 
7531 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7532 %{
7533   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7534   effect(KILL cr);
7535 
7536   format %{ "negl    $dst\t# int" %}
7537   opcode(0xF7, 0x03);  // Opcode F7 /3
7538   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7539   ins_pipe(ialu_reg);
7540 %}
7541 
7542 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7543 %{
7544   match(Set dst (SubL zero dst));
7545   effect(KILL cr);
7546 
7547   format %{ "negq    $dst\t# long" %}
7548   opcode(0xF7, 0x03);  // Opcode F7 /3
7549   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7550   ins_pipe(ialu_reg);
7551 %}
7552 
7553 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7554 %{
7555   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7556   effect(KILL cr);
7557 
7558   format %{ "negq    $dst\t# long" %}
7559   opcode(0xF7, 0x03);  // Opcode F7 /3
7560   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7561   ins_pipe(ialu_reg);
7562 %}
7563 
7564 //----------Multiplication/Division Instructions-------------------------------
7565 // Integer Multiplication Instructions
7566 // Multiply Register
7567 
7568 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7569 %{
7570   match(Set dst (MulI dst src));
7571   effect(KILL cr);
7572 
7573   ins_cost(300);
7574   format %{ "imull   $dst, $src\t# int" %}
7575   opcode(0x0F, 0xAF);
7576   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7577   ins_pipe(ialu_reg_reg_alu0);
7578 %}
7579 
7580 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7581 %{
7582   match(Set dst (MulI src imm));
7583   effect(KILL cr);
7584 
7585   ins_cost(300);
7586   format %{ "imull   $dst, $src, $imm\t# int" %}
7587   opcode(0x69); /* 69 /r id */
7588   ins_encode(REX_reg_reg(dst, src),
7589              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7590   ins_pipe(ialu_reg_reg_alu0);
7591 %}
7592 
7593 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7594 %{
7595   match(Set dst (MulI dst (LoadI src)));
7596   effect(KILL cr);
7597 
7598   ins_cost(350);
7599   format %{ "imull   $dst, $src\t# int" %}
7600   opcode(0x0F, 0xAF);
7601   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7602   ins_pipe(ialu_reg_mem_alu0);
7603 %}
7604 
7605 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7606 %{
7607   match(Set dst (MulI (LoadI src) imm));
7608   effect(KILL cr);
7609 
7610   ins_cost(300);
7611   format %{ "imull   $dst, $src, $imm\t# int" %}
7612   opcode(0x69); /* 69 /r id */
7613   ins_encode(REX_reg_mem(dst, src),
7614              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7615   ins_pipe(ialu_reg_mem_alu0);
7616 %}
7617 
7618 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7619 %{
7620   match(Set dst (MulL dst src));
7621   effect(KILL cr);
7622 
7623   ins_cost(300);
7624   format %{ "imulq   $dst, $src\t# long" %}
7625   opcode(0x0F, 0xAF);
7626   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7627   ins_pipe(ialu_reg_reg_alu0);
7628 %}
7629 
7630 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7631 %{
7632   match(Set dst (MulL src imm));
7633   effect(KILL cr);
7634 
7635   ins_cost(300);
7636   format %{ "imulq   $dst, $src, $imm\t# long" %}
7637   opcode(0x69); /* 69 /r id */
7638   ins_encode(REX_reg_reg_wide(dst, src),
7639              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7640   ins_pipe(ialu_reg_reg_alu0);
7641 %}
7642 
7643 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7644 %{
7645   match(Set dst (MulL dst (LoadL src)));
7646   effect(KILL cr);
7647 
7648   ins_cost(350);
7649   format %{ "imulq   $dst, $src\t# long" %}
7650   opcode(0x0F, 0xAF);
7651   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7652   ins_pipe(ialu_reg_mem_alu0);
7653 %}
7654 
7655 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7656 %{
7657   match(Set dst (MulL (LoadL src) imm));
7658   effect(KILL cr);
7659 
7660   ins_cost(300);
7661   format %{ "imulq   $dst, $src, $imm\t# long" %}
7662   opcode(0x69); /* 69 /r id */
7663   ins_encode(REX_reg_mem_wide(dst, src),
7664              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7665   ins_pipe(ialu_reg_mem_alu0);
7666 %}
7667 
7668 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7669 %{
7670   match(Set dst (MulHiL src rax));
7671   effect(USE_KILL rax, KILL cr);
7672 
7673   ins_cost(300);
7674   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7675   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7676   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7677   ins_pipe(ialu_reg_reg_alu0);
7678 %}
7679 
7680 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7681                    rFlagsReg cr)
7682 %{
7683   match(Set rax (DivI rax div));
7684   effect(KILL rdx, KILL cr);
7685 
7686   ins_cost(30*100+10*100); // XXX
7687   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7688             "jne,s   normal\n\t"
7689             "xorl    rdx, rdx\n\t"
7690             "cmpl    $div, -1\n\t"
7691             "je,s    done\n"
7692     "normal: cdql\n\t"
7693             "idivl   $div\n"
7694     "done:"        %}
7695   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7696   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7697   ins_pipe(ialu_reg_reg_alu0);
7698 %}
7699 
7700 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7701                    rFlagsReg cr)
7702 %{
7703   match(Set rax (DivL rax div));
7704   effect(KILL rdx, KILL cr);
7705 
7706   ins_cost(30*100+10*100); // XXX
7707   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7708             "cmpq    rax, rdx\n\t"
7709             "jne,s   normal\n\t"
7710             "xorl    rdx, rdx\n\t"
7711             "cmpq    $div, -1\n\t"
7712             "je,s    done\n"
7713     "normal: cdqq\n\t"
7714             "idivq   $div\n"
7715     "done:"        %}
7716   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7717   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7718   ins_pipe(ialu_reg_reg_alu0);
7719 %}
7720 
7721 // Integer DIVMOD with Register, both quotient and mod results
7722 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7723                              rFlagsReg cr)
7724 %{
7725   match(DivModI rax div);
7726   effect(KILL cr);
7727 
7728   ins_cost(30*100+10*100); // XXX
7729   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7730             "jne,s   normal\n\t"
7731             "xorl    rdx, rdx\n\t"
7732             "cmpl    $div, -1\n\t"
7733             "je,s    done\n"
7734     "normal: cdql\n\t"
7735             "idivl   $div\n"
7736     "done:"        %}
7737   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7738   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7739   ins_pipe(pipe_slow);
7740 %}
7741 
7742 // Long DIVMOD with Register, both quotient and mod results
7743 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7744                              rFlagsReg cr)
7745 %{
7746   match(DivModL rax div);
7747   effect(KILL cr);
7748 
7749   ins_cost(30*100+10*100); // XXX
7750   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7751             "cmpq    rax, rdx\n\t"
7752             "jne,s   normal\n\t"
7753             "xorl    rdx, rdx\n\t"
7754             "cmpq    $div, -1\n\t"
7755             "je,s    done\n"
7756     "normal: cdqq\n\t"
7757             "idivq   $div\n"
7758     "done:"        %}
7759   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7760   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7761   ins_pipe(pipe_slow);
7762 %}
7763 
7764 //----------- DivL-By-Constant-Expansions--------------------------------------
7765 // DivI cases are handled by the compiler
7766 
7767 // Magic constant, reciprocal of 10
7768 instruct loadConL_0x6666666666666667(rRegL dst)
7769 %{
7770   effect(DEF dst);
7771 
7772   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
7773   ins_encode(load_immL(dst, 0x6666666666666667));
7774   ins_pipe(ialu_reg);
7775 %}
7776 
7777 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7778 %{
7779   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
7780 
7781   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
7782   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7783   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7784   ins_pipe(ialu_reg_reg_alu0);
7785 %}
7786 
7787 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
7788 %{
7789   effect(USE_DEF dst, KILL cr);
7790 
7791   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
7792   opcode(0xC1, 0x7); /* C1 /7 ib */
7793   ins_encode(reg_opc_imm_wide(dst, 0x3F));
7794   ins_pipe(ialu_reg);
7795 %}
7796 
7797 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
7798 %{
7799   effect(USE_DEF dst, KILL cr);
7800 
7801   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
7802   opcode(0xC1, 0x7); /* C1 /7 ib */
7803   ins_encode(reg_opc_imm_wide(dst, 0x2));
7804   ins_pipe(ialu_reg);
7805 %}
7806 
7807 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
7808 %{
7809   match(Set dst (DivL src div));
7810 
7811   ins_cost((5+8)*100);
7812   expand %{
7813     rax_RegL rax;                     // Killed temp
7814     rFlagsReg cr;                     // Killed
7815     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
7816     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
7817     sarL_rReg_63(src, cr);            // sarq  src, 63
7818     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
7819     subL_rReg(dst, src, cr);          // subl  rdx, src
7820   %}
7821 %}
7822 
7823 //-----------------------------------------------------------------------------
7824 
7825 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
7826                    rFlagsReg cr)
7827 %{
7828   match(Set rdx (ModI rax div));
7829   effect(KILL rax, KILL cr);
7830 
7831   ins_cost(300); // XXX
7832   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
7833             "jne,s   normal\n\t"
7834             "xorl    rdx, rdx\n\t"
7835             "cmpl    $div, -1\n\t"
7836             "je,s    done\n"
7837     "normal: cdql\n\t"
7838             "idivl   $div\n"
7839     "done:"        %}
7840   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7841   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7842   ins_pipe(ialu_reg_reg_alu0);
7843 %}
7844 
7845 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
7846                    rFlagsReg cr)
7847 %{
7848   match(Set rdx (ModL rax div));
7849   effect(KILL rax, KILL cr);
7850 
7851   ins_cost(300); // XXX
7852   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
7853             "cmpq    rax, rdx\n\t"
7854             "jne,s   normal\n\t"
7855             "xorl    rdx, rdx\n\t"
7856             "cmpq    $div, -1\n\t"
7857             "je,s    done\n"
7858     "normal: cdqq\n\t"
7859             "idivq   $div\n"
7860     "done:"        %}
7861   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7862   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7863   ins_pipe(ialu_reg_reg_alu0);
7864 %}
7865 
7866 // Integer Shift Instructions
7867 // Shift Left by one
7868 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7869 %{
7870   match(Set dst (LShiftI dst shift));
7871   effect(KILL cr);
7872 
7873   format %{ "sall    $dst, $shift" %}
7874   opcode(0xD1, 0x4); /* D1 /4 */
7875   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7876   ins_pipe(ialu_reg);
7877 %}
7878 
7879 // Shift Left by one
7880 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7881 %{
7882   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7883   effect(KILL cr);
7884 
7885   format %{ "sall    $dst, $shift\t" %}
7886   opcode(0xD1, 0x4); /* D1 /4 */
7887   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7888   ins_pipe(ialu_mem_imm);
7889 %}
7890 
7891 // Shift Left by 8-bit immediate
7892 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7893 %{
7894   match(Set dst (LShiftI dst shift));
7895   effect(KILL cr);
7896 
7897   format %{ "sall    $dst, $shift" %}
7898   opcode(0xC1, 0x4); /* C1 /4 ib */
7899   ins_encode(reg_opc_imm(dst, shift));
7900   ins_pipe(ialu_reg);
7901 %}
7902 
7903 // Shift Left by 8-bit immediate
7904 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7905 %{
7906   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7907   effect(KILL cr);
7908 
7909   format %{ "sall    $dst, $shift" %}
7910   opcode(0xC1, 0x4); /* C1 /4 ib */
7911   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7912   ins_pipe(ialu_mem_imm);
7913 %}
7914 
7915 // Shift Left by variable
7916 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7917 %{
7918   match(Set dst (LShiftI dst shift));
7919   effect(KILL cr);
7920 
7921   format %{ "sall    $dst, $shift" %}
7922   opcode(0xD3, 0x4); /* D3 /4 */
7923   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7924   ins_pipe(ialu_reg_reg);
7925 %}
7926 
7927 // Shift Left by variable
7928 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7929 %{
7930   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7931   effect(KILL cr);
7932 
7933   format %{ "sall    $dst, $shift" %}
7934   opcode(0xD3, 0x4); /* D3 /4 */
7935   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7936   ins_pipe(ialu_mem_reg);
7937 %}
7938 
7939 // Arithmetic shift right by one
7940 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7941 %{
7942   match(Set dst (RShiftI dst shift));
7943   effect(KILL cr);
7944 
7945   format %{ "sarl    $dst, $shift" %}
7946   opcode(0xD1, 0x7); /* D1 /7 */
7947   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7948   ins_pipe(ialu_reg);
7949 %}
7950 
7951 // Arithmetic shift right by one
7952 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7953 %{
7954   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
7955   effect(KILL cr);
7956 
7957   format %{ "sarl    $dst, $shift" %}
7958   opcode(0xD1, 0x7); /* D1 /7 */
7959   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7960   ins_pipe(ialu_mem_imm);
7961 %}
7962 
7963 // Arithmetic Shift Right by 8-bit immediate
7964 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7965 %{
7966   match(Set dst (RShiftI dst shift));
7967   effect(KILL cr);
7968 
7969   format %{ "sarl    $dst, $shift" %}
7970   opcode(0xC1, 0x7); /* C1 /7 ib */
7971   ins_encode(reg_opc_imm(dst, shift));
7972   ins_pipe(ialu_mem_imm);
7973 %}
7974 
7975 // Arithmetic Shift Right by 8-bit immediate
7976 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7977 %{
7978   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
7979   effect(KILL cr);
7980 
7981   format %{ "sarl    $dst, $shift" %}
7982   opcode(0xC1, 0x7); /* C1 /7 ib */
7983   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7984   ins_pipe(ialu_mem_imm);
7985 %}
7986 
7987 // Arithmetic Shift Right by variable
7988 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7989 %{
7990   match(Set dst (RShiftI dst shift));
7991   effect(KILL cr);
7992 
7993   format %{ "sarl    $dst, $shift" %}
7994   opcode(0xD3, 0x7); /* D3 /7 */
7995   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7996   ins_pipe(ialu_reg_reg);
7997 %}
7998 
7999 // Arithmetic Shift Right by variable
8000 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8001 %{
8002   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8003   effect(KILL cr);
8004 
8005   format %{ "sarl    $dst, $shift" %}
8006   opcode(0xD3, 0x7); /* D3 /7 */
8007   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8008   ins_pipe(ialu_mem_reg);
8009 %}
8010 
8011 // Logical shift right by one
8012 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8013 %{
8014   match(Set dst (URShiftI dst shift));
8015   effect(KILL cr);
8016 
8017   format %{ "shrl    $dst, $shift" %}
8018   opcode(0xD1, 0x5); /* D1 /5 */
8019   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8020   ins_pipe(ialu_reg);
8021 %}
8022 
8023 // Logical shift right by one
8024 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8025 %{
8026   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8027   effect(KILL cr);
8028 
8029   format %{ "shrl    $dst, $shift" %}
8030   opcode(0xD1, 0x5); /* D1 /5 */
8031   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8032   ins_pipe(ialu_mem_imm);
8033 %}
8034 
8035 // Logical Shift Right by 8-bit immediate
8036 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8037 %{
8038   match(Set dst (URShiftI dst shift));
8039   effect(KILL cr);
8040 
8041   format %{ "shrl    $dst, $shift" %}
8042   opcode(0xC1, 0x5); /* C1 /5 ib */
8043   ins_encode(reg_opc_imm(dst, shift));
8044   ins_pipe(ialu_reg);
8045 %}
8046 
8047 // Logical Shift Right by 8-bit immediate
8048 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8049 %{
8050   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8051   effect(KILL cr);
8052 
8053   format %{ "shrl    $dst, $shift" %}
8054   opcode(0xC1, 0x5); /* C1 /5 ib */
8055   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8056   ins_pipe(ialu_mem_imm);
8057 %}
8058 
8059 // Logical Shift Right by variable
8060 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8061 %{
8062   match(Set dst (URShiftI dst shift));
8063   effect(KILL cr);
8064 
8065   format %{ "shrl    $dst, $shift" %}
8066   opcode(0xD3, 0x5); /* D3 /5 */
8067   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8068   ins_pipe(ialu_reg_reg);
8069 %}
8070 
8071 // Logical Shift Right by variable
8072 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8073 %{
8074   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8075   effect(KILL cr);
8076 
8077   format %{ "shrl    $dst, $shift" %}
8078   opcode(0xD3, 0x5); /* D3 /5 */
8079   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8080   ins_pipe(ialu_mem_reg);
8081 %}
8082 
8083 // Long Shift Instructions
8084 // Shift Left by one
8085 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8086 %{
8087   match(Set dst (LShiftL dst shift));
8088   effect(KILL cr);
8089 
8090   format %{ "salq    $dst, $shift" %}
8091   opcode(0xD1, 0x4); /* D1 /4 */
8092   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8093   ins_pipe(ialu_reg);
8094 %}
8095 
8096 // Shift Left by one
8097 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8098 %{
8099   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8100   effect(KILL cr);
8101 
8102   format %{ "salq    $dst, $shift" %}
8103   opcode(0xD1, 0x4); /* D1 /4 */
8104   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8105   ins_pipe(ialu_mem_imm);
8106 %}
8107 
8108 // Shift Left by 8-bit immediate
8109 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8110 %{
8111   match(Set dst (LShiftL dst shift));
8112   effect(KILL cr);
8113 
8114   format %{ "salq    $dst, $shift" %}
8115   opcode(0xC1, 0x4); /* C1 /4 ib */
8116   ins_encode(reg_opc_imm_wide(dst, shift));
8117   ins_pipe(ialu_reg);
8118 %}
8119 
8120 // Shift Left by 8-bit immediate
8121 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8122 %{
8123   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8124   effect(KILL cr);
8125 
8126   format %{ "salq    $dst, $shift" %}
8127   opcode(0xC1, 0x4); /* C1 /4 ib */
8128   ins_encode(REX_mem_wide(dst), OpcP,
8129              RM_opc_mem(secondary, dst), Con8or32(shift));
8130   ins_pipe(ialu_mem_imm);
8131 %}
8132 
8133 // Shift Left by variable
8134 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8135 %{
8136   match(Set dst (LShiftL dst shift));
8137   effect(KILL cr);
8138 
8139   format %{ "salq    $dst, $shift" %}
8140   opcode(0xD3, 0x4); /* D3 /4 */
8141   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8142   ins_pipe(ialu_reg_reg);
8143 %}
8144 
8145 // Shift Left by variable
8146 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8147 %{
8148   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8149   effect(KILL cr);
8150 
8151   format %{ "salq    $dst, $shift" %}
8152   opcode(0xD3, 0x4); /* D3 /4 */
8153   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8154   ins_pipe(ialu_mem_reg);
8155 %}
8156 
8157 // Arithmetic shift right by one
8158 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8159 %{
8160   match(Set dst (RShiftL dst shift));
8161   effect(KILL cr);
8162 
8163   format %{ "sarq    $dst, $shift" %}
8164   opcode(0xD1, 0x7); /* D1 /7 */
8165   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8166   ins_pipe(ialu_reg);
8167 %}
8168 
8169 // Arithmetic shift right by one
8170 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8171 %{
8172   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8173   effect(KILL cr);
8174 
8175   format %{ "sarq    $dst, $shift" %}
8176   opcode(0xD1, 0x7); /* D1 /7 */
8177   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8178   ins_pipe(ialu_mem_imm);
8179 %}
8180 
8181 // Arithmetic Shift Right by 8-bit immediate
8182 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8183 %{
8184   match(Set dst (RShiftL dst shift));
8185   effect(KILL cr);
8186 
8187   format %{ "sarq    $dst, $shift" %}
8188   opcode(0xC1, 0x7); /* C1 /7 ib */
8189   ins_encode(reg_opc_imm_wide(dst, shift));
8190   ins_pipe(ialu_mem_imm);
8191 %}
8192 
8193 // Arithmetic Shift Right by 8-bit immediate
8194 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8195 %{
8196   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8197   effect(KILL cr);
8198 
8199   format %{ "sarq    $dst, $shift" %}
8200   opcode(0xC1, 0x7); /* C1 /7 ib */
8201   ins_encode(REX_mem_wide(dst), OpcP,
8202              RM_opc_mem(secondary, dst), Con8or32(shift));
8203   ins_pipe(ialu_mem_imm);
8204 %}
8205 
8206 // Arithmetic Shift Right by variable
8207 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8208 %{
8209   match(Set dst (RShiftL dst shift));
8210   effect(KILL cr);
8211 
8212   format %{ "sarq    $dst, $shift" %}
8213   opcode(0xD3, 0x7); /* D3 /7 */
8214   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8215   ins_pipe(ialu_reg_reg);
8216 %}
8217 
8218 // Arithmetic Shift Right by variable
8219 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8220 %{
8221   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8222   effect(KILL cr);
8223 
8224   format %{ "sarq    $dst, $shift" %}
8225   opcode(0xD3, 0x7); /* D3 /7 */
8226   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8227   ins_pipe(ialu_mem_reg);
8228 %}
8229 
8230 // Logical shift right by one
8231 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8232 %{
8233   match(Set dst (URShiftL dst shift));
8234   effect(KILL cr);
8235 
8236   format %{ "shrq    $dst, $shift" %}
8237   opcode(0xD1, 0x5); /* D1 /5 */
8238   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8239   ins_pipe(ialu_reg);
8240 %}
8241 
8242 // Logical shift right by one
8243 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8244 %{
8245   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8246   effect(KILL cr);
8247 
8248   format %{ "shrq    $dst, $shift" %}
8249   opcode(0xD1, 0x5); /* D1 /5 */
8250   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8251   ins_pipe(ialu_mem_imm);
8252 %}
8253 
8254 // Logical Shift Right by 8-bit immediate
8255 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8256 %{
8257   match(Set dst (URShiftL dst shift));
8258   effect(KILL cr);
8259 
8260   format %{ "shrq    $dst, $shift" %}
8261   opcode(0xC1, 0x5); /* C1 /5 ib */
8262   ins_encode(reg_opc_imm_wide(dst, shift));
8263   ins_pipe(ialu_reg);
8264 %}
8265 
8266 
8267 // Logical Shift Right by 8-bit immediate
8268 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8269 %{
8270   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8271   effect(KILL cr);
8272 
8273   format %{ "shrq    $dst, $shift" %}
8274   opcode(0xC1, 0x5); /* C1 /5 ib */
8275   ins_encode(REX_mem_wide(dst), OpcP,
8276              RM_opc_mem(secondary, dst), Con8or32(shift));
8277   ins_pipe(ialu_mem_imm);
8278 %}
8279 
8280 // Logical Shift Right by variable
8281 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8282 %{
8283   match(Set dst (URShiftL dst shift));
8284   effect(KILL cr);
8285 
8286   format %{ "shrq    $dst, $shift" %}
8287   opcode(0xD3, 0x5); /* D3 /5 */
8288   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8289   ins_pipe(ialu_reg_reg);
8290 %}
8291 
8292 // Logical Shift Right by variable
8293 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8294 %{
8295   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8296   effect(KILL cr);
8297 
8298   format %{ "shrq    $dst, $shift" %}
8299   opcode(0xD3, 0x5); /* D3 /5 */
8300   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8301   ins_pipe(ialu_mem_reg);
8302 %}
8303 
8304 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8305 // This idiom is used by the compiler for the i2b bytecode.
8306 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8307 %{
8308   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8309 
8310   format %{ "movsbl  $dst, $src\t# i2b" %}
8311   opcode(0x0F, 0xBE);
8312   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8313   ins_pipe(ialu_reg_reg);
8314 %}
8315 
8316 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8317 // This idiom is used by the compiler the i2s bytecode.
8318 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8319 %{
8320   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8321 
8322   format %{ "movswl  $dst, $src\t# i2s" %}
8323   opcode(0x0F, 0xBF);
8324   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8325   ins_pipe(ialu_reg_reg);
8326 %}
8327 
8328 // ROL/ROR instructions
8329 
8330 // ROL expand
8331 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8332   effect(KILL cr, USE_DEF dst);
8333 
8334   format %{ "roll    $dst" %}
8335   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8336   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8337   ins_pipe(ialu_reg);
8338 %}
8339 
8340 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8341   effect(USE_DEF dst, USE shift, KILL cr);
8342 
8343   format %{ "roll    $dst, $shift" %}
8344   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8345   ins_encode( reg_opc_imm(dst, shift) );
8346   ins_pipe(ialu_reg);
8347 %}
8348 
8349 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8350 %{
8351   effect(USE_DEF dst, USE shift, KILL cr);
8352 
8353   format %{ "roll    $dst, $shift" %}
8354   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8355   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8356   ins_pipe(ialu_reg_reg);
8357 %}
8358 // end of ROL expand
8359 
8360 // Rotate Left by one
8361 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8362 %{
8363   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8364 
8365   expand %{
8366     rolI_rReg_imm1(dst, cr);
8367   %}
8368 %}
8369 
8370 // Rotate Left by 8-bit immediate
8371 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8372 %{
8373   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8374   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8375 
8376   expand %{
8377     rolI_rReg_imm8(dst, lshift, cr);
8378   %}
8379 %}
8380 
8381 // Rotate Left by variable
8382 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8383 %{
8384   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8385 
8386   expand %{
8387     rolI_rReg_CL(dst, shift, cr);
8388   %}
8389 %}
8390 
8391 // Rotate Left by variable
8392 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8393 %{
8394   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8395 
8396   expand %{
8397     rolI_rReg_CL(dst, shift, cr);
8398   %}
8399 %}
8400 
8401 // ROR expand
8402 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8403 %{
8404   effect(USE_DEF dst, KILL cr);
8405 
8406   format %{ "rorl    $dst" %}
8407   opcode(0xD1, 0x1); /* D1 /1 */
8408   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8409   ins_pipe(ialu_reg);
8410 %}
8411 
8412 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8413 %{
8414   effect(USE_DEF dst, USE shift, KILL cr);
8415 
8416   format %{ "rorl    $dst, $shift" %}
8417   opcode(0xC1, 0x1); /* C1 /1 ib */
8418   ins_encode(reg_opc_imm(dst, shift));
8419   ins_pipe(ialu_reg);
8420 %}
8421 
8422 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8423 %{
8424   effect(USE_DEF dst, USE shift, KILL cr);
8425 
8426   format %{ "rorl    $dst, $shift" %}
8427   opcode(0xD3, 0x1); /* D3 /1 */
8428   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8429   ins_pipe(ialu_reg_reg);
8430 %}
8431 // end of ROR expand
8432 
8433 // Rotate Right by one
8434 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8435 %{
8436   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8437 
8438   expand %{
8439     rorI_rReg_imm1(dst, cr);
8440   %}
8441 %}
8442 
8443 // Rotate Right by 8-bit immediate
8444 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8445 %{
8446   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8447   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8448 
8449   expand %{
8450     rorI_rReg_imm8(dst, rshift, cr);
8451   %}
8452 %}
8453 
8454 // Rotate Right by variable
8455 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8456 %{
8457   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8458 
8459   expand %{
8460     rorI_rReg_CL(dst, shift, cr);
8461   %}
8462 %}
8463 
8464 // Rotate Right by variable
8465 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8466 %{
8467   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8468 
8469   expand %{
8470     rorI_rReg_CL(dst, shift, cr);
8471   %}
8472 %}
8473 
8474 // for long rotate
8475 // ROL expand
8476 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8477   effect(USE_DEF dst, KILL cr);
8478 
8479   format %{ "rolq    $dst" %}
8480   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8481   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8482   ins_pipe(ialu_reg);
8483 %}
8484 
8485 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8486   effect(USE_DEF dst, USE shift, KILL cr);
8487 
8488   format %{ "rolq    $dst, $shift" %}
8489   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8490   ins_encode( reg_opc_imm_wide(dst, shift) );
8491   ins_pipe(ialu_reg);
8492 %}
8493 
8494 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8495 %{
8496   effect(USE_DEF dst, USE shift, KILL cr);
8497 
8498   format %{ "rolq    $dst, $shift" %}
8499   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8500   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8501   ins_pipe(ialu_reg_reg);
8502 %}
8503 // end of ROL expand
8504 
8505 // Rotate Left by one
8506 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8507 %{
8508   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8509 
8510   expand %{
8511     rolL_rReg_imm1(dst, cr);
8512   %}
8513 %}
8514 
8515 // Rotate Left by 8-bit immediate
8516 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8517 %{
8518   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8519   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8520 
8521   expand %{
8522     rolL_rReg_imm8(dst, lshift, cr);
8523   %}
8524 %}
8525 
8526 // Rotate Left by variable
8527 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8528 %{
8529   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8530 
8531   expand %{
8532     rolL_rReg_CL(dst, shift, cr);
8533   %}
8534 %}
8535 
8536 // Rotate Left by variable
8537 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8538 %{
8539   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8540 
8541   expand %{
8542     rolL_rReg_CL(dst, shift, cr);
8543   %}
8544 %}
8545 
8546 // ROR expand
8547 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8548 %{
8549   effect(USE_DEF dst, KILL cr);
8550 
8551   format %{ "rorq    $dst" %}
8552   opcode(0xD1, 0x1); /* D1 /1 */
8553   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8554   ins_pipe(ialu_reg);
8555 %}
8556 
8557 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8558 %{
8559   effect(USE_DEF dst, USE shift, KILL cr);
8560 
8561   format %{ "rorq    $dst, $shift" %}
8562   opcode(0xC1, 0x1); /* C1 /1 ib */
8563   ins_encode(reg_opc_imm_wide(dst, shift));
8564   ins_pipe(ialu_reg);
8565 %}
8566 
8567 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8568 %{
8569   effect(USE_DEF dst, USE shift, KILL cr);
8570 
8571   format %{ "rorq    $dst, $shift" %}
8572   opcode(0xD3, 0x1); /* D3 /1 */
8573   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8574   ins_pipe(ialu_reg_reg);
8575 %}
8576 // end of ROR expand
8577 
8578 // Rotate Right by one
8579 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8580 %{
8581   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8582 
8583   expand %{
8584     rorL_rReg_imm1(dst, cr);
8585   %}
8586 %}
8587 
8588 // Rotate Right by 8-bit immediate
8589 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8590 %{
8591   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8592   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8593 
8594   expand %{
8595     rorL_rReg_imm8(dst, rshift, cr);
8596   %}
8597 %}
8598 
8599 // Rotate Right by variable
8600 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8601 %{
8602   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8603 
8604   expand %{
8605     rorL_rReg_CL(dst, shift, cr);
8606   %}
8607 %}
8608 
8609 // Rotate Right by variable
8610 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8611 %{
8612   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8613 
8614   expand %{
8615     rorL_rReg_CL(dst, shift, cr);
8616   %}
8617 %}
8618 
8619 // Logical Instructions
8620 
8621 // Integer Logical Instructions
8622 
8623 // And Instructions
8624 // And Register with Register
8625 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8626 %{
8627   match(Set dst (AndI dst src));
8628   effect(KILL cr);
8629 
8630   format %{ "andl    $dst, $src\t# int" %}
8631   opcode(0x23);
8632   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8633   ins_pipe(ialu_reg_reg);
8634 %}
8635 
8636 // And Register with Immediate 255
8637 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8638 %{
8639   match(Set dst (AndI dst src));
8640 
8641   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8642   opcode(0x0F, 0xB6);
8643   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8644   ins_pipe(ialu_reg);
8645 %}
8646 
8647 // And Register with Immediate 255 and promote to long
8648 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8649 %{
8650   match(Set dst (ConvI2L (AndI src mask)));
8651 
8652   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8653   opcode(0x0F, 0xB6);
8654   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8655   ins_pipe(ialu_reg);
8656 %}
8657 
8658 // And Register with Immediate 65535
8659 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
8660 %{
8661   match(Set dst (AndI dst src));
8662 
8663   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
8664   opcode(0x0F, 0xB7);
8665   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8666   ins_pipe(ialu_reg);
8667 %}
8668 
8669 // And Register with Immediate 65535 and promote to long
8670 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8671 %{
8672   match(Set dst (ConvI2L (AndI src mask)));
8673 
8674   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
8675   opcode(0x0F, 0xB7);
8676   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8677   ins_pipe(ialu_reg);
8678 %}
8679 
8680 // And Register with Immediate
8681 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8682 %{
8683   match(Set dst (AndI dst src));
8684   effect(KILL cr);
8685 
8686   format %{ "andl    $dst, $src\t# int" %}
8687   opcode(0x81, 0x04); /* Opcode 81 /4 */
8688   ins_encode(OpcSErm(dst, src), Con8or32(src));
8689   ins_pipe(ialu_reg);
8690 %}
8691 
8692 // And Register with Memory
8693 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8694 %{
8695   match(Set dst (AndI dst (LoadI src)));
8696   effect(KILL cr);
8697 
8698   ins_cost(125);
8699   format %{ "andl    $dst, $src\t# int" %}
8700   opcode(0x23);
8701   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8702   ins_pipe(ialu_reg_mem);
8703 %}
8704 
8705 // And Memory with Register
8706 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8707 %{
8708   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8709   effect(KILL cr);
8710 
8711   ins_cost(150);
8712   format %{ "andl    $dst, $src\t# int" %}
8713   opcode(0x21); /* Opcode 21 /r */
8714   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8715   ins_pipe(ialu_mem_reg);
8716 %}
8717 
8718 // And Memory with Immediate
8719 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
8720 %{
8721   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8722   effect(KILL cr);
8723 
8724   ins_cost(125);
8725   format %{ "andl    $dst, $src\t# int" %}
8726   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8727   ins_encode(REX_mem(dst), OpcSE(src),
8728              RM_opc_mem(secondary, dst), Con8or32(src));
8729   ins_pipe(ialu_mem_imm);
8730 %}
8731 
8732 // BMI1 instructions
8733 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
8734   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
8735   predicate(UseBMI1Instructions);
8736   effect(KILL cr);
8737 
8738   ins_cost(125);
8739   format %{ "andnl  $dst, $src1, $src2" %}
8740 
8741   ins_encode %{
8742     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
8743   %}
8744   ins_pipe(ialu_reg_mem);
8745 %}
8746 
8747 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
8748   match(Set dst (AndI (XorI src1 minus_1) src2));
8749   predicate(UseBMI1Instructions);
8750   effect(KILL cr);
8751 
8752   format %{ "andnl  $dst, $src1, $src2" %}
8753 
8754   ins_encode %{
8755     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
8756   %}
8757   ins_pipe(ialu_reg);
8758 %}
8759 
8760 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
8761   match(Set dst (AndI (SubI imm_zero src) src));
8762   predicate(UseBMI1Instructions);
8763   effect(KILL cr);
8764 
8765   format %{ "blsil  $dst, $src" %}
8766 
8767   ins_encode %{
8768     __ blsil($dst$$Register, $src$$Register);
8769   %}
8770   ins_pipe(ialu_reg);
8771 %}
8772 
8773 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
8774   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
8775   predicate(UseBMI1Instructions);
8776   effect(KILL cr);
8777 
8778   ins_cost(125);
8779   format %{ "blsil  $dst, $src" %}
8780 
8781   ins_encode %{
8782     __ blsil($dst$$Register, $src$$Address);
8783   %}
8784   ins_pipe(ialu_reg_mem);
8785 %}
8786 
8787 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8788 %{
8789   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
8790   predicate(UseBMI1Instructions);
8791   effect(KILL cr);
8792 
8793   ins_cost(125);
8794   format %{ "blsmskl $dst, $src" %}
8795 
8796   ins_encode %{
8797     __ blsmskl($dst$$Register, $src$$Address);
8798   %}
8799   ins_pipe(ialu_reg_mem);
8800 %}
8801 
8802 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8803 %{
8804   match(Set dst (XorI (AddI src minus_1) src));
8805   predicate(UseBMI1Instructions);
8806   effect(KILL cr);
8807 
8808   format %{ "blsmskl $dst, $src" %}
8809 
8810   ins_encode %{
8811     __ blsmskl($dst$$Register, $src$$Register);
8812   %}
8813 
8814   ins_pipe(ialu_reg);
8815 %}
8816 
8817 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8818 %{
8819   match(Set dst (AndI (AddI src minus_1) src) );
8820   predicate(UseBMI1Instructions);
8821   effect(KILL cr);
8822 
8823   format %{ "blsrl  $dst, $src" %}
8824 
8825   ins_encode %{
8826     __ blsrl($dst$$Register, $src$$Register);
8827   %}
8828 
8829   ins_pipe(ialu_reg_mem);
8830 %}
8831 
8832 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8833 %{
8834   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
8835   predicate(UseBMI1Instructions);
8836   effect(KILL cr);
8837 
8838   ins_cost(125);
8839   format %{ "blsrl  $dst, $src" %}
8840 
8841   ins_encode %{
8842     __ blsrl($dst$$Register, $src$$Address);
8843   %}
8844 
8845   ins_pipe(ialu_reg);
8846 %}
8847 
8848 // Or Instructions
8849 // Or Register with Register
8850 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8851 %{
8852   match(Set dst (OrI dst src));
8853   effect(KILL cr);
8854 
8855   format %{ "orl     $dst, $src\t# int" %}
8856   opcode(0x0B);
8857   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8858   ins_pipe(ialu_reg_reg);
8859 %}
8860 
8861 // Or Register with Immediate
8862 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8863 %{
8864   match(Set dst (OrI dst src));
8865   effect(KILL cr);
8866 
8867   format %{ "orl     $dst, $src\t# int" %}
8868   opcode(0x81, 0x01); /* Opcode 81 /1 id */
8869   ins_encode(OpcSErm(dst, src), Con8or32(src));
8870   ins_pipe(ialu_reg);
8871 %}
8872 
8873 // Or Register with Memory
8874 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8875 %{
8876   match(Set dst (OrI dst (LoadI src)));
8877   effect(KILL cr);
8878 
8879   ins_cost(125);
8880   format %{ "orl     $dst, $src\t# int" %}
8881   opcode(0x0B);
8882   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8883   ins_pipe(ialu_reg_mem);
8884 %}
8885 
8886 // Or Memory with Register
8887 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8888 %{
8889   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8890   effect(KILL cr);
8891 
8892   ins_cost(150);
8893   format %{ "orl     $dst, $src\t# int" %}
8894   opcode(0x09); /* Opcode 09 /r */
8895   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8896   ins_pipe(ialu_mem_reg);
8897 %}
8898 
8899 // Or Memory with Immediate
8900 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
8901 %{
8902   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8903   effect(KILL cr);
8904 
8905   ins_cost(125);
8906   format %{ "orl     $dst, $src\t# int" %}
8907   opcode(0x81, 0x1); /* Opcode 81 /1 id */
8908   ins_encode(REX_mem(dst), OpcSE(src),
8909              RM_opc_mem(secondary, dst), Con8or32(src));
8910   ins_pipe(ialu_mem_imm);
8911 %}
8912 
8913 // Xor Instructions
8914 // Xor Register with Register
8915 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8916 %{
8917   match(Set dst (XorI dst src));
8918   effect(KILL cr);
8919 
8920   format %{ "xorl    $dst, $src\t# int" %}
8921   opcode(0x33);
8922   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8923   ins_pipe(ialu_reg_reg);
8924 %}
8925 
8926 // Xor Register with Immediate -1
8927 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
8928   match(Set dst (XorI dst imm));
8929 
8930   format %{ "not    $dst" %}
8931   ins_encode %{
8932      __ notl($dst$$Register);
8933   %}
8934   ins_pipe(ialu_reg);
8935 %}
8936 
8937 // Xor Register with Immediate
8938 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8939 %{
8940   match(Set dst (XorI dst src));
8941   effect(KILL cr);
8942 
8943   format %{ "xorl    $dst, $src\t# int" %}
8944   opcode(0x81, 0x06); /* Opcode 81 /6 id */
8945   ins_encode(OpcSErm(dst, src), Con8or32(src));
8946   ins_pipe(ialu_reg);
8947 %}
8948 
8949 // Xor Register with Memory
8950 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8951 %{
8952   match(Set dst (XorI dst (LoadI src)));
8953   effect(KILL cr);
8954 
8955   ins_cost(125);
8956   format %{ "xorl    $dst, $src\t# int" %}
8957   opcode(0x33);
8958   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8959   ins_pipe(ialu_reg_mem);
8960 %}
8961 
8962 // Xor Memory with Register
8963 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8964 %{
8965   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
8966   effect(KILL cr);
8967 
8968   ins_cost(150);
8969   format %{ "xorl    $dst, $src\t# int" %}
8970   opcode(0x31); /* Opcode 31 /r */
8971   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8972   ins_pipe(ialu_mem_reg);
8973 %}
8974 
8975 // Xor Memory with Immediate
8976 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
8977 %{
8978   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
8979   effect(KILL cr);
8980 
8981   ins_cost(125);
8982   format %{ "xorl    $dst, $src\t# int" %}
8983   opcode(0x81, 0x6); /* Opcode 81 /6 id */
8984   ins_encode(REX_mem(dst), OpcSE(src),
8985              RM_opc_mem(secondary, dst), Con8or32(src));
8986   ins_pipe(ialu_mem_imm);
8987 %}
8988 
8989 
8990 // Long Logical Instructions
8991 
8992 // And Instructions
8993 // And Register with Register
8994 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8995 %{
8996   match(Set dst (AndL dst src));
8997   effect(KILL cr);
8998 
8999   format %{ "andq    $dst, $src\t# long" %}
9000   opcode(0x23);
9001   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9002   ins_pipe(ialu_reg_reg);
9003 %}
9004 
9005 // And Register with Immediate 255
9006 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9007 %{
9008   match(Set dst (AndL dst src));
9009 
9010   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9011   opcode(0x0F, 0xB6);
9012   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9013   ins_pipe(ialu_reg);
9014 %}
9015 
9016 // And Register with Immediate 65535
9017 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9018 %{
9019   match(Set dst (AndL dst src));
9020 
9021   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9022   opcode(0x0F, 0xB7);
9023   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9024   ins_pipe(ialu_reg);
9025 %}
9026 
9027 // And Register with Immediate
9028 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9029 %{
9030   match(Set dst (AndL dst src));
9031   effect(KILL cr);
9032 
9033   format %{ "andq    $dst, $src\t# long" %}
9034   opcode(0x81, 0x04); /* Opcode 81 /4 */
9035   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9036   ins_pipe(ialu_reg);
9037 %}
9038 
9039 // And Register with Memory
9040 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9041 %{
9042   match(Set dst (AndL dst (LoadL src)));
9043   effect(KILL cr);
9044 
9045   ins_cost(125);
9046   format %{ "andq    $dst, $src\t# long" %}
9047   opcode(0x23);
9048   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9049   ins_pipe(ialu_reg_mem);
9050 %}
9051 
9052 // And Memory with Register
9053 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9054 %{
9055   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9056   effect(KILL cr);
9057 
9058   ins_cost(150);
9059   format %{ "andq    $dst, $src\t# long" %}
9060   opcode(0x21); /* Opcode 21 /r */
9061   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9062   ins_pipe(ialu_mem_reg);
9063 %}
9064 
9065 // And Memory with Immediate
9066 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9067 %{
9068   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9069   effect(KILL cr);
9070 
9071   ins_cost(125);
9072   format %{ "andq    $dst, $src\t# long" %}
9073   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9074   ins_encode(REX_mem_wide(dst), OpcSE(src),
9075              RM_opc_mem(secondary, dst), Con8or32(src));
9076   ins_pipe(ialu_mem_imm);
9077 %}
9078 
9079 // BMI1 instructions
9080 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9081   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9082   predicate(UseBMI1Instructions);
9083   effect(KILL cr);
9084 
9085   ins_cost(125);
9086   format %{ "andnq  $dst, $src1, $src2" %}
9087 
9088   ins_encode %{
9089     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9090   %}
9091   ins_pipe(ialu_reg_mem);
9092 %}
9093 
9094 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9095   match(Set dst (AndL (XorL src1 minus_1) src2));
9096   predicate(UseBMI1Instructions);
9097   effect(KILL cr);
9098 
9099   format %{ "andnq  $dst, $src1, $src2" %}
9100 
9101   ins_encode %{
9102   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9103   %}
9104   ins_pipe(ialu_reg_mem);
9105 %}
9106 
9107 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9108   match(Set dst (AndL (SubL imm_zero src) src));
9109   predicate(UseBMI1Instructions);
9110   effect(KILL cr);
9111 
9112   format %{ "blsiq  $dst, $src" %}
9113 
9114   ins_encode %{
9115     __ blsiq($dst$$Register, $src$$Register);
9116   %}
9117   ins_pipe(ialu_reg);
9118 %}
9119 
9120 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9121   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9122   predicate(UseBMI1Instructions);
9123   effect(KILL cr);
9124 
9125   ins_cost(125);
9126   format %{ "blsiq  $dst, $src" %}
9127 
9128   ins_encode %{
9129     __ blsiq($dst$$Register, $src$$Address);
9130   %}
9131   ins_pipe(ialu_reg_mem);
9132 %}
9133 
9134 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9135 %{
9136   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9137   predicate(UseBMI1Instructions);
9138   effect(KILL cr);
9139 
9140   ins_cost(125);
9141   format %{ "blsmskq $dst, $src" %}
9142 
9143   ins_encode %{
9144     __ blsmskq($dst$$Register, $src$$Address);
9145   %}
9146   ins_pipe(ialu_reg_mem);
9147 %}
9148 
9149 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9150 %{
9151   match(Set dst (XorL (AddL src minus_1) src));
9152   predicate(UseBMI1Instructions);
9153   effect(KILL cr);
9154 
9155   format %{ "blsmskq $dst, $src" %}
9156 
9157   ins_encode %{
9158     __ blsmskq($dst$$Register, $src$$Register);
9159   %}
9160 
9161   ins_pipe(ialu_reg);
9162 %}
9163 
9164 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9165 %{
9166   match(Set dst (AndL (AddL src minus_1) src) );
9167   predicate(UseBMI1Instructions);
9168   effect(KILL cr);
9169 
9170   format %{ "blsrq  $dst, $src" %}
9171 
9172   ins_encode %{
9173     __ blsrq($dst$$Register, $src$$Register);
9174   %}
9175 
9176   ins_pipe(ialu_reg);
9177 %}
9178 
9179 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9180 %{
9181   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9182   predicate(UseBMI1Instructions);
9183   effect(KILL cr);
9184 
9185   ins_cost(125);
9186   format %{ "blsrq  $dst, $src" %}
9187 
9188   ins_encode %{
9189     __ blsrq($dst$$Register, $src$$Address);
9190   %}
9191 
9192   ins_pipe(ialu_reg);
9193 %}
9194 
9195 // Or Instructions
9196 // Or Register with Register
9197 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9198 %{
9199   match(Set dst (OrL dst src));
9200   effect(KILL cr);
9201 
9202   format %{ "orq     $dst, $src\t# long" %}
9203   opcode(0x0B);
9204   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9205   ins_pipe(ialu_reg_reg);
9206 %}
9207 
9208 // Use any_RegP to match R15 (TLS register) without spilling.
9209 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9210   match(Set dst (OrL dst (CastP2X src)));
9211   effect(KILL cr);
9212 
9213   format %{ "orq     $dst, $src\t# long" %}
9214   opcode(0x0B);
9215   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9216   ins_pipe(ialu_reg_reg);
9217 %}
9218 
9219 
9220 // Or Register with Immediate
9221 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9222 %{
9223   match(Set dst (OrL dst src));
9224   effect(KILL cr);
9225 
9226   format %{ "orq     $dst, $src\t# long" %}
9227   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9228   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9229   ins_pipe(ialu_reg);
9230 %}
9231 
9232 // Or Register with Memory
9233 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9234 %{
9235   match(Set dst (OrL dst (LoadL src)));
9236   effect(KILL cr);
9237 
9238   ins_cost(125);
9239   format %{ "orq     $dst, $src\t# long" %}
9240   opcode(0x0B);
9241   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9242   ins_pipe(ialu_reg_mem);
9243 %}
9244 
9245 // Or Memory with Register
9246 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9247 %{
9248   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9249   effect(KILL cr);
9250 
9251   ins_cost(150);
9252   format %{ "orq     $dst, $src\t# long" %}
9253   opcode(0x09); /* Opcode 09 /r */
9254   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9255   ins_pipe(ialu_mem_reg);
9256 %}
9257 
9258 // Or Memory with Immediate
9259 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9260 %{
9261   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9262   effect(KILL cr);
9263 
9264   ins_cost(125);
9265   format %{ "orq     $dst, $src\t# long" %}
9266   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9267   ins_encode(REX_mem_wide(dst), OpcSE(src),
9268              RM_opc_mem(secondary, dst), Con8or32(src));
9269   ins_pipe(ialu_mem_imm);
9270 %}
9271 
9272 // Xor Instructions
9273 // Xor Register with Register
9274 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9275 %{
9276   match(Set dst (XorL dst src));
9277   effect(KILL cr);
9278 
9279   format %{ "xorq    $dst, $src\t# long" %}
9280   opcode(0x33);
9281   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9282   ins_pipe(ialu_reg_reg);
9283 %}
9284 
9285 // Xor Register with Immediate -1
9286 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9287   match(Set dst (XorL dst imm));
9288 
9289   format %{ "notq   $dst" %}
9290   ins_encode %{
9291      __ notq($dst$$Register);
9292   %}
9293   ins_pipe(ialu_reg);
9294 %}
9295 
9296 // Xor Register with Immediate
9297 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9298 %{
9299   match(Set dst (XorL dst src));
9300   effect(KILL cr);
9301 
9302   format %{ "xorq    $dst, $src\t# long" %}
9303   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9304   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9305   ins_pipe(ialu_reg);
9306 %}
9307 
9308 // Xor Register with Memory
9309 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9310 %{
9311   match(Set dst (XorL dst (LoadL src)));
9312   effect(KILL cr);
9313 
9314   ins_cost(125);
9315   format %{ "xorq    $dst, $src\t# long" %}
9316   opcode(0x33);
9317   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9318   ins_pipe(ialu_reg_mem);
9319 %}
9320 
9321 // Xor Memory with Register
9322 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9323 %{
9324   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9325   effect(KILL cr);
9326 
9327   ins_cost(150);
9328   format %{ "xorq    $dst, $src\t# long" %}
9329   opcode(0x31); /* Opcode 31 /r */
9330   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9331   ins_pipe(ialu_mem_reg);
9332 %}
9333 
9334 // Xor Memory with Immediate
9335 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9336 %{
9337   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9338   effect(KILL cr);
9339 
9340   ins_cost(125);
9341   format %{ "xorq    $dst, $src\t# long" %}
9342   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9343   ins_encode(REX_mem_wide(dst), OpcSE(src),
9344              RM_opc_mem(secondary, dst), Con8or32(src));
9345   ins_pipe(ialu_mem_imm);
9346 %}
9347 
9348 // Convert Int to Boolean
9349 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9350 %{
9351   match(Set dst (Conv2B src));
9352   effect(KILL cr);
9353 
9354   format %{ "testl   $src, $src\t# ci2b\n\t"
9355             "setnz   $dst\n\t"
9356             "movzbl  $dst, $dst" %}
9357   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9358              setNZ_reg(dst),
9359              REX_reg_breg(dst, dst), // movzbl
9360              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9361   ins_pipe(pipe_slow); // XXX
9362 %}
9363 
9364 // Convert Pointer to Boolean
9365 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9366 %{
9367   match(Set dst (Conv2B src));
9368   effect(KILL cr);
9369 
9370   format %{ "testq   $src, $src\t# cp2b\n\t"
9371             "setnz   $dst\n\t"
9372             "movzbl  $dst, $dst" %}
9373   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9374              setNZ_reg(dst),
9375              REX_reg_breg(dst, dst), // movzbl
9376              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9377   ins_pipe(pipe_slow); // XXX
9378 %}
9379 
9380 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9381 %{
9382   match(Set dst (CmpLTMask p q));
9383   effect(KILL cr);
9384 
9385   ins_cost(400);
9386   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9387             "setlt   $dst\n\t"
9388             "movzbl  $dst, $dst\n\t"
9389             "negl    $dst" %}
9390   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9391              setLT_reg(dst),
9392              REX_reg_breg(dst, dst), // movzbl
9393              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9394              neg_reg(dst));
9395   ins_pipe(pipe_slow);
9396 %}
9397 
9398 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9399 %{
9400   match(Set dst (CmpLTMask dst zero));
9401   effect(KILL cr);
9402 
9403   ins_cost(100);
9404   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9405   ins_encode %{
9406   __ sarl($dst$$Register, 31);
9407   %}
9408   ins_pipe(ialu_reg);
9409 %}
9410 
9411 /* Better to save a register than avoid a branch */
9412 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9413 %{
9414   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9415   effect(KILL cr);
9416   ins_cost(300);
9417   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9418             "jge    done\n\t"
9419             "addl   $p,$y\n"
9420             "done:  " %}
9421   ins_encode %{
9422     Register Rp = $p$$Register;
9423     Register Rq = $q$$Register;
9424     Register Ry = $y$$Register;
9425     Label done;
9426     __ subl(Rp, Rq);
9427     __ jccb(Assembler::greaterEqual, done);
9428     __ addl(Rp, Ry);
9429     __ bind(done);
9430   %}
9431   ins_pipe(pipe_cmplt);
9432 %}
9433 
9434 /* Better to save a register than avoid a branch */
9435 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9436 %{
9437   match(Set y (AndI (CmpLTMask p q) y));
9438   effect(KILL cr);
9439 
9440   ins_cost(300);
9441 
9442   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9443             "jlt      done\n\t"
9444             "xorl     $y, $y\n"
9445             "done:  " %}
9446   ins_encode %{
9447     Register Rp = $p$$Register;
9448     Register Rq = $q$$Register;
9449     Register Ry = $y$$Register;
9450     Label done;
9451     __ cmpl(Rp, Rq);
9452     __ jccb(Assembler::less, done);
9453     __ xorl(Ry, Ry);
9454     __ bind(done);
9455   %}
9456   ins_pipe(pipe_cmplt);
9457 %}
9458 
9459 
9460 //---------- FP Instructions------------------------------------------------
9461 
9462 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9463 %{
9464   match(Set cr (CmpF src1 src2));
9465 
9466   ins_cost(145);
9467   format %{ "ucomiss $src1, $src2\n\t"
9468             "jnp,s   exit\n\t"
9469             "pushfq\t# saw NaN, set CF\n\t"
9470             "andq    [rsp], #0xffffff2b\n\t"
9471             "popfq\n"
9472     "exit:" %}
9473   ins_encode %{
9474     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9475     emit_cmpfp_fixup(_masm);
9476   %}
9477   ins_pipe(pipe_slow);
9478 %}
9479 
9480 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9481   match(Set cr (CmpF src1 src2));
9482 
9483   ins_cost(100);
9484   format %{ "ucomiss $src1, $src2" %}
9485   ins_encode %{
9486     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9487   %}
9488   ins_pipe(pipe_slow);
9489 %}
9490 
9491 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9492 %{
9493   match(Set cr (CmpF src1 (LoadF src2)));
9494 
9495   ins_cost(145);
9496   format %{ "ucomiss $src1, $src2\n\t"
9497             "jnp,s   exit\n\t"
9498             "pushfq\t# saw NaN, set CF\n\t"
9499             "andq    [rsp], #0xffffff2b\n\t"
9500             "popfq\n"
9501     "exit:" %}
9502   ins_encode %{
9503     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9504     emit_cmpfp_fixup(_masm);
9505   %}
9506   ins_pipe(pipe_slow);
9507 %}
9508 
9509 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9510   match(Set cr (CmpF src1 (LoadF src2)));
9511 
9512   ins_cost(100);
9513   format %{ "ucomiss $src1, $src2" %}
9514   ins_encode %{
9515     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9516   %}
9517   ins_pipe(pipe_slow);
9518 %}
9519 
9520 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9521   match(Set cr (CmpF src con));
9522 
9523   ins_cost(145);
9524   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9525             "jnp,s   exit\n\t"
9526             "pushfq\t# saw NaN, set CF\n\t"
9527             "andq    [rsp], #0xffffff2b\n\t"
9528             "popfq\n"
9529     "exit:" %}
9530   ins_encode %{
9531     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9532     emit_cmpfp_fixup(_masm);
9533   %}
9534   ins_pipe(pipe_slow);
9535 %}
9536 
9537 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9538   match(Set cr (CmpF src con));
9539   ins_cost(100);
9540   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9541   ins_encode %{
9542     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9543   %}
9544   ins_pipe(pipe_slow);
9545 %}
9546 
9547 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9548 %{
9549   match(Set cr (CmpD src1 src2));
9550 
9551   ins_cost(145);
9552   format %{ "ucomisd $src1, $src2\n\t"
9553             "jnp,s   exit\n\t"
9554             "pushfq\t# saw NaN, set CF\n\t"
9555             "andq    [rsp], #0xffffff2b\n\t"
9556             "popfq\n"
9557     "exit:" %}
9558   ins_encode %{
9559     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9560     emit_cmpfp_fixup(_masm);
9561   %}
9562   ins_pipe(pipe_slow);
9563 %}
9564 
9565 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9566   match(Set cr (CmpD src1 src2));
9567 
9568   ins_cost(100);
9569   format %{ "ucomisd $src1, $src2 test" %}
9570   ins_encode %{
9571     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9572   %}
9573   ins_pipe(pipe_slow);
9574 %}
9575 
9576 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9577 %{
9578   match(Set cr (CmpD src1 (LoadD src2)));
9579 
9580   ins_cost(145);
9581   format %{ "ucomisd $src1, $src2\n\t"
9582             "jnp,s   exit\n\t"
9583             "pushfq\t# saw NaN, set CF\n\t"
9584             "andq    [rsp], #0xffffff2b\n\t"
9585             "popfq\n"
9586     "exit:" %}
9587   ins_encode %{
9588     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9589     emit_cmpfp_fixup(_masm);
9590   %}
9591   ins_pipe(pipe_slow);
9592 %}
9593 
9594 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9595   match(Set cr (CmpD src1 (LoadD src2)));
9596 
9597   ins_cost(100);
9598   format %{ "ucomisd $src1, $src2" %}
9599   ins_encode %{
9600     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9601   %}
9602   ins_pipe(pipe_slow);
9603 %}
9604 
9605 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9606   match(Set cr (CmpD src con));
9607 
9608   ins_cost(145);
9609   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9610             "jnp,s   exit\n\t"
9611             "pushfq\t# saw NaN, set CF\n\t"
9612             "andq    [rsp], #0xffffff2b\n\t"
9613             "popfq\n"
9614     "exit:" %}
9615   ins_encode %{
9616     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9617     emit_cmpfp_fixup(_masm);
9618   %}
9619   ins_pipe(pipe_slow);
9620 %}
9621 
9622 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9623   match(Set cr (CmpD src con));
9624   ins_cost(100);
9625   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9626   ins_encode %{
9627     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9628   %}
9629   ins_pipe(pipe_slow);
9630 %}
9631 
9632 // Compare into -1,0,1
9633 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9634 %{
9635   match(Set dst (CmpF3 src1 src2));
9636   effect(KILL cr);
9637 
9638   ins_cost(275);
9639   format %{ "ucomiss $src1, $src2\n\t"
9640             "movl    $dst, #-1\n\t"
9641             "jp,s    done\n\t"
9642             "jb,s    done\n\t"
9643             "setne   $dst\n\t"
9644             "movzbl  $dst, $dst\n"
9645     "done:" %}
9646   ins_encode %{
9647     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9648     emit_cmpfp3(_masm, $dst$$Register);
9649   %}
9650   ins_pipe(pipe_slow);
9651 %}
9652 
9653 // Compare into -1,0,1
9654 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9655 %{
9656   match(Set dst (CmpF3 src1 (LoadF src2)));
9657   effect(KILL cr);
9658 
9659   ins_cost(275);
9660   format %{ "ucomiss $src1, $src2\n\t"
9661             "movl    $dst, #-1\n\t"
9662             "jp,s    done\n\t"
9663             "jb,s    done\n\t"
9664             "setne   $dst\n\t"
9665             "movzbl  $dst, $dst\n"
9666     "done:" %}
9667   ins_encode %{
9668     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9669     emit_cmpfp3(_masm, $dst$$Register);
9670   %}
9671   ins_pipe(pipe_slow);
9672 %}
9673 
9674 // Compare into -1,0,1
9675 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9676   match(Set dst (CmpF3 src con));
9677   effect(KILL cr);
9678 
9679   ins_cost(275);
9680   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9681             "movl    $dst, #-1\n\t"
9682             "jp,s    done\n\t"
9683             "jb,s    done\n\t"
9684             "setne   $dst\n\t"
9685             "movzbl  $dst, $dst\n"
9686     "done:" %}
9687   ins_encode %{
9688     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9689     emit_cmpfp3(_masm, $dst$$Register);
9690   %}
9691   ins_pipe(pipe_slow);
9692 %}
9693 
9694 // Compare into -1,0,1
9695 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9696 %{
9697   match(Set dst (CmpD3 src1 src2));
9698   effect(KILL cr);
9699 
9700   ins_cost(275);
9701   format %{ "ucomisd $src1, $src2\n\t"
9702             "movl    $dst, #-1\n\t"
9703             "jp,s    done\n\t"
9704             "jb,s    done\n\t"
9705             "setne   $dst\n\t"
9706             "movzbl  $dst, $dst\n"
9707     "done:" %}
9708   ins_encode %{
9709     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9710     emit_cmpfp3(_masm, $dst$$Register);
9711   %}
9712   ins_pipe(pipe_slow);
9713 %}
9714 
9715 // Compare into -1,0,1
9716 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9717 %{
9718   match(Set dst (CmpD3 src1 (LoadD src2)));
9719   effect(KILL cr);
9720 
9721   ins_cost(275);
9722   format %{ "ucomisd $src1, $src2\n\t"
9723             "movl    $dst, #-1\n\t"
9724             "jp,s    done\n\t"
9725             "jb,s    done\n\t"
9726             "setne   $dst\n\t"
9727             "movzbl  $dst, $dst\n"
9728     "done:" %}
9729   ins_encode %{
9730     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9731     emit_cmpfp3(_masm, $dst$$Register);
9732   %}
9733   ins_pipe(pipe_slow);
9734 %}
9735 
9736 // Compare into -1,0,1
9737 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
9738   match(Set dst (CmpD3 src con));
9739   effect(KILL cr);
9740 
9741   ins_cost(275);
9742   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9743             "movl    $dst, #-1\n\t"
9744             "jp,s    done\n\t"
9745             "jb,s    done\n\t"
9746             "setne   $dst\n\t"
9747             "movzbl  $dst, $dst\n"
9748     "done:" %}
9749   ins_encode %{
9750     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9751     emit_cmpfp3(_masm, $dst$$Register);
9752   %}
9753   ins_pipe(pipe_slow);
9754 %}
9755 
9756 // -----------Trig and Trancendental Instructions------------------------------
9757 instruct cosD_reg(regD dst) %{
9758   match(Set dst (CosD dst));
9759 
9760   format %{ "dcos   $dst\n\t" %}
9761   opcode(0xD9, 0xFF);
9762   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9763   ins_pipe( pipe_slow );
9764 %}
9765 
9766 instruct sinD_reg(regD dst) %{
9767   match(Set dst (SinD dst));
9768 
9769   format %{ "dsin   $dst\n\t" %}
9770   opcode(0xD9, 0xFE);
9771   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9772   ins_pipe( pipe_slow );
9773 %}
9774 
9775 instruct tanD_reg(regD dst) %{
9776   match(Set dst (TanD dst));
9777 
9778   format %{ "dtan   $dst\n\t" %}
9779   ins_encode( Push_SrcXD(dst),
9780               Opcode(0xD9), Opcode(0xF2),   //fptan
9781               Opcode(0xDD), Opcode(0xD8),   //fstp st
9782               Push_ResultXD(dst) );
9783   ins_pipe( pipe_slow );
9784 %}
9785 
9786 instruct log10D_reg(regD dst) %{
9787   // The source and result Double operands in XMM registers
9788   match(Set dst (Log10D dst));
9789   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
9790   // fyl2x        ; compute log_10(2) * log_2(x)
9791   format %{ "fldlg2\t\t\t#Log10\n\t"
9792             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
9793          %}
9794    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
9795               Push_SrcXD(dst),
9796               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9797               Push_ResultXD(dst));
9798 
9799   ins_pipe( pipe_slow );
9800 %}
9801 
9802 instruct logD_reg(regD dst) %{
9803   // The source and result Double operands in XMM registers
9804   match(Set dst (LogD dst));
9805   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
9806   // fyl2x        ; compute log_e(2) * log_2(x)
9807   format %{ "fldln2\t\t\t#Log_e\n\t"
9808             "fyl2x\t\t\t# Q=Log_e*Log_2(x)\n\t"
9809          %}
9810   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
9811               Push_SrcXD(dst),
9812               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9813               Push_ResultXD(dst));
9814   ins_pipe( pipe_slow );
9815 %}
9816 
9817 instruct powD_reg(regD dst, regD src0, regD src1, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9818   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
9819   effect(KILL rax, KILL rdx, KILL rcx, KILL cr);
9820   format %{ "fast_pow $src0 $src1 -> $dst  // KILL $rax, $rcx, $rdx" %}
9821   ins_encode %{
9822     __ subptr(rsp, 8);
9823     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);
9824     __ fld_d(Address(rsp, 0));
9825     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);
9826     __ fld_d(Address(rsp, 0));
9827     __ fast_pow();
9828     __ fstp_d(Address(rsp, 0));
9829     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9830     __ addptr(rsp, 8);
9831   %}
9832   ins_pipe( pipe_slow );
9833 %}
9834 
9835 instruct expD_reg(regD dst, regD src, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9836   match(Set dst (ExpD src));
9837   effect(KILL rax, KILL rcx, KILL rdx, KILL cr);
9838   format %{ "fast_exp $dst -> $src  // KILL $rax, $rcx, $rdx" %}
9839   ins_encode %{
9840     __ subptr(rsp, 8);
9841     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9842     __ fld_d(Address(rsp, 0));
9843     __ fast_exp();
9844     __ fstp_d(Address(rsp, 0));
9845     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9846     __ addptr(rsp, 8);
9847   %}
9848   ins_pipe( pipe_slow );
9849 %}
9850 
9851 //----------Arithmetic Conversion Instructions---------------------------------
9852 
9853 instruct roundFloat_nop(regF dst)
9854 %{
9855   match(Set dst (RoundFloat dst));
9856 
9857   ins_cost(0);
9858   ins_encode();
9859   ins_pipe(empty);
9860 %}
9861 
9862 instruct roundDouble_nop(regD dst)
9863 %{
9864   match(Set dst (RoundDouble dst));
9865 
9866   ins_cost(0);
9867   ins_encode();
9868   ins_pipe(empty);
9869 %}
9870 
9871 instruct convF2D_reg_reg(regD dst, regF src)
9872 %{
9873   match(Set dst (ConvF2D src));
9874 
9875   format %{ "cvtss2sd $dst, $src" %}
9876   ins_encode %{
9877     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
9878   %}
9879   ins_pipe(pipe_slow); // XXX
9880 %}
9881 
9882 instruct convF2D_reg_mem(regD dst, memory src)
9883 %{
9884   match(Set dst (ConvF2D (LoadF src)));
9885 
9886   format %{ "cvtss2sd $dst, $src" %}
9887   ins_encode %{
9888     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
9889   %}
9890   ins_pipe(pipe_slow); // XXX
9891 %}
9892 
9893 instruct convD2F_reg_reg(regF dst, regD src)
9894 %{
9895   match(Set dst (ConvD2F src));
9896 
9897   format %{ "cvtsd2ss $dst, $src" %}
9898   ins_encode %{
9899     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
9900   %}
9901   ins_pipe(pipe_slow); // XXX
9902 %}
9903 
9904 instruct convD2F_reg_mem(regF dst, memory src)
9905 %{
9906   match(Set dst (ConvD2F (LoadD src)));
9907 
9908   format %{ "cvtsd2ss $dst, $src" %}
9909   ins_encode %{
9910     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
9911   %}
9912   ins_pipe(pipe_slow); // XXX
9913 %}
9914 
9915 // XXX do mem variants
9916 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
9917 %{
9918   match(Set dst (ConvF2I src));
9919   effect(KILL cr);
9920 
9921   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
9922             "cmpl    $dst, #0x80000000\n\t"
9923             "jne,s   done\n\t"
9924             "subq    rsp, #8\n\t"
9925             "movss   [rsp], $src\n\t"
9926             "call    f2i_fixup\n\t"
9927             "popq    $dst\n"
9928     "done:   "%}
9929   ins_encode %{
9930     Label done;
9931     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
9932     __ cmpl($dst$$Register, 0x80000000);
9933     __ jccb(Assembler::notEqual, done);
9934     __ subptr(rsp, 8);
9935     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9936     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
9937     __ pop($dst$$Register);
9938     __ bind(done);
9939   %}
9940   ins_pipe(pipe_slow);
9941 %}
9942 
9943 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
9944 %{
9945   match(Set dst (ConvF2L src));
9946   effect(KILL cr);
9947 
9948   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
9949             "cmpq    $dst, [0x8000000000000000]\n\t"
9950             "jne,s   done\n\t"
9951             "subq    rsp, #8\n\t"
9952             "movss   [rsp], $src\n\t"
9953             "call    f2l_fixup\n\t"
9954             "popq    $dst\n"
9955     "done:   "%}
9956   ins_encode %{
9957     Label done;
9958     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
9959     __ cmp64($dst$$Register,
9960              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9961     __ jccb(Assembler::notEqual, done);
9962     __ subptr(rsp, 8);
9963     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9964     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
9965     __ pop($dst$$Register);
9966     __ bind(done);
9967   %}
9968   ins_pipe(pipe_slow);
9969 %}
9970 
9971 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
9972 %{
9973   match(Set dst (ConvD2I src));
9974   effect(KILL cr);
9975 
9976   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
9977             "cmpl    $dst, #0x80000000\n\t"
9978             "jne,s   done\n\t"
9979             "subq    rsp, #8\n\t"
9980             "movsd   [rsp], $src\n\t"
9981             "call    d2i_fixup\n\t"
9982             "popq    $dst\n"
9983     "done:   "%}
9984   ins_encode %{
9985     Label done;
9986     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
9987     __ cmpl($dst$$Register, 0x80000000);
9988     __ jccb(Assembler::notEqual, done);
9989     __ subptr(rsp, 8);
9990     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9991     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
9992     __ pop($dst$$Register);
9993     __ bind(done);
9994   %}
9995   ins_pipe(pipe_slow);
9996 %}
9997 
9998 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
9999 %{
10000   match(Set dst (ConvD2L src));
10001   effect(KILL cr);
10002 
10003   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10004             "cmpq    $dst, [0x8000000000000000]\n\t"
10005             "jne,s   done\n\t"
10006             "subq    rsp, #8\n\t"
10007             "movsd   [rsp], $src\n\t"
10008             "call    d2l_fixup\n\t"
10009             "popq    $dst\n"
10010     "done:   "%}
10011   ins_encode %{
10012     Label done;
10013     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10014     __ cmp64($dst$$Register,
10015              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10016     __ jccb(Assembler::notEqual, done);
10017     __ subptr(rsp, 8);
10018     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10019     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10020     __ pop($dst$$Register);
10021     __ bind(done);
10022   %}
10023   ins_pipe(pipe_slow);
10024 %}
10025 
10026 instruct convI2F_reg_reg(regF dst, rRegI src)
10027 %{
10028   predicate(!UseXmmI2F);
10029   match(Set dst (ConvI2F src));
10030 
10031   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10032   ins_encode %{
10033     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10034   %}
10035   ins_pipe(pipe_slow); // XXX
10036 %}
10037 
10038 instruct convI2F_reg_mem(regF dst, memory src)
10039 %{
10040   match(Set dst (ConvI2F (LoadI src)));
10041 
10042   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10043   ins_encode %{
10044     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10045   %}
10046   ins_pipe(pipe_slow); // XXX
10047 %}
10048 
10049 instruct convI2D_reg_reg(regD dst, rRegI src)
10050 %{
10051   predicate(!UseXmmI2D);
10052   match(Set dst (ConvI2D src));
10053 
10054   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10055   ins_encode %{
10056     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10057   %}
10058   ins_pipe(pipe_slow); // XXX
10059 %}
10060 
10061 instruct convI2D_reg_mem(regD dst, memory src)
10062 %{
10063   match(Set dst (ConvI2D (LoadI src)));
10064 
10065   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10066   ins_encode %{
10067     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10068   %}
10069   ins_pipe(pipe_slow); // XXX
10070 %}
10071 
10072 instruct convXI2F_reg(regF dst, rRegI src)
10073 %{
10074   predicate(UseXmmI2F);
10075   match(Set dst (ConvI2F src));
10076 
10077   format %{ "movdl $dst, $src\n\t"
10078             "cvtdq2psl $dst, $dst\t# i2f" %}
10079   ins_encode %{
10080     __ movdl($dst$$XMMRegister, $src$$Register);
10081     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10082   %}
10083   ins_pipe(pipe_slow); // XXX
10084 %}
10085 
10086 instruct convXI2D_reg(regD dst, rRegI src)
10087 %{
10088   predicate(UseXmmI2D);
10089   match(Set dst (ConvI2D src));
10090 
10091   format %{ "movdl $dst, $src\n\t"
10092             "cvtdq2pdl $dst, $dst\t# i2d" %}
10093   ins_encode %{
10094     __ movdl($dst$$XMMRegister, $src$$Register);
10095     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10096   %}
10097   ins_pipe(pipe_slow); // XXX
10098 %}
10099 
10100 instruct convL2F_reg_reg(regF dst, rRegL src)
10101 %{
10102   match(Set dst (ConvL2F src));
10103 
10104   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10105   ins_encode %{
10106     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10107   %}
10108   ins_pipe(pipe_slow); // XXX
10109 %}
10110 
10111 instruct convL2F_reg_mem(regF dst, memory src)
10112 %{
10113   match(Set dst (ConvL2F (LoadL src)));
10114 
10115   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10116   ins_encode %{
10117     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10118   %}
10119   ins_pipe(pipe_slow); // XXX
10120 %}
10121 
10122 instruct convL2D_reg_reg(regD dst, rRegL src)
10123 %{
10124   match(Set dst (ConvL2D src));
10125 
10126   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10127   ins_encode %{
10128     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10129   %}
10130   ins_pipe(pipe_slow); // XXX
10131 %}
10132 
10133 instruct convL2D_reg_mem(regD dst, memory src)
10134 %{
10135   match(Set dst (ConvL2D (LoadL src)));
10136 
10137   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10138   ins_encode %{
10139     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10140   %}
10141   ins_pipe(pipe_slow); // XXX
10142 %}
10143 
10144 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10145 %{
10146   match(Set dst (ConvI2L src));
10147 
10148   ins_cost(125);
10149   format %{ "movslq  $dst, $src\t# i2l" %}
10150   ins_encode %{
10151     __ movslq($dst$$Register, $src$$Register);
10152   %}
10153   ins_pipe(ialu_reg_reg);
10154 %}
10155 
10156 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10157 // %{
10158 //   match(Set dst (ConvI2L src));
10159 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10160 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10161 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10162 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10163 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10164 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10165 
10166 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10167 //   ins_encode(enc_copy(dst, src));
10168 // //   opcode(0x63); // needs REX.W
10169 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10170 //   ins_pipe(ialu_reg_reg);
10171 // %}
10172 
10173 // Zero-extend convert int to long
10174 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10175 %{
10176   match(Set dst (AndL (ConvI2L src) mask));
10177 
10178   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10179   ins_encode %{
10180     if ($dst$$reg != $src$$reg) {
10181       __ movl($dst$$Register, $src$$Register);
10182     }
10183   %}
10184   ins_pipe(ialu_reg_reg);
10185 %}
10186 
10187 // Zero-extend convert int to long
10188 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10189 %{
10190   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10191 
10192   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10193   ins_encode %{
10194     __ movl($dst$$Register, $src$$Address);
10195   %}
10196   ins_pipe(ialu_reg_mem);
10197 %}
10198 
10199 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10200 %{
10201   match(Set dst (AndL src mask));
10202 
10203   format %{ "movl    $dst, $src\t# zero-extend long" %}
10204   ins_encode %{
10205     __ movl($dst$$Register, $src$$Register);
10206   %}
10207   ins_pipe(ialu_reg_reg);
10208 %}
10209 
10210 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10211 %{
10212   match(Set dst (ConvL2I src));
10213 
10214   format %{ "movl    $dst, $src\t# l2i" %}
10215   ins_encode %{
10216     __ movl($dst$$Register, $src$$Register);
10217   %}
10218   ins_pipe(ialu_reg_reg);
10219 %}
10220 
10221 
10222 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10223   match(Set dst (MoveF2I src));
10224   effect(DEF dst, USE src);
10225 
10226   ins_cost(125);
10227   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10228   ins_encode %{
10229     __ movl($dst$$Register, Address(rsp, $src$$disp));
10230   %}
10231   ins_pipe(ialu_reg_mem);
10232 %}
10233 
10234 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10235   match(Set dst (MoveI2F src));
10236   effect(DEF dst, USE src);
10237 
10238   ins_cost(125);
10239   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10240   ins_encode %{
10241     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10242   %}
10243   ins_pipe(pipe_slow);
10244 %}
10245 
10246 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10247   match(Set dst (MoveD2L src));
10248   effect(DEF dst, USE src);
10249 
10250   ins_cost(125);
10251   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10252   ins_encode %{
10253     __ movq($dst$$Register, Address(rsp, $src$$disp));
10254   %}
10255   ins_pipe(ialu_reg_mem);
10256 %}
10257 
10258 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10259   predicate(!UseXmmLoadAndClearUpper);
10260   match(Set dst (MoveL2D src));
10261   effect(DEF dst, USE src);
10262 
10263   ins_cost(125);
10264   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10265   ins_encode %{
10266     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10267   %}
10268   ins_pipe(pipe_slow);
10269 %}
10270 
10271 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10272   predicate(UseXmmLoadAndClearUpper);
10273   match(Set dst (MoveL2D src));
10274   effect(DEF dst, USE src);
10275 
10276   ins_cost(125);
10277   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10278   ins_encode %{
10279     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10280   %}
10281   ins_pipe(pipe_slow);
10282 %}
10283 
10284 
10285 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10286   match(Set dst (MoveF2I src));
10287   effect(DEF dst, USE src);
10288 
10289   ins_cost(95); // XXX
10290   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10291   ins_encode %{
10292     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10293   %}
10294   ins_pipe(pipe_slow);
10295 %}
10296 
10297 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10298   match(Set dst (MoveI2F src));
10299   effect(DEF dst, USE src);
10300 
10301   ins_cost(100);
10302   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10303   ins_encode %{
10304     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10305   %}
10306   ins_pipe( ialu_mem_reg );
10307 %}
10308 
10309 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10310   match(Set dst (MoveD2L src));
10311   effect(DEF dst, USE src);
10312 
10313   ins_cost(95); // XXX
10314   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10315   ins_encode %{
10316     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10317   %}
10318   ins_pipe(pipe_slow);
10319 %}
10320 
10321 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10322   match(Set dst (MoveL2D src));
10323   effect(DEF dst, USE src);
10324 
10325   ins_cost(100);
10326   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10327   ins_encode %{
10328     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10329   %}
10330   ins_pipe(ialu_mem_reg);
10331 %}
10332 
10333 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10334   match(Set dst (MoveF2I src));
10335   effect(DEF dst, USE src);
10336   ins_cost(85);
10337   format %{ "movd    $dst,$src\t# MoveF2I" %}
10338   ins_encode %{
10339     __ movdl($dst$$Register, $src$$XMMRegister);
10340   %}
10341   ins_pipe( pipe_slow );
10342 %}
10343 
10344 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10345   match(Set dst (MoveD2L src));
10346   effect(DEF dst, USE src);
10347   ins_cost(85);
10348   format %{ "movd    $dst,$src\t# MoveD2L" %}
10349   ins_encode %{
10350     __ movdq($dst$$Register, $src$$XMMRegister);
10351   %}
10352   ins_pipe( pipe_slow );
10353 %}
10354 
10355 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10356   match(Set dst (MoveI2F src));
10357   effect(DEF dst, USE src);
10358   ins_cost(100);
10359   format %{ "movd    $dst,$src\t# MoveI2F" %}
10360   ins_encode %{
10361     __ movdl($dst$$XMMRegister, $src$$Register);
10362   %}
10363   ins_pipe( pipe_slow );
10364 %}
10365 
10366 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10367   match(Set dst (MoveL2D src));
10368   effect(DEF dst, USE src);
10369   ins_cost(100);
10370   format %{ "movd    $dst,$src\t# MoveL2D" %}
10371   ins_encode %{
10372      __ movdq($dst$$XMMRegister, $src$$Register);
10373   %}
10374   ins_pipe( pipe_slow );
10375 %}
10376 
10377 
10378 // =======================================================================
10379 // fast clearing of an array
10380 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10381                   rFlagsReg cr)
10382 %{
10383   predicate(!UseFastStosb);
10384   match(Set dummy (ClearArray cnt base));
10385   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10386 
10387   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10388             "rep     stosq\t# Store rax to *rdi++ while rcx--" %}
10389   ins_encode %{ 
10390     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10391   %}
10392   ins_pipe(pipe_slow);
10393 %}
10394 
10395 instruct rep_fast_stosb(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10396                         rFlagsReg cr)
10397 %{
10398   predicate(UseFastStosb);
10399   match(Set dummy (ClearArray cnt base));
10400   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10401   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10402             "shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10403             "rep     stosb\t# Store rax to *rdi++ while rcx--" %}
10404   ins_encode %{ 
10405     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10406   %}
10407   ins_pipe( pipe_slow );
10408 %}
10409 
10410 instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10411                         rax_RegI result, regD tmp1, rFlagsReg cr)
10412 %{
10413   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10414   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10415 
10416   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10417   ins_encode %{
10418     __ string_compare($str1$$Register, $str2$$Register,
10419                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10420                       $tmp1$$XMMRegister);
10421   %}
10422   ins_pipe( pipe_slow );
10423 %}
10424 
10425 // fast search of substring with known size.
10426 instruct string_indexof_con(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10427                             rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10428 %{
10429   predicate(UseSSE42Intrinsics);
10430   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10431   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10432 
10433   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10434   ins_encode %{
10435     int icnt2 = (int)$int_cnt2$$constant;
10436     if (icnt2 >= 8) {
10437       // IndexOf for constant substrings with size >= 8 elements
10438       // which don't need to be loaded through stack.
10439       __ string_indexofC8($str1$$Register, $str2$$Register,
10440                           $cnt1$$Register, $cnt2$$Register,
10441                           icnt2, $result$$Register,
10442                           $vec$$XMMRegister, $tmp$$Register);
10443     } else {
10444       // Small strings are loaded through stack if they cross page boundary.
10445       __ string_indexof($str1$$Register, $str2$$Register,
10446                         $cnt1$$Register, $cnt2$$Register,
10447                         icnt2, $result$$Register,
10448                         $vec$$XMMRegister, $tmp$$Register);
10449     }
10450   %}
10451   ins_pipe( pipe_slow );
10452 %}
10453 
10454 instruct string_indexof(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10455                         rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10456 %{
10457   predicate(UseSSE42Intrinsics);
10458   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10459   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10460 
10461   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10462   ins_encode %{
10463     __ string_indexof($str1$$Register, $str2$$Register,
10464                       $cnt1$$Register, $cnt2$$Register,
10465                       (-1), $result$$Register,
10466                       $vec$$XMMRegister, $tmp$$Register);
10467   %}
10468   ins_pipe( pipe_slow );
10469 %}
10470 
10471 // fast string equals
10472 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10473                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10474 %{
10475   match(Set result (StrEquals (Binary str1 str2) cnt));
10476   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10477 
10478   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10479   ins_encode %{
10480     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
10481                           $cnt$$Register, $result$$Register, $tmp3$$Register,
10482                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10483   %}
10484   ins_pipe( pipe_slow );
10485 %}
10486 
10487 // fast array equals
10488 instruct array_equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10489                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10490 %{
10491   match(Set result (AryEq ary1 ary2));
10492   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10493   //ins_cost(300);
10494 
10495   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10496   ins_encode %{
10497     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
10498                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
10499                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10500   %}
10501   ins_pipe( pipe_slow );
10502 %}
10503 
10504 // encode char[] to byte[] in ISO_8859_1
10505 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10506                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10507                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10508   match(Set result (EncodeISOArray src (Binary dst len)));
10509   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10510 
10511   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10512   ins_encode %{
10513     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10514                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10515                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10516   %}
10517   ins_pipe( pipe_slow );
10518 %}
10519 
10520 //----------Overflow Math Instructions-----------------------------------------
10521 
10522 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10523 %{
10524   match(Set cr (OverflowAddI op1 op2));
10525   effect(DEF cr, USE_KILL op1, USE op2);
10526 
10527   format %{ "addl    $op1, $op2\t# overflow check int" %}
10528 
10529   ins_encode %{
10530     __ addl($op1$$Register, $op2$$Register);
10531   %}
10532   ins_pipe(ialu_reg_reg);
10533 %}
10534 
10535 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
10536 %{
10537   match(Set cr (OverflowAddI op1 op2));
10538   effect(DEF cr, USE_KILL op1, USE op2);
10539 
10540   format %{ "addl    $op1, $op2\t# overflow check int" %}
10541 
10542   ins_encode %{
10543     __ addl($op1$$Register, $op2$$constant);
10544   %}
10545   ins_pipe(ialu_reg_reg);
10546 %}
10547 
10548 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10549 %{
10550   match(Set cr (OverflowAddL op1 op2));
10551   effect(DEF cr, USE_KILL op1, USE op2);
10552 
10553   format %{ "addq    $op1, $op2\t# overflow check long" %}
10554   ins_encode %{
10555     __ addq($op1$$Register, $op2$$Register);
10556   %}
10557   ins_pipe(ialu_reg_reg);
10558 %}
10559 
10560 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
10561 %{
10562   match(Set cr (OverflowAddL op1 op2));
10563   effect(DEF cr, USE_KILL op1, USE op2);
10564 
10565   format %{ "addq    $op1, $op2\t# overflow check long" %}
10566   ins_encode %{
10567     __ addq($op1$$Register, $op2$$constant);
10568   %}
10569   ins_pipe(ialu_reg_reg);
10570 %}
10571 
10572 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10573 %{
10574   match(Set cr (OverflowSubI op1 op2));
10575 
10576   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10577   ins_encode %{
10578     __ cmpl($op1$$Register, $op2$$Register);
10579   %}
10580   ins_pipe(ialu_reg_reg);
10581 %}
10582 
10583 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10584 %{
10585   match(Set cr (OverflowSubI op1 op2));
10586 
10587   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10588   ins_encode %{
10589     __ cmpl($op1$$Register, $op2$$constant);
10590   %}
10591   ins_pipe(ialu_reg_reg);
10592 %}
10593 
10594 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10595 %{
10596   match(Set cr (OverflowSubL op1 op2));
10597 
10598   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10599   ins_encode %{
10600     __ cmpq($op1$$Register, $op2$$Register);
10601   %}
10602   ins_pipe(ialu_reg_reg);
10603 %}
10604 
10605 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10606 %{
10607   match(Set cr (OverflowSubL op1 op2));
10608 
10609   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10610   ins_encode %{
10611     __ cmpq($op1$$Register, $op2$$constant);
10612   %}
10613   ins_pipe(ialu_reg_reg);
10614 %}
10615 
10616 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
10617 %{
10618   match(Set cr (OverflowSubI zero op2));
10619   effect(DEF cr, USE_KILL op2);
10620 
10621   format %{ "negl    $op2\t# overflow check int" %}
10622   ins_encode %{
10623     __ negl($op2$$Register);
10624   %}
10625   ins_pipe(ialu_reg_reg);
10626 %}
10627 
10628 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
10629 %{
10630   match(Set cr (OverflowSubL zero op2));
10631   effect(DEF cr, USE_KILL op2);
10632 
10633   format %{ "negq    $op2\t# overflow check long" %}
10634   ins_encode %{
10635     __ negq($op2$$Register);
10636   %}
10637   ins_pipe(ialu_reg_reg);
10638 %}
10639 
10640 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10641 %{
10642   match(Set cr (OverflowMulI op1 op2));
10643   effect(DEF cr, USE_KILL op1, USE op2);
10644 
10645   format %{ "imull    $op1, $op2\t# overflow check int" %}
10646   ins_encode %{
10647     __ imull($op1$$Register, $op2$$Register);
10648   %}
10649   ins_pipe(ialu_reg_reg_alu0);
10650 %}
10651 
10652 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
10653 %{
10654   match(Set cr (OverflowMulI op1 op2));
10655   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10656 
10657   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
10658   ins_encode %{
10659     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
10660   %}
10661   ins_pipe(ialu_reg_reg_alu0);
10662 %}
10663 
10664 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10665 %{
10666   match(Set cr (OverflowMulL op1 op2));
10667   effect(DEF cr, USE_KILL op1, USE op2);
10668 
10669   format %{ "imulq    $op1, $op2\t# overflow check long" %}
10670   ins_encode %{
10671     __ imulq($op1$$Register, $op2$$Register);
10672   %}
10673   ins_pipe(ialu_reg_reg_alu0);
10674 %}
10675 
10676 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
10677 %{
10678   match(Set cr (OverflowMulL op1 op2));
10679   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10680 
10681   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
10682   ins_encode %{
10683     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
10684   %}
10685   ins_pipe(ialu_reg_reg_alu0);
10686 %}
10687 
10688 
10689 //----------Control Flow Instructions------------------------------------------
10690 // Signed compare Instructions
10691 
10692 // XXX more variants!!
10693 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10694 %{
10695   match(Set cr (CmpI op1 op2));
10696   effect(DEF cr, USE op1, USE op2);
10697 
10698   format %{ "cmpl    $op1, $op2" %}
10699   opcode(0x3B);  /* Opcode 3B /r */
10700   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10701   ins_pipe(ialu_cr_reg_reg);
10702 %}
10703 
10704 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10705 %{
10706   match(Set cr (CmpI op1 op2));
10707 
10708   format %{ "cmpl    $op1, $op2" %}
10709   opcode(0x81, 0x07); /* Opcode 81 /7 */
10710   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10711   ins_pipe(ialu_cr_reg_imm);
10712 %}
10713 
10714 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
10715 %{
10716   match(Set cr (CmpI op1 (LoadI op2)));
10717 
10718   ins_cost(500); // XXX
10719   format %{ "cmpl    $op1, $op2" %}
10720   opcode(0x3B); /* Opcode 3B /r */
10721   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10722   ins_pipe(ialu_cr_reg_mem);
10723 %}
10724 
10725 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
10726 %{
10727   match(Set cr (CmpI src zero));
10728 
10729   format %{ "testl   $src, $src" %}
10730   opcode(0x85);
10731   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10732   ins_pipe(ialu_cr_reg_imm);
10733 %}
10734 
10735 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
10736 %{
10737   match(Set cr (CmpI (AndI src con) zero));
10738 
10739   format %{ "testl   $src, $con" %}
10740   opcode(0xF7, 0x00);
10741   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
10742   ins_pipe(ialu_cr_reg_imm);
10743 %}
10744 
10745 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
10746 %{
10747   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
10748 
10749   format %{ "testl   $src, $mem" %}
10750   opcode(0x85);
10751   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
10752   ins_pipe(ialu_cr_reg_mem);
10753 %}
10754 
10755 // Unsigned compare Instructions; really, same as signed except they
10756 // produce an rFlagsRegU instead of rFlagsReg.
10757 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
10758 %{
10759   match(Set cr (CmpU op1 op2));
10760 
10761   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10762   opcode(0x3B); /* Opcode 3B /r */
10763   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10764   ins_pipe(ialu_cr_reg_reg);
10765 %}
10766 
10767 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
10768 %{
10769   match(Set cr (CmpU op1 op2));
10770 
10771   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10772   opcode(0x81,0x07); /* Opcode 81 /7 */
10773   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10774   ins_pipe(ialu_cr_reg_imm);
10775 %}
10776 
10777 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
10778 %{
10779   match(Set cr (CmpU op1 (LoadI op2)));
10780 
10781   ins_cost(500); // XXX
10782   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10783   opcode(0x3B); /* Opcode 3B /r */
10784   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10785   ins_pipe(ialu_cr_reg_mem);
10786 %}
10787 
10788 // // // Cisc-spilled version of cmpU_rReg
10789 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
10790 // //%{
10791 // //  match(Set cr (CmpU (LoadI op1) op2));
10792 // //
10793 // //  format %{ "CMPu   $op1,$op2" %}
10794 // //  ins_cost(500);
10795 // //  opcode(0x39);  /* Opcode 39 /r */
10796 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10797 // //%}
10798 
10799 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
10800 %{
10801   match(Set cr (CmpU src zero));
10802 
10803   format %{ "testl  $src, $src\t# unsigned" %}
10804   opcode(0x85);
10805   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10806   ins_pipe(ialu_cr_reg_imm);
10807 %}
10808 
10809 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
10810 %{
10811   match(Set cr (CmpP op1 op2));
10812 
10813   format %{ "cmpq    $op1, $op2\t# ptr" %}
10814   opcode(0x3B); /* Opcode 3B /r */
10815   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10816   ins_pipe(ialu_cr_reg_reg);
10817 %}
10818 
10819 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
10820 %{
10821   match(Set cr (CmpP op1 (LoadP op2)));
10822 
10823   ins_cost(500); // XXX
10824   format %{ "cmpq    $op1, $op2\t# ptr" %}
10825   opcode(0x3B); /* Opcode 3B /r */
10826   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10827   ins_pipe(ialu_cr_reg_mem);
10828 %}
10829 
10830 // // // Cisc-spilled version of cmpP_rReg
10831 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
10832 // //%{
10833 // //  match(Set cr (CmpP (LoadP op1) op2));
10834 // //
10835 // //  format %{ "CMPu   $op1,$op2" %}
10836 // //  ins_cost(500);
10837 // //  opcode(0x39);  /* Opcode 39 /r */
10838 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10839 // //%}
10840 
10841 // XXX this is generalized by compP_rReg_mem???
10842 // Compare raw pointer (used in out-of-heap check).
10843 // Only works because non-oop pointers must be raw pointers
10844 // and raw pointers have no anti-dependencies.
10845 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
10846 %{
10847   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
10848   match(Set cr (CmpP op1 (LoadP op2)));
10849 
10850   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
10851   opcode(0x3B); /* Opcode 3B /r */
10852   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10853   ins_pipe(ialu_cr_reg_mem);
10854 %}
10855 
10856 // This will generate a signed flags result. This should be OK since
10857 // any compare to a zero should be eq/neq.
10858 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
10859 %{
10860   match(Set cr (CmpP src zero));
10861 
10862   format %{ "testq   $src, $src\t# ptr" %}
10863   opcode(0x85);
10864   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
10865   ins_pipe(ialu_cr_reg_imm);
10866 %}
10867 
10868 // This will generate a signed flags result. This should be OK since
10869 // any compare to a zero should be eq/neq.
10870 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
10871 %{
10872   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
10873   match(Set cr (CmpP (LoadP op) zero));
10874 
10875   ins_cost(500); // XXX
10876   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
10877   opcode(0xF7); /* Opcode F7 /0 */
10878   ins_encode(REX_mem_wide(op),
10879              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
10880   ins_pipe(ialu_cr_reg_imm);
10881 %}
10882 
10883 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
10884 %{
10885   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
10886   match(Set cr (CmpP (LoadP mem) zero));
10887 
10888   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
10889   ins_encode %{
10890     __ cmpq(r12, $mem$$Address);
10891   %}
10892   ins_pipe(ialu_cr_reg_mem);
10893 %}
10894 
10895 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
10896 %{
10897   match(Set cr (CmpN op1 op2));
10898 
10899   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10900   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
10901   ins_pipe(ialu_cr_reg_reg);
10902 %}
10903 
10904 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
10905 %{
10906   match(Set cr (CmpN src (LoadN mem)));
10907 
10908   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
10909   ins_encode %{
10910     __ cmpl($src$$Register, $mem$$Address);
10911   %}
10912   ins_pipe(ialu_cr_reg_mem);
10913 %}
10914 
10915 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
10916   match(Set cr (CmpN op1 op2));
10917 
10918   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10919   ins_encode %{
10920     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
10921   %}
10922   ins_pipe(ialu_cr_reg_imm);
10923 %}
10924 
10925 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
10926 %{
10927   match(Set cr (CmpN src (LoadN mem)));
10928 
10929   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
10930   ins_encode %{
10931     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
10932   %}
10933   ins_pipe(ialu_cr_reg_mem);
10934 %}
10935 
10936 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
10937   match(Set cr (CmpN op1 op2));
10938 
10939   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
10940   ins_encode %{
10941     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
10942   %}
10943   ins_pipe(ialu_cr_reg_imm);
10944 %}
10945 
10946 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
10947 %{
10948   match(Set cr (CmpN src (LoadNKlass mem)));
10949 
10950   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
10951   ins_encode %{
10952     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
10953   %}
10954   ins_pipe(ialu_cr_reg_mem);
10955 %}
10956 
10957 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
10958   match(Set cr (CmpN src zero));
10959 
10960   format %{ "testl   $src, $src\t# compressed ptr" %}
10961   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
10962   ins_pipe(ialu_cr_reg_imm);
10963 %}
10964 
10965 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
10966 %{
10967   predicate(Universe::narrow_oop_base() != NULL);
10968   match(Set cr (CmpN (LoadN mem) zero));
10969 
10970   ins_cost(500); // XXX
10971   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
10972   ins_encode %{
10973     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
10974   %}
10975   ins_pipe(ialu_cr_reg_mem);
10976 %}
10977 
10978 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
10979 %{
10980   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
10981   match(Set cr (CmpN (LoadN mem) zero));
10982 
10983   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
10984   ins_encode %{
10985     __ cmpl(r12, $mem$$Address);
10986   %}
10987   ins_pipe(ialu_cr_reg_mem);
10988 %}
10989 
10990 // Yanked all unsigned pointer compare operations.
10991 // Pointer compares are done with CmpP which is already unsigned.
10992 
10993 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10994 %{
10995   match(Set cr (CmpL op1 op2));
10996 
10997   format %{ "cmpq    $op1, $op2" %}
10998   opcode(0x3B);  /* Opcode 3B /r */
10999   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11000   ins_pipe(ialu_cr_reg_reg);
11001 %}
11002 
11003 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11004 %{
11005   match(Set cr (CmpL op1 op2));
11006 
11007   format %{ "cmpq    $op1, $op2" %}
11008   opcode(0x81, 0x07); /* Opcode 81 /7 */
11009   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11010   ins_pipe(ialu_cr_reg_imm);
11011 %}
11012 
11013 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11014 %{
11015   match(Set cr (CmpL op1 (LoadL op2)));
11016 
11017   format %{ "cmpq    $op1, $op2" %}
11018   opcode(0x3B); /* Opcode 3B /r */
11019   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11020   ins_pipe(ialu_cr_reg_mem);
11021 %}
11022 
11023 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11024 %{
11025   match(Set cr (CmpL src zero));
11026 
11027   format %{ "testq   $src, $src" %}
11028   opcode(0x85);
11029   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11030   ins_pipe(ialu_cr_reg_imm);
11031 %}
11032 
11033 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11034 %{
11035   match(Set cr (CmpL (AndL src con) zero));
11036 
11037   format %{ "testq   $src, $con\t# long" %}
11038   opcode(0xF7, 0x00);
11039   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11040   ins_pipe(ialu_cr_reg_imm);
11041 %}
11042 
11043 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11044 %{
11045   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11046 
11047   format %{ "testq   $src, $mem" %}
11048   opcode(0x85);
11049   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11050   ins_pipe(ialu_cr_reg_mem);
11051 %}
11052 
11053 // Manifest a CmpL result in an integer register.  Very painful.
11054 // This is the test to avoid.
11055 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11056 %{
11057   match(Set dst (CmpL3 src1 src2));
11058   effect(KILL flags);
11059 
11060   ins_cost(275); // XXX
11061   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11062             "movl    $dst, -1\n\t"
11063             "jl,s    done\n\t"
11064             "setne   $dst\n\t"
11065             "movzbl  $dst, $dst\n\t"
11066     "done:" %}
11067   ins_encode(cmpl3_flag(src1, src2, dst));
11068   ins_pipe(pipe_slow);
11069 %}
11070 
11071 //----------Max and Min--------------------------------------------------------
11072 // Min Instructions
11073 
11074 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11075 %{
11076   effect(USE_DEF dst, USE src, USE cr);
11077 
11078   format %{ "cmovlgt $dst, $src\t# min" %}
11079   opcode(0x0F, 0x4F);
11080   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11081   ins_pipe(pipe_cmov_reg);
11082 %}
11083 
11084 
11085 instruct minI_rReg(rRegI dst, rRegI src)
11086 %{
11087   match(Set dst (MinI dst src));
11088 
11089   ins_cost(200);
11090   expand %{
11091     rFlagsReg cr;
11092     compI_rReg(cr, dst, src);
11093     cmovI_reg_g(dst, src, cr);
11094   %}
11095 %}
11096 
11097 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11098 %{
11099   effect(USE_DEF dst, USE src, USE cr);
11100 
11101   format %{ "cmovllt $dst, $src\t# max" %}
11102   opcode(0x0F, 0x4C);
11103   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11104   ins_pipe(pipe_cmov_reg);
11105 %}
11106 
11107 
11108 instruct maxI_rReg(rRegI dst, rRegI src)
11109 %{
11110   match(Set dst (MaxI dst src));
11111 
11112   ins_cost(200);
11113   expand %{
11114     rFlagsReg cr;
11115     compI_rReg(cr, dst, src);
11116     cmovI_reg_l(dst, src, cr);
11117   %}
11118 %}
11119 
11120 // ============================================================================
11121 // Branch Instructions
11122 
11123 // Jump Direct - Label defines a relative address from JMP+1
11124 instruct jmpDir(label labl)
11125 %{
11126   match(Goto);
11127   effect(USE labl);
11128 
11129   ins_cost(300);
11130   format %{ "jmp     $labl" %}
11131   size(5);
11132   ins_encode %{
11133     Label* L = $labl$$label;
11134     __ jmp(*L, false); // Always long jump
11135   %}
11136   ins_pipe(pipe_jmp);
11137 %}
11138 
11139 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11140 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11141 %{
11142   match(If cop cr);
11143   effect(USE labl);
11144 
11145   ins_cost(300);
11146   format %{ "j$cop     $labl" %}
11147   size(6);
11148   ins_encode %{
11149     Label* L = $labl$$label;
11150     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11151   %}
11152   ins_pipe(pipe_jcc);
11153 %}
11154 
11155 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11156 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11157 %{
11158   match(CountedLoopEnd cop cr);
11159   effect(USE labl);
11160 
11161   ins_cost(300);
11162   format %{ "j$cop     $labl\t# loop end" %}
11163   size(6);
11164   ins_encode %{
11165     Label* L = $labl$$label;
11166     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11167   %}
11168   ins_pipe(pipe_jcc);
11169 %}
11170 
11171 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11172 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11173   match(CountedLoopEnd cop cmp);
11174   effect(USE labl);
11175 
11176   ins_cost(300);
11177   format %{ "j$cop,u   $labl\t# loop end" %}
11178   size(6);
11179   ins_encode %{
11180     Label* L = $labl$$label;
11181     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11182   %}
11183   ins_pipe(pipe_jcc);
11184 %}
11185 
11186 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11187   match(CountedLoopEnd cop cmp);
11188   effect(USE labl);
11189 
11190   ins_cost(200);
11191   format %{ "j$cop,u   $labl\t# loop end" %}
11192   size(6);
11193   ins_encode %{
11194     Label* L = $labl$$label;
11195     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11196   %}
11197   ins_pipe(pipe_jcc);
11198 %}
11199 
11200 // Jump Direct Conditional - using unsigned comparison
11201 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11202   match(If cop cmp);
11203   effect(USE labl);
11204 
11205   ins_cost(300);
11206   format %{ "j$cop,u  $labl" %}
11207   size(6);
11208   ins_encode %{
11209     Label* L = $labl$$label;
11210     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11211   %}
11212   ins_pipe(pipe_jcc);
11213 %}
11214 
11215 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11216   match(If cop cmp);
11217   effect(USE labl);
11218 
11219   ins_cost(200);
11220   format %{ "j$cop,u  $labl" %}
11221   size(6);
11222   ins_encode %{
11223     Label* L = $labl$$label;
11224     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11225   %}
11226   ins_pipe(pipe_jcc);
11227 %}
11228 
11229 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11230   match(If cop cmp);
11231   effect(USE labl);
11232 
11233   ins_cost(200);
11234   format %{ $$template
11235     if ($cop$$cmpcode == Assembler::notEqual) {
11236       $$emit$$"jp,u   $labl\n\t"
11237       $$emit$$"j$cop,u   $labl"
11238     } else {
11239       $$emit$$"jp,u   done\n\t"
11240       $$emit$$"j$cop,u   $labl\n\t"
11241       $$emit$$"done:"
11242     }
11243   %}
11244   ins_encode %{
11245     Label* l = $labl$$label;
11246     if ($cop$$cmpcode == Assembler::notEqual) {
11247       __ jcc(Assembler::parity, *l, false);
11248       __ jcc(Assembler::notEqual, *l, false);
11249     } else if ($cop$$cmpcode == Assembler::equal) {
11250       Label done;
11251       __ jccb(Assembler::parity, done);
11252       __ jcc(Assembler::equal, *l, false);
11253       __ bind(done);
11254     } else {
11255        ShouldNotReachHere();
11256     }
11257   %}
11258   ins_pipe(pipe_jcc);
11259 %}
11260 
11261 // ============================================================================
11262 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11263 // superklass array for an instance of the superklass.  Set a hidden
11264 // internal cache on a hit (cache is checked with exposed code in
11265 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11266 // encoding ALSO sets flags.
11267 
11268 instruct partialSubtypeCheck(rdi_RegP result,
11269                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11270                              rFlagsReg cr)
11271 %{
11272   match(Set result (PartialSubtypeCheck sub super));
11273   effect(KILL rcx, KILL cr);
11274 
11275   ins_cost(1100);  // slightly larger than the next version
11276   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11277             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11278             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11279             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11280             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11281             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11282             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11283     "miss:\t" %}
11284 
11285   opcode(0x1); // Force a XOR of RDI
11286   ins_encode(enc_PartialSubtypeCheck());
11287   ins_pipe(pipe_slow);
11288 %}
11289 
11290 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11291                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11292                                      immP0 zero,
11293                                      rdi_RegP result)
11294 %{
11295   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11296   effect(KILL rcx, KILL result);
11297 
11298   ins_cost(1000);
11299   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11300             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11301             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11302             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11303             "jne,s   miss\t\t# Missed: flags nz\n\t"
11304             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11305     "miss:\t" %}
11306 
11307   opcode(0x0); // No need to XOR RDI
11308   ins_encode(enc_PartialSubtypeCheck());
11309   ins_pipe(pipe_slow);
11310 %}
11311 
11312 // ============================================================================
11313 // Branch Instructions -- short offset versions
11314 //
11315 // These instructions are used to replace jumps of a long offset (the default
11316 // match) with jumps of a shorter offset.  These instructions are all tagged
11317 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11318 // match rules in general matching.  Instead, the ADLC generates a conversion
11319 // method in the MachNode which can be used to do in-place replacement of the
11320 // long variant with the shorter variant.  The compiler will determine if a
11321 // branch can be taken by the is_short_branch_offset() predicate in the machine
11322 // specific code section of the file.
11323 
11324 // Jump Direct - Label defines a relative address from JMP+1
11325 instruct jmpDir_short(label labl) %{
11326   match(Goto);
11327   effect(USE labl);
11328 
11329   ins_cost(300);
11330   format %{ "jmp,s   $labl" %}
11331   size(2);
11332   ins_encode %{
11333     Label* L = $labl$$label;
11334     __ jmpb(*L);
11335   %}
11336   ins_pipe(pipe_jmp);
11337   ins_short_branch(1);
11338 %}
11339 
11340 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11341 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11342   match(If cop cr);
11343   effect(USE labl);
11344 
11345   ins_cost(300);
11346   format %{ "j$cop,s   $labl" %}
11347   size(2);
11348   ins_encode %{
11349     Label* L = $labl$$label;
11350     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11351   %}
11352   ins_pipe(pipe_jcc);
11353   ins_short_branch(1);
11354 %}
11355 
11356 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11357 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11358   match(CountedLoopEnd cop cr);
11359   effect(USE labl);
11360 
11361   ins_cost(300);
11362   format %{ "j$cop,s   $labl\t# loop end" %}
11363   size(2);
11364   ins_encode %{
11365     Label* L = $labl$$label;
11366     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11367   %}
11368   ins_pipe(pipe_jcc);
11369   ins_short_branch(1);
11370 %}
11371 
11372 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11373 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11374   match(CountedLoopEnd cop cmp);
11375   effect(USE labl);
11376 
11377   ins_cost(300);
11378   format %{ "j$cop,us  $labl\t# loop end" %}
11379   size(2);
11380   ins_encode %{
11381     Label* L = $labl$$label;
11382     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11383   %}
11384   ins_pipe(pipe_jcc);
11385   ins_short_branch(1);
11386 %}
11387 
11388 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11389   match(CountedLoopEnd cop cmp);
11390   effect(USE labl);
11391 
11392   ins_cost(300);
11393   format %{ "j$cop,us  $labl\t# loop end" %}
11394   size(2);
11395   ins_encode %{
11396     Label* L = $labl$$label;
11397     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11398   %}
11399   ins_pipe(pipe_jcc);
11400   ins_short_branch(1);
11401 %}
11402 
11403 // Jump Direct Conditional - using unsigned comparison
11404 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11405   match(If cop cmp);
11406   effect(USE labl);
11407 
11408   ins_cost(300);
11409   format %{ "j$cop,us  $labl" %}
11410   size(2);
11411   ins_encode %{
11412     Label* L = $labl$$label;
11413     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11414   %}
11415   ins_pipe(pipe_jcc);
11416   ins_short_branch(1);
11417 %}
11418 
11419 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11420   match(If cop cmp);
11421   effect(USE labl);
11422 
11423   ins_cost(300);
11424   format %{ "j$cop,us  $labl" %}
11425   size(2);
11426   ins_encode %{
11427     Label* L = $labl$$label;
11428     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11429   %}
11430   ins_pipe(pipe_jcc);
11431   ins_short_branch(1);
11432 %}
11433 
11434 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11435   match(If cop cmp);
11436   effect(USE labl);
11437 
11438   ins_cost(300);
11439   format %{ $$template
11440     if ($cop$$cmpcode == Assembler::notEqual) {
11441       $$emit$$"jp,u,s   $labl\n\t"
11442       $$emit$$"j$cop,u,s   $labl"
11443     } else {
11444       $$emit$$"jp,u,s   done\n\t"
11445       $$emit$$"j$cop,u,s  $labl\n\t"
11446       $$emit$$"done:"
11447     }
11448   %}
11449   size(4);
11450   ins_encode %{
11451     Label* l = $labl$$label;
11452     if ($cop$$cmpcode == Assembler::notEqual) {
11453       __ jccb(Assembler::parity, *l);
11454       __ jccb(Assembler::notEqual, *l);
11455     } else if ($cop$$cmpcode == Assembler::equal) {
11456       Label done;
11457       __ jccb(Assembler::parity, done);
11458       __ jccb(Assembler::equal, *l);
11459       __ bind(done);
11460     } else {
11461        ShouldNotReachHere();
11462     }
11463   %}
11464   ins_pipe(pipe_jcc);
11465   ins_short_branch(1);
11466 %}
11467 
11468 // ============================================================================
11469 // inlined locking and unlocking
11470 
11471 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
11472   predicate(Compile::current()->use_rtm());
11473   match(Set cr (FastLock object box));
11474   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
11475   ins_cost(300);
11476   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
11477   ins_encode %{
11478     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11479                  $scr$$Register, $cx1$$Register, $cx2$$Register,
11480                  _counters, _rtm_counters, _stack_rtm_counters,
11481                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
11482                  true, ra_->C->profile_rtm());
11483   %}
11484   ins_pipe(pipe_slow);
11485 %}
11486 
11487 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
11488   predicate(!Compile::current()->use_rtm());
11489   match(Set cr (FastLock object box));
11490   effect(TEMP tmp, TEMP scr, USE_KILL box);
11491   ins_cost(300);
11492   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11493   ins_encode %{
11494     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11495                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
11496   %}
11497   ins_pipe(pipe_slow);
11498 %}
11499 
11500 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
11501   match(Set cr (FastUnlock object box));
11502   effect(TEMP tmp, USE_KILL box);
11503   ins_cost(300);
11504   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11505   ins_encode %{
11506     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
11507   %}
11508   ins_pipe(pipe_slow);
11509 %}
11510 
11511 
11512 // ============================================================================
11513 // Safepoint Instructions
11514 instruct safePoint_poll(rFlagsReg cr)
11515 %{
11516   predicate(!Assembler::is_polling_page_far());
11517   match(SafePoint);
11518   effect(KILL cr);
11519 
11520   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
11521             "# Safepoint: poll for GC" %}
11522   ins_cost(125);
11523   ins_encode %{
11524     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11525     __ testl(rax, addr);
11526   %}
11527   ins_pipe(ialu_reg_mem);
11528 %}
11529 
11530 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
11531 %{
11532   predicate(Assembler::is_polling_page_far());
11533   match(SafePoint poll);
11534   effect(KILL cr, USE poll);
11535 
11536   format %{ "testl  rax, [$poll]\t"
11537             "# Safepoint: poll for GC" %}
11538   ins_cost(125);
11539   ins_encode %{
11540     __ relocate(relocInfo::poll_type);
11541     __ testl(rax, Address($poll$$Register, 0));
11542   %}
11543   ins_pipe(ialu_reg_mem);
11544 %}
11545 
11546 // ============================================================================
11547 // Procedure Call/Return Instructions
11548 // Call Java Static Instruction
11549 // Note: If this code changes, the corresponding ret_addr_offset() and
11550 //       compute_padding() functions will have to be adjusted.
11551 instruct CallStaticJavaDirect(method meth) %{
11552   match(CallStaticJava);
11553   effect(USE meth);
11554 
11555   ins_cost(300);
11556   format %{ "call,static " %}
11557   opcode(0xE8); /* E8 cd */
11558   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
11559   ins_pipe(pipe_slow);
11560   ins_alignment(4);
11561 %}
11562 
11563 // Call Java Dynamic Instruction
11564 // Note: If this code changes, the corresponding ret_addr_offset() and
11565 //       compute_padding() functions will have to be adjusted.
11566 instruct CallDynamicJavaDirect(method meth)
11567 %{
11568   match(CallDynamicJava);
11569   effect(USE meth);
11570 
11571   ins_cost(300);
11572   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
11573             "call,dynamic " %}
11574   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
11575   ins_pipe(pipe_slow);
11576   ins_alignment(4);
11577 %}
11578 
11579 // Call Runtime Instruction
11580 instruct CallRuntimeDirect(method meth)
11581 %{
11582   match(CallRuntime);
11583   effect(USE meth);
11584 
11585   ins_cost(300);
11586   format %{ "call,runtime " %}
11587   ins_encode(clear_avx, Java_To_Runtime(meth));
11588   ins_pipe(pipe_slow);
11589 %}
11590 
11591 // Call runtime without safepoint
11592 instruct CallLeafDirect(method meth)
11593 %{
11594   match(CallLeaf);
11595   effect(USE meth);
11596 
11597   ins_cost(300);
11598   format %{ "call_leaf,runtime " %}
11599   ins_encode(clear_avx, Java_To_Runtime(meth));
11600   ins_pipe(pipe_slow);
11601 %}
11602 
11603 // Call runtime without safepoint
11604 instruct CallLeafNoFPDirect(method meth)
11605 %{
11606   match(CallLeafNoFP);
11607   effect(USE meth);
11608 
11609   ins_cost(300);
11610   format %{ "call_leaf_nofp,runtime " %}
11611   ins_encode(Java_To_Runtime(meth));
11612   ins_pipe(pipe_slow);
11613 %}
11614 
11615 // Return Instruction
11616 // Remove the return address & jump to it.
11617 // Notice: We always emit a nop after a ret to make sure there is room
11618 // for safepoint patching
11619 instruct Ret()
11620 %{
11621   match(Return);
11622 
11623   format %{ "ret" %}
11624   opcode(0xC3);
11625   ins_encode(OpcP);
11626   ins_pipe(pipe_jmp);
11627 %}
11628 
11629 // Tail Call; Jump from runtime stub to Java code.
11630 // Also known as an 'interprocedural jump'.
11631 // Target of jump will eventually return to caller.
11632 // TailJump below removes the return address.
11633 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
11634 %{
11635   match(TailCall jump_target method_oop);
11636 
11637   ins_cost(300);
11638   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
11639   opcode(0xFF, 0x4); /* Opcode FF /4 */
11640   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
11641   ins_pipe(pipe_jmp);
11642 %}
11643 
11644 // Tail Jump; remove the return address; jump to target.
11645 // TailCall above leaves the return address around.
11646 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
11647 %{
11648   match(TailJump jump_target ex_oop);
11649 
11650   ins_cost(300);
11651   format %{ "popq    rdx\t# pop return address\n\t"
11652             "jmp     $jump_target" %}
11653   opcode(0xFF, 0x4); /* Opcode FF /4 */
11654   ins_encode(Opcode(0x5a), // popq rdx
11655              REX_reg(jump_target), OpcP, reg_opc(jump_target));
11656   ins_pipe(pipe_jmp);
11657 %}
11658 
11659 // Create exception oop: created by stack-crawling runtime code.
11660 // Created exception is now available to this handler, and is setup
11661 // just prior to jumping to this handler.  No code emitted.
11662 instruct CreateException(rax_RegP ex_oop)
11663 %{
11664   match(Set ex_oop (CreateEx));
11665 
11666   size(0);
11667   // use the following format syntax
11668   format %{ "# exception oop is in rax; no code emitted" %}
11669   ins_encode();
11670   ins_pipe(empty);
11671 %}
11672 
11673 // Rethrow exception:
11674 // The exception oop will come in the first argument position.
11675 // Then JUMP (not call) to the rethrow stub code.
11676 instruct RethrowException()
11677 %{
11678   match(Rethrow);
11679 
11680   // use the following format syntax
11681   format %{ "jmp     rethrow_stub" %}
11682   ins_encode(enc_rethrow);
11683   ins_pipe(pipe_jmp);
11684 %}
11685 
11686 
11687 // ============================================================================
11688 // This name is KNOWN by the ADLC and cannot be changed.
11689 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
11690 // for this guy.
11691 instruct tlsLoadP(r15_RegP dst) %{
11692   match(Set dst (ThreadLocal));
11693   effect(DEF dst);
11694 
11695   size(0);
11696   format %{ "# TLS is in R15" %}
11697   ins_encode( /*empty encoding*/ );
11698   ins_pipe(ialu_reg_reg);
11699 %}
11700 
11701 
11702 //----------PEEPHOLE RULES-----------------------------------------------------
11703 // These must follow all instruction definitions as they use the names
11704 // defined in the instructions definitions.
11705 //
11706 // peepmatch ( root_instr_name [preceding_instruction]* );
11707 //
11708 // peepconstraint %{
11709 // (instruction_number.operand_name relational_op instruction_number.operand_name
11710 //  [, ...] );
11711 // // instruction numbers are zero-based using left to right order in peepmatch
11712 //
11713 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
11714 // // provide an instruction_number.operand_name for each operand that appears
11715 // // in the replacement instruction's match rule
11716 //
11717 // ---------VM FLAGS---------------------------------------------------------
11718 //
11719 // All peephole optimizations can be turned off using -XX:-OptoPeephole
11720 //
11721 // Each peephole rule is given an identifying number starting with zero and
11722 // increasing by one in the order seen by the parser.  An individual peephole
11723 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
11724 // on the command-line.
11725 //
11726 // ---------CURRENT LIMITATIONS----------------------------------------------
11727 //
11728 // Only match adjacent instructions in same basic block
11729 // Only equality constraints
11730 // Only constraints between operands, not (0.dest_reg == RAX_enc)
11731 // Only one replacement instruction
11732 //
11733 // ---------EXAMPLE----------------------------------------------------------
11734 //
11735 // // pertinent parts of existing instructions in architecture description
11736 // instruct movI(rRegI dst, rRegI src)
11737 // %{
11738 //   match(Set dst (CopyI src));
11739 // %}
11740 //
11741 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
11742 // %{
11743 //   match(Set dst (AddI dst src));
11744 //   effect(KILL cr);
11745 // %}
11746 //
11747 // // Change (inc mov) to lea
11748 // peephole %{
11749 //   // increment preceeded by register-register move
11750 //   peepmatch ( incI_rReg movI );
11751 //   // require that the destination register of the increment
11752 //   // match the destination register of the move
11753 //   peepconstraint ( 0.dst == 1.dst );
11754 //   // construct a replacement instruction that sets
11755 //   // the destination to ( move's source register + one )
11756 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
11757 // %}
11758 //
11759 
11760 // Implementation no longer uses movX instructions since
11761 // machine-independent system no longer uses CopyX nodes.
11762 //
11763 // peephole
11764 // %{
11765 //   peepmatch (incI_rReg movI);
11766 //   peepconstraint (0.dst == 1.dst);
11767 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11768 // %}
11769 
11770 // peephole
11771 // %{
11772 //   peepmatch (decI_rReg movI);
11773 //   peepconstraint (0.dst == 1.dst);
11774 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11775 // %}
11776 
11777 // peephole
11778 // %{
11779 //   peepmatch (addI_rReg_imm movI);
11780 //   peepconstraint (0.dst == 1.dst);
11781 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11782 // %}
11783 
11784 // peephole
11785 // %{
11786 //   peepmatch (incL_rReg movL);
11787 //   peepconstraint (0.dst == 1.dst);
11788 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11789 // %}
11790 
11791 // peephole
11792 // %{
11793 //   peepmatch (decL_rReg movL);
11794 //   peepconstraint (0.dst == 1.dst);
11795 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11796 // %}
11797 
11798 // peephole
11799 // %{
11800 //   peepmatch (addL_rReg_imm movL);
11801 //   peepconstraint (0.dst == 1.dst);
11802 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11803 // %}
11804 
11805 // peephole
11806 // %{
11807 //   peepmatch (addP_rReg_imm movP);
11808 //   peepconstraint (0.dst == 1.dst);
11809 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
11810 // %}
11811 
11812 // // Change load of spilled value to only a spill
11813 // instruct storeI(memory mem, rRegI src)
11814 // %{
11815 //   match(Set mem (StoreI mem src));
11816 // %}
11817 //
11818 // instruct loadI(rRegI dst, memory mem)
11819 // %{
11820 //   match(Set dst (LoadI mem));
11821 // %}
11822 //
11823 
11824 peephole
11825 %{
11826   peepmatch (loadI storeI);
11827   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11828   peepreplace (storeI(1.mem 1.mem 1.src));
11829 %}
11830 
11831 peephole
11832 %{
11833   peepmatch (loadL storeL);
11834   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11835   peepreplace (storeL(1.mem 1.mem 1.src));
11836 %}
11837 
11838 //----------SMARTSPILL RULES---------------------------------------------------
11839 // These must follow all instruction definitions as they use the names
11840 // defined in the instructions definitions.