1 //
   2 // Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 // AArch64 Architecture Description File
  27 
  28 //----------REGISTER DEFINITION BLOCK------------------------------------------
  29 // This information is used by the matcher and the register allocator to
  30 // describe individual registers and classes of registers within the target
  31 // archtecture.
  32 
  33 register %{
  34 //----------Architecture Description Register Definitions----------------------
  35 // General Registers
  36 // "reg_def"  name ( register save type, C convention save type,
  37 //                   ideal register type, encoding );
  38 // Register Save Types:
  39 //
  40 // NS  = No-Save:       The register allocator assumes that these registers
  41 //                      can be used without saving upon entry to the method, &
  42 //                      that they do not need to be saved at call sites.
  43 //
  44 // SOC = Save-On-Call:  The register allocator assumes that these registers
  45 //                      can be used without saving upon entry to the method,
  46 //                      but that they must be saved at call sites.
  47 //
  48 // SOE = Save-On-Entry: The register allocator assumes that these registers
  49 //                      must be saved before using them upon entry to the
  50 //                      method, but they do not need to be saved at call
  51 //                      sites.
  52 //
  53 // AS  = Always-Save:   The register allocator assumes that these registers
  54 //                      must be saved before using them upon entry to the
  55 //                      method, & that they must be saved at call sites.
  56 //
  57 // Ideal Register Type is used to determine how to save & restore a
  58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  60 //
  61 // The encoding number is the actual bit-pattern placed into the opcodes.
  62 
  63 // We must define the 64 bit int registers in two 32 bit halves, the
  64 // real lower register and a virtual upper half register. upper halves
  65 // are used by the register allocator but are not actually supplied as
  66 // operands to memory ops.
  67 //
  68 // follow the C1 compiler in making registers
  69 //
  70 //   r0-r7,r10-r26 volatile (caller save)
  71 //   r27-r32 system (no save, no allocate)
  72 //   r8-r9 invisible to the allocator (so we can use them as scratch regs)
  73 //
  74 // as regards Java usage. we don't use any callee save registers
  75 // because this makes it difficult to de-optimise a frame (see comment
  76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
  77 //
  78 
  79 // General Registers
  80 
  81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
  82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
  83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
  84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
  85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
  86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
  87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
  88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
  89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
  90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
  91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
  92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
  93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
  94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
  95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
  96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
  97 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  98 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  99 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
 100 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 101 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
 102 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
 103 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
 104 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
 105 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
 106 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
 107 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
 108 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
 109 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
 110 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
 111 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
 112 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
 113 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()        );
 114 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
 115 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
 116 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
 117 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
 118 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
 119 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
 120 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
 121 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
 122 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
 123 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
 124 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
 125 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
 126 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
 127 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
 128 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
 129 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
 130 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
 131 reg_def R27     (  NS, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
 132 reg_def R27_H   (  NS, SOE, Op_RegI, 27, r27->as_VMReg()->next());
 133 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
 134 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
 135 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
 136 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
 137 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
 138 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
 139 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
 140 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
 141 
 142 // ----------------------------
 143 // Float/Double Registers
 144 // ----------------------------
 145 
 146 // Double Registers
 147 
 148 // The rules of ADL require that double registers be defined in pairs.
 149 // Each pair must be two 32-bit values, but not necessarily a pair of
 150 // single float registers. In each pair, ADLC-assigned register numbers
 151 // must be adjacent, with the lower number even. Finally, when the
 152 // CPU stores such a register pair to memory, the word associated with
 153 // the lower ADLC-assigned number must be stored to the lower address.
 154 
 155 // AArch64 has 32 floating-point registers. Each can store a vector of
 156 // single or double precision floating-point values up to 8 * 32
 157 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
 158 // use the first float or double element of the vector.
 159 
 160 // for Java use float registers v0-v15 are always save on call whereas
 161 // the platform ABI treats v8-v15 as callee save). float registers
 162 // v16-v31 are SOC as per the platform spec
 163 
 164   reg_def V0   ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()         );
 165   reg_def V0_H ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next() );
 166   reg_def V1   ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()         );
 167   reg_def V1_H ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next() );
 168   reg_def V2   ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()         );
 169   reg_def V2_H ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next() );
 170   reg_def V3   ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()         );
 171   reg_def V3_H ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next() );
 172   reg_def V4   ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()         );
 173   reg_def V4_H ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next() );
 174   reg_def V5   ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()         );
 175   reg_def V5_H ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next() );
 176   reg_def V6   ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()         );
 177   reg_def V6_H ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next() );
 178   reg_def V7   ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()         );
 179   reg_def V7_H ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next() );
 180   reg_def V8   ( SOC, SOE, Op_RegF,  8, v8->as_VMReg()         );
 181   reg_def V8_H ( SOC, SOE, Op_RegF,  8, v8->as_VMReg()->next() );
 182   reg_def V9   ( SOC, SOE, Op_RegF,  9, v9->as_VMReg()         );
 183   reg_def V9_H ( SOC, SOE, Op_RegF,  9, v9->as_VMReg()->next() );
 184   reg_def V10  ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()        );
 185   reg_def V10_H( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next());
 186   reg_def V11  ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()        );
 187   reg_def V11_H( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next());
 188   reg_def V12  ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()        );
 189   reg_def V12_H( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next());
 190   reg_def V13  ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()        );
 191   reg_def V13_H( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next());
 192   reg_def V14  ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()        );
 193   reg_def V14_H( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next());
 194   reg_def V15  ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()        );
 195   reg_def V15_H( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next());
 196   reg_def V16  ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()        );
 197   reg_def V16_H( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next());
 198   reg_def V17  ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()        );
 199   reg_def V17_H( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next());
 200   reg_def V18  ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()        );
 201   reg_def V18_H( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next());
 202   reg_def V19  ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()        );
 203   reg_def V19_H( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next());
 204   reg_def V20  ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()        );
 205   reg_def V20_H( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next());
 206   reg_def V21  ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()        );
 207   reg_def V21_H( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next());
 208   reg_def V22  ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()        );
 209   reg_def V22_H( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next());
 210   reg_def V23  ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()        );
 211   reg_def V23_H( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next());
 212   reg_def V24  ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()        );
 213   reg_def V24_H( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next());
 214   reg_def V25  ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()        );
 215   reg_def V25_H( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next());
 216   reg_def V26  ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()        );
 217   reg_def V26_H( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next());
 218   reg_def V27  ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()        );
 219   reg_def V27_H( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next());
 220   reg_def V28  ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()        );
 221   reg_def V28_H( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next());
 222   reg_def V29  ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()        );
 223   reg_def V29_H( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next());
 224   reg_def V30  ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()        );
 225   reg_def V30_H( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next());
 226   reg_def V31  ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()        );
 227   reg_def V31_H( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next());
 228 
 229 // ----------------------------
 230 // Special Registers
 231 // ----------------------------
 232 
 233 // the AArch64 CSPR status flag register is not directly acessible as
 234 // instruction operand. the FPSR status flag register is a system
 235 // register which can be written/read using MSR/MRS but again does not
 236 // appear as an operand (a code identifying the FSPR occurs as an
 237 // immediate value in the instruction).
 238 
 239 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
 240 
 241 
 242 // Specify priority of register selection within phases of register
 243 // allocation.  Highest priority is first.  A useful heuristic is to
 244 // give registers a low priority when they are required by machine
 245 // instructions, like EAX and EDX on I486, and choose no-save registers
 246 // before save-on-call, & save-on-call before save-on-entry.  Registers
 247 // which participate in fixed calling sequences should come last.
 248 // Registers which are used as pairs must fall on an even boundary.
 249 
 250 alloc_class chunk0(
 251     // volatiles
 252     R10, R10_H,
 253     R11, R11_H,
 254     R12, R12_H,
 255     R13, R13_H,
 256     R14, R14_H,
 257     R15, R15_H,
 258     R16, R16_H,
 259     R17, R17_H,
 260     R18, R18_H,
 261 
 262     // arg registers
 263     R0, R0_H,
 264     R1, R1_H,
 265     R2, R2_H,
 266     R3, R3_H,
 267     R4, R4_H,
 268     R5, R5_H,
 269     R6, R6_H,
 270     R7, R7_H,
 271 
 272     // non-volatiles
 273     R19, R19_H,
 274     R20, R20_H,
 275     R21, R21_H,
 276     R22, R22_H,
 277     R23, R23_H,
 278     R24, R24_H,
 279     R25, R25_H,
 280     R26, R26_H,
 281 
 282     // non-allocatable registers
 283 
 284     R27, R27_H, // heapbase
 285     R28, R28_H, // thread
 286     R29, R29_H, // fp
 287     R30, R30_H, // lr
 288     R31, R31_H, // sp
 289 );
 290 
 291 alloc_class chunk1(
 292 
 293     // no save
 294     V16, V16_H,
 295     V17, V17_H,
 296     V18, V18_H,
 297     V19, V19_H,
 298     V20, V20_H,
 299     V21, V21_H,
 300     V22, V22_H,
 301     V23, V23_H,
 302     V24, V24_H,
 303     V25, V25_H,
 304     V26, V26_H,
 305     V27, V27_H,
 306     V28, V28_H,
 307     V29, V29_H,
 308     V30, V30_H,
 309     V31, V31_H,
 310 
 311     // arg registers
 312     V0, V0_H,
 313     V1, V1_H,
 314     V2, V2_H,
 315     V3, V3_H,
 316     V4, V4_H,
 317     V5, V5_H,
 318     V6, V6_H,
 319     V7, V7_H,
 320 
 321     // non-volatiles
 322     V8, V8_H,
 323     V9, V9_H,
 324     V10, V10_H,
 325     V11, V11_H,
 326     V12, V12_H,
 327     V13, V13_H,
 328     V14, V14_H,
 329     V15, V15_H,
 330 );
 331 
 332 alloc_class chunk2(RFLAGS);
 333 
 334 //----------Architecture Description Register Classes--------------------------
 335 // Several register classes are automatically defined based upon information in
 336 // this architecture description.
 337 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 338 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 339 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 340 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 341 //
 342 
 343 // Class for all 32 bit integer registers -- excludes SP which will
 344 // never be used as an integer register
 345 reg_class any_reg32(
 346     R0,
 347     R1,
 348     R2,
 349     R3,
 350     R4,
 351     R5,
 352     R6,
 353     R7,
 354     R10,
 355     R11,
 356     R12,
 357     R13,
 358     R14,
 359     R15,
 360     R16,
 361     R17,
 362     R18,
 363     R19,
 364     R20,
 365     R21,
 366     R22,
 367     R23,
 368     R24,
 369     R25,
 370     R26,
 371     R27,
 372     R28,
 373     R29,
 374     R30
 375 );
 376 
 377 // Singleton class for R0 int register
 378 reg_class int_r0_reg(R0);
 379 
 380 // Singleton class for R2 int register
 381 reg_class int_r2_reg(R2);
 382 
 383 // Singleton class for R3 int register
 384 reg_class int_r3_reg(R3);
 385 
 386 // Singleton class for R4 int register
 387 reg_class int_r4_reg(R4);
 388 
 389 // Class for all long integer registers (including RSP)
 390 reg_class any_reg(
 391     R0, R0_H,
 392     R1, R1_H,
 393     R2, R2_H,
 394     R3, R3_H,
 395     R4, R4_H,
 396     R5, R5_H,
 397     R6, R6_H,
 398     R7, R7_H,
 399     R10, R10_H,
 400     R11, R11_H,
 401     R12, R12_H,
 402     R13, R13_H,
 403     R14, R14_H,
 404     R15, R15_H,
 405     R16, R16_H,
 406     R17, R17_H,
 407     R18, R18_H,
 408     R19, R19_H,
 409     R20, R20_H,
 410     R21, R21_H,
 411     R22, R22_H,
 412     R23, R23_H,
 413     R24, R24_H,
 414     R25, R25_H,
 415     R26, R26_H,
 416     R27, R27_H,
 417     R28, R28_H,
 418     R29, R29_H,
 419     R30, R30_H,
 420     R31, R31_H
 421 );
 422 
 423 // Class for all non-special integer registers
 424 reg_class no_special_reg32(
 425     R0,
 426     R1,
 427     R2,
 428     R3,
 429     R4,
 430     R5,
 431     R6,
 432     R7,
 433     R10,
 434     R11,
 435     R12,                        // rmethod
 436     R13,
 437     R14,
 438     R15,
 439     R16,
 440     R17,
 441     R18,
 442     R19,
 443     R20,
 444     R21,
 445     R22,
 446     R23,
 447     R24,
 448     R25,
 449     R26
 450  /* R27, */                     // heapbase
 451  /* R28, */                     // thread
 452  /* R29, */                     // fp
 453  /* R30, */                     // lr
 454  /* R31 */                      // sp
 455 );
 456 
 457 // Class for all non-special long integer registers
 458 reg_class no_special_reg(
 459     R0, R0_H,
 460     R1, R1_H,
 461     R2, R2_H,
 462     R3, R3_H,
 463     R4, R4_H,
 464     R5, R5_H,
 465     R6, R6_H,
 466     R7, R7_H,
 467     R10, R10_H,
 468     R11, R11_H,
 469     R12, R12_H,                 // rmethod
 470     R13, R13_H,
 471     R14, R14_H,
 472     R15, R15_H,
 473     R16, R16_H,
 474     R17, R17_H,
 475     R18, R18_H,
 476     R19, R19_H,
 477     R20, R20_H,
 478     R21, R21_H,
 479     R22, R22_H,
 480     R23, R23_H,
 481     R24, R24_H,
 482     R25, R25_H,
 483     R26, R26_H,
 484  /* R27, R27_H, */              // heapbase
 485  /* R28, R28_H, */              // thread
 486  /* R29, R29_H, */              // fp
 487  /* R30, R30_H, */              // lr
 488  /* R31, R31_H */               // sp
 489 );
 490 
 491 // Class for 64 bit register r0
 492 reg_class r0_reg(
 493     R0, R0_H
 494 );
 495 
 496 // Class for 64 bit register r1
 497 reg_class r1_reg(
 498     R1, R1_H
 499 );
 500 
 501 // Class for 64 bit register r2
 502 reg_class r2_reg(
 503     R2, R2_H
 504 );
 505 
 506 // Class for 64 bit register r3
 507 reg_class r3_reg(
 508     R3, R3_H
 509 );
 510 
 511 // Class for 64 bit register r4
 512 reg_class r4_reg(
 513     R4, R4_H
 514 );
 515 
 516 // Class for 64 bit register r5
 517 reg_class r5_reg(
 518     R5, R5_H
 519 );
 520 
 521 // Class for 64 bit register r10
 522 reg_class r10_reg(
 523     R10, R10_H
 524 );
 525 
 526 // Class for 64 bit register r11
 527 reg_class r11_reg(
 528     R11, R11_H
 529 );
 530 
 531 // Class for method register
 532 reg_class method_reg(
 533     R12, R12_H
 534 );
 535 
 536 // Class for heapbase register
 537 reg_class heapbase_reg(
 538     R27, R27_H
 539 );
 540 
 541 // Class for thread register
 542 reg_class thread_reg(
 543     R28, R28_H
 544 );
 545 
 546 // Class for frame pointer register
 547 reg_class fp_reg(
 548     R29, R29_H
 549 );
 550 
 551 // Class for link register
 552 reg_class lr_reg(
 553     R30, R30_H
 554 );
 555 
 556 // Class for long sp register
 557 reg_class sp_reg(
 558   R31, R31_H
 559 );
 560 
 561 // Class for all pointer registers
 562 reg_class ptr_reg(
 563     R0, R0_H,
 564     R1, R1_H,
 565     R2, R2_H,
 566     R3, R3_H,
 567     R4, R4_H,
 568     R5, R5_H,
 569     R6, R6_H,
 570     R7, R7_H,
 571     R10, R10_H,
 572     R11, R11_H,
 573     R12, R12_H,
 574     R13, R13_H,
 575     R14, R14_H,
 576     R15, R15_H,
 577     R16, R16_H,
 578     R17, R17_H,
 579     R18, R18_H,
 580     R19, R19_H,
 581     R20, R20_H,
 582     R21, R21_H,
 583     R22, R22_H,
 584     R23, R23_H,
 585     R24, R24_H,
 586     R25, R25_H,
 587     R26, R26_H,
 588     R27, R27_H,
 589     R28, R28_H,
 590     R29, R29_H,
 591     R30, R30_H,
 592     R31, R31_H
 593 );
 594 
 595 // Class for all non_special pointer registers
 596 reg_class no_special_ptr_reg(
 597     R0, R0_H,
 598     R1, R1_H,
 599     R2, R2_H,
 600     R3, R3_H,
 601     R4, R4_H,
 602     R5, R5_H,
 603     R6, R6_H,
 604     R7, R7_H,
 605     R10, R10_H,
 606     R11, R11_H,
 607     R12, R12_H,
 608     R13, R13_H,
 609     R14, R14_H,
 610     R15, R15_H,
 611     R16, R16_H,
 612     R17, R17_H,
 613     R18, R18_H,
 614     R19, R19_H,
 615     R20, R20_H,
 616     R21, R21_H,
 617     R22, R22_H,
 618     R23, R23_H,
 619     R24, R24_H,
 620     R25, R25_H,
 621     R26, R26_H,
 622  /* R27, R27_H, */              // heapbase
 623  /* R28, R28_H, */              // thread
 624  /* R29, R29_H, */              // fp
 625  /* R30, R30_H, */              // lr
 626  /* R31, R31_H */               // sp
 627 );
 628 
 629 // Class for all float registers
 630 reg_class float_reg(
 631     V0,
 632     V1,
 633     V2,
 634     V3,
 635     V4,
 636     V5,
 637     V6,
 638     V7,
 639     V8,
 640     V9,
 641     V10,
 642     V11,
 643     V12,
 644     V13,
 645     V14,
 646     V15,
 647     V16,
 648     V17,
 649     V18,
 650     V19,
 651     V20,
 652     V21,
 653     V22,
 654     V23,
 655     V24,
 656     V25,
 657     V26,
 658     V27,
 659     V28,
 660     V29,
 661     V30,
 662     V31
 663 );
 664 
 665 // Double precision float registers have virtual `high halves' that
 666 // are needed by the allocator.
 667 // Class for all double registers
 668 reg_class double_reg(
 669     V0, V0_H,
 670     V1, V1_H,
 671     V2, V2_H,
 672     V3, V3_H,
 673     V4, V4_H,
 674     V5, V5_H,
 675     V6, V6_H,
 676     V7, V7_H,
 677     V8, V8_H,
 678     V9, V9_H,
 679     V10, V10_H,
 680     V11, V11_H,
 681     V12, V12_H,
 682     V13, V13_H,
 683     V14, V14_H,
 684     V15, V15_H,
 685     V16, V16_H,
 686     V17, V17_H,
 687     V18, V18_H,
 688     V19, V19_H,
 689     V20, V20_H,
 690     V21, V21_H,
 691     V22, V22_H,
 692     V23, V23_H,
 693     V24, V24_H,
 694     V25, V25_H,
 695     V26, V26_H,
 696     V27, V27_H,
 697     V28, V28_H,
 698     V29, V29_H,
 699     V30, V30_H,
 700     V31, V31_H
 701 );
 702 
 703 // Class for 128 bit register v0
 704 reg_class v0_reg(
 705     V0, V0_H
 706 );
 707 
 708 // Class for 128 bit register v1
 709 reg_class v1_reg(
 710     V1, V1_H
 711 );
 712 
 713 // Class for 128 bit register v2
 714 reg_class v2_reg(
 715     V2, V2_H
 716 );
 717 
 718 // Class for 128 bit register v3
 719 reg_class v3_reg(
 720     V3, V3_H
 721 );
 722 
 723 // Singleton class for condition codes
 724 reg_class int_flags(RFLAGS);
 725 
 726 %}
 727 
 728 //----------DEFINITION BLOCK---------------------------------------------------
 729 // Define name --> value mappings to inform the ADLC of an integer valued name
 730 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 731 // Format:
 732 //        int_def  <name>         ( <int_value>, <expression>);
 733 // Generated Code in ad_<arch>.hpp
 734 //        #define  <name>   (<expression>)
 735 //        // value == <int_value>
 736 // Generated code in ad_<arch>.cpp adlc_verification()
 737 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 738 //
 739 
 740 // we follow the ppc-aix port in using a simple cost model which ranks
 741 // register operations as cheap, memory ops as more expensive and
 742 // branches as most expensive. the first two have a low as well as a
 743 // normal cost. huge cost appears to be a way of saying don't do
 744 // something
 745 
 746 definitions %{
 747   // The default cost (of a register move instruction).
 748   int_def INSN_COST            (    100,     100);
 749   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 750   int_def CALL_COST            (    200,     2 * INSN_COST);
 751   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 752 %}
 753 
 754 
 755 //----------SOURCE BLOCK-------------------------------------------------------
 756 // This is a block of C++ code which provides values, functions, and
 757 // definitions necessary in the rest of the architecture description
 758 
 759 source_hpp %{
 760 
 761 class CallStubImpl {
 762 
 763   //--------------------------------------------------------------
 764   //---<  Used for optimization in Compile::shorten_branches  >---
 765   //--------------------------------------------------------------
 766 
 767  public:
 768   // Size of call trampoline stub.
 769   static uint size_call_trampoline() {
 770     return 0; // no call trampolines on this platform
 771   }
 772 
 773   // number of relocations needed by a call trampoline stub
 774   static uint reloc_call_trampoline() {
 775     return 0; // no call trampolines on this platform
 776   }
 777 };
 778 
 779 class HandlerImpl {
 780 
 781  public:
 782 
 783   static int emit_exception_handler(CodeBuffer &cbuf);
 784   static int emit_deopt_handler(CodeBuffer& cbuf);
 785 
 786   static uint size_exception_handler() {
 787     return MacroAssembler::far_branch_size();
 788   }
 789 
 790   static uint size_deopt_handler() {
 791     // count one adr and one far branch instruction
 792     return 4 * NativeInstruction::instruction_size;
 793   }
 794 };
 795 
 796   bool preceded_by_ordered_load(const Node *barrier);
 797 
 798   // Use barrier instructions rather than load acquire / store
 799   // release.
 800   const bool UseBarriersForVolatile = true;
 801 %}
 802 
 803 source %{
 804 
 805   // AArch64 has load acquire and store release instructions which we
 806   // use for ordered memory accesses, e.g. for volatiles.  The ideal
 807   // graph generator also inserts memory barriers around volatile
 808   // accesses, and we don't want to generate both barriers and acq/rel
 809   // instructions.  So, when we emit a MemBarAcquire we look back in
 810   // the ideal graph for an ordered load and only emit the barrier if
 811   // we don't find one.
 812 
 813 bool preceded_by_ordered_load(const Node *barrier) {
 814   Node *x = barrier->lookup(TypeFunc::Parms);
 815 
 816   if (! x)
 817     return false;
 818 
 819   if (x->is_DecodeNarrowPtr())
 820     x = x->in(1);
 821 
 822   if (x->is_Load())
 823     return ! x->as_Load()->is_unordered();
 824 
 825   return false;
 826 }
 827 
 828 #define __ _masm.
 829 
 830 // advance declarations for helper functions to convert register
 831 // indices to register objects
 832 
 833 // the ad file has to provide implementations of certain methods
 834 // expected by the generic code
 835 //
 836 // REQUIRED FUNCTIONALITY
 837 
 838 //=============================================================================
 839 
 840 // !!!!! Special hack to get all types of calls to specify the byte offset
 841 //       from the start of the call to the point where the return address
 842 //       will point.
 843 
 844 int MachCallStaticJavaNode::ret_addr_offset()
 845 {
 846   // call should be a simple bl
 847   // unless this is a method handle invoke in which case it is
 848   // mov(rfp, sp), bl, mov(sp, rfp)
 849   int off = 4;
 850   if (_method_handle_invoke) {
 851     off += 4;
 852   }
 853   return off;
 854 }
 855 
 856 int MachCallDynamicJavaNode::ret_addr_offset()
 857 {
 858   return 16; // movz, movk, movk, bl
 859 }
 860 
 861 int MachCallRuntimeNode::ret_addr_offset() {
 862   // for generated stubs the call will be
 863   //   far_call(addr)
 864   // for real runtime callouts it will be six instructions
 865   // see aarch64_enc_java_to_runtime
 866   //   adr(rscratch2, retaddr)
 867   //   lea(rscratch1, RuntimeAddress(addr)
 868   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 869   //   blrt rscratch1
 870   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 871   if (cb) {
 872     return MacroAssembler::far_branch_size();
 873   } else {
 874     return 6 * NativeInstruction::instruction_size;
 875   }
 876 }
 877 
 878 // Indicate if the safepoint node needs the polling page as an input
 879 
 880 // the shared code plants the oop data at the start of the generated
 881 // code for the safepoint node and that needs ot be at the load
 882 // instruction itself. so we cannot plant a mov of the safepoint poll
 883 // address followed by a load. setting this to true means the mov is
 884 // scheduled as a prior instruction. that's better for scheduling
 885 // anyway.
 886 
 887 bool SafePointNode::needs_polling_address_input()
 888 {
 889   return true;
 890 }
 891 
 892 //=============================================================================
 893 
 894 #ifndef PRODUCT
 895 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 896   st->print("BREAKPOINT");
 897 }
 898 #endif
 899 
 900 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 901   MacroAssembler _masm(&cbuf);
 902   __ brk(0);
 903 }
 904 
 905 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 906   return MachNode::size(ra_);
 907 }
 908 
 909 //=============================================================================
 910 
 911 #ifndef PRODUCT
 912   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 913     st->print("nop \t# %d bytes pad for loops and calls", _count);
 914   }
 915 #endif
 916 
 917   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 918     MacroAssembler _masm(&cbuf);
 919     for (int i = 0; i < _count; i++) {
 920       __ nop();
 921     }
 922   }
 923 
 924   uint MachNopNode::size(PhaseRegAlloc*) const {
 925     return _count * NativeInstruction::instruction_size;
 926   }
 927 
 928 //=============================================================================
 929 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 930 
 931 int Compile::ConstantTable::calculate_table_base_offset() const {
 932   return 0;  // absolute addressing, no offset
 933 }
 934 
 935 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 936 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 937   ShouldNotReachHere();
 938 }
 939 
 940 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 941   // Empty encoding
 942 }
 943 
 944 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 945   return 0;
 946 }
 947 
 948 #ifndef PRODUCT
 949 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 950   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 951 }
 952 #endif
 953 
 954 #ifndef PRODUCT
 955 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 956   Compile* C = ra_->C;
 957 
 958   int framesize = C->frame_slots() << LogBytesPerInt;
 959 
 960   if (C->need_stack_bang(framesize))
 961     st->print("# stack bang size=%d\n\t", framesize);
 962 
 963   if (framesize == 0) {
 964     // Is this even possible?
 965     st->print("stp  lr, rfp, [sp, #%d]!", -(2 * wordSize));
 966   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 967     st->print("sub  sp, sp, #%d\n\t", framesize);
 968     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 969   } else {
 970     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 971     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 972     st->print("sub  sp, sp, rscratch1");
 973   }
 974 }
 975 #endif
 976 
 977 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 978   Compile* C = ra_->C;
 979   MacroAssembler _masm(&cbuf);
 980 
 981   // n.b. frame size includes space for return pc and rfp
 982   const long framesize = C->frame_size_in_bytes();
 983   assert(framesize%(2*wordSize) == 0, "must preserve 2*wordSize alignment");
 984 
 985   // insert a nop at the start of the prolog so we can patch in a
 986   // branch if we need to invalidate the method later
 987   __ nop();
 988 
 989   int bangsize = C->bang_size_in_bytes();
 990   if (C->need_stack_bang(bangsize) && UseStackBanging)
 991     __ generate_stack_overflow_check(bangsize);
 992 
 993   __ build_frame(framesize);
 994 
 995   if (NotifySimulator) {
 996     __ notify(Assembler::method_entry);
 997   }
 998 
 999   if (VerifyStackAtCalls) {
1000     Unimplemented();
1001   }
1002 
1003   C->set_frame_complete(cbuf.insts_size());
1004 
1005   if (C->has_mach_constant_base_node()) {
1006     // NOTE: We set the table base offset here because users might be
1007     // emitted before MachConstantBaseNode.
1008     Compile::ConstantTable& constant_table = C->constant_table();
1009     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1010   }
1011 }
1012 
1013 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1014 {
1015   return MachNode::size(ra_); // too many variables; just compute it
1016                               // the hard way
1017 }
1018 
1019 int MachPrologNode::reloc() const
1020 {
1021   return 0;
1022 }
1023 
1024 //=============================================================================
1025 
1026 #ifndef PRODUCT
1027 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1028   Compile* C = ra_->C;
1029   int framesize = C->frame_slots() << LogBytesPerInt;
1030 
1031   st->print("# pop frame %d\n\t",framesize);
1032 
1033   if (framesize == 0) {
1034     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1035   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1036     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1037     st->print("add  sp, sp, #%d\n\t", framesize);
1038   } else {
1039     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1040     st->print("add  sp, sp, rscratch1\n\t");
1041     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1042   }
1043 
1044   if (do_polling() && C->is_method_compilation()) {
1045     st->print("# touch polling page\n\t");
1046     st->print("mov  rscratch1, #0x%lx\n\t", p2i(os::get_polling_page()));
1047     st->print("ldr zr, [rscratch1]");
1048   }
1049 }
1050 #endif
1051 
1052 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1053   Compile* C = ra_->C;
1054   MacroAssembler _masm(&cbuf);
1055   int framesize = C->frame_slots() << LogBytesPerInt;
1056 
1057   __ remove_frame(framesize);
1058 
1059   if (NotifySimulator) {
1060     __ notify(Assembler::method_reentry);
1061   }
1062 
1063   if (do_polling() && C->is_method_compilation()) {
1064     __ read_polling_page(rscratch1, os::get_polling_page(), relocInfo::poll_return_type);
1065   }
1066 }
1067 
1068 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1069   // Variable size. Determine dynamically.
1070   return MachNode::size(ra_);
1071 }
1072 
1073 int MachEpilogNode::reloc() const {
1074   // Return number of relocatable values contained in this instruction.
1075   return 1; // 1 for polling page.
1076 }
1077 
1078 const Pipeline * MachEpilogNode::pipeline() const {
1079   return MachNode::pipeline_class();
1080 }
1081 
1082 // This method seems to be obsolete. It is declared in machnode.hpp
1083 // and defined in all *.ad files, but it is never called. Should we
1084 // get rid of it?
1085 int MachEpilogNode::safepoint_offset() const {
1086   assert(do_polling(), "no return for this epilog node");
1087   return 4;
1088 }
1089 
1090 //=============================================================================
1091 
1092 // Figure out which register class each belongs in: rc_int, rc_float or
1093 // rc_stack.
1094 enum RC { rc_bad, rc_int, rc_float, rc_stack };
1095 
1096 static enum RC rc_class(OptoReg::Name reg) {
1097 
1098   if (reg == OptoReg::Bad) {
1099     return rc_bad;
1100   }
1101 
1102   // we have 30 int registers * 2 halves
1103   // (rscratch1 and rscratch2 are omitted)
1104 
1105   if (reg < 60) {
1106     return rc_int;
1107   }
1108 
1109   // we have 32 float register * 2 halves
1110   if (reg < 60 + 64) {
1111     return rc_float;
1112   }
1113 
1114   // Between float regs & stack is the flags regs.
1115   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1116 
1117   return rc_stack;
1118 }
1119 
1120 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1121   Compile* C = ra_->C;
1122 
1123   // Get registers to move.
1124   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1125   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1126   OptoReg::Name dst_hi = ra_->get_reg_second(this);
1127   OptoReg::Name dst_lo = ra_->get_reg_first(this);
1128 
1129   enum RC src_hi_rc = rc_class(src_hi);
1130   enum RC src_lo_rc = rc_class(src_lo);
1131   enum RC dst_hi_rc = rc_class(dst_hi);
1132   enum RC dst_lo_rc = rc_class(dst_lo);
1133 
1134   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1135 
1136   if (src_hi != OptoReg::Bad) {
1137     assert((src_lo&1)==0 && src_lo+1==src_hi &&
1138            (dst_lo&1)==0 && dst_lo+1==dst_hi,
1139            "expected aligned-adjacent pairs");
1140   }
1141 
1142   if (src_lo == dst_lo && src_hi == dst_hi) {
1143     return 0;            // Self copy, no move.
1144   }
1145 
1146   switch (src_lo_rc) {
1147   case rc_int:
1148     if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
1149       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1150           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1151           // 64 bit
1152         if (cbuf) {
1153           MacroAssembler _masm(cbuf);
1154           __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1155                  as_Register(Matcher::_regEncode[src_lo]));
1156         } else if (st) {
1157           st->print("mov  %s, %s\t# shuffle",
1158                     Matcher::regName[dst_lo],
1159                     Matcher::regName[src_lo]);
1160         }
1161       } else {
1162         // 32 bit
1163         if (cbuf) {
1164           MacroAssembler _masm(cbuf);
1165           __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1166                   as_Register(Matcher::_regEncode[src_lo]));
1167         } else if (st) {
1168           st->print("movw  %s, %s\t# shuffle",
1169                     Matcher::regName[dst_lo],
1170                     Matcher::regName[src_lo]);
1171         }
1172       }
1173     } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1174       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1175           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1176           // 64 bit
1177         if (cbuf) {
1178           MacroAssembler _masm(cbuf);
1179           __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1180                    as_Register(Matcher::_regEncode[src_lo]));
1181         } else if (st) {
1182           st->print("fmovd  %s, %s\t# shuffle",
1183                     Matcher::regName[dst_lo],
1184                     Matcher::regName[src_lo]);
1185         }
1186       } else {
1187         // 32 bit
1188         if (cbuf) {
1189           MacroAssembler _masm(cbuf);
1190           __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1191                    as_Register(Matcher::_regEncode[src_lo]));
1192         } else if (st) {
1193           st->print("fmovs  %s, %s\t# shuffle",
1194                     Matcher::regName[dst_lo],
1195                     Matcher::regName[src_lo]);
1196         }
1197       }
1198     } else {                    // gpr --> stack spill
1199       assert(dst_lo_rc == rc_stack, "spill to bad register class");
1200       int dst_offset = ra_->reg2offset(dst_lo);
1201       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1202           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1203           // 64 bit
1204         if (cbuf) {
1205           MacroAssembler _masm(cbuf);
1206           __ str(as_Register(Matcher::_regEncode[src_lo]),
1207                  Address(sp, dst_offset));
1208         } else if (st) {
1209           st->print("str  %s, [sp, #%d]\t# spill",
1210                     Matcher::regName[src_lo],
1211                     dst_offset);
1212         }
1213       } else {
1214         // 32 bit
1215         if (cbuf) {
1216           MacroAssembler _masm(cbuf);
1217           __ strw(as_Register(Matcher::_regEncode[src_lo]),
1218                  Address(sp, dst_offset));
1219         } else if (st) {
1220           st->print("strw  %s, [sp, #%d]\t# spill",
1221                     Matcher::regName[src_lo],
1222                     dst_offset);
1223         }
1224       }
1225     }
1226     return 4;
1227   case rc_float:
1228     if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
1229       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1230           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1231           // 64 bit
1232         if (cbuf) {
1233           MacroAssembler _masm(cbuf);
1234           __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
1235                    as_FloatRegister(Matcher::_regEncode[src_lo]));
1236         } else if (st) {
1237           st->print("fmovd  %s, %s\t# shuffle",
1238                     Matcher::regName[dst_lo],
1239                     Matcher::regName[src_lo]);
1240         }
1241       } else {
1242         // 32 bit
1243         if (cbuf) {
1244           MacroAssembler _masm(cbuf);
1245           __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
1246                    as_FloatRegister(Matcher::_regEncode[src_lo]));
1247         } else if (st) {
1248           st->print("fmovs  %s, %s\t# shuffle",
1249                     Matcher::regName[dst_lo],
1250                     Matcher::regName[src_lo]);
1251         }
1252       }
1253     } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
1254       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1255           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1256           // 64 bit
1257         if (cbuf) {
1258           MacroAssembler _masm(cbuf);
1259           __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1260                    as_FloatRegister(Matcher::_regEncode[src_lo]));
1261         } else if (st) {
1262           st->print("fmovd  %s, %s\t# shuffle",
1263                     Matcher::regName[dst_lo],
1264                     Matcher::regName[src_lo]);
1265         }
1266       } else {
1267         // 32 bit
1268         if (cbuf) {
1269           MacroAssembler _masm(cbuf);
1270           __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1271                    as_FloatRegister(Matcher::_regEncode[src_lo]));
1272         } else if (st) {
1273           st->print("fmovs  %s, %s\t# shuffle",
1274                     Matcher::regName[dst_lo],
1275                     Matcher::regName[src_lo]);
1276         }
1277       }
1278     } else {                    // fpr --> stack spill
1279       assert(dst_lo_rc == rc_stack, "spill to bad register class");
1280       int dst_offset = ra_->reg2offset(dst_lo);
1281       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1282           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1283           // 64 bit
1284         if (cbuf) {
1285           MacroAssembler _masm(cbuf);
1286           __ strd(as_FloatRegister(Matcher::_regEncode[src_lo]),
1287                  Address(sp, dst_offset));
1288         } else if (st) {
1289           st->print("strd  %s, [sp, #%d]\t# spill",
1290                     Matcher::regName[src_lo],
1291                     dst_offset);
1292         }
1293       } else {
1294         // 32 bit
1295         if (cbuf) {
1296           MacroAssembler _masm(cbuf);
1297           __ strs(as_FloatRegister(Matcher::_regEncode[src_lo]),
1298                  Address(sp, dst_offset));
1299         } else if (st) {
1300           st->print("strs  %s, [sp, #%d]\t# spill",
1301                     Matcher::regName[src_lo],
1302                     dst_offset);
1303         }
1304       }
1305     }
1306     return 4;
1307   case rc_stack:
1308     int src_offset = ra_->reg2offset(src_lo);
1309     if (dst_lo_rc == rc_int) {  // stack --> gpr load
1310       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1311           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1312           // 64 bit
1313         if (cbuf) {
1314           MacroAssembler _masm(cbuf);
1315           __ ldr(as_Register(Matcher::_regEncode[dst_lo]),
1316                  Address(sp, src_offset));
1317         } else if (st) {
1318           st->print("ldr  %s, [sp, %d]\t# restore",
1319                     Matcher::regName[dst_lo],
1320                     src_offset);
1321         }
1322       } else {
1323         // 32 bit
1324         if (cbuf) {
1325           MacroAssembler _masm(cbuf);
1326           __ ldrw(as_Register(Matcher::_regEncode[dst_lo]),
1327                   Address(sp, src_offset));
1328         } else if (st) {
1329           st->print("ldr  %s, [sp, %d]\t# restore",
1330                     Matcher::regName[dst_lo],
1331                    src_offset);
1332         }
1333       }
1334       return 4;
1335     } else if (dst_lo_rc == rc_float) { // stack --> fpr load
1336       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1337           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1338           // 64 bit
1339         if (cbuf) {
1340           MacroAssembler _masm(cbuf);
1341           __ ldrd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1342                  Address(sp, src_offset));
1343         } else if (st) {
1344           st->print("ldrd  %s, [sp, %d]\t# restore",
1345                     Matcher::regName[dst_lo],
1346                     src_offset);
1347         }
1348       } else {
1349         // 32 bit
1350         if (cbuf) {
1351           MacroAssembler _masm(cbuf);
1352           __ ldrs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1353                   Address(sp, src_offset));
1354         } else if (st) {
1355           st->print("ldrs  %s, [sp, %d]\t# restore",
1356                     Matcher::regName[dst_lo],
1357                    src_offset);
1358         }
1359       }
1360       return 4;
1361     } else {                    // stack --> stack copy
1362       assert(dst_lo_rc == rc_stack, "spill to bad register class");
1363       int dst_offset = ra_->reg2offset(dst_lo);
1364       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1365           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1366           // 64 bit
1367         if (cbuf) {
1368           MacroAssembler _masm(cbuf);
1369           __ ldr(rscratch1, Address(sp, src_offset));
1370           __ str(rscratch1, Address(sp, dst_offset));
1371         } else if (st) {
1372           st->print("ldr  rscratch1, [sp, %d]\t# mem-mem spill",
1373                     src_offset);
1374           st->print("\n\t");
1375           st->print("str  rscratch1, [sp, %d]",
1376                     dst_offset);
1377         }
1378       } else {
1379         // 32 bit
1380         if (cbuf) {
1381           MacroAssembler _masm(cbuf);
1382           __ ldrw(rscratch1, Address(sp, src_offset));
1383           __ strw(rscratch1, Address(sp, dst_offset));
1384         } else if (st) {
1385           st->print("ldrw  rscratch1, [sp, %d]\t# mem-mem spill",
1386                     src_offset);
1387           st->print("\n\t");
1388           st->print("strw  rscratch1, [sp, %d]",
1389                     dst_offset);
1390         }
1391       }
1392       return 8;
1393     }
1394   }
1395 
1396   assert(false," bad rc_class for spill ");
1397   Unimplemented();
1398   return 0;
1399 
1400 }
1401 
1402 #ifndef PRODUCT
1403 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1404   if (!ra_)
1405     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
1406   else
1407     implementation(NULL, ra_, false, st);
1408 }
1409 #endif
1410 
1411 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1412   implementation(&cbuf, ra_, false, NULL);
1413 }
1414 
1415 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1416   return implementation(NULL, ra_, true, NULL);
1417 }
1418 
1419 //=============================================================================
1420 
1421 #ifndef PRODUCT
1422 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1423   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1424   int reg = ra_->get_reg_first(this);
1425   st->print("add %s, rsp, #%d]\t# box lock",
1426             Matcher::regName[reg], offset);
1427 }
1428 #endif
1429 
1430 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1431   MacroAssembler _masm(&cbuf);
1432 
1433   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1434   int reg    = ra_->get_encode(this);
1435 
1436   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
1437     __ add(as_Register(reg), sp, offset);
1438   } else {
1439     ShouldNotReachHere();
1440   }
1441 }
1442 
1443 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
1444   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
1445   return 4;
1446 }
1447 
1448 //=============================================================================
1449 
1450 #ifndef PRODUCT
1451 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1452 {
1453   st->print_cr("# MachUEPNode");
1454   if (UseCompressedClassPointers) {
1455     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1456     if (Universe::narrow_klass_shift() != 0) {
1457       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1458     }
1459   } else {
1460    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1461   }
1462   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
1463   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
1464 }
1465 #endif
1466 
1467 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1468 {
1469   // This is the unverified entry point.
1470   MacroAssembler _masm(&cbuf);
1471 
1472   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
1473   Label skip;
1474   // TODO
1475   // can we avoid this skip and still use a reloc?
1476   __ br(Assembler::EQ, skip);
1477   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1478   __ bind(skip);
1479 }
1480 
1481 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1482 {
1483   return MachNode::size(ra_);
1484 }
1485 
1486 // REQUIRED EMIT CODE
1487 
1488 //=============================================================================
1489 
1490 // Emit exception handler code.
1491 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
1492 {
1493   // mov rscratch1 #exception_blob_entry_point
1494   // br rscratch1
1495   // Note that the code buffer's insts_mark is always relative to insts.
1496   // That's why we must use the macroassembler to generate a handler.
1497   MacroAssembler _masm(&cbuf);
1498   address base =
1499   __ start_a_stub(size_exception_handler());
1500   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1501   int offset = __ offset();
1502   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
1503   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1504   __ end_a_stub();
1505   return offset;
1506 }
1507 
1508 // Emit deopt handler code.
1509 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
1510 {
1511   // Note that the code buffer's insts_mark is always relative to insts.
1512   // That's why we must use the macroassembler to generate a handler.
1513   MacroAssembler _masm(&cbuf);
1514   address base =
1515   __ start_a_stub(size_deopt_handler());
1516   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1517   int offset = __ offset();
1518 
1519   __ adr(lr, __ pc());
1520   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
1521 
1522   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1523   __ end_a_stub();
1524   return offset;
1525 }
1526 
1527 // REQUIRED MATCHER CODE
1528 
1529 //=============================================================================
1530 
1531 const bool Matcher::match_rule_supported(int opcode) {
1532 
1533   // TODO
1534   // identify extra cases that we might want to provide match rules for
1535   // e.g. Op_StrEquals and other intrinsics
1536   if (!has_match_rule(opcode)) {
1537     return false;
1538   }
1539 
1540   return true;  // Per default match rules are supported.
1541 }
1542 
1543 int Matcher::regnum_to_fpu_offset(int regnum)
1544 {
1545   Unimplemented();
1546   return 0;
1547 }
1548 
1549 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset)
1550 {
1551   Unimplemented();
1552   return false;
1553 }
1554 
1555 const bool Matcher::isSimpleConstant64(jlong value) {
1556   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1557   // Probably always true, even if a temp register is required.
1558   return true;
1559 }
1560 
1561 // true just means we have fast l2f conversion
1562 const bool Matcher::convL2FSupported(void) {
1563   return true;
1564 }
1565 
1566 // Vector width in bytes.
1567 const int Matcher::vector_width_in_bytes(BasicType bt) {
1568   // TODO fixme
1569   return 0;
1570 }
1571 
1572 // Limits on vector size (number of elements) loaded into vector.
1573 const int Matcher::max_vector_size(const BasicType bt) {
1574   return vector_width_in_bytes(bt)/type2aelembytes(bt);
1575 }
1576 const int Matcher::min_vector_size(const BasicType bt) {
1577   int max_size = max_vector_size(bt);
1578   // Min size which can be loaded into vector is 4 bytes.
1579   int size = (type2aelembytes(bt) == 1) ? 4 : 2;
1580   return MIN2(size,max_size);
1581 }
1582 
1583 // Vector ideal reg.
1584 const int Matcher::vector_ideal_reg(int len) {
1585   // TODO fixme
1586   return Op_RegD;
1587 }
1588 
1589 // Only lowest bits of xmm reg are used for vector shift count.
1590 const int Matcher::vector_shift_count_ideal_reg(int size) {
1591   // TODO fixme
1592   return Op_RegL;
1593 }
1594 
1595 // AES support not yet implemented
1596 const bool Matcher::pass_original_key_for_aes() {
1597   return false;
1598 }
1599 
1600 // x86 supports misaligned vectors store/load.
1601 const bool Matcher::misaligned_vectors_ok() {
1602   // TODO fixme
1603   // return !AlignVector; // can be changed by flag
1604   return false;
1605 }
1606 
1607 // false => size gets scaled to BytesPerLong, ok.
1608 const bool Matcher::init_array_count_is_in_bytes = false;
1609 
1610 // Threshold size for cleararray.
1611 const int Matcher::init_array_short_size = 18 * BytesPerLong;
1612 
1613 // Use conditional move (CMOVL)
1614 const int Matcher::long_cmove_cost() {
1615   // long cmoves are no more expensive than int cmoves
1616   return 0;
1617 }
1618 
1619 const int Matcher::float_cmove_cost() {
1620   // float cmoves are no more expensive than int cmoves
1621   return 0;
1622 }
1623 
1624 // Does the CPU require late expand (see block.cpp for description of late expand)?
1625 const bool Matcher::require_postalloc_expand = false;
1626 
1627 // Should the Matcher clone shifts on addressing modes, expecting them
1628 // to be subsumed into complex addressing expressions or compute them
1629 // into registers?  True for Intel but false for most RISCs
1630 const bool Matcher::clone_shift_expressions = false;
1631 
1632 // Do we need to mask the count passed to shift instructions or does
1633 // the cpu only look at the lower 5/6 bits anyway?
1634 const bool Matcher::need_masked_shift_count = false;
1635 
1636 // This affects two different things:
1637 //  - how Decode nodes are matched
1638 //  - how ImplicitNullCheck opportunities are recognized
1639 // If true, the matcher will try to remove all Decodes and match them
1640 // (as operands) into nodes. NullChecks are not prepared to deal with
1641 // Decodes by final_graph_reshaping().
1642 // If false, final_graph_reshaping() forces the decode behind the Cmp
1643 // for a NullCheck. The matcher matches the Decode node into a register.
1644 // Implicit_null_check optimization moves the Decode along with the
1645 // memory operation back up before the NullCheck.
1646 bool Matcher::narrow_oop_use_complex_address() {
1647   return Universe::narrow_oop_shift() == 0;
1648 }
1649 
1650 bool Matcher::narrow_klass_use_complex_address() {
1651 // TODO
1652 // decide whether we need to set this to true
1653   return false;
1654 }
1655 
1656 // Is it better to copy float constants, or load them directly from
1657 // memory?  Intel can load a float constant from a direct address,
1658 // requiring no extra registers.  Most RISCs will have to materialize
1659 // an address into a register first, so they would do better to copy
1660 // the constant from stack.
1661 const bool Matcher::rematerialize_float_constants = false;
1662 
1663 // If CPU can load and store mis-aligned doubles directly then no
1664 // fixup is needed.  Else we split the double into 2 integer pieces
1665 // and move it piece-by-piece.  Only happens when passing doubles into
1666 // C code as the Java calling convention forces doubles to be aligned.
1667 const bool Matcher::misaligned_doubles_ok = true;
1668 
1669 // No-op on amd64
1670 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
1671   Unimplemented();
1672 }
1673 
1674 // Advertise here if the CPU requires explicit rounding operations to
1675 // implement the UseStrictFP mode.
1676 const bool Matcher::strict_fp_requires_explicit_rounding = false;
1677 
1678 // Are floats converted to double when stored to stack during
1679 // deoptimization?
1680 bool Matcher::float_in_double() { return true; }
1681 
1682 // Do ints take an entire long register or just half?
1683 // The relevant question is how the int is callee-saved:
1684 // the whole long is written but de-opt'ing will have to extract
1685 // the relevant 32 bits.
1686 const bool Matcher::int_in_long = true;
1687 
1688 // Return whether or not this register is ever used as an argument.
1689 // This function is used on startup to build the trampoline stubs in
1690 // generateOptoStub.  Registers not mentioned will be killed by the VM
1691 // call in the trampoline, and arguments in those registers not be
1692 // available to the callee.
1693 bool Matcher::can_be_java_arg(int reg)
1694 {
1695   return
1696     reg ==  R0_num || reg == R0_H_num ||
1697     reg ==  R1_num || reg == R1_H_num ||
1698     reg ==  R2_num || reg == R2_H_num ||
1699     reg ==  R3_num || reg == R3_H_num ||
1700     reg ==  R4_num || reg == R4_H_num ||
1701     reg ==  R5_num || reg == R5_H_num ||
1702     reg ==  R6_num || reg == R6_H_num ||
1703     reg ==  R7_num || reg == R7_H_num ||
1704     reg ==  V0_num || reg == V0_H_num ||
1705     reg ==  V1_num || reg == V1_H_num ||
1706     reg ==  V2_num || reg == V2_H_num ||
1707     reg ==  V3_num || reg == V3_H_num ||
1708     reg ==  V4_num || reg == V4_H_num ||
1709     reg ==  V5_num || reg == V5_H_num ||
1710     reg ==  V6_num || reg == V6_H_num ||
1711     reg ==  V7_num || reg == V7_H_num;
1712 }
1713 
1714 bool Matcher::is_spillable_arg(int reg)
1715 {
1716   return can_be_java_arg(reg);
1717 }
1718 
1719 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
1720   return false;
1721 }
1722 
1723 RegMask Matcher::divI_proj_mask() {
1724   ShouldNotReachHere();
1725   return RegMask();
1726 }
1727 
1728 // Register for MODI projection of divmodI.
1729 RegMask Matcher::modI_proj_mask() {
1730   ShouldNotReachHere();
1731   return RegMask();
1732 }
1733 
1734 // Register for DIVL projection of divmodL.
1735 RegMask Matcher::divL_proj_mask() {
1736   ShouldNotReachHere();
1737   return RegMask();
1738 }
1739 
1740 // Register for MODL projection of divmodL.
1741 RegMask Matcher::modL_proj_mask() {
1742   ShouldNotReachHere();
1743   return RegMask();
1744 }
1745 
1746 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1747   return RegMask();
1748 }
1749 
1750 // helper for encoding java_to_runtime calls on sim
1751 //
1752 // this is needed to compute the extra arguments required when
1753 // planting a call to the simulator blrt instruction. the TypeFunc
1754 // can be queried to identify the counts for integral, and floating
1755 // arguments and the return type
1756 
1757 static void getCallInfo(const TypeFunc *tf, int &gpcnt, int &fpcnt, int &rtype)
1758 {
1759   int gps = 0;
1760   int fps = 0;
1761   const TypeTuple *domain = tf->domain();
1762   int max = domain->cnt();
1763   for (int i = TypeFunc::Parms; i < max; i++) {
1764     const Type *t = domain->field_at(i);
1765     switch(t->basic_type()) {
1766     case T_FLOAT:
1767     case T_DOUBLE:
1768       fps++;
1769     default:
1770       gps++;
1771     }
1772   }
1773   gpcnt = gps;
1774   fpcnt = fps;
1775   BasicType rt = tf->return_type();
1776   switch (rt) {
1777   case T_VOID:
1778     rtype = MacroAssembler::ret_type_void;
1779     break;
1780   default:
1781     rtype = MacroAssembler::ret_type_integral;
1782     break;
1783   case T_FLOAT:
1784     rtype = MacroAssembler::ret_type_float;
1785     break;
1786   case T_DOUBLE:
1787     rtype = MacroAssembler::ret_type_double;
1788     break;
1789   }
1790 }
1791 
1792 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
1793   MacroAssembler _masm(&cbuf);                                          \
1794   {                                                                     \
1795     guarantee(INDEX == -1, "mode not permitted for volatile");          \
1796     guarantee(DISP == 0, "mode not permitted for volatile");            \
1797     guarantee(SCALE == 0, "mode not permitted for volatile");           \
1798     __ INSN(REG, as_Register(BASE));                                    \
1799   }
1800 
1801 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
1802 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
1803 
1804   // Used for all non-volatile memory accesses.  The use of
1805   // $mem->opcode() to discover whether this pattern uses sign-extended
1806   // offsets is something of a kludge.
1807   static void loadStore(MacroAssembler masm, mem_insn insn,
1808                          Register reg, int opcode,
1809                          Register base, int index, int size, int disp)
1810   {
1811     Address::extend scale;
1812 
1813     // Hooboy, this is fugly.  We need a way to communicate to the
1814     // encoder that the index needs to be sign extended, so we have to
1815     // enumerate all the cases.
1816     switch (opcode) {
1817     case INDINDEXSCALEDOFFSETI2L:
1818     case INDINDEXSCALEDI2L:
1819     case INDINDEXSCALEDOFFSETI2LN:
1820     case INDINDEXSCALEDI2LN:
1821     case INDINDEXOFFSETI2L:
1822     case INDINDEXOFFSETI2LN:
1823       scale = Address::sxtw(size);
1824       break;
1825     default:
1826       scale = Address::lsl(size);
1827     }
1828 
1829     if (index == -1) {
1830       (masm.*insn)(reg, Address(base, disp));
1831     } else {
1832       if (disp == 0) {
1833         (masm.*insn)(reg, Address(base, as_Register(index), scale));
1834       } else {
1835         masm.lea(rscratch1, Address(base, disp));
1836         (masm.*insn)(reg, Address(rscratch1, as_Register(index), scale));
1837       }
1838     }
1839   }
1840 
1841   static void loadStore(MacroAssembler masm, mem_float_insn insn,
1842                          FloatRegister reg, int opcode,
1843                          Register base, int index, int size, int disp)
1844   {
1845     Address::extend scale;
1846 
1847     switch (opcode) {
1848     case INDINDEXSCALEDOFFSETI2L:
1849     case INDINDEXSCALEDI2L:
1850     case INDINDEXSCALEDOFFSETI2LN:
1851     case INDINDEXSCALEDI2LN:
1852       scale = Address::sxtw(size);
1853       break;
1854     default:
1855       scale = Address::lsl(size);
1856     }
1857 
1858      if (index == -1) {
1859       (masm.*insn)(reg, Address(base, disp));
1860     } else {
1861       if (disp == 0) {
1862         (masm.*insn)(reg, Address(base, as_Register(index), scale));
1863       } else {
1864         masm.lea(rscratch1, Address(base, disp));
1865         (masm.*insn)(reg, Address(rscratch1, as_Register(index), scale));
1866       }
1867     }
1868   }
1869 
1870 %}
1871 
1872 
1873 
1874 //----------ENCODING BLOCK-----------------------------------------------------
1875 // This block specifies the encoding classes used by the compiler to
1876 // output byte streams.  Encoding classes are parameterized macros
1877 // used by Machine Instruction Nodes in order to generate the bit
1878 // encoding of the instruction.  Operands specify their base encoding
1879 // interface with the interface keyword.  There are currently
1880 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1881 // COND_INTER.  REG_INTER causes an operand to generate a function
1882 // which returns its register number when queried.  CONST_INTER causes
1883 // an operand to generate a function which returns the value of the
1884 // constant when queried.  MEMORY_INTER causes an operand to generate
1885 // four functions which return the Base Register, the Index Register,
1886 // the Scale Value, and the Offset Value of the operand when queried.
1887 // COND_INTER causes an operand to generate six functions which return
1888 // the encoding code (ie - encoding bits for the instruction)
1889 // associated with each basic boolean condition for a conditional
1890 // instruction.
1891 //
1892 // Instructions specify two basic values for encoding.  Again, a
1893 // function is available to check if the constant displacement is an
1894 // oop. They use the ins_encode keyword to specify their encoding
1895 // classes (which must be a sequence of enc_class names, and their
1896 // parameters, specified in the encoding block), and they use the
1897 // opcode keyword to specify, in order, their primary, secondary, and
1898 // tertiary opcode.  Only the opcode sections which a particular
1899 // instruction needs for encoding need to be specified.
1900 encode %{
1901   // Build emit functions for each basic byte or larger field in the
1902   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1903   // from C++ code in the enc_class source block.  Emit functions will
1904   // live in the main source block for now.  In future, we can
1905   // generalize this by adding a syntax that specifies the sizes of
1906   // fields in an order, so that the adlc can build the emit functions
1907   // automagically
1908 
1909   // catch all for unimplemented encodings
1910   enc_class enc_unimplemented %{
1911     MacroAssembler _masm(&cbuf);
1912     __ unimplemented("C2 catch all");
1913   %}
1914 
1915   // BEGIN Non-volatile memory access
1916 
1917   enc_class aarch64_enc_ldrsbw(iRegI dst, memory mem) %{
1918     Register dst_reg = as_Register($dst$$reg);
1919     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
1920                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1921   %}
1922 
1923   enc_class aarch64_enc_ldrsb(iRegI dst, memory mem) %{
1924     Register dst_reg = as_Register($dst$$reg);
1925     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
1926                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1927   %}
1928 
1929   enc_class aarch64_enc_ldrb(iRegI dst, memory mem) %{
1930     Register dst_reg = as_Register($dst$$reg);
1931     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
1932                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1933   %}
1934 
1935   enc_class aarch64_enc_ldrb(iRegL dst, memory mem) %{
1936     Register dst_reg = as_Register($dst$$reg);
1937     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
1938                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1939   %}
1940 
1941   enc_class aarch64_enc_ldrshw(iRegI dst, memory mem) %{
1942     Register dst_reg = as_Register($dst$$reg);
1943     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
1944                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1945   %}
1946 
1947   enc_class aarch64_enc_ldrsh(iRegI dst, memory mem) %{
1948     Register dst_reg = as_Register($dst$$reg);
1949     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
1950                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1951   %}
1952 
1953   enc_class aarch64_enc_ldrh(iRegI dst, memory mem) %{
1954     Register dst_reg = as_Register($dst$$reg);
1955     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
1956                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1957   %}
1958 
1959   enc_class aarch64_enc_ldrh(iRegL dst, memory mem) %{
1960     Register dst_reg = as_Register($dst$$reg);
1961     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
1962                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1963   %}
1964 
1965   enc_class aarch64_enc_ldrw(iRegI dst, memory mem) %{
1966     Register dst_reg = as_Register($dst$$reg);
1967     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
1968                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1969   %}
1970 
1971   enc_class aarch64_enc_ldrw(iRegL dst, memory mem) %{
1972     Register dst_reg = as_Register($dst$$reg);
1973     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
1974                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1975   %}
1976 
1977   enc_class aarch64_enc_ldrsw(iRegL dst, memory mem) %{
1978     Register dst_reg = as_Register($dst$$reg);
1979     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
1980                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1981   %}
1982 
1983   enc_class aarch64_enc_ldr(iRegL dst, memory mem) %{
1984     Register dst_reg = as_Register($dst$$reg);
1985     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
1986                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1987   %}
1988 
1989   enc_class aarch64_enc_ldrs(vRegF dst, memory mem) %{
1990     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
1991     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
1992                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1993   %}
1994 
1995   enc_class aarch64_enc_ldrd(vRegD dst, memory mem) %{
1996     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
1997     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
1998                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1999   %}
2000 
2001   enc_class aarch64_enc_strb(iRegI src, memory mem) %{
2002     Register src_reg = as_Register($src$$reg);
2003     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2004                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2005   %}
2006 
2007   enc_class aarch64_enc_strb0(memory mem) %{
2008     MacroAssembler _masm(&cbuf);
2009     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2010                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2011   %}
2012 
2013   enc_class aarch64_enc_strh(iRegI src, memory mem) %{
2014     Register src_reg = as_Register($src$$reg);
2015     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2016                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2017   %}
2018 
2019   enc_class aarch64_enc_strh0(memory mem) %{
2020     MacroAssembler _masm(&cbuf);
2021     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2022                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2023   %}
2024 
2025   enc_class aarch64_enc_strw(iRegI src, memory mem) %{
2026     Register src_reg = as_Register($src$$reg);
2027     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2028                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2029   %}
2030 
2031   enc_class aarch64_enc_strw0(memory mem) %{
2032     MacroAssembler _masm(&cbuf);
2033     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2034                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2035   %}
2036 
2037   enc_class aarch64_enc_str(iRegL src, memory mem) %{
2038     Register src_reg = as_Register($src$$reg);
2039     // we sometimes get asked to store the stack pointer into the
2040     // current thread -- we cannot do that directly on AArch64
2041     if (src_reg == r31_sp) {
2042       MacroAssembler _masm(&cbuf);
2043       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2044       __ mov(rscratch2, sp);
2045       src_reg = rscratch2;
2046     }
2047     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2048                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2049   %}
2050 
2051   enc_class aarch64_enc_str0(memory mem) %{
2052     MacroAssembler _masm(&cbuf);
2053     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2054                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2055   %}
2056 
2057   enc_class aarch64_enc_strs(vRegF src, memory mem) %{
2058     FloatRegister src_reg = as_FloatRegister($src$$reg);
2059     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2060                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2061   %}
2062 
2063   enc_class aarch64_enc_strd(vRegD src, memory mem) %{
2064     FloatRegister src_reg = as_FloatRegister($src$$reg);
2065     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2066                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2067   %}
2068 
2069   // END Non-volatile memory access
2070 
2071   // volatile loads and stores
2072 
2073   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
2074     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2075                  rscratch1, stlrb);
2076   %}
2077 
2078   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
2079     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2080                  rscratch1, stlrh);
2081   %}
2082 
2083   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
2084     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2085                  rscratch1, stlrw);
2086   %}
2087 
2088 
2089   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
2090     Register dst_reg = as_Register($dst$$reg);
2091     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2092              rscratch1, ldarb);
2093     __ sxtbw(dst_reg, dst_reg);
2094   %}
2095 
2096   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
2097     Register dst_reg = as_Register($dst$$reg);
2098     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2099              rscratch1, ldarb);
2100     __ sxtb(dst_reg, dst_reg);
2101   %}
2102 
2103   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
2104     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2105              rscratch1, ldarb);
2106   %}
2107 
2108   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
2109     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2110              rscratch1, ldarb);
2111   %}
2112 
2113   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
2114     Register dst_reg = as_Register($dst$$reg);
2115     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2116              rscratch1, ldarh);
2117     __ sxthw(dst_reg, dst_reg);
2118   %}
2119 
2120   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
2121     Register dst_reg = as_Register($dst$$reg);
2122     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2123              rscratch1, ldarh);
2124     __ sxth(dst_reg, dst_reg);
2125   %}
2126 
2127   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
2128     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2129              rscratch1, ldarh);
2130   %}
2131 
2132   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
2133     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2134              rscratch1, ldarh);
2135   %}
2136 
2137   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
2138     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2139              rscratch1, ldarw);
2140   %}
2141 
2142   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
2143     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2144              rscratch1, ldarw);
2145   %}
2146 
2147   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
2148     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2149              rscratch1, ldar);
2150   %}
2151 
2152   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
2153     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2154              rscratch1, ldarw);
2155     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
2156   %}
2157 
2158   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
2159     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2160              rscratch1, ldar);
2161     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
2162   %}
2163 
2164   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
2165     Register src_reg = as_Register($src$$reg);
2166     // we sometimes get asked to store the stack pointer into the
2167     // current thread -- we cannot do that directly on AArch64
2168     if (src_reg == r31_sp) {
2169         MacroAssembler _masm(&cbuf);
2170       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2171       __ mov(rscratch2, sp);
2172       src_reg = rscratch2;
2173     }
2174     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2175                  rscratch1, stlr);
2176   %}
2177 
2178   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
2179     {
2180       MacroAssembler _masm(&cbuf);
2181       FloatRegister src_reg = as_FloatRegister($src$$reg);
2182       __ fmovs(rscratch2, src_reg);
2183     }
2184     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2185                  rscratch1, stlrw);
2186   %}
2187 
2188   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
2189     {
2190       MacroAssembler _masm(&cbuf);
2191       FloatRegister src_reg = as_FloatRegister($src$$reg);
2192       __ fmovd(rscratch2, src_reg);
2193     }
2194     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2195                  rscratch1, stlr);
2196   %}
2197 
2198   // synchronized read/update encodings
2199 
2200   enc_class aarch64_enc_ldaxr(iRegL dst, memory mem) %{
2201     MacroAssembler _masm(&cbuf);
2202     Register dst_reg = as_Register($dst$$reg);
2203     Register base = as_Register($mem$$base);
2204     int index = $mem$$index;
2205     int scale = $mem$$scale;
2206     int disp = $mem$$disp;
2207     if (index == -1) {
2208        if (disp != 0) {
2209         __ lea(rscratch1, Address(base, disp));
2210         __ ldaxr(dst_reg, rscratch1);
2211       } else {
2212         // TODO
2213         // should we ever get anything other than this case?
2214         __ ldaxr(dst_reg, base);
2215       }
2216     } else {
2217       Register index_reg = as_Register(index);
2218       if (disp == 0) {
2219         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
2220         __ ldaxr(dst_reg, rscratch1);
2221       } else {
2222         __ lea(rscratch1, Address(base, disp));
2223         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
2224         __ ldaxr(dst_reg, rscratch1);
2225       }
2226     }
2227   %}
2228 
2229   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory mem) %{
2230     MacroAssembler _masm(&cbuf);
2231     Register src_reg = as_Register($src$$reg);
2232     Register base = as_Register($mem$$base);
2233     int index = $mem$$index;
2234     int scale = $mem$$scale;
2235     int disp = $mem$$disp;
2236     if (index == -1) {
2237        if (disp != 0) {
2238         __ lea(rscratch2, Address(base, disp));
2239         __ stlxr(rscratch1, src_reg, rscratch2);
2240       } else {
2241         // TODO
2242         // should we ever get anything other than this case?
2243         __ stlxr(rscratch1, src_reg, base);
2244       }
2245     } else {
2246       Register index_reg = as_Register(index);
2247       if (disp == 0) {
2248         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
2249         __ stlxr(rscratch1, src_reg, rscratch2);
2250       } else {
2251         __ lea(rscratch2, Address(base, disp));
2252         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
2253         __ stlxr(rscratch1, src_reg, rscratch2);
2254       }
2255     }
2256     __ cmpw(rscratch1, zr);
2257   %}
2258 
2259   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2260     MacroAssembler _masm(&cbuf);
2261     Register old_reg = as_Register($oldval$$reg);
2262     Register new_reg = as_Register($newval$$reg);
2263     Register base = as_Register($mem$$base);
2264     Register addr_reg;
2265     int index = $mem$$index;
2266     int scale = $mem$$scale;
2267     int disp = $mem$$disp;
2268     if (index == -1) {
2269        if (disp != 0) {
2270         __ lea(rscratch2, Address(base, disp));
2271         addr_reg = rscratch2;
2272       } else {
2273         // TODO
2274         // should we ever get anything other than this case?
2275         addr_reg = base;
2276       }
2277     } else {
2278       Register index_reg = as_Register(index);
2279       if (disp == 0) {
2280         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
2281         addr_reg = rscratch2;
2282       } else {
2283         __ lea(rscratch2, Address(base, disp));
2284         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
2285         addr_reg = rscratch2;
2286       }
2287     }
2288     Label retry_load, done;
2289     __ bind(retry_load);
2290     __ ldxr(rscratch1, addr_reg);
2291     __ cmp(rscratch1, old_reg);
2292     __ br(Assembler::NE, done);
2293     __ stlxr(rscratch1, new_reg, addr_reg);
2294     __ cbnzw(rscratch1, retry_load);
2295     __ bind(done);
2296   %}
2297 
2298   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2299     MacroAssembler _masm(&cbuf);
2300     Register old_reg = as_Register($oldval$$reg);
2301     Register new_reg = as_Register($newval$$reg);
2302     Register base = as_Register($mem$$base);
2303     Register addr_reg;
2304     int index = $mem$$index;
2305     int scale = $mem$$scale;
2306     int disp = $mem$$disp;
2307     if (index == -1) {
2308        if (disp != 0) {
2309         __ lea(rscratch2, Address(base, disp));
2310         addr_reg = rscratch2;
2311       } else {
2312         // TODO
2313         // should we ever get anything other than this case?
2314         addr_reg = base;
2315       }
2316     } else {
2317       Register index_reg = as_Register(index);
2318       if (disp == 0) {
2319         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
2320         addr_reg = rscratch2;
2321       } else {
2322         __ lea(rscratch2, Address(base, disp));
2323         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
2324         addr_reg = rscratch2;
2325       }
2326     }
2327     Label retry_load, done;
2328     __ bind(retry_load);
2329     __ ldxrw(rscratch1, addr_reg);
2330     __ cmpw(rscratch1, old_reg);
2331     __ br(Assembler::NE, done);
2332     __ stlxrw(rscratch1, new_reg, addr_reg);
2333     __ cbnzw(rscratch1, retry_load);
2334     __ bind(done);
2335   %}
2336 
2337   // auxiliary used for CompareAndSwapX to set result register
2338   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
2339     MacroAssembler _masm(&cbuf);
2340     Register res_reg = as_Register($res$$reg);
2341     __ cset(res_reg, Assembler::EQ);
2342   %}
2343 
2344   // prefetch encodings
2345 
2346   enc_class aarch64_enc_prefetchw(memory mem) %{
2347     MacroAssembler _masm(&cbuf);
2348     Register base = as_Register($mem$$base);
2349     int index = $mem$$index;
2350     int scale = $mem$$scale;
2351     int disp = $mem$$disp;
2352     if (index == -1) {
2353       __ prfm(Address(base, disp), PSTL1KEEP);
2354       __ nop();
2355     } else {
2356       Register index_reg = as_Register(index);
2357       if (disp == 0) {
2358         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
2359       } else {
2360         __ lea(rscratch1, Address(base, disp));
2361         __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
2362       }
2363     }
2364   %}
2365 
2366   enc_class aarch64_enc_clear_array_reg_reg(iRegL_R11 cnt, iRegP_R10 base) %{
2367     MacroAssembler _masm(&cbuf);
2368     Register cnt_reg = as_Register($cnt$$reg);
2369     Register base_reg = as_Register($base$$reg);
2370     // base is word aligned
2371     // cnt is count of words
2372 
2373     Label loop;
2374     Label entry;
2375 
2376 //  Algorithm:
2377 //
2378 //    scratch1 = cnt & 7;
2379 //    cnt -= scratch1;
2380 //    p += scratch1;
2381 //    switch (scratch1) {
2382 //      do {
2383 //        cnt -= 8;
2384 //          p[-8] = 0;
2385 //        case 7:
2386 //          p[-7] = 0;
2387 //        case 6:
2388 //          p[-6] = 0;
2389 //          // ...
2390 //        case 1:
2391 //          p[-1] = 0;
2392 //        case 0:
2393 //          p += 8;
2394 //      } while (cnt);
2395 //    }
2396 
2397     const int unroll = 8; // Number of str(zr) instructions we'll unroll
2398 
2399     __ andr(rscratch1, cnt_reg, unroll - 1);  // tmp1 = cnt % unroll
2400     __ sub(cnt_reg, cnt_reg, rscratch1);      // cnt -= unroll
2401     // base_reg always points to the end of the region we're about to zero
2402     __ add(base_reg, base_reg, rscratch1, Assembler::LSL, exact_log2(wordSize));
2403     __ adr(rscratch2, entry);
2404     __ sub(rscratch2, rscratch2, rscratch1, Assembler::LSL, 2);
2405     __ br(rscratch2);
2406     __ bind(loop);
2407     __ sub(cnt_reg, cnt_reg, unroll);
2408     for (int i = -unroll; i < 0; i++)
2409       __ str(zr, Address(base_reg, i * wordSize));
2410     __ bind(entry);
2411     __ add(base_reg, base_reg, unroll * wordSize);
2412     __ cbnz(cnt_reg, loop);
2413   %}
2414 
2415   /// mov envcodings
2416 
2417   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
2418     MacroAssembler _masm(&cbuf);
2419     u_int32_t con = (u_int32_t)$src$$constant;
2420     Register dst_reg = as_Register($dst$$reg);
2421     if (con == 0) {
2422       __ movw(dst_reg, zr);
2423     } else {
2424       __ movw(dst_reg, con);
2425     }
2426   %}
2427 
2428   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
2429     MacroAssembler _masm(&cbuf);
2430     Register dst_reg = as_Register($dst$$reg);
2431     u_int64_t con = (u_int64_t)$src$$constant;
2432     if (con == 0) {
2433       __ mov(dst_reg, zr);
2434     } else {
2435       __ mov(dst_reg, con);
2436     }
2437   %}
2438 
2439   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
2440     MacroAssembler _masm(&cbuf);
2441     Register dst_reg = as_Register($dst$$reg);
2442     address con = (address)$src$$constant;
2443     if (con == NULL || con == (address)1) {
2444       ShouldNotReachHere();
2445     } else {
2446       relocInfo::relocType rtype = $src->constant_reloc();
2447       if (rtype == relocInfo::oop_type) {
2448         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
2449       } else if (rtype == relocInfo::metadata_type) {
2450         __ mov_metadata(dst_reg, (Metadata*)con);
2451       } else {
2452         assert(rtype == relocInfo::none, "unexpected reloc type");
2453         if (con < (address)(uintptr_t)os::vm_page_size()) {
2454           __ mov(dst_reg, con);
2455         } else {
2456           unsigned long offset;
2457           __ adrp(dst_reg, con, offset);
2458           __ add(dst_reg, dst_reg, offset);
2459         }
2460       }
2461     }
2462   %}
2463 
2464   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
2465     MacroAssembler _masm(&cbuf);
2466     Register dst_reg = as_Register($dst$$reg);
2467     __ mov(dst_reg, zr);
2468   %}
2469 
2470   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
2471     MacroAssembler _masm(&cbuf);
2472     Register dst_reg = as_Register($dst$$reg);
2473     __ mov(dst_reg, (u_int64_t)1);
2474   %}
2475 
2476   enc_class aarch64_enc_mov_poll_page(iRegP dst, immPollPage src) %{
2477     MacroAssembler _masm(&cbuf);
2478     address page = (address)$src$$constant;
2479     Register dst_reg = as_Register($dst$$reg);
2480     unsigned long off;
2481     __ adrp(dst_reg, Address(page, relocInfo::poll_type), off);
2482     assert(off == 0, "assumed offset == 0");
2483   %}
2484 
2485   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
2486     MacroAssembler _masm(&cbuf);
2487     address page = (address)$src$$constant;
2488     Register dst_reg = as_Register($dst$$reg);
2489     unsigned long off;
2490     __ adrp(dst_reg, ExternalAddress(page), off);
2491     assert(off == 0, "assumed offset == 0");
2492   %}
2493 
2494   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
2495     MacroAssembler _masm(&cbuf);
2496     Register dst_reg = as_Register($dst$$reg);
2497     address con = (address)$src$$constant;
2498     if (con == NULL) {
2499       ShouldNotReachHere();
2500     } else {
2501       relocInfo::relocType rtype = $src->constant_reloc();
2502       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
2503       __ set_narrow_oop(dst_reg, (jobject)con);
2504     }
2505   %}
2506 
2507   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
2508     MacroAssembler _masm(&cbuf);
2509     Register dst_reg = as_Register($dst$$reg);
2510     __ mov(dst_reg, zr);
2511   %}
2512 
2513   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
2514     MacroAssembler _masm(&cbuf);
2515     Register dst_reg = as_Register($dst$$reg);
2516     address con = (address)$src$$constant;
2517     if (con == NULL) {
2518       ShouldNotReachHere();
2519     } else {
2520       relocInfo::relocType rtype = $src->constant_reloc();
2521       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
2522       __ set_narrow_klass(dst_reg, (Klass *)con);
2523     }
2524   %}
2525 
2526   // arithmetic encodings
2527 
2528   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
2529     MacroAssembler _masm(&cbuf);
2530     Register dst_reg = as_Register($dst$$reg);
2531     Register src_reg = as_Register($src1$$reg);
2532     int32_t con = (int32_t)$src2$$constant;
2533     // add has primary == 0, subtract has primary == 1
2534     if ($primary) { con = -con; }
2535     if (con < 0) {
2536       __ subw(dst_reg, src_reg, -con);
2537     } else {
2538       __ addw(dst_reg, src_reg, con);
2539     }
2540   %}
2541 
2542   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
2543     MacroAssembler _masm(&cbuf);
2544     Register dst_reg = as_Register($dst$$reg);
2545     Register src_reg = as_Register($src1$$reg);
2546     int32_t con = (int32_t)$src2$$constant;
2547     // add has primary == 0, subtract has primary == 1
2548     if ($primary) { con = -con; }
2549     if (con < 0) {
2550       __ sub(dst_reg, src_reg, -con);
2551     } else {
2552       __ add(dst_reg, src_reg, con);
2553     }
2554   %}
2555 
2556   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
2557     MacroAssembler _masm(&cbuf);
2558    Register dst_reg = as_Register($dst$$reg);
2559    Register src1_reg = as_Register($src1$$reg);
2560    Register src2_reg = as_Register($src2$$reg);
2561     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
2562   %}
2563 
2564   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
2565     MacroAssembler _masm(&cbuf);
2566    Register dst_reg = as_Register($dst$$reg);
2567    Register src1_reg = as_Register($src1$$reg);
2568    Register src2_reg = as_Register($src2$$reg);
2569     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
2570   %}
2571 
2572   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
2573     MacroAssembler _masm(&cbuf);
2574    Register dst_reg = as_Register($dst$$reg);
2575    Register src1_reg = as_Register($src1$$reg);
2576    Register src2_reg = as_Register($src2$$reg);
2577     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
2578   %}
2579 
2580   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
2581     MacroAssembler _masm(&cbuf);
2582    Register dst_reg = as_Register($dst$$reg);
2583    Register src1_reg = as_Register($src1$$reg);
2584    Register src2_reg = as_Register($src2$$reg);
2585     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
2586   %}
2587 
2588   // compare instruction encodings
2589 
2590   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
2591     MacroAssembler _masm(&cbuf);
2592     Register reg1 = as_Register($src1$$reg);
2593     Register reg2 = as_Register($src2$$reg);
2594     __ cmpw(reg1, reg2);
2595   %}
2596 
2597   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
2598     MacroAssembler _masm(&cbuf);
2599     Register reg = as_Register($src1$$reg);
2600     int32_t val = $src2$$constant;
2601     if (val >= 0) {
2602       __ subsw(zr, reg, val);
2603     } else {
2604       __ addsw(zr, reg, -val);
2605     }
2606   %}
2607 
2608   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
2609     MacroAssembler _masm(&cbuf);
2610     Register reg1 = as_Register($src1$$reg);
2611     u_int32_t val = (u_int32_t)$src2$$constant;
2612     __ movw(rscratch1, val);
2613     __ cmpw(reg1, rscratch1);
2614   %}
2615 
2616   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
2617     MacroAssembler _masm(&cbuf);
2618     Register reg1 = as_Register($src1$$reg);
2619     Register reg2 = as_Register($src2$$reg);
2620     __ cmp(reg1, reg2);
2621   %}
2622 
2623   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
2624     MacroAssembler _masm(&cbuf);
2625     Register reg = as_Register($src1$$reg);
2626     int64_t val = $src2$$constant;
2627     if (val >= 0) {
2628       __ subs(zr, reg, val);
2629     } else if (val != -val) {
2630       __ adds(zr, reg, -val);
2631     } else {
2632     // aargh, Long.MIN_VALUE is a special case
2633       __ orr(rscratch1, zr, (u_int64_t)val);
2634       __ subs(zr, reg, rscratch1);
2635     }
2636   %}
2637 
2638   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
2639     MacroAssembler _masm(&cbuf);
2640     Register reg1 = as_Register($src1$$reg);
2641     u_int64_t val = (u_int64_t)$src2$$constant;
2642     __ mov(rscratch1, val);
2643     __ cmp(reg1, rscratch1);
2644   %}
2645 
2646   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
2647     MacroAssembler _masm(&cbuf);
2648     Register reg1 = as_Register($src1$$reg);
2649     Register reg2 = as_Register($src2$$reg);
2650     __ cmp(reg1, reg2);
2651   %}
2652 
2653   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
2654     MacroAssembler _masm(&cbuf);
2655     Register reg1 = as_Register($src1$$reg);
2656     Register reg2 = as_Register($src2$$reg);
2657     __ cmpw(reg1, reg2);
2658   %}
2659 
2660   enc_class aarch64_enc_testp(iRegP src) %{
2661     MacroAssembler _masm(&cbuf);
2662     Register reg = as_Register($src$$reg);
2663     __ cmp(reg, zr);
2664   %}
2665 
2666   enc_class aarch64_enc_testn(iRegN src) %{
2667     MacroAssembler _masm(&cbuf);
2668     Register reg = as_Register($src$$reg);
2669     __ cmpw(reg, zr);
2670   %}
2671 
2672   enc_class aarch64_enc_b(label lbl) %{
2673     MacroAssembler _masm(&cbuf);
2674     Label *L = $lbl$$label;
2675     __ b(*L);
2676   %}
2677 
2678   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
2679     MacroAssembler _masm(&cbuf);
2680     Label *L = $lbl$$label;
2681     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
2682   %}
2683 
2684   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
2685     MacroAssembler _masm(&cbuf);
2686     Label *L = $lbl$$label;
2687     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
2688   %}
2689 
2690   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
2691   %{
2692      Register sub_reg = as_Register($sub$$reg);
2693      Register super_reg = as_Register($super$$reg);
2694      Register temp_reg = as_Register($temp$$reg);
2695      Register result_reg = as_Register($result$$reg);
2696 
2697      Label miss;
2698      MacroAssembler _masm(&cbuf);
2699      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
2700                                      NULL, &miss,
2701                                      /*set_cond_codes:*/ true);
2702      if ($primary) {
2703        __ mov(result_reg, zr);
2704      }
2705      __ bind(miss);
2706   %}
2707 
2708   enc_class aarch64_enc_java_static_call(method meth) %{
2709     MacroAssembler _masm(&cbuf);
2710 
2711     address addr = (address)$meth$$method;
2712     if (!_method) {
2713       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
2714       __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
2715     } else if (_optimized_virtual) {
2716       __ trampoline_call(Address(addr, relocInfo::opt_virtual_call_type), &cbuf);
2717     } else {
2718       __ trampoline_call(Address(addr, relocInfo::static_call_type), &cbuf);
2719     }
2720 
2721     if (_method) {
2722       // Emit stub for static call
2723       CompiledStaticCall::emit_to_interp_stub(cbuf);
2724     }
2725   %}
2726 
2727   enc_class aarch64_enc_java_handle_call(method meth) %{
2728     MacroAssembler _masm(&cbuf);
2729     relocInfo::relocType reloc;
2730 
2731     // RFP is preserved across all calls, even compiled calls.
2732     // Use it to preserve SP.
2733     __ mov(rfp, sp);
2734 
2735     const int start_offset = __ offset();
2736     address addr = (address)$meth$$method;
2737     if (!_method) {
2738       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
2739       __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
2740     } else if (_optimized_virtual) {
2741       __ trampoline_call(Address(addr, relocInfo::opt_virtual_call_type), &cbuf);
2742     } else {
2743       __ trampoline_call(Address(addr, relocInfo::static_call_type), &cbuf);
2744     }
2745 
2746     if (_method) {
2747       // Emit stub for static call
2748       CompiledStaticCall::emit_to_interp_stub(cbuf);
2749     }
2750 
2751     // now restore sp
2752     __ mov(sp, rfp);
2753   %}
2754 
2755   enc_class aarch64_enc_java_dynamic_call(method meth) %{
2756     MacroAssembler _masm(&cbuf);
2757     __ ic_call((address)$meth$$method);
2758   %}
2759 
2760   enc_class aarch64_enc_call_epilog() %{
2761     MacroAssembler _masm(&cbuf);
2762     if (VerifyStackAtCalls) {
2763       // Check that stack depth is unchanged: find majik cookie on stack
2764       __ call_Unimplemented();
2765     }
2766   %}
2767 
2768   enc_class aarch64_enc_java_to_runtime(method meth) %{
2769     MacroAssembler _masm(&cbuf);
2770 
2771     // some calls to generated routines (arraycopy code) are scheduled
2772     // by C2 as runtime calls. if so we can call them using a br (they
2773     // will be in a reachable segment) otherwise we have to use a blrt
2774     // which loads the absolute address into a register.
2775     address entry = (address)$meth$$method;
2776     CodeBlob *cb = CodeCache::find_blob(entry);
2777     if (cb) {
2778       __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
2779     } else {
2780       int gpcnt;
2781       int fpcnt;
2782       int rtype;
2783       getCallInfo(tf(), gpcnt, fpcnt, rtype);
2784       Label retaddr;
2785       __ adr(rscratch2, retaddr);
2786       __ lea(rscratch1, RuntimeAddress(entry));
2787       // Leave a breadcrumb for JavaThread::pd_last_frame().
2788       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
2789       __ blrt(rscratch1, gpcnt, fpcnt, rtype);
2790       __ bind(retaddr);
2791       __ add(sp, sp, 2 * wordSize);
2792     }
2793   %}
2794 
2795   enc_class aarch64_enc_rethrow() %{
2796     MacroAssembler _masm(&cbuf);
2797     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
2798   %}
2799 
2800   enc_class aarch64_enc_ret() %{
2801     MacroAssembler _masm(&cbuf);
2802     __ ret(lr);
2803   %}
2804 
2805   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
2806     MacroAssembler _masm(&cbuf);
2807     Register target_reg = as_Register($jump_target$$reg);
2808     __ br(target_reg);
2809   %}
2810 
2811   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
2812     MacroAssembler _masm(&cbuf);
2813     Register target_reg = as_Register($jump_target$$reg);
2814     // exception oop should be in r0
2815     // ret addr has been popped into lr
2816     // callee expects it in r3
2817     __ mov(r3, lr);
2818     __ br(target_reg);
2819   %}
2820 
2821   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
2822     MacroAssembler _masm(&cbuf);
2823     Register oop = as_Register($object$$reg);
2824     Register box = as_Register($box$$reg);
2825     Register disp_hdr = as_Register($tmp$$reg);
2826     Register tmp = as_Register($tmp2$$reg);
2827     Label cont;
2828     Label object_has_monitor;
2829     Label cas_failed;
2830 
2831     assert_different_registers(oop, box, tmp, disp_hdr);
2832 
2833     // Load markOop from object into displaced_header.
2834     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
2835 
2836     // Always do locking in runtime.
2837     if (EmitSync & 0x01) {
2838       __ cmp(oop, zr);
2839       return;
2840     }
2841 
2842     if (UseBiasedLocking) {
2843       __ biased_locking_enter(disp_hdr, oop, box, tmp, true, cont);
2844     }
2845 
2846     // Handle existing monitor
2847     if (EmitSync & 0x02) {
2848       // we can use AArch64's bit test and branch here but
2849       // markoopDesc does not define a bit index just the bit value
2850       // so assert in case the bit pos changes
2851 #     define __monitor_value_log2 1
2852       assert(markOopDesc::monitor_value == (1 << __monitor_value_log2), "incorrect bit position");
2853       __ tbnz(disp_hdr, __monitor_value_log2, object_has_monitor);
2854 #     undef __monitor_value_log2
2855     }
2856 
2857     // Set displaced_header to be (markOop of object | UNLOCK_VALUE).
2858     __ orr(disp_hdr, disp_hdr, markOopDesc::unlocked_value);
2859 
2860     // Load Compare Value application register.
2861 
2862     // Initialize the box. (Must happen before we update the object mark!)
2863     __ str(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
2864 
2865     // Compare object markOop with mark and if equal exchange scratch1
2866     // with object markOop.
2867     // Note that this is simply a CAS: it does not generate any
2868     // barriers.  These are separately generated by
2869     // membar_acquire_lock().
2870     {
2871       Label retry_load;
2872       __ bind(retry_load);
2873       __ ldxr(tmp, oop);
2874       __ cmp(tmp, disp_hdr);
2875       __ br(Assembler::NE, cas_failed);
2876       // use stlxr to ensure update is immediately visible
2877       __ stlxr(tmp, box, oop);
2878       __ cbzw(tmp, cont);
2879       __ b(retry_load);
2880     }
2881 
2882     // Formerly:
2883     // __ cmpxchgptr(/*oldv=*/disp_hdr,
2884     //               /*newv=*/box,
2885     //               /*addr=*/oop,
2886     //               /*tmp=*/tmp,
2887     //               cont,
2888     //               /*fail*/NULL);
2889 
2890     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2891 
2892     // If the compare-and-exchange succeeded, then we found an unlocked
2893     // object, will have now locked it will continue at label cont
2894 
2895     __ bind(cas_failed);
2896     // We did not see an unlocked object so try the fast recursive case.
2897 
2898     // Check if the owner is self by comparing the value in the
2899     // markOop of object (disp_hdr) with the stack pointer.
2900     __ mov(rscratch1, sp);
2901     __ sub(disp_hdr, disp_hdr, rscratch1);
2902     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place));
2903     // If condition is true we are cont and hence we can store 0 as the
2904     // displaced header in the box, which indicates that it is a recursive lock.
2905     __ ands(tmp/*==0?*/, disp_hdr, tmp);
2906     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
2907 
2908     // Handle existing monitor.
2909     if ((EmitSync & 0x02) == 0) {
2910       __ b(cont);
2911 
2912       __ bind(object_has_monitor);
2913       // The object's monitor m is unlocked iff m->owner == NULL,
2914       // otherwise m->owner may contain a thread or a stack address.
2915       //
2916       // Try to CAS m->owner from NULL to current thread.
2917       __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
2918       __ mov(disp_hdr, zr);
2919 
2920       {
2921         Label retry_load, fail;
2922         __ bind(retry_load);
2923         __ ldxr(rscratch1, tmp);
2924         __ cmp(disp_hdr, rscratch1);
2925         __ br(Assembler::NE, fail);
2926         // use stlxr to ensure update is immediately visible
2927         __ stlxr(rscratch1, rthread, tmp);
2928         __ cbnzw(rscratch1, retry_load);
2929         __ bind(fail);
2930       }
2931 
2932       // Label next;
2933       // __ cmpxchgptr(/*oldv=*/disp_hdr,
2934       //               /*newv=*/rthread,
2935       //               /*addr=*/tmp,
2936       //               /*tmp=*/rscratch1,
2937       //               /*succeed*/next,
2938       //               /*fail*/NULL);
2939       // __ bind(next);
2940 
2941       // store a non-null value into the box.
2942       __ str(box, Address(box, BasicLock::displaced_header_offset_in_bytes()));
2943 
2944       // PPC port checks the following invariants
2945       // #ifdef ASSERT
2946       // bne(flag, cont);
2947       // We have acquired the monitor, check some invariants.
2948       // addw(/*monitor=*/tmp, tmp, -ObjectMonitor::owner_offset_in_bytes());
2949       // Invariant 1: _recursions should be 0.
2950       // assert(ObjectMonitor::recursions_size_in_bytes() == 8, "unexpected size");
2951       // assert_mem8_is_zero(ObjectMonitor::recursions_offset_in_bytes(), tmp,
2952       //                        "monitor->_recursions should be 0", -1);
2953       // Invariant 2: OwnerIsThread shouldn't be 0.
2954       // assert(ObjectMonitor::OwnerIsThread_size_in_bytes() == 4, "unexpected size");
2955       //assert_mem4_isnot_zero(ObjectMonitor::OwnerIsThread_offset_in_bytes(), tmp,
2956       //                           "monitor->OwnerIsThread shouldn't be 0", -1);
2957       // #endif
2958     }
2959 
2960     __ bind(cont);
2961     // flag == EQ indicates success
2962     // flag == NE indicates failure
2963 
2964   %}
2965 
2966   // TODO
2967   // reimplement this with custom cmpxchgptr code
2968   // which avoids some of the unnecessary branching
2969   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
2970     MacroAssembler _masm(&cbuf);
2971     Register oop = as_Register($object$$reg);
2972     Register box = as_Register($box$$reg);
2973     Register disp_hdr = as_Register($tmp$$reg);
2974     Register tmp = as_Register($tmp2$$reg);
2975     Label cont;
2976     Label object_has_monitor;
2977     Label cas_failed;
2978 
2979     assert_different_registers(oop, box, tmp, disp_hdr);
2980 
2981     // Always do locking in runtime.
2982     if (EmitSync & 0x01) {
2983       __ cmp(oop, zr); // Oop can't be 0 here => always false.
2984       return;
2985     }
2986 
2987     if (UseBiasedLocking) {
2988       __ biased_locking_exit(oop, tmp, cont);
2989     }
2990 
2991     // Find the lock address and load the displaced header from the stack.
2992     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
2993 
2994     // If the displaced header is 0, we have a recursive unlock.
2995     __ cmp(disp_hdr, zr);
2996     __ br(Assembler::EQ, cont);
2997 
2998 
2999     // Handle existing monitor.
3000     if ((EmitSync & 0x02) == 0) {
3001       __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3002       __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3003     }
3004 
3005     // Check if it is still a light weight lock, this is is true if we
3006     // see the stack address of the basicLock in the markOop of the
3007     // object.
3008 
3009       {
3010         Label retry_load;
3011         __ bind(retry_load);
3012         __ ldxr(tmp, oop);
3013         __ cmp(box, tmp);
3014         __ br(Assembler::NE, cas_failed);
3015         // use stlxr to ensure update is immediately visible
3016         __ stlxr(tmp, disp_hdr, oop);
3017         __ cbzw(tmp, cont);
3018         __ b(retry_load);
3019       }
3020 
3021     // __ cmpxchgptr(/*compare_value=*/box,
3022     //               /*exchange_value=*/disp_hdr,
3023     //               /*where=*/oop,
3024     //               /*result=*/tmp,
3025     //               cont,
3026     //               /*cas_failed*/NULL);
3027     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3028 
3029     __ bind(cas_failed);
3030 
3031     // Handle existing monitor.
3032     if ((EmitSync & 0x02) == 0) {
3033       __ b(cont);
3034 
3035       __ bind(object_has_monitor);
3036       __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor
3037       __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3038       __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3039       __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3040       __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3041       __ cmp(rscratch1, zr);
3042       __ br(Assembler::NE, cont);
3043 
3044       __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3045       __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3046       __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3047       __ cmp(rscratch1, zr);
3048       __ cbnz(rscratch1, cont);
3049       // need a release store here
3050       __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3051       __ stlr(rscratch1, tmp); // rscratch1 is zero
3052     }
3053 
3054     __ bind(cont);
3055     // flag == EQ indicates success
3056     // flag == NE indicates failure
3057   %}
3058 
3059 %}
3060 
3061 //----------FRAME--------------------------------------------------------------
3062 // Definition of frame structure and management information.
3063 //
3064 //  S T A C K   L A Y O U T    Allocators stack-slot number
3065 //                             |   (to get allocators register number
3066 //  G  Owned by    |        |  v    add OptoReg::stack0())
3067 //  r   CALLER     |        |
3068 //  o     |        +--------+      pad to even-align allocators stack-slot
3069 //  w     V        |  pad0  |        numbers; owned by CALLER
3070 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
3071 //  h     ^        |   in   |  5
3072 //        |        |  args  |  4   Holes in incoming args owned by SELF
3073 //  |     |        |        |  3
3074 //  |     |        +--------+
3075 //  V     |        | old out|      Empty on Intel, window on Sparc
3076 //        |    old |preserve|      Must be even aligned.
3077 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
3078 //        |        |   in   |  3   area for Intel ret address
3079 //     Owned by    |preserve|      Empty on Sparc.
3080 //       SELF      +--------+
3081 //        |        |  pad2  |  2   pad to align old SP
3082 //        |        +--------+  1
3083 //        |        | locks  |  0
3084 //        |        +--------+----> OptoReg::stack0(), even aligned
3085 //        |        |  pad1  | 11   pad to align new SP
3086 //        |        +--------+
3087 //        |        |        | 10
3088 //        |        | spills |  9   spills
3089 //        V        |        |  8   (pad0 slot for callee)
3090 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
3091 //        ^        |  out   |  7
3092 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
3093 //     Owned by    +--------+
3094 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
3095 //        |    new |preserve|      Must be even-aligned.
3096 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
3097 //        |        |        |
3098 //
3099 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
3100 //         known from SELF's arguments and the Java calling convention.
3101 //         Region 6-7 is determined per call site.
3102 // Note 2: If the calling convention leaves holes in the incoming argument
3103 //         area, those holes are owned by SELF.  Holes in the outgoing area
3104 //         are owned by the CALLEE.  Holes should not be nessecary in the
3105 //         incoming area, as the Java calling convention is completely under
3106 //         the control of the AD file.  Doubles can be sorted and packed to
3107 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
3108 //         varargs C calling conventions.
3109 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
3110 //         even aligned with pad0 as needed.
3111 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
3112 //           (the latter is true on Intel but is it false on AArch64?)
3113 //         region 6-11 is even aligned; it may be padded out more so that
3114 //         the region from SP to FP meets the minimum stack alignment.
3115 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3116 //         alignment.  Region 11, pad1, may be dynamically extended so that
3117 //         SP meets the minimum alignment.
3118 
3119 frame %{
3120   // What direction does stack grow in (assumed to be same for C & Java)
3121   stack_direction(TOWARDS_LOW);
3122 
3123   // These three registers define part of the calling convention
3124   // between compiled code and the interpreter.
3125 
3126   // Inline Cache Register or methodOop for I2C.
3127   inline_cache_reg(R12);
3128 
3129   // Method Oop Register when calling interpreter.
3130   interpreter_method_oop_reg(R12);
3131 
3132   // Number of stack slots consumed by locking an object
3133   sync_stack_slots(2);
3134 
3135   // Compiled code's Frame Pointer
3136   frame_pointer(R31);
3137 
3138   // Interpreter stores its frame pointer in a register which is
3139   // stored to the stack by I2CAdaptors.
3140   // I2CAdaptors convert from interpreted java to compiled java.
3141   interpreter_frame_pointer(R29);
3142 
3143   // Stack alignment requirement
3144   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3145 
3146   // Number of stack slots between incoming argument block and the start of
3147   // a new frame.  The PROLOG must add this many slots to the stack.  The
3148   // EPILOG must remove this many slots. aarch64 needs two slots for
3149   // return address and fp.
3150   // TODO think this is correct but check
3151   in_preserve_stack_slots(4);
3152 
3153   // Number of outgoing stack slots killed above the out_preserve_stack_slots
3154   // for calls to C.  Supports the var-args backing area for register parms.
3155   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3156 
3157   // The after-PROLOG location of the return address.  Location of
3158   // return address specifies a type (REG or STACK) and a number
3159   // representing the register number (i.e. - use a register name) or
3160   // stack slot.
3161   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3162   // Otherwise, it is above the locks and verification slot and alignment word
3163   // TODO this may well be correct but need to check why that - 2 is there
3164   // ppc port uses 0 but we definitely need to allow for fixed_slots
3165   // which folds in the space used for monitors
3166   return_addr(STACK - 2 +
3167               round_to((Compile::current()->in_preserve_stack_slots() +
3168                         Compile::current()->fixed_slots()),
3169                        stack_alignment_in_slots()));
3170 
3171   // Body of function which returns an integer array locating
3172   // arguments either in registers or in stack slots.  Passed an array
3173   // of ideal registers called "sig" and a "length" count.  Stack-slot
3174   // offsets are based on outgoing arguments, i.e. a CALLER setting up
3175   // arguments for a CALLEE.  Incoming stack arguments are
3176   // automatically biased by the preserve_stack_slots field above.
3177 
3178   calling_convention
3179   %{
3180     // No difference between ingoing/outgoing just pass false
3181     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3182   %}
3183 
3184   c_calling_convention
3185   %{
3186     // This is obviously always outgoing
3187     (void) SharedRuntime::c_calling_convention(sig_bt, regs, NULL, length);
3188   %}
3189 
3190   // Location of compiled Java return values.  Same as C for now.
3191   return_value
3192   %{
3193     // TODO do we allow ideal_reg == Op_RegN???
3194     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3195            "only return normal values");
3196 
3197     static const int lo[Op_RegL + 1] = { // enum name
3198       0,                                 // Op_Node
3199       0,                                 // Op_Set
3200       R0_num,                            // Op_RegN
3201       R0_num,                            // Op_RegI
3202       R0_num,                            // Op_RegP
3203       V0_num,                            // Op_RegF
3204       V0_num,                            // Op_RegD
3205       R0_num                             // Op_RegL
3206     };
3207 
3208     static const int hi[Op_RegL + 1] = { // enum name
3209       0,                                 // Op_Node
3210       0,                                 // Op_Set
3211       OptoReg::Bad,                       // Op_RegN
3212       OptoReg::Bad,                      // Op_RegI
3213       R0_H_num,                          // Op_RegP
3214       OptoReg::Bad,                      // Op_RegF
3215       V0_H_num,                          // Op_RegD
3216       R0_H_num                           // Op_RegL
3217     };
3218 
3219     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3220   %}
3221 %}
3222 
3223 //----------ATTRIBUTES---------------------------------------------------------
3224 //----------Operand Attributes-------------------------------------------------
3225 op_attrib op_cost(1);        // Required cost attribute
3226 
3227 //----------Instruction Attributes---------------------------------------------
3228 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3229 ins_attrib ins_size(32);        // Required size attribute (in bits)
3230 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3231                                 // a non-matching short branch variant
3232                                 // of some long branch?
3233 ins_attrib ins_alignment(4);    // Required alignment attribute (must
3234                                 // be a power of 2) specifies the
3235                                 // alignment that some part of the
3236                                 // instruction (not necessarily the
3237                                 // start) requires.  If > 1, a
3238                                 // compute_padding() function must be
3239                                 // provided for the instruction
3240 
3241 //----------OPERANDS-----------------------------------------------------------
3242 // Operand definitions must precede instruction definitions for correct parsing
3243 // in the ADLC because operands constitute user defined types which are used in
3244 // instruction definitions.
3245 
3246 //----------Simple Operands----------------------------------------------------
3247 
3248 // Integer operands 32 bit
3249 // 32 bit immediate
3250 operand immI()
3251 %{
3252   match(ConI);
3253 
3254   op_cost(0);
3255   format %{ %}
3256   interface(CONST_INTER);
3257 %}
3258 
3259 // 32 bit zero
3260 operand immI0()
3261 %{
3262   predicate(n->get_int() == 0);
3263   match(ConI);
3264 
3265   op_cost(0);
3266   format %{ %}
3267   interface(CONST_INTER);
3268 %}
3269 
3270 // 32 bit unit increment
3271 operand immI_1()
3272 %{
3273   predicate(n->get_int() == 1);
3274   match(ConI);
3275 
3276   op_cost(0);
3277   format %{ %}
3278   interface(CONST_INTER);
3279 %}
3280 
3281 // 32 bit unit decrement
3282 operand immI_M1()
3283 %{
3284   predicate(n->get_int() == -1);
3285   match(ConI);
3286 
3287   op_cost(0);
3288   format %{ %}
3289   interface(CONST_INTER);
3290 %}
3291 
3292 operand immI_le_4()
3293 %{
3294   predicate(n->get_int() <= 4);
3295   match(ConI);
3296 
3297   op_cost(0);
3298   format %{ %}
3299   interface(CONST_INTER);
3300 %}
3301 
3302 operand immI_31()
3303 %{
3304   predicate(n->get_int() == 31);
3305   match(ConI);
3306 
3307   op_cost(0);
3308   format %{ %}
3309   interface(CONST_INTER);
3310 %}
3311 
3312 operand immI_8()
3313 %{
3314   predicate(n->get_int() == 8);
3315   match(ConI);
3316 
3317   op_cost(0);
3318   format %{ %}
3319   interface(CONST_INTER);
3320 %}
3321 
3322 operand immI_16()
3323 %{
3324   predicate(n->get_int() == 16);
3325   match(ConI);
3326 
3327   op_cost(0);
3328   format %{ %}
3329   interface(CONST_INTER);
3330 %}
3331 
3332 operand immI_24()
3333 %{
3334   predicate(n->get_int() == 24);
3335   match(ConI);
3336 
3337   op_cost(0);
3338   format %{ %}
3339   interface(CONST_INTER);
3340 %}
3341 
3342 operand immI_32()
3343 %{
3344   predicate(n->get_int() == 32);
3345   match(ConI);
3346 
3347   op_cost(0);
3348   format %{ %}
3349   interface(CONST_INTER);
3350 %}
3351 
3352 operand immI_48()
3353 %{
3354   predicate(n->get_int() == 48);
3355   match(ConI);
3356 
3357   op_cost(0);
3358   format %{ %}
3359   interface(CONST_INTER);
3360 %}
3361 
3362 operand immI_56()
3363 %{
3364   predicate(n->get_int() == 56);
3365   match(ConI);
3366 
3367   op_cost(0);
3368   format %{ %}
3369   interface(CONST_INTER);
3370 %}
3371 
3372 operand immI_64()
3373 %{
3374   predicate(n->get_int() == 64);
3375   match(ConI);
3376 
3377   op_cost(0);
3378   format %{ %}
3379   interface(CONST_INTER);
3380 %}
3381 
3382 operand immI_255()
3383 %{
3384   predicate(n->get_int() == 255);
3385   match(ConI);
3386 
3387   op_cost(0);
3388   format %{ %}
3389   interface(CONST_INTER);
3390 %}
3391 
3392 operand immI_65535()
3393 %{
3394   predicate(n->get_int() == 65535);
3395   match(ConI);
3396 
3397   op_cost(0);
3398   format %{ %}
3399   interface(CONST_INTER);
3400 %}
3401 
3402 operand immL_63()
3403 %{
3404   predicate(n->get_int() == 63);
3405   match(ConI);
3406 
3407   op_cost(0);
3408   format %{ %}
3409   interface(CONST_INTER);
3410 %}
3411 
3412 operand immL_255()
3413 %{
3414   predicate(n->get_int() == 255);
3415   match(ConI);
3416 
3417   op_cost(0);
3418   format %{ %}
3419   interface(CONST_INTER);
3420 %}
3421 
3422 operand immL_65535()
3423 %{
3424   predicate(n->get_long() == 65535L);
3425   match(ConL);
3426 
3427   op_cost(0);
3428   format %{ %}
3429   interface(CONST_INTER);
3430 %}
3431 
3432 operand immL_4294967295()
3433 %{
3434   predicate(n->get_long() == 4294967295L);
3435   match(ConL);
3436 
3437   op_cost(0);
3438   format %{ %}
3439   interface(CONST_INTER);
3440 %}
3441 
3442 operand immL_bitmask()
3443 %{
3444   predicate(((n->get_long() & 0xc000000000000000l) == 0)
3445             && is_power_of_2(n->get_long() + 1));
3446   match(ConL);
3447 
3448   op_cost(0);
3449   format %{ %}
3450   interface(CONST_INTER);
3451 %}
3452 
3453 operand immI_bitmask()
3454 %{
3455   predicate(((n->get_int() & 0xc0000000) == 0)
3456             && is_power_of_2(n->get_int() + 1));
3457   match(ConI);
3458 
3459   op_cost(0);
3460   format %{ %}
3461   interface(CONST_INTER);
3462 %}
3463 
3464 // Scale values for scaled offset addressing modes (up to long but not quad)
3465 operand immIScale()
3466 %{
3467   predicate(0 <= n->get_int() && (n->get_int() <= 3));
3468   match(ConI);
3469 
3470   op_cost(0);
3471   format %{ %}
3472   interface(CONST_INTER);
3473 %}
3474 
3475 // 26 bit signed offset -- for pc-relative branches
3476 operand immI26()
3477 %{
3478   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
3479   match(ConI);
3480 
3481   op_cost(0);
3482   format %{ %}
3483   interface(CONST_INTER);
3484 %}
3485 
3486 // 19 bit signed offset -- for pc-relative loads
3487 operand immI19()
3488 %{
3489   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
3490   match(ConI);
3491 
3492   op_cost(0);
3493   format %{ %}
3494   interface(CONST_INTER);
3495 %}
3496 
3497 // 12 bit unsigned offset -- for base plus immediate loads
3498 operand immIU12()
3499 %{
3500   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
3501   match(ConI);
3502 
3503   op_cost(0);
3504   format %{ %}
3505   interface(CONST_INTER);
3506 %}
3507 
3508 operand immLU12()
3509 %{
3510   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
3511   match(ConL);
3512 
3513   op_cost(0);
3514   format %{ %}
3515   interface(CONST_INTER);
3516 %}
3517 
3518 // Offset for scaled or unscaled immediate loads and stores
3519 operand immIOffset()
3520 %{
3521   predicate(Address::offset_ok_for_immed(n->get_int()));
3522   match(ConI);
3523 
3524   op_cost(0);
3525   format %{ %}
3526   interface(CONST_INTER);
3527 %}
3528 
3529 operand immLoffset()
3530 %{
3531   predicate(Address::offset_ok_for_immed(n->get_long()));
3532   match(ConL);
3533 
3534   op_cost(0);
3535   format %{ %}
3536   interface(CONST_INTER);
3537 %}
3538 
3539 // 32 bit integer valid for add sub immediate
3540 operand immIAddSub()
3541 %{
3542   predicate(Assembler::operand_valid_for_add_sub_immediate((long)n->get_int()));
3543   match(ConI);
3544   op_cost(0);
3545   format %{ %}
3546   interface(CONST_INTER);
3547 %}
3548 
3549 // 32 bit unsigned integer valid for logical immediate
3550 // TODO -- check this is right when e.g the mask is 0x80000000
3551 operand immILog()
3552 %{
3553   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (unsigned long)n->get_int()));
3554   match(ConI);
3555 
3556   op_cost(0);
3557   format %{ %}
3558   interface(CONST_INTER);
3559 %}
3560 
3561 // Integer operands 64 bit
3562 // 64 bit immediate
3563 operand immL()
3564 %{
3565   match(ConL);
3566 
3567   op_cost(0);
3568   format %{ %}
3569   interface(CONST_INTER);
3570 %}
3571 
3572 // 64 bit zero
3573 operand immL0()
3574 %{
3575   predicate(n->get_long() == 0);
3576   match(ConL);
3577 
3578   op_cost(0);
3579   format %{ %}
3580   interface(CONST_INTER);
3581 %}
3582 
3583 // 64 bit unit increment
3584 operand immL_1()
3585 %{
3586   predicate(n->get_long() == 1);
3587   match(ConL);
3588 
3589   op_cost(0);
3590   format %{ %}
3591   interface(CONST_INTER);
3592 %}
3593 
3594 // 64 bit unit decrement
3595 operand immL_M1()
3596 %{
3597   predicate(n->get_long() == -1);
3598   match(ConL);
3599 
3600   op_cost(0);
3601   format %{ %}
3602   interface(CONST_INTER);
3603 %}
3604 
3605 // 32 bit offset of pc in thread anchor
3606 
3607 operand immL_pc_off()
3608 %{
3609   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
3610                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
3611   match(ConL);
3612 
3613   op_cost(0);
3614   format %{ %}
3615   interface(CONST_INTER);
3616 %}
3617 
3618 // 64 bit integer valid for add sub immediate
3619 operand immLAddSub()
3620 %{
3621   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
3622   match(ConL);
3623   op_cost(0);
3624   format %{ %}
3625   interface(CONST_INTER);
3626 %}
3627 
3628 // 64 bit integer valid for logical immediate
3629 operand immLLog()
3630 %{
3631   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (unsigned long)n->get_long()));
3632   match(ConL);
3633   op_cost(0);
3634   format %{ %}
3635   interface(CONST_INTER);
3636 %}
3637 
3638 // Long Immediate: low 32-bit mask
3639 operand immL_32bits()
3640 %{
3641   predicate(n->get_long() == 0xFFFFFFFFL);
3642   match(ConL);
3643   op_cost(0);
3644   format %{ %}
3645   interface(CONST_INTER);
3646 %}
3647 
3648 // Pointer operands
3649 // Pointer Immediate
3650 operand immP()
3651 %{
3652   match(ConP);
3653 
3654   op_cost(0);
3655   format %{ %}
3656   interface(CONST_INTER);
3657 %}
3658 
3659 // NULL Pointer Immediate
3660 operand immP0()
3661 %{
3662   predicate(n->get_ptr() == 0);
3663   match(ConP);
3664 
3665   op_cost(0);
3666   format %{ %}
3667   interface(CONST_INTER);
3668 %}
3669 
3670 // Pointer Immediate One
3671 // this is used in object initialization (initial object header)
3672 operand immP_1()
3673 %{
3674   predicate(n->get_ptr() == 1);
3675   match(ConP);
3676 
3677   op_cost(0);
3678   format %{ %}
3679   interface(CONST_INTER);
3680 %}
3681 
3682 // Polling Page Pointer Immediate
3683 operand immPollPage()
3684 %{
3685   predicate((address)n->get_ptr() == os::get_polling_page());
3686   match(ConP);
3687 
3688   op_cost(0);
3689   format %{ %}
3690   interface(CONST_INTER);
3691 %}
3692 
3693 // Card Table Byte Map Base
3694 operand immByteMapBase()
3695 %{
3696   // Get base of card map
3697   predicate((jbyte*)n->get_ptr() ==
3698         ((CardTableModRefBS*)(Universe::heap()->barrier_set()))->byte_map_base);
3699   match(ConP);
3700 
3701   op_cost(0);
3702   format %{ %}
3703   interface(CONST_INTER);
3704 %}
3705 
3706 // Pointer Immediate Minus One
3707 // this is used when we want to write the current PC to the thread anchor
3708 operand immP_M1()
3709 %{
3710   predicate(n->get_ptr() == -1);
3711   match(ConP);
3712 
3713   op_cost(0);
3714   format %{ %}
3715   interface(CONST_INTER);
3716 %}
3717 
3718 // Pointer Immediate Minus Two
3719 // this is used when we want to write the current PC to the thread anchor
3720 operand immP_M2()
3721 %{
3722   predicate(n->get_ptr() == -2);
3723   match(ConP);
3724 
3725   op_cost(0);
3726   format %{ %}
3727   interface(CONST_INTER);
3728 %}
3729 
3730 // Float and Double operands
3731 // Double Immediate
3732 operand immD()
3733 %{
3734   match(ConD);
3735   op_cost(0);
3736   format %{ %}
3737   interface(CONST_INTER);
3738 %}
3739 
3740 // Double Immediate: +0.0d
3741 operand immD0()
3742 %{
3743   predicate(jlong_cast(n->getd()) == 0);
3744   match(ConD);
3745 
3746   op_cost(0);
3747   format %{ %}
3748   interface(CONST_INTER);
3749 %}
3750 
3751 // constant 'double +0.0'.
3752 operand immDPacked()
3753 %{
3754   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
3755   match(ConD);
3756   op_cost(0);
3757   format %{ %}
3758   interface(CONST_INTER);
3759 %}
3760 
3761 // Float Immediate
3762 operand immF()
3763 %{
3764   match(ConF);
3765   op_cost(0);
3766   format %{ %}
3767   interface(CONST_INTER);
3768 %}
3769 
3770 // Float Immediate: +0.0f.
3771 operand immF0()
3772 %{
3773   predicate(jint_cast(n->getf()) == 0);
3774   match(ConF);
3775 
3776   op_cost(0);
3777   format %{ %}
3778   interface(CONST_INTER);
3779 %}
3780 
3781 //
3782 operand immFPacked()
3783 %{
3784   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
3785   match(ConF);
3786   op_cost(0);
3787   format %{ %}
3788   interface(CONST_INTER);
3789 %}
3790 
3791 // Narrow pointer operands
3792 // Narrow Pointer Immediate
3793 operand immN()
3794 %{
3795   match(ConN);
3796 
3797   op_cost(0);
3798   format %{ %}
3799   interface(CONST_INTER);
3800 %}
3801 
3802 // Narrow NULL Pointer Immediate
3803 operand immN0()
3804 %{
3805   predicate(n->get_narrowcon() == 0);
3806   match(ConN);
3807 
3808   op_cost(0);
3809   format %{ %}
3810   interface(CONST_INTER);
3811 %}
3812 
3813 operand immNKlass()
3814 %{
3815   match(ConNKlass);
3816 
3817   op_cost(0);
3818   format %{ %}
3819   interface(CONST_INTER);
3820 %}
3821 
3822 // Integer 32 bit Register Operands
3823 // Integer 32 bitRegister (excludes SP)
3824 operand iRegI()
3825 %{
3826   constraint(ALLOC_IN_RC(any_reg32));
3827   match(RegI);
3828   match(iRegINoSp);
3829   op_cost(0);
3830   format %{ %}
3831   interface(REG_INTER);
3832 %}
3833 
3834 // Integer 32 bit Register not Special
3835 operand iRegINoSp()
3836 %{
3837   constraint(ALLOC_IN_RC(no_special_reg32));
3838   match(RegI);
3839   op_cost(0);
3840   format %{ %}
3841   interface(REG_INTER);
3842 %}
3843 
3844 // Integer 64 bit Register Operands
3845 // Integer 64 bit Register (includes SP)
3846 operand iRegL()
3847 %{
3848   constraint(ALLOC_IN_RC(any_reg));
3849   match(RegL);
3850   match(iRegLNoSp);
3851   op_cost(0);
3852   format %{ %}
3853   interface(REG_INTER);
3854 %}
3855 
3856 // Integer 64 bit Register not Special
3857 operand iRegLNoSp()
3858 %{
3859   constraint(ALLOC_IN_RC(no_special_reg));
3860   match(RegL);
3861   format %{ %}
3862   interface(REG_INTER);
3863 %}
3864 
3865 // Pointer Register Operands
3866 // Pointer Register
3867 operand iRegP()
3868 %{
3869   constraint(ALLOC_IN_RC(ptr_reg));
3870   match(RegP);
3871   match(iRegPNoSp);
3872   match(iRegP_R0);
3873   //match(iRegP_R2);
3874   //match(iRegP_R4);
3875   //match(iRegP_R5);
3876   match(thread_RegP);
3877   op_cost(0);
3878   format %{ %}
3879   interface(REG_INTER);
3880 %}
3881 
3882 // Pointer 64 bit Register not Special
3883 operand iRegPNoSp()
3884 %{
3885   constraint(ALLOC_IN_RC(no_special_ptr_reg));
3886   match(RegP);
3887   // match(iRegP);
3888   // match(iRegP_R0);
3889   // match(iRegP_R2);
3890   // match(iRegP_R4);
3891   // match(iRegP_R5);
3892   // match(thread_RegP);
3893   op_cost(0);
3894   format %{ %}
3895   interface(REG_INTER);
3896 %}
3897 
3898 // Pointer 64 bit Register R0 only
3899 operand iRegP_R0()
3900 %{
3901   constraint(ALLOC_IN_RC(r0_reg));
3902   match(RegP);
3903   // match(iRegP);
3904   match(iRegPNoSp);
3905   op_cost(0);
3906   format %{ %}
3907   interface(REG_INTER);
3908 %}
3909 
3910 // Pointer 64 bit Register R1 only
3911 operand iRegP_R1()
3912 %{
3913   constraint(ALLOC_IN_RC(r1_reg));
3914   match(RegP);
3915   // match(iRegP);
3916   match(iRegPNoSp);
3917   op_cost(0);
3918   format %{ %}
3919   interface(REG_INTER);
3920 %}
3921 
3922 // Pointer 64 bit Register R2 only
3923 operand iRegP_R2()
3924 %{
3925   constraint(ALLOC_IN_RC(r2_reg));
3926   match(RegP);
3927   // match(iRegP);
3928   match(iRegPNoSp);
3929   op_cost(0);
3930   format %{ %}
3931   interface(REG_INTER);
3932 %}
3933 
3934 // Pointer 64 bit Register R3 only
3935 operand iRegP_R3()
3936 %{
3937   constraint(ALLOC_IN_RC(r3_reg));
3938   match(RegP);
3939   // match(iRegP);
3940   match(iRegPNoSp);
3941   op_cost(0);
3942   format %{ %}
3943   interface(REG_INTER);
3944 %}
3945 
3946 // Pointer 64 bit Register R4 only
3947 operand iRegP_R4()
3948 %{
3949   constraint(ALLOC_IN_RC(r4_reg));
3950   match(RegP);
3951   // match(iRegP);
3952   match(iRegPNoSp);
3953   op_cost(0);
3954   format %{ %}
3955   interface(REG_INTER);
3956 %}
3957 
3958 // Pointer 64 bit Register R5 only
3959 operand iRegP_R5()
3960 %{
3961   constraint(ALLOC_IN_RC(r5_reg));
3962   match(RegP);
3963   // match(iRegP);
3964   match(iRegPNoSp);
3965   op_cost(0);
3966   format %{ %}
3967   interface(REG_INTER);
3968 %}
3969 
3970 // Pointer 64 bit Register R10 only
3971 operand iRegP_R10()
3972 %{
3973   constraint(ALLOC_IN_RC(r10_reg));
3974   match(RegP);
3975   // match(iRegP);
3976   match(iRegPNoSp);
3977   op_cost(0);
3978   format %{ %}
3979   interface(REG_INTER);
3980 %}
3981 
3982 // Long 64 bit Register R11 only
3983 operand iRegL_R11()
3984 %{
3985   constraint(ALLOC_IN_RC(r11_reg));
3986   match(RegL);
3987   match(iRegLNoSp);
3988   op_cost(0);
3989   format %{ %}
3990   interface(REG_INTER);
3991 %}
3992 
3993 // Pointer 64 bit Register FP only
3994 operand iRegP_FP()
3995 %{
3996   constraint(ALLOC_IN_RC(fp_reg));
3997   match(RegP);
3998   // match(iRegP);
3999   op_cost(0);
4000   format %{ %}
4001   interface(REG_INTER);
4002 %}
4003 
4004 // Register R0 only
4005 operand iRegI_R0()
4006 %{
4007   constraint(ALLOC_IN_RC(int_r0_reg));
4008   match(RegI);
4009   match(iRegINoSp);
4010   op_cost(0);
4011   format %{ %}
4012   interface(REG_INTER);
4013 %}
4014 
4015 // Register R2 only
4016 operand iRegI_R2()
4017 %{
4018   constraint(ALLOC_IN_RC(int_r2_reg));
4019   match(RegI);
4020   match(iRegINoSp);
4021   op_cost(0);
4022   format %{ %}
4023   interface(REG_INTER);
4024 %}
4025 
4026 // Register R3 only
4027 operand iRegI_R3()
4028 %{
4029   constraint(ALLOC_IN_RC(int_r3_reg));
4030   match(RegI);
4031   match(iRegINoSp);
4032   op_cost(0);
4033   format %{ %}
4034   interface(REG_INTER);
4035 %}
4036 
4037 
4038 // Register R2 only
4039 operand iRegI_R4()
4040 %{
4041   constraint(ALLOC_IN_RC(int_r4_reg));
4042   match(RegI);
4043   match(iRegINoSp);
4044   op_cost(0);
4045   format %{ %}
4046   interface(REG_INTER);
4047 %}
4048 
4049 
4050 // Pointer Register Operands
4051 // Narrow Pointer Register
4052 operand iRegN()
4053 %{
4054   constraint(ALLOC_IN_RC(any_reg32));
4055   match(RegN);
4056   match(iRegNNoSp);
4057   op_cost(0);
4058   format %{ %}
4059   interface(REG_INTER);
4060 %}
4061 
4062 // Integer 64 bit Register not Special
4063 operand iRegNNoSp()
4064 %{
4065   constraint(ALLOC_IN_RC(no_special_reg32));
4066   match(RegN);
4067   op_cost(0);
4068   format %{ %}
4069   interface(REG_INTER);
4070 %}
4071 
4072 // heap base register -- used for encoding immN0
4073 
4074 operand iRegIHeapbase()
4075 %{
4076   constraint(ALLOC_IN_RC(heapbase_reg));
4077   match(RegI);
4078   op_cost(0);
4079   format %{ %}
4080   interface(REG_INTER);
4081 %}
4082 
4083 // Float Register
4084 // Float register operands
4085 operand vRegF()
4086 %{
4087   constraint(ALLOC_IN_RC(float_reg));
4088   match(RegF);
4089 
4090   op_cost(0);
4091   format %{ %}
4092   interface(REG_INTER);
4093 %}
4094 
4095 // Double Register
4096 // Double register operands
4097 operand vRegD()
4098 %{
4099   constraint(ALLOC_IN_RC(double_reg));
4100   match(RegD);
4101 
4102   op_cost(0);
4103   format %{ %}
4104   interface(REG_INTER);
4105 %}
4106 
4107 operand vRegD_V0()
4108 %{
4109   constraint(ALLOC_IN_RC(v0_reg));
4110   match(RegD);
4111   op_cost(0);
4112   format %{ %}
4113   interface(REG_INTER);
4114 %}
4115 
4116 operand vRegD_V1()
4117 %{
4118   constraint(ALLOC_IN_RC(v1_reg));
4119   match(RegD);
4120   op_cost(0);
4121   format %{ %}
4122   interface(REG_INTER);
4123 %}
4124 
4125 operand vRegD_V2()
4126 %{
4127   constraint(ALLOC_IN_RC(v2_reg));
4128   match(RegD);
4129   op_cost(0);
4130   format %{ %}
4131   interface(REG_INTER);
4132 %}
4133 
4134 operand vRegD_V3()
4135 %{
4136   constraint(ALLOC_IN_RC(v3_reg));
4137   match(RegD);
4138   op_cost(0);
4139   format %{ %}
4140   interface(REG_INTER);
4141 %}
4142 
4143 // Flags register, used as output of signed compare instructions
4144 
4145 // note that on AArch64 we also use this register as the output for
4146 // for floating point compare instructions (CmpF CmpD). this ensures
4147 // that ordered inequality tests use GT, GE, LT or LE none of which
4148 // pass through cases where the result is unordered i.e. one or both
4149 // inputs to the compare is a NaN. this means that the ideal code can
4150 // replace e.g. a GT with an LE and not end up capturing the NaN case
4151 // (where the comparison should always fail). EQ and NE tests are
4152 // always generated in ideal code so that unordered folds into the NE
4153 // case, matching the behaviour of AArch64 NE.
4154 //
4155 // This differs from x86 where the outputs of FP compares use a
4156 // special FP flags registers and where compares based on this
4157 // register are distinguished into ordered inequalities (cmpOpUCF) and
4158 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
4159 // to explicitly handle the unordered case in branches. x86 also has
4160 // to include extra CMoveX rules to accept a cmpOpUCF input.
4161 
4162 operand rFlagsReg()
4163 %{
4164   constraint(ALLOC_IN_RC(int_flags));
4165   match(RegFlags);
4166 
4167   op_cost(0);
4168   format %{ "RFLAGS" %}
4169   interface(REG_INTER);
4170 %}
4171 
4172 // Flags register, used as output of unsigned compare instructions
4173 operand rFlagsRegU()
4174 %{
4175   constraint(ALLOC_IN_RC(int_flags));
4176   match(RegFlags);
4177 
4178   op_cost(0);
4179   format %{ "RFLAGSU" %}
4180   interface(REG_INTER);
4181 %}
4182 
4183 // Special Registers
4184 
4185 // Method Register
4186 operand inline_cache_RegP(iRegP reg)
4187 %{
4188   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
4189   match(reg);
4190   match(iRegPNoSp);
4191   op_cost(0);
4192   format %{ %}
4193   interface(REG_INTER);
4194 %}
4195 
4196 operand interpreter_method_oop_RegP(iRegP reg)
4197 %{
4198   constraint(ALLOC_IN_RC(method_reg)); // interpreter_method_oop_reg
4199   match(reg);
4200   match(iRegPNoSp);
4201   op_cost(0);
4202   format %{ %}
4203   interface(REG_INTER);
4204 %}
4205 
4206 // Thread Register
4207 operand thread_RegP(iRegP reg)
4208 %{
4209   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
4210   match(reg);
4211   op_cost(0);
4212   format %{ %}
4213   interface(REG_INTER);
4214 %}
4215 
4216 operand lr_RegP(iRegP reg)
4217 %{
4218   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
4219   match(reg);
4220   op_cost(0);
4221   format %{ %}
4222   interface(REG_INTER);
4223 %}
4224 
4225 //----------Memory Operands----------------------------------------------------
4226 
4227 operand indirect(iRegP reg)
4228 %{
4229   constraint(ALLOC_IN_RC(ptr_reg));
4230   match(reg);
4231   op_cost(0);
4232   format %{ "[$reg]" %}
4233   interface(MEMORY_INTER) %{
4234     base($reg);
4235     index(0xffffffff);
4236     scale(0x0);
4237     disp(0x0);
4238   %}
4239 %}
4240 
4241 operand indIndexScaledOffsetI(iRegP reg, iRegL lreg, immIScale scale, immIU12 off)
4242 %{
4243   constraint(ALLOC_IN_RC(ptr_reg));
4244   match(AddP (AddP reg (LShiftL lreg scale)) off);
4245   op_cost(INSN_COST);
4246   format %{ "$reg, $lreg lsl($scale), $off" %}
4247   interface(MEMORY_INTER) %{
4248     base($reg);
4249     index($lreg);
4250     scale($scale);
4251     disp($off);
4252   %}
4253 %}
4254 
4255 operand indIndexScaledOffsetL(iRegP reg, iRegL lreg, immIScale scale, immLU12 off)
4256 %{
4257   constraint(ALLOC_IN_RC(ptr_reg));
4258   match(AddP (AddP reg (LShiftL lreg scale)) off);
4259   op_cost(INSN_COST);
4260   format %{ "$reg, $lreg lsl($scale), $off" %}
4261   interface(MEMORY_INTER) %{
4262     base($reg);
4263     index($lreg);
4264     scale($scale);
4265     disp($off);
4266   %}
4267 %}
4268 
4269 operand indIndexOffsetI2L(iRegP reg, iRegI ireg, immLU12 off)
4270 %{
4271   constraint(ALLOC_IN_RC(ptr_reg));
4272   match(AddP (AddP reg (ConvI2L ireg)) off);
4273   op_cost(INSN_COST);
4274   format %{ "$reg, $ireg, $off I2L" %}
4275   interface(MEMORY_INTER) %{
4276     base($reg);
4277     index($ireg);
4278     scale(0x0);
4279     disp($off);
4280   %}
4281 %}
4282 
4283 operand indIndexScaledOffsetI2L(iRegP reg, iRegI ireg, immIScale scale, immLU12 off)
4284 %{
4285   constraint(ALLOC_IN_RC(ptr_reg));
4286   match(AddP (AddP reg (LShiftL (ConvI2L ireg) scale)) off);
4287   op_cost(INSN_COST);
4288   format %{ "$reg, $ireg sxtw($scale), $off I2L" %}
4289   interface(MEMORY_INTER) %{
4290     base($reg);
4291     index($ireg);
4292     scale($scale);
4293     disp($off);
4294   %}
4295 %}
4296 
4297 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
4298 %{
4299   constraint(ALLOC_IN_RC(ptr_reg));
4300   match(AddP reg (LShiftL (ConvI2L ireg) scale));
4301   op_cost(0);
4302   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
4303   interface(MEMORY_INTER) %{
4304     base($reg);
4305     index($ireg);
4306     scale($scale);
4307     disp(0x0);
4308   %}
4309 %}
4310 
4311 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
4312 %{
4313   constraint(ALLOC_IN_RC(ptr_reg));
4314   match(AddP reg (LShiftL lreg scale));
4315   op_cost(0);
4316   format %{ "$reg, $lreg lsl($scale)" %}
4317   interface(MEMORY_INTER) %{
4318     base($reg);
4319     index($lreg);
4320     scale($scale);
4321     disp(0x0);
4322   %}
4323 %}
4324 
4325 operand indIndex(iRegP reg, iRegL lreg)
4326 %{
4327   constraint(ALLOC_IN_RC(ptr_reg));
4328   match(AddP reg lreg);
4329   op_cost(0);
4330   format %{ "$reg, $lreg" %}
4331   interface(MEMORY_INTER) %{
4332     base($reg);
4333     index($lreg);
4334     scale(0x0);
4335     disp(0x0);
4336   %}
4337 %}
4338 
4339 operand indOffI(iRegP reg, immIOffset off)
4340 %{
4341   constraint(ALLOC_IN_RC(ptr_reg));
4342   match(AddP reg off);
4343   op_cost(INSN_COST);
4344   format %{ "[$reg, $off]" %}
4345   interface(MEMORY_INTER) %{
4346     base($reg);
4347     index(0xffffffff);
4348     scale(0x0);
4349     disp($off);
4350   %}
4351 %}
4352 
4353 operand indOffL(iRegP reg, immLoffset off)
4354 %{
4355   constraint(ALLOC_IN_RC(ptr_reg));
4356   match(AddP reg off);
4357   op_cost(0);
4358   format %{ "[$reg, $off]" %}
4359   interface(MEMORY_INTER) %{
4360     base($reg);
4361     index(0xffffffff);
4362     scale(0x0);
4363     disp($off);
4364   %}
4365 %}
4366 
4367 
4368 operand indirectN(iRegN reg)
4369 %{
4370   predicate(Universe::narrow_oop_shift() == 0);
4371   constraint(ALLOC_IN_RC(ptr_reg));
4372   match(DecodeN reg);
4373   op_cost(0);
4374   format %{ "[$reg]\t# narrow" %}
4375   interface(MEMORY_INTER) %{
4376     base($reg);
4377     index(0xffffffff);
4378     scale(0x0);
4379     disp(0x0);
4380   %}
4381 %}
4382 
4383 operand indIndexScaledOffsetIN(iRegN reg, iRegL lreg, immIScale scale, immIU12 off)
4384 %{
4385   predicate(Universe::narrow_oop_shift() == 0);
4386   constraint(ALLOC_IN_RC(ptr_reg));
4387   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4388   op_cost(0);
4389   format %{ "$reg, $lreg lsl($scale), $off\t# narrow" %}
4390   interface(MEMORY_INTER) %{
4391     base($reg);
4392     index($lreg);
4393     scale($scale);
4394     disp($off);
4395   %}
4396 %}
4397 
4398 operand indIndexScaledOffsetLN(iRegN reg, iRegL lreg, immIScale scale, immLU12 off)
4399 %{
4400   predicate(Universe::narrow_oop_shift() == 0);
4401   constraint(ALLOC_IN_RC(ptr_reg));
4402   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4403   op_cost(INSN_COST);
4404   format %{ "$reg, $lreg lsl($scale), $off\t# narrow" %}
4405   interface(MEMORY_INTER) %{
4406     base($reg);
4407     index($lreg);
4408     scale($scale);
4409     disp($off);
4410   %}
4411 %}
4412 
4413 operand indIndexOffsetI2LN(iRegN reg, iRegI ireg, immLU12 off)
4414 %{
4415   predicate(Universe::narrow_oop_shift() == 0);
4416   constraint(ALLOC_IN_RC(ptr_reg));
4417   match(AddP (AddP (DecodeN reg) (ConvI2L ireg)) off);
4418   op_cost(INSN_COST);
4419   format %{ "$reg, $ireg, $off I2L\t# narrow" %}
4420   interface(MEMORY_INTER) %{
4421     base($reg);
4422     index($ireg);
4423     scale(0x0);
4424     disp($off);
4425   %}
4426 %}
4427 
4428 operand indIndexScaledOffsetI2LN(iRegN reg, iRegI ireg, immIScale scale, immLU12 off)
4429 %{
4430   predicate(Universe::narrow_oop_shift() == 0);
4431   constraint(ALLOC_IN_RC(ptr_reg));
4432   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale)) off);
4433   op_cost(INSN_COST);
4434   format %{ "$reg, $ireg sxtw($scale), $off I2L\t# narrow" %}
4435   interface(MEMORY_INTER) %{
4436     base($reg);
4437     index($ireg);
4438     scale($scale);
4439     disp($off);
4440   %}
4441 %}
4442 
4443 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
4444 %{
4445   predicate(Universe::narrow_oop_shift() == 0);
4446   constraint(ALLOC_IN_RC(ptr_reg));
4447   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
4448   op_cost(0);
4449   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
4450   interface(MEMORY_INTER) %{
4451     base($reg);
4452     index($ireg);
4453     scale($scale);
4454     disp(0x0);
4455   %}
4456 %}
4457 
4458 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
4459 %{
4460   predicate(Universe::narrow_oop_shift() == 0);
4461   constraint(ALLOC_IN_RC(ptr_reg));
4462   match(AddP (DecodeN reg) (LShiftL lreg scale));
4463   op_cost(0);
4464   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
4465   interface(MEMORY_INTER) %{
4466     base($reg);
4467     index($lreg);
4468     scale($scale);
4469     disp(0x0);
4470   %}
4471 %}
4472 
4473 operand indIndexN(iRegN reg, iRegL lreg)
4474 %{
4475   predicate(Universe::narrow_oop_shift() == 0);
4476   constraint(ALLOC_IN_RC(ptr_reg));
4477   match(AddP (DecodeN reg) lreg);
4478   op_cost(0);
4479   format %{ "$reg, $lreg\t# narrow" %}
4480   interface(MEMORY_INTER) %{
4481     base($reg);
4482     index($lreg);
4483     scale(0x0);
4484     disp(0x0);
4485   %}
4486 %}
4487 
4488 operand indOffIN(iRegN reg, immIOffset off)
4489 %{
4490   predicate(Universe::narrow_oop_shift() == 0);
4491   constraint(ALLOC_IN_RC(ptr_reg));
4492   match(AddP (DecodeN reg) off);
4493   op_cost(0);
4494   format %{ "[$reg, $off]\t# narrow" %}
4495   interface(MEMORY_INTER) %{
4496     base($reg);
4497     index(0xffffffff);
4498     scale(0x0);
4499     disp($off);
4500   %}
4501 %}
4502 
4503 operand indOffLN(iRegN reg, immLoffset off)
4504 %{
4505   predicate(Universe::narrow_oop_shift() == 0);
4506   constraint(ALLOC_IN_RC(ptr_reg));
4507   match(AddP (DecodeN reg) off);
4508   op_cost(0);
4509   format %{ "[$reg, $off]\t# narrow" %}
4510   interface(MEMORY_INTER) %{
4511     base($reg);
4512     index(0xffffffff);
4513     scale(0x0);
4514     disp($off);
4515   %}
4516 %}
4517 
4518 
4519 
4520 // AArch64 opto stubs need to write to the pc slot in the thread anchor
4521 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
4522 %{
4523   constraint(ALLOC_IN_RC(ptr_reg));
4524   match(AddP reg off);
4525   op_cost(0);
4526   format %{ "[$reg, $off]" %}
4527   interface(MEMORY_INTER) %{
4528     base($reg);
4529     index(0xffffffff);
4530     scale(0x0);
4531     disp($off);
4532   %}
4533 %}
4534 
4535 //----------Special Memory Operands--------------------------------------------
4536 // Stack Slot Operand - This operand is used for loading and storing temporary
4537 //                      values on the stack where a match requires a value to
4538 //                      flow through memory.
4539 operand stackSlotP(sRegP reg)
4540 %{
4541   constraint(ALLOC_IN_RC(stack_slots));
4542   op_cost(100);
4543   // No match rule because this operand is only generated in matching
4544   // match(RegP);
4545   format %{ "[$reg]" %}
4546   interface(MEMORY_INTER) %{
4547     base(0x1e);  // RSP
4548     index(0x0);  // No Index
4549     scale(0x0);  // No Scale
4550     disp($reg);  // Stack Offset
4551   %}
4552 %}
4553 
4554 operand stackSlotI(sRegI reg)
4555 %{
4556   constraint(ALLOC_IN_RC(stack_slots));
4557   // No match rule because this operand is only generated in matching
4558   // match(RegI);
4559   format %{ "[$reg]" %}
4560   interface(MEMORY_INTER) %{
4561     base(0x1e);  // RSP
4562     index(0x0);  // No Index
4563     scale(0x0);  // No Scale
4564     disp($reg);  // Stack Offset
4565   %}
4566 %}
4567 
4568 operand stackSlotF(sRegF reg)
4569 %{
4570   constraint(ALLOC_IN_RC(stack_slots));
4571   // No match rule because this operand is only generated in matching
4572   // match(RegF);
4573   format %{ "[$reg]" %}
4574   interface(MEMORY_INTER) %{
4575     base(0x1e);  // RSP
4576     index(0x0);  // No Index
4577     scale(0x0);  // No Scale
4578     disp($reg);  // Stack Offset
4579   %}
4580 %}
4581 
4582 operand stackSlotD(sRegD reg)
4583 %{
4584   constraint(ALLOC_IN_RC(stack_slots));
4585   // No match rule because this operand is only generated in matching
4586   // match(RegD);
4587   format %{ "[$reg]" %}
4588   interface(MEMORY_INTER) %{
4589     base(0x1e);  // RSP
4590     index(0x0);  // No Index
4591     scale(0x0);  // No Scale
4592     disp($reg);  // Stack Offset
4593   %}
4594 %}
4595 
4596 operand stackSlotL(sRegL reg)
4597 %{
4598   constraint(ALLOC_IN_RC(stack_slots));
4599   // No match rule because this operand is only generated in matching
4600   // match(RegL);
4601   format %{ "[$reg]" %}
4602   interface(MEMORY_INTER) %{
4603     base(0x1e);  // RSP
4604     index(0x0);  // No Index
4605     scale(0x0);  // No Scale
4606     disp($reg);  // Stack Offset
4607   %}
4608 %}
4609 
4610 // Operands for expressing Control Flow
4611 // NOTE: Label is a predefined operand which should not be redefined in
4612 //       the AD file. It is generically handled within the ADLC.
4613 
4614 //----------Conditional Branch Operands----------------------------------------
4615 // Comparison Op  - This is the operation of the comparison, and is limited to
4616 //                  the following set of codes:
4617 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4618 //
4619 // Other attributes of the comparison, such as unsignedness, are specified
4620 // by the comparison instruction that sets a condition code flags register.
4621 // That result is represented by a flags operand whose subtype is appropriate
4622 // to the unsignedness (etc.) of the comparison.
4623 //
4624 // Later, the instruction which matches both the Comparison Op (a Bool) and
4625 // the flags (produced by the Cmp) specifies the coding of the comparison op
4626 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4627 
4628 // used for signed integral comparisons and fp comparisons
4629 
4630 operand cmpOp()
4631 %{
4632   match(Bool);
4633 
4634   format %{ "" %}
4635   interface(COND_INTER) %{
4636     equal(0x0, "eq");
4637     not_equal(0x1, "ne");
4638     less(0xb, "lt");
4639     greater_equal(0xa, "ge");
4640     less_equal(0xd, "le");
4641     greater(0xc, "gt");
4642     overflow(0x6, "vs");
4643     no_overflow(0x7, "vc");
4644   %}
4645 %}
4646 
4647 // used for unsigned integral comparisons
4648 
4649 operand cmpOpU()
4650 %{
4651   match(Bool);
4652 
4653   format %{ "" %}
4654   interface(COND_INTER) %{
4655     equal(0x0, "eq");
4656     not_equal(0x1, "ne");
4657     less(0x3, "lo");
4658     greater_equal(0x2, "hs");
4659     less_equal(0x9, "ls");
4660     greater(0x8, "hi");
4661     overflow(0x6, "vs");
4662     no_overflow(0x7, "vc");
4663   %}
4664 %}
4665 
4666 // Special operand allowing long args to int ops to be truncated for free
4667 
4668 operand iRegL2I(iRegL reg) %{
4669 
4670   op_cost(0);
4671 
4672   match(ConvL2I reg);
4673 
4674   format %{ "l2i($reg)" %}
4675 
4676   interface(REG_INTER)
4677 %}
4678 
4679 
4680 //----------OPERAND CLASSES----------------------------------------------------
4681 // Operand Classes are groups of operands that are used as to simplify
4682 // instruction definitions by not requiring the AD writer to specify
4683 // separate instructions for every form of operand when the
4684 // instruction accepts multiple operand types with the same basic
4685 // encoding and format. The classic case of this is memory operands.
4686 
4687 // memory is used to define read/write location for load/store
4688 // instruction defs. we can turn a memory op into an Address
4689 
4690 opclass memory(indirect, indIndexScaledOffsetI, indIndexScaledOffsetL, indIndexOffsetI2L, indIndexScaledOffsetI2L, indIndexScaled, indIndexScaledI2L, indIndex, indOffI, indOffL,
4691                indirectN, indIndexScaledOffsetIN, indIndexScaledOffsetLN, indIndexOffsetI2LN, indIndexScaledOffsetI2LN, indIndexScaledN, indIndexScaledI2LN, indIndexN, indOffIN, indOffLN);
4692 
4693 
4694 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
4695 // operations. it allows the src to be either an iRegI or a (ConvL2I
4696 // iRegL). in the latter case the l2i normally planted for a ConvL2I
4697 // can be elided because the 32-bit instruction will just employ the
4698 // lower 32 bits anyway.
4699 //
4700 // n.b. this does not elide all L2I conversions. if the truncated
4701 // value is consumed by more than one operation then the ConvL2I
4702 // cannot be bundled into the consuming nodes so an l2i gets planted
4703 // (actually a movw $dst $src) and the downstream instructions consume
4704 // the result of the l2i as an iRegI input. That's a shame since the
4705 // movw is actually redundant but its not too costly.
4706 
4707 opclass iRegIorL2I(iRegI, iRegL2I);
4708 
4709 //----------PIPELINE-----------------------------------------------------------
4710 // Rules which define the behavior of the target architectures pipeline.
4711 // Integer ALU reg operation
4712 pipeline %{
4713 
4714 attributes %{
4715   // ARM instructions are of fixed length
4716   fixed_size_instructions;        // Fixed size instructions TODO does
4717   max_instructions_per_bundle = 2;   // A53 = 2, A57 = 4
4718   // ARM instructions come in 32-bit word units
4719   instruction_unit_size = 4;         // An instruction is 4 bytes long
4720   instruction_fetch_unit_size = 64;  // The processor fetches one line
4721   instruction_fetch_units = 1;       // of 64 bytes
4722 
4723   // List of nop instructions
4724   nops( MachNop );
4725 %}
4726 
4727 // We don't use an actual pipeline model so don't care about resources
4728 // or description. we do use pipeline classes to introduce fixed
4729 // latencies
4730 
4731 //----------RESOURCES----------------------------------------------------------
4732 // Resources are the functional units available to the machine
4733 
4734 resources( INS0, INS1, INS01 = INS0 | INS1,
4735            ALU0, ALU1, ALU = ALU0 | ALU1,
4736            MAC,
4737            DIV,
4738            BRANCH,
4739            LDST,
4740            NEON_FP);
4741 
4742 //----------PIPELINE DESCRIPTION-----------------------------------------------
4743 // Pipeline Description specifies the stages in the machine's pipeline
4744 
4745 pipe_desc(ISS, EX1, EX2, WR);
4746 
4747 //----------PIPELINE CLASSES---------------------------------------------------
4748 // Pipeline Classes describe the stages in which input and output are
4749 // referenced by the hardware pipeline.
4750 
4751 //------- Integer ALU operations --------------------------
4752 
4753 // Integer ALU reg-reg operation
4754 // Operands needed in EX1, result generated in EX2
4755 // Eg.  ADD     x0, x1, x2
4756 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
4757 %{
4758   single_instruction;
4759   dst    : EX2(write);
4760   src1   : EX1(read);
4761   src2   : EX1(read);
4762   INS01  : ISS; // Dual issue as instruction 0 or 1
4763   ALU    : EX2;
4764 %}
4765 
4766 // Integer ALU reg-reg operation with constant shift
4767 // Shifted register must be available in LATE_ISS instead of EX1
4768 // Eg.  ADD     x0, x1, x2, LSL #2
4769 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
4770 %{
4771   single_instruction;
4772   dst    : EX2(write);
4773   src1   : EX1(read);
4774   src2   : ISS(read);
4775   INS01  : ISS;
4776   ALU    : EX2;
4777 %}
4778 
4779 // Integer ALU reg operation with constant shift
4780 // Eg.  LSL     x0, x1, #shift
4781 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
4782 %{
4783   single_instruction;
4784   dst    : EX2(write);
4785   src1   : ISS(read);
4786   INS01  : ISS;
4787   ALU    : EX2;
4788 %}
4789 
4790 // Integer ALU reg-reg operation with variable shift
4791 // Both operands must be available in LATE_ISS instead of EX1
4792 // Result is available in EX1 instead of EX2
4793 // Eg.  LSLV    x0, x1, x2
4794 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
4795 %{
4796   single_instruction;
4797   dst    : EX1(write);
4798   src1   : ISS(read);
4799   src2   : ISS(read);
4800   INS01  : ISS;
4801   ALU    : EX1;
4802 %}
4803 
4804 // Integer ALU reg-reg operation with extract
4805 // As for _vshift above, but result generated in EX2
4806 // Eg.  EXTR    x0, x1, x2, #N
4807 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
4808 %{
4809   single_instruction;
4810   dst    : EX2(write);
4811   src1   : ISS(read);
4812   src2   : ISS(read);
4813   INS1   : ISS; // Can only dual issue as Instruction 1
4814   ALU    : EX1;
4815 %}
4816 
4817 // Integer ALU reg operation
4818 // Eg.  NEG     x0, x1
4819 pipe_class ialu_reg(iRegI dst, iRegI src)
4820 %{
4821   single_instruction;
4822   dst    : EX2(write);
4823   src    : EX1(read);
4824   INS01  : ISS;
4825   ALU    : EX2;
4826 %}
4827 
4828 // Integer ALU reg mmediate operation
4829 // Eg.  ADD     x0, x1, #N
4830 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
4831 %{
4832   single_instruction;
4833   dst    : EX2(write);
4834   src1   : EX1(read);
4835   INS01  : ISS;
4836   ALU    : EX2;
4837 %}
4838 
4839 // Integer ALU immediate operation (no source operands)
4840 // Eg.  MOV     x0, #N
4841 pipe_class ialu_imm(iRegI dst)
4842 %{
4843   single_instruction;
4844   dst    : EX1(write);
4845   INS01  : ISS;
4846   ALU    : EX1;
4847 %}
4848 
4849 //------- Compare operation -------------------------------
4850 
4851 // Compare reg-reg
4852 // Eg.  CMP     x0, x1
4853 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
4854 %{
4855   single_instruction;
4856 //  fixed_latency(16);
4857   cr     : EX2(write);
4858   op1    : EX1(read);
4859   op2    : EX1(read);
4860   INS01  : ISS;
4861   ALU    : EX2;
4862 %}
4863 
4864 // Compare reg-reg
4865 // Eg.  CMP     x0, #N
4866 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
4867 %{
4868   single_instruction;
4869 //  fixed_latency(16);
4870   cr     : EX2(write);
4871   op1    : EX1(read);
4872   INS01  : ISS;
4873   ALU    : EX2;
4874 %}
4875 
4876 //------- Conditional instructions ------------------------
4877 
4878 // Conditional no operands
4879 // Eg.  CSINC   x0, zr, zr, <cond>
4880 pipe_class icond_none(iRegI dst, rFlagsReg cr)
4881 %{
4882   single_instruction;
4883   cr     : EX1(read);
4884   dst    : EX2(write);
4885   INS01  : ISS;
4886   ALU    : EX2;
4887 %}
4888 
4889 // Conditional 2 operand
4890 // EG.  CSEL    X0, X1, X2, <cond>
4891 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
4892 %{
4893   single_instruction;
4894   cr     : EX1(read);
4895   src1   : EX1(read);
4896   src2   : EX1(read);
4897   dst    : EX2(write);
4898   INS01  : ISS;
4899   ALU    : EX2;
4900 %}
4901 
4902 // Conditional 2 operand
4903 // EG.  CSEL    X0, X1, X2, <cond>
4904 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
4905 %{
4906   single_instruction;
4907   cr     : EX1(read);
4908   src    : EX1(read);
4909   dst    : EX2(write);
4910   INS01  : ISS;
4911   ALU    : EX2;
4912 %}
4913 
4914 //------- Multiply pipeline operations --------------------
4915 
4916 // Multiply reg-reg
4917 // Eg.  MUL     w0, w1, w2
4918 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
4919 %{
4920   single_instruction;
4921   dst    : WR(write);
4922   src1   : ISS(read);
4923   src2   : ISS(read);
4924   INS01  : ISS;
4925   MAC    : WR;
4926 %}
4927 
4928 // Multiply accumulate
4929 // Eg.  MADD    w0, w1, w2, w3
4930 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
4931 %{
4932   single_instruction;
4933   dst    : WR(write);
4934   src1   : ISS(read);
4935   src2   : ISS(read);
4936   src3   : ISS(read);
4937   INS01  : ISS;
4938   MAC    : WR;
4939 %}
4940 
4941 // Eg.  MUL     w0, w1, w2
4942 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
4943 %{
4944   single_instruction;
4945   fixed_latency(3); // Maximum latency for 64 bit mul
4946   dst    : WR(write);
4947   src1   : ISS(read);
4948   src2   : ISS(read);
4949   INS01  : ISS;
4950   MAC    : WR;
4951 %}
4952 
4953 // Multiply accumulate
4954 // Eg.  MADD    w0, w1, w2, w3
4955 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
4956 %{
4957   single_instruction;
4958   fixed_latency(3); // Maximum latency for 64 bit mul
4959   dst    : WR(write);
4960   src1   : ISS(read);
4961   src2   : ISS(read);
4962   src3   : ISS(read);
4963   INS01  : ISS;
4964   MAC    : WR;
4965 %}
4966 
4967 //------- Divide pipeline operations --------------------
4968 
4969 // Eg.  SDIV    w0, w1, w2
4970 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
4971 %{
4972   single_instruction;
4973   fixed_latency(8); // Maximum latency for 32 bit divide
4974   dst    : WR(write);
4975   src1   : ISS(read);
4976   src2   : ISS(read);
4977   INS0   : ISS; // Can only dual issue as instruction 0
4978   DIV    : WR;
4979 %}
4980 
4981 // Eg.  SDIV    x0, x1, x2
4982 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
4983 %{
4984   single_instruction;
4985   fixed_latency(16); // Maximum latency for 64 bit divide
4986   dst    : WR(write);
4987   src1   : ISS(read);
4988   src2   : ISS(read);
4989   INS0   : ISS; // Can only dual issue as instruction 0
4990   DIV    : WR;
4991 %}
4992 
4993 //------- Load pipeline operations ------------------------
4994 
4995 // Load - prefetch
4996 // Eg.  PFRM    <mem>
4997 pipe_class iload_prefetch(memory mem)
4998 %{
4999   single_instruction;
5000   mem    : ISS(read);
5001   INS01  : ISS;
5002   LDST   : WR;
5003 %}
5004 
5005 // Load - reg, mem
5006 // Eg.  LDR     x0, <mem>
5007 pipe_class iload_reg_mem(iRegI dst, memory mem)
5008 %{
5009   single_instruction;
5010   dst    : WR(write);
5011   mem    : ISS(read);
5012   INS01  : ISS;
5013   LDST   : WR;
5014 %}
5015 
5016 // Load - reg, reg
5017 // Eg.  LDR     x0, [sp, x1]
5018 pipe_class iload_reg_reg(iRegI dst, iRegI src)
5019 %{
5020   single_instruction;
5021   dst    : WR(write);
5022   src    : ISS(read);
5023   INS01  : ISS;
5024   LDST   : WR;
5025 %}
5026 
5027 //------- Store pipeline operations -----------------------
5028 
5029 // Store - zr, mem
5030 // Eg.  STR     zr, <mem>
5031 pipe_class istore_mem(memory mem)
5032 %{
5033   single_instruction;
5034   mem    : ISS(read);
5035   INS01  : ISS;
5036   LDST   : WR;
5037 %}
5038 
5039 // Store - reg, mem
5040 // Eg.  STR     x0, <mem>
5041 pipe_class istore_reg_mem(iRegI src, memory mem)
5042 %{
5043   single_instruction;
5044   mem    : ISS(read);
5045   src    : EX2(read);
5046   INS01  : ISS;
5047   LDST   : WR;
5048 %}
5049 
5050 // Store - reg, reg
5051 // Eg. STR      x0, [sp, x1]
5052 pipe_class istore_reg_reg(iRegI dst, iRegI src)
5053 %{
5054   single_instruction;
5055   dst    : ISS(read);
5056   src    : EX2(read);
5057   INS01  : ISS;
5058   LDST   : WR;
5059 %}
5060 
5061 //------- Store pipeline operations -----------------------
5062 
5063 // Branch
5064 pipe_class pipe_branch()
5065 %{
5066   single_instruction;
5067   INS01  : ISS;
5068   BRANCH : EX1;
5069 %}
5070 
5071 // Conditional branch
5072 pipe_class pipe_branch_cond(rFlagsReg cr)
5073 %{
5074   single_instruction;
5075   cr     : EX1(read);
5076   INS01  : ISS;
5077   BRANCH : EX1;
5078 %}
5079 
5080 // Compare & Branch
5081 // EG.  CBZ/CBNZ
5082 pipe_class pipe_cmp_branch(iRegI op1)
5083 %{
5084   single_instruction;
5085   op1    : EX1(read);
5086   INS01  : ISS;
5087   BRANCH : EX1;
5088 %}
5089 
5090 //------- Synchronisation operations ----------------------
5091 
5092 // Any operation requiring serialization.
5093 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
5094 pipe_class pipe_serial()
5095 %{
5096   single_instruction;
5097   force_serialization;
5098   fixed_latency(16);
5099   INS01  : ISS(2); // Cannot dual issue with any other instruction
5100   LDST   : WR;
5101 %}
5102 
5103 // Generic big/slow expanded idiom - also serialized
5104 pipe_class pipe_slow()
5105 %{
5106   instruction_count(10);
5107   multiple_bundles;
5108   force_serialization;
5109   fixed_latency(16);
5110   INS01  : ISS(2); // Cannot dual issue with any other instruction
5111   LDST   : WR;
5112 %}
5113 
5114 // Empty pipeline class
5115 pipe_class pipe_class_empty()
5116 %{
5117   single_instruction;
5118   fixed_latency(0);
5119 %}
5120 
5121 // Default pipeline class.
5122 pipe_class pipe_class_default()
5123 %{
5124   single_instruction;
5125   fixed_latency(2);
5126 %}
5127 
5128 // Pipeline class for compares.
5129 pipe_class pipe_class_compare()
5130 %{
5131   single_instruction;
5132   fixed_latency(16);
5133 %}
5134 
5135 // Pipeline class for memory operations.
5136 pipe_class pipe_class_memory()
5137 %{
5138   single_instruction;
5139   fixed_latency(16);
5140 %}
5141 
5142 // Pipeline class for call.
5143 pipe_class pipe_class_call()
5144 %{
5145   single_instruction;
5146   fixed_latency(100);
5147 %}
5148 
5149 // Define the class for the Nop node.
5150 define %{
5151    MachNop = pipe_class_empty;
5152 %}
5153 
5154 %}
5155 //----------INSTRUCTIONS-------------------------------------------------------
5156 //
5157 // match      -- States which machine-independent subtree may be replaced
5158 //               by this instruction.
5159 // ins_cost   -- The estimated cost of this instruction is used by instruction
5160 //               selection to identify a minimum cost tree of machine
5161 //               instructions that matches a tree of machine-independent
5162 //               instructions.
5163 // format     -- A string providing the disassembly for this instruction.
5164 //               The value of an instruction's operand may be inserted
5165 //               by referring to it with a '$' prefix.
5166 // opcode     -- Three instruction opcodes may be provided.  These are referred
5167 //               to within an encode class as $primary, $secondary, and $tertiary
5168 //               rrspectively.  The primary opcode is commonly used to
5169 //               indicate the type of machine instruction, while secondary
5170 //               and tertiary are often used for prefix options or addressing
5171 //               modes.
5172 // ins_encode -- A list of encode classes with parameters. The encode class
5173 //               name must have been defined in an 'enc_class' specification
5174 //               in the encode section of the architecture description.
5175 
5176 // ============================================================================
5177 // Memory (Load/Store) Instructions
5178 
5179 // Load Instructions
5180 
5181 // Load Byte (8 bit signed)
5182 instruct loadB(iRegINoSp dst, memory mem)
5183 %{
5184   match(Set dst (LoadB mem));
5185   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5186 
5187   ins_cost(4 * INSN_COST);
5188   format %{ "ldrsbw  $dst, $mem\t# byte" %}
5189 
5190   ins_encode(aarch64_enc_ldrsbw(dst, mem));
5191 
5192   ins_pipe(iload_reg_mem);
5193 %}
5194 
5195 // Load Byte (8 bit signed) into long
5196 instruct loadB2L(iRegLNoSp dst, memory mem)
5197 %{
5198   match(Set dst (ConvI2L (LoadB mem)));
5199   predicate(UseBarriersForVolatile || n->in(1)->as_Load()->is_unordered());
5200 
5201   ins_cost(4 * INSN_COST);
5202   format %{ "ldrsb  $dst, $mem\t# byte" %}
5203 
5204   ins_encode(aarch64_enc_ldrsb(dst, mem));
5205 
5206   ins_pipe(iload_reg_mem);
5207 %}
5208 
5209 // Load Byte (8 bit unsigned)
5210 instruct loadUB(iRegINoSp dst, memory mem)
5211 %{
5212   match(Set dst (LoadUB mem));
5213   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5214 
5215   ins_cost(4 * INSN_COST);
5216   format %{ "ldrbw  $dst, $mem\t# byte" %}
5217 
5218   ins_encode(aarch64_enc_ldrb(dst, mem));
5219 
5220   ins_pipe(iload_reg_mem);
5221 %}
5222 
5223 // Load Byte (8 bit unsigned) into long
5224 instruct loadUB2L(iRegLNoSp dst, memory mem)
5225 %{
5226   match(Set dst (ConvI2L (LoadUB mem)));
5227   predicate(UseBarriersForVolatile || n->in(1)->as_Load()->is_unordered());
5228 
5229   ins_cost(4 * INSN_COST);
5230   format %{ "ldrb  $dst, $mem\t# byte" %}
5231 
5232   ins_encode(aarch64_enc_ldrb(dst, mem));
5233 
5234   ins_pipe(iload_reg_mem);
5235 %}
5236 
5237 // Load Short (16 bit signed)
5238 instruct loadS(iRegINoSp dst, memory mem)
5239 %{
5240   match(Set dst (LoadS mem));
5241   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5242 
5243   ins_cost(4 * INSN_COST);
5244   format %{ "ldrshw  $dst, $mem\t# short" %}
5245 
5246   ins_encode(aarch64_enc_ldrshw(dst, mem));
5247 
5248   ins_pipe(iload_reg_mem);
5249 %}
5250 
5251 // Load Short (16 bit signed) into long
5252 instruct loadS2L(iRegLNoSp dst, memory mem)
5253 %{
5254   match(Set dst (ConvI2L (LoadS mem)));
5255   predicate(UseBarriersForVolatile || n->in(1)->as_Load()->is_unordered());
5256 
5257   ins_cost(4 * INSN_COST);
5258   format %{ "ldrsh  $dst, $mem\t# short" %}
5259 
5260   ins_encode(aarch64_enc_ldrsh(dst, mem));
5261 
5262   ins_pipe(iload_reg_mem);
5263 %}
5264 
5265 // Load Char (16 bit unsigned)
5266 instruct loadUS(iRegINoSp dst, memory mem)
5267 %{
5268   match(Set dst (LoadUS mem));
5269   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5270 
5271   ins_cost(4 * INSN_COST);
5272   format %{ "ldrh  $dst, $mem\t# short" %}
5273 
5274   ins_encode(aarch64_enc_ldrh(dst, mem));
5275 
5276   ins_pipe(iload_reg_mem);
5277 %}
5278 
5279 // Load Short/Char (16 bit unsigned) into long
5280 instruct loadUS2L(iRegLNoSp dst, memory mem)
5281 %{
5282   match(Set dst (ConvI2L (LoadUS mem)));
5283   predicate(UseBarriersForVolatile || n->in(1)->as_Load()->is_unordered());
5284 
5285   ins_cost(4 * INSN_COST);
5286   format %{ "ldrh  $dst, $mem\t# short" %}
5287 
5288   ins_encode(aarch64_enc_ldrh(dst, mem));
5289 
5290   ins_pipe(iload_reg_mem);
5291 %}
5292 
5293 // Load Integer (32 bit signed)
5294 instruct loadI(iRegINoSp dst, memory mem)
5295 %{
5296   match(Set dst (LoadI mem));
5297   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5298 
5299   ins_cost(4 * INSN_COST);
5300   format %{ "ldrw  $dst, $mem\t# int" %}
5301 
5302   ins_encode(aarch64_enc_ldrw(dst, mem));
5303 
5304   ins_pipe(iload_reg_mem);
5305 %}
5306 
5307 // Load Integer (32 bit signed) into long
5308 instruct loadI2L(iRegLNoSp dst, memory mem)
5309 %{
5310   match(Set dst (ConvI2L (LoadI mem)));
5311   predicate(UseBarriersForVolatile || n->in(1)->as_Load()->is_unordered());
5312 
5313   ins_cost(4 * INSN_COST);
5314   format %{ "ldrsw  $dst, $mem\t# int" %}
5315 
5316   ins_encode(aarch64_enc_ldrsw(dst, mem));
5317 
5318   ins_pipe(iload_reg_mem);
5319 %}
5320 
5321 // Load Integer (32 bit unsigned) into long
5322 instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
5323 %{
5324   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5325   predicate(UseBarriersForVolatile || n->in(1)->in(1)->as_Load()->is_unordered());
5326 
5327   ins_cost(4 * INSN_COST);
5328   format %{ "ldrw  $dst, $mem\t# int" %}
5329 
5330   ins_encode(aarch64_enc_ldrw(dst, mem));
5331 
5332   ins_pipe(iload_reg_mem);
5333 %}
5334 
5335 // Load Long (64 bit signed)
5336 instruct loadL(iRegLNoSp dst, memory mem)
5337 %{
5338   match(Set dst (LoadL mem));
5339   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5340 
5341   ins_cost(4 * INSN_COST);
5342   format %{ "ldr  $dst, $mem\t# int" %}
5343 
5344   ins_encode(aarch64_enc_ldr(dst, mem));
5345 
5346   ins_pipe(iload_reg_mem);
5347 %}
5348 
5349 // Load Range
5350 instruct loadRange(iRegINoSp dst, memory mem)
5351 %{
5352   match(Set dst (LoadRange mem));
5353 
5354   ins_cost(4 * INSN_COST);
5355   format %{ "ldrw  $dst, $mem\t# range" %}
5356 
5357   ins_encode(aarch64_enc_ldrw(dst, mem));
5358 
5359   ins_pipe(iload_reg_mem);
5360 %}
5361 
5362 // Load Pointer
5363 instruct loadP(iRegPNoSp dst, memory mem)
5364 %{
5365   match(Set dst (LoadP mem));
5366   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5367 
5368   ins_cost(4 * INSN_COST);
5369   format %{ "ldr  $dst, $mem\t# ptr" %}
5370 
5371   ins_encode(aarch64_enc_ldr(dst, mem));
5372 
5373   ins_pipe(iload_reg_mem);
5374 %}
5375 
5376 // Load Compressed Pointer
5377 instruct loadN(iRegNNoSp dst, memory mem)
5378 %{
5379   match(Set dst (LoadN mem));
5380   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5381 
5382   ins_cost(4 * INSN_COST);
5383   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
5384 
5385   ins_encode(aarch64_enc_ldrw(dst, mem));
5386 
5387   ins_pipe(iload_reg_mem);
5388 %}
5389 
5390 // Load Klass Pointer
5391 instruct loadKlass(iRegPNoSp dst, memory mem)
5392 %{
5393   match(Set dst (LoadKlass mem));
5394   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5395 
5396   ins_cost(4 * INSN_COST);
5397   format %{ "ldr  $dst, $mem\t# class" %}
5398 
5399   ins_encode(aarch64_enc_ldr(dst, mem));
5400 
5401   ins_pipe(iload_reg_mem);
5402 %}
5403 
5404 // Load Narrow Klass Pointer
5405 instruct loadNKlass(iRegNNoSp dst, memory mem)
5406 %{
5407   match(Set dst (LoadNKlass mem));
5408   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5409 
5410   ins_cost(4 * INSN_COST);
5411   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
5412 
5413   ins_encode(aarch64_enc_ldrw(dst, mem));
5414 
5415   ins_pipe(iload_reg_mem);
5416 %}
5417 
5418 // Load Float
5419 instruct loadF(vRegF dst, memory mem)
5420 %{
5421   match(Set dst (LoadF mem));
5422   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5423 
5424   ins_cost(4 * INSN_COST);
5425   format %{ "ldrs  $dst, $mem\t# float" %}
5426 
5427   ins_encode( aarch64_enc_ldrs(dst, mem) );
5428 
5429   ins_pipe(pipe_class_memory);
5430 %}
5431 
5432 // Load Double
5433 instruct loadD(vRegD dst, memory mem)
5434 %{
5435   match(Set dst (LoadD mem));
5436   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5437 
5438   ins_cost(4 * INSN_COST);
5439   format %{ "ldrd  $dst, $mem\t# double" %}
5440 
5441   ins_encode( aarch64_enc_ldrd(dst, mem) );
5442 
5443   ins_pipe(pipe_class_memory);
5444 %}
5445 
5446 
5447 // Load Int Constant
5448 instruct loadConI(iRegINoSp dst, immI src)
5449 %{
5450   match(Set dst src);
5451 
5452   ins_cost(INSN_COST);
5453   format %{ "mov $dst, $src\t# int" %}
5454 
5455   ins_encode( aarch64_enc_movw_imm(dst, src) );
5456 
5457   ins_pipe(ialu_imm);
5458 %}
5459 
5460 // Load Long Constant
5461 instruct loadConL(iRegLNoSp dst, immL src)
5462 %{
5463   match(Set dst src);
5464 
5465   ins_cost(INSN_COST);
5466   format %{ "mov $dst, $src\t# long" %}
5467 
5468   ins_encode( aarch64_enc_mov_imm(dst, src) );
5469 
5470   ins_pipe(ialu_imm);
5471 %}
5472 
5473 // Load Pointer Constant
5474 
5475 instruct loadConP(iRegPNoSp dst, immP con)
5476 %{
5477   match(Set dst con);
5478 
5479   ins_cost(INSN_COST * 4);
5480   format %{
5481     "mov  $dst, $con\t# ptr\n\t"
5482   %}
5483 
5484   ins_encode(aarch64_enc_mov_p(dst, con));
5485 
5486   ins_pipe(ialu_imm);
5487 %}
5488 
5489 // Load Null Pointer Constant
5490 
5491 instruct loadConP0(iRegPNoSp dst, immP0 con)
5492 %{
5493   match(Set dst con);
5494 
5495   ins_cost(INSN_COST);
5496   format %{ "mov  $dst, $con\t# NULL ptr" %}
5497 
5498   ins_encode(aarch64_enc_mov_p0(dst, con));
5499 
5500   ins_pipe(ialu_imm);
5501 %}
5502 
5503 // Load Pointer Constant One
5504 
5505 instruct loadConP1(iRegPNoSp dst, immP_1 con)
5506 %{
5507   match(Set dst con);
5508 
5509   ins_cost(INSN_COST);
5510   format %{ "mov  $dst, $con\t# NULL ptr" %}
5511 
5512   ins_encode(aarch64_enc_mov_p1(dst, con));
5513 
5514   ins_pipe(ialu_imm);
5515 %}
5516 
5517 // Load Poll Page Constant
5518 
5519 instruct loadConPollPage(iRegPNoSp dst, immPollPage con)
5520 %{
5521   match(Set dst con);
5522 
5523   ins_cost(INSN_COST);
5524   format %{ "adr  $dst, $con\t# Poll Page Ptr" %}
5525 
5526   ins_encode(aarch64_enc_mov_poll_page(dst, con));
5527 
5528   ins_pipe(ialu_imm);
5529 %}
5530 
5531 // Load Byte Map Base Constant
5532 
5533 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
5534 %{
5535   match(Set dst con);
5536 
5537   ins_cost(INSN_COST);
5538   format %{ "adr  $dst, $con\t# Byte Map Base" %}
5539 
5540   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
5541 
5542   ins_pipe(ialu_imm);
5543 %}
5544 
5545 // Load Narrow Pointer Constant
5546 
5547 instruct loadConN(iRegNNoSp dst, immN con)
5548 %{
5549   match(Set dst con);
5550 
5551   ins_cost(INSN_COST * 4);
5552   format %{ "mov  $dst, $con\t# compressed ptr" %}
5553 
5554   ins_encode(aarch64_enc_mov_n(dst, con));
5555 
5556   ins_pipe(ialu_imm);
5557 %}
5558 
5559 // Load Narrow Null Pointer Constant
5560 
5561 instruct loadConN0(iRegNNoSp dst, immN0 con)
5562 %{
5563   match(Set dst con);
5564 
5565   ins_cost(INSN_COST);
5566   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
5567 
5568   ins_encode(aarch64_enc_mov_n0(dst, con));
5569 
5570   ins_pipe(ialu_imm);
5571 %}
5572 
5573 // Load Narrow Klass Constant
5574 
5575 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
5576 %{
5577   match(Set dst con);
5578 
5579   ins_cost(INSN_COST);
5580   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
5581 
5582   ins_encode(aarch64_enc_mov_nk(dst, con));
5583 
5584   ins_pipe(ialu_imm);
5585 %}
5586 
5587 // Load Packed Float Constant
5588 
5589 instruct loadConF_packed(vRegF dst, immFPacked con) %{
5590   match(Set dst con);
5591   ins_cost(INSN_COST * 4);
5592   format %{ "fmovs  $dst, $con"%}
5593   ins_encode %{
5594     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
5595   %}
5596 
5597   ins_pipe(pipe_class_default);
5598 %}
5599 
5600 // Load Float Constant
5601 
5602 instruct loadConF(vRegF dst, immF con) %{
5603   match(Set dst con);
5604 
5605   ins_cost(INSN_COST * 4);
5606 
5607   format %{
5608     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
5609   %}
5610 
5611   ins_encode %{
5612     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
5613   %}
5614 
5615   ins_pipe(pipe_class_default);
5616 %}
5617 
5618 // Load Packed Double Constant
5619 
5620 instruct loadConD_packed(vRegD dst, immDPacked con) %{
5621   match(Set dst con);
5622   ins_cost(INSN_COST);
5623   format %{ "fmovd  $dst, $con"%}
5624   ins_encode %{
5625     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
5626   %}
5627 
5628   ins_pipe(pipe_class_default);
5629 %}
5630 
5631 // Load Double Constant
5632 
5633 instruct loadConD(vRegD dst, immD con) %{
5634   match(Set dst con);
5635 
5636   ins_cost(INSN_COST * 5);
5637   format %{
5638     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
5639   %}
5640 
5641   ins_encode %{
5642     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
5643   %}
5644 
5645   ins_pipe(pipe_class_default);
5646 %}
5647 
5648 // Store Instructions
5649 
5650 // Store CMS card-mark Immediate
5651 instruct storeimmCM0(immI0 zero, memory mem)
5652 %{
5653   match(Set mem (StoreCM mem zero));
5654 
5655   ins_cost(INSN_COST);
5656   format %{ "strb zr, $mem\t# byte" %}
5657 
5658   ins_encode(aarch64_enc_strb0(mem));
5659 
5660   ins_pipe(istore_mem);
5661 %}
5662 
5663 // Store Byte
5664 instruct storeB(iRegIorL2I src, memory mem)
5665 %{
5666   match(Set mem (StoreB mem src));
5667   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5668 
5669   ins_cost(INSN_COST);
5670   format %{ "strb  $src, $mem\t# byte" %}
5671 
5672   ins_encode(aarch64_enc_strb(src, mem));
5673 
5674   ins_pipe(istore_reg_mem);
5675 %}
5676 
5677 
5678 instruct storeimmB0(immI0 zero, memory mem)
5679 %{
5680   match(Set mem (StoreB mem zero));
5681   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5682 
5683   ins_cost(INSN_COST);
5684   format %{ "strb zr, $mem\t# byte" %}
5685 
5686   ins_encode(aarch64_enc_strb0(mem));
5687 
5688   ins_pipe(istore_mem);
5689 %}
5690 
5691 // Store Char/Short
5692 instruct storeC(iRegIorL2I src, memory mem)
5693 %{
5694   match(Set mem (StoreC mem src));
5695   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5696 
5697   ins_cost(INSN_COST);
5698   format %{ "strh  $src, $mem\t# short" %}
5699 
5700   ins_encode(aarch64_enc_strh(src, mem));
5701 
5702   ins_pipe(istore_reg_mem);
5703 %}
5704 
5705 instruct storeimmC0(immI0 zero, memory mem)
5706 %{
5707   match(Set mem (StoreC mem zero));
5708   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5709 
5710   ins_cost(INSN_COST);
5711   format %{ "strh  zr, $mem\t# short" %}
5712 
5713   ins_encode(aarch64_enc_strh0(mem));
5714 
5715   ins_pipe(istore_mem);
5716 %}
5717 
5718 // Store Integer
5719 
5720 instruct storeI(iRegIorL2I src, memory mem)
5721 %{
5722   match(Set mem(StoreI mem src));
5723   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5724 
5725   ins_cost(INSN_COST);
5726   format %{ "strw  $src, $mem\t# int" %}
5727 
5728   ins_encode(aarch64_enc_strw(src, mem));
5729 
5730   ins_pipe(istore_reg_mem);
5731 %}
5732 
5733 instruct storeimmI0(immI0 zero, memory mem)
5734 %{
5735   match(Set mem(StoreI mem zero));
5736   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5737 
5738   ins_cost(INSN_COST);
5739   format %{ "strw  zr, $mem\t# int" %}
5740 
5741   ins_encode(aarch64_enc_strw0(mem));
5742 
5743   ins_pipe(istore_mem);
5744 %}
5745 
5746 // Store Long (64 bit signed)
5747 instruct storeL(iRegL src, memory mem)
5748 %{
5749   match(Set mem (StoreL mem src));
5750   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5751 
5752   ins_cost(INSN_COST);
5753   format %{ "str  $src, $mem\t# int" %}
5754 
5755   ins_encode(aarch64_enc_str(src, mem));
5756 
5757   ins_pipe(istore_reg_mem);
5758 %}
5759 
5760 // Store Long (64 bit signed)
5761 instruct storeimmL0(immL0 zero, memory mem)
5762 %{
5763   match(Set mem (StoreL mem zero));
5764   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5765 
5766   ins_cost(INSN_COST);
5767   format %{ "str  zr, $mem\t# int" %}
5768 
5769   ins_encode(aarch64_enc_str0(mem));
5770 
5771   ins_pipe(istore_mem);
5772 %}
5773 
5774 // Store Pointer
5775 instruct storeP(iRegP src, memory mem)
5776 %{
5777   match(Set mem (StoreP mem src));
5778   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5779 
5780   ins_cost(INSN_COST);
5781   format %{ "str  $src, $mem\t# ptr" %}
5782 
5783   ins_encode(aarch64_enc_str(src, mem));
5784 
5785   ins_pipe(istore_reg_mem);
5786 %}
5787 
5788 // Store Pointer
5789 instruct storeimmP0(immP0 zero, memory mem)
5790 %{
5791   match(Set mem (StoreP mem zero));
5792   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5793 
5794   ins_cost(INSN_COST);
5795   format %{ "str zr, $mem\t# ptr" %}
5796 
5797   ins_encode(aarch64_enc_str0(mem));
5798 
5799   ins_pipe(istore_mem);
5800 %}
5801 
5802 // Store Compressed Pointer
5803 instruct storeN(iRegN src, memory mem)
5804 %{
5805   match(Set mem (StoreN mem src));
5806   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5807 
5808   ins_cost(INSN_COST);
5809   format %{ "strw  $src, $mem\t# compressed ptr" %}
5810 
5811   ins_encode(aarch64_enc_strw(src, mem));
5812 
5813   ins_pipe(istore_reg_mem);
5814 %}
5815 
5816 instruct storeImmN0(iRegIHeapbase heapbase, immN0 zero, memory mem)
5817 %{
5818   match(Set mem (StoreN mem zero));
5819   predicate(Universe::narrow_oop_base() == NULL &&
5820             Universe::narrow_klass_base() == NULL &&
5821             (UseBarriersForVolatile || n->as_Store()->is_unordered()));
5822 
5823   ins_cost(INSN_COST);
5824   format %{ "strw  rheapbase, $mem\t# compressed ptr (rheapbase==0)" %}
5825 
5826   ins_encode(aarch64_enc_strw(heapbase, mem));
5827 
5828   ins_pipe(istore_reg_mem);
5829 %}
5830 
5831 // Store Float
5832 instruct storeF(vRegF src, memory mem)
5833 %{
5834   match(Set mem (StoreF mem src));
5835   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5836 
5837   ins_cost(INSN_COST);
5838   format %{ "strs  $src, $mem\t# float" %}
5839 
5840   ins_encode( aarch64_enc_strs(src, mem) );
5841 
5842   ins_pipe(pipe_class_memory);
5843 %}
5844 
5845 // TODO
5846 // implement storeImmF0 and storeFImmPacked
5847 
5848 // Store Double
5849 instruct storeD(vRegD src, memory mem)
5850 %{
5851   match(Set mem (StoreD mem src));
5852   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5853 
5854   ins_cost(INSN_COST);
5855   format %{ "strd  $src, $mem\t# double" %}
5856 
5857   ins_encode( aarch64_enc_strd(src, mem) );
5858 
5859   ins_pipe(pipe_class_memory);
5860 %}
5861 
5862 // Store Compressed Klass Pointer
5863 instruct storeNKlass(iRegN src, memory mem)
5864 %{
5865   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5866   match(Set mem (StoreNKlass mem src));
5867 
5868   ins_cost(INSN_COST);
5869   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
5870 
5871   ins_encode(aarch64_enc_strw(src, mem));
5872 
5873   ins_pipe(istore_reg_mem);
5874 %}
5875 
5876 // TODO
5877 // implement storeImmD0 and storeDImmPacked
5878 
5879 // prefetch instructions
5880 // Must be safe to execute with invalid address (cannot fault).
5881 
5882 instruct prefetchalloc( memory mem ) %{
5883   match(PrefetchAllocation mem);
5884 
5885   ins_cost(INSN_COST);
5886   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
5887 
5888   ins_encode( aarch64_enc_prefetchw(mem) );
5889 
5890   ins_pipe(iload_prefetch);
5891 %}
5892 
5893 //  ---------------- volatile loads and stores ----------------
5894 
5895 // Load Byte (8 bit signed)
5896 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
5897 %{
5898   match(Set dst (LoadB mem));
5899 
5900   ins_cost(VOLATILE_REF_COST);
5901   format %{ "ldarsb  $dst, $mem\t# byte" %}
5902 
5903   ins_encode(aarch64_enc_ldarsb(dst, mem));
5904 
5905   ins_pipe(pipe_serial);
5906 %}
5907 
5908 // Load Byte (8 bit signed) into long
5909 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
5910 %{
5911   match(Set dst (ConvI2L (LoadB mem)));
5912 
5913   ins_cost(VOLATILE_REF_COST);
5914   format %{ "ldarsb  $dst, $mem\t# byte" %}
5915 
5916   ins_encode(aarch64_enc_ldarsb(dst, mem));
5917 
5918   ins_pipe(pipe_serial);
5919 %}
5920 
5921 // Load Byte (8 bit unsigned)
5922 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
5923 %{
5924   match(Set dst (LoadUB mem));
5925 
5926   ins_cost(VOLATILE_REF_COST);
5927   format %{ "ldarb  $dst, $mem\t# byte" %}
5928 
5929   ins_encode(aarch64_enc_ldarb(dst, mem));
5930 
5931   ins_pipe(pipe_serial);
5932 %}
5933 
5934 // Load Byte (8 bit unsigned) into long
5935 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
5936 %{
5937   match(Set dst (ConvI2L (LoadUB mem)));
5938 
5939   ins_cost(VOLATILE_REF_COST);
5940   format %{ "ldarb  $dst, $mem\t# byte" %}
5941 
5942   ins_encode(aarch64_enc_ldarb(dst, mem));
5943 
5944   ins_pipe(pipe_serial);
5945 %}
5946 
5947 // Load Short (16 bit signed)
5948 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
5949 %{
5950   match(Set dst (LoadS mem));
5951 
5952   ins_cost(VOLATILE_REF_COST);
5953   format %{ "ldarshw  $dst, $mem\t# short" %}
5954 
5955   ins_encode(aarch64_enc_ldarshw(dst, mem));
5956 
5957   ins_pipe(pipe_serial);
5958 %}
5959 
5960 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
5961 %{
5962   match(Set dst (LoadUS mem));
5963 
5964   ins_cost(VOLATILE_REF_COST);
5965   format %{ "ldarhw  $dst, $mem\t# short" %}
5966 
5967   ins_encode(aarch64_enc_ldarhw(dst, mem));
5968 
5969   ins_pipe(pipe_serial);
5970 %}
5971 
5972 // Load Short/Char (16 bit unsigned) into long
5973 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
5974 %{
5975   match(Set dst (ConvI2L (LoadUS mem)));
5976 
5977   ins_cost(VOLATILE_REF_COST);
5978   format %{ "ldarh  $dst, $mem\t# short" %}
5979 
5980   ins_encode(aarch64_enc_ldarh(dst, mem));
5981 
5982   ins_pipe(pipe_serial);
5983 %}
5984 
5985 // Load Short/Char (16 bit signed) into long
5986 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
5987 %{
5988   match(Set dst (ConvI2L (LoadS mem)));
5989 
5990   ins_cost(VOLATILE_REF_COST);
5991   format %{ "ldarh  $dst, $mem\t# short" %}
5992 
5993   ins_encode(aarch64_enc_ldarsh(dst, mem));
5994 
5995   ins_pipe(pipe_serial);
5996 %}
5997 
5998 // Load Integer (32 bit signed)
5999 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
6000 %{
6001   match(Set dst (LoadI mem));
6002 
6003   ins_cost(VOLATILE_REF_COST);
6004   format %{ "ldarw  $dst, $mem\t# int" %}
6005 
6006   ins_encode(aarch64_enc_ldarw(dst, mem));
6007 
6008   ins_pipe(pipe_serial);
6009 %}
6010 
6011 // Load Integer (32 bit unsigned) into long
6012 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
6013 %{
6014   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6015 
6016   ins_cost(VOLATILE_REF_COST);
6017   format %{ "ldarw  $dst, $mem\t# int" %}
6018 
6019   ins_encode(aarch64_enc_ldarw(dst, mem));
6020 
6021   ins_pipe(pipe_serial);
6022 %}
6023 
6024 // Load Long (64 bit signed)
6025 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
6026 %{
6027   match(Set dst (LoadL mem));
6028 
6029   ins_cost(VOLATILE_REF_COST);
6030   format %{ "ldar  $dst, $mem\t# int" %}
6031 
6032   ins_encode(aarch64_enc_ldar(dst, mem));
6033 
6034   ins_pipe(pipe_serial);
6035 %}
6036 
6037 // Load Pointer
6038 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
6039 %{
6040   match(Set dst (LoadP mem));
6041 
6042   ins_cost(VOLATILE_REF_COST);
6043   format %{ "ldar  $dst, $mem\t# ptr" %}
6044 
6045   ins_encode(aarch64_enc_ldar(dst, mem));
6046 
6047   ins_pipe(pipe_serial);
6048 %}
6049 
6050 // Load Compressed Pointer
6051 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
6052 %{
6053   match(Set dst (LoadN mem));
6054 
6055   ins_cost(VOLATILE_REF_COST);
6056   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
6057 
6058   ins_encode(aarch64_enc_ldarw(dst, mem));
6059 
6060   ins_pipe(pipe_serial);
6061 %}
6062 
6063 // Load Float
6064 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
6065 %{
6066   match(Set dst (LoadF mem));
6067 
6068   ins_cost(VOLATILE_REF_COST);
6069   format %{ "ldars  $dst, $mem\t# float" %}
6070 
6071   ins_encode( aarch64_enc_fldars(dst, mem) );
6072 
6073   ins_pipe(pipe_serial);
6074 %}
6075 
6076 // Load Double
6077 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
6078 %{
6079   match(Set dst (LoadD mem));
6080 
6081   ins_cost(VOLATILE_REF_COST);
6082   format %{ "ldard  $dst, $mem\t# double" %}
6083 
6084   ins_encode( aarch64_enc_fldard(dst, mem) );
6085 
6086   ins_pipe(pipe_serial);
6087 %}
6088 
6089 // Store Byte
6090 instruct storeB_volatile(iRegI src, /* sync_memory*/indirect mem)
6091 %{
6092   match(Set mem (StoreB mem src));
6093 
6094   ins_cost(VOLATILE_REF_COST);
6095   format %{ "stlrb  $src, $mem\t# byte" %}
6096 
6097   ins_encode(aarch64_enc_stlrb(src, mem));
6098 
6099   ins_pipe(pipe_class_memory);
6100 %}
6101 
6102 // Store Char/Short
6103 instruct storeC_volatile(iRegI src, /* sync_memory*/indirect mem)
6104 %{
6105   match(Set mem (StoreC mem src));
6106 
6107   ins_cost(VOLATILE_REF_COST);
6108   format %{ "stlrh  $src, $mem\t# short" %}
6109 
6110   ins_encode(aarch64_enc_stlrh(src, mem));
6111 
6112   ins_pipe(pipe_class_memory);
6113 %}
6114 
6115 // Store Integer
6116 
6117 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
6118 %{
6119   match(Set mem(StoreI mem src));
6120 
6121   ins_cost(VOLATILE_REF_COST);
6122   format %{ "stlrw  $src, $mem\t# int" %}
6123 
6124   ins_encode(aarch64_enc_stlrw(src, mem));
6125 
6126   ins_pipe(pipe_class_memory);
6127 %}
6128 
6129 // Store Long (64 bit signed)
6130 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
6131 %{
6132   match(Set mem (StoreL mem src));
6133 
6134   ins_cost(VOLATILE_REF_COST);
6135   format %{ "stlr  $src, $mem\t# int" %}
6136 
6137   ins_encode(aarch64_enc_stlr(src, mem));
6138 
6139   ins_pipe(pipe_class_memory);
6140 %}
6141 
6142 // Store Pointer
6143 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
6144 %{
6145   match(Set mem (StoreP mem src));
6146 
6147   ins_cost(VOLATILE_REF_COST);
6148   format %{ "stlr  $src, $mem\t# ptr" %}
6149 
6150   ins_encode(aarch64_enc_stlr(src, mem));
6151 
6152   ins_pipe(pipe_class_memory);
6153 %}
6154 
6155 // Store Compressed Pointer
6156 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
6157 %{
6158   match(Set mem (StoreN mem src));
6159 
6160   ins_cost(VOLATILE_REF_COST);
6161   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
6162 
6163   ins_encode(aarch64_enc_stlrw(src, mem));
6164 
6165   ins_pipe(pipe_class_memory);
6166 %}
6167 
6168 // Store Float
6169 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
6170 %{
6171   match(Set mem (StoreF mem src));
6172 
6173   ins_cost(VOLATILE_REF_COST);
6174   format %{ "stlrs  $src, $mem\t# float" %}
6175 
6176   ins_encode( aarch64_enc_fstlrs(src, mem) );
6177 
6178   ins_pipe(pipe_class_memory);
6179 %}
6180 
6181 // TODO
6182 // implement storeImmF0 and storeFImmPacked
6183 
6184 // Store Double
6185 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
6186 %{
6187   match(Set mem (StoreD mem src));
6188 
6189   ins_cost(VOLATILE_REF_COST);
6190   format %{ "stlrd  $src, $mem\t# double" %}
6191 
6192   ins_encode( aarch64_enc_fstlrd(src, mem) );
6193 
6194   ins_pipe(pipe_class_memory);
6195 %}
6196 
6197 //  ---------------- end of volatile loads and stores ----------------
6198 
6199 // ============================================================================
6200 // BSWAP Instructions
6201 
6202 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
6203   match(Set dst (ReverseBytesI src));
6204 
6205   ins_cost(INSN_COST);
6206   format %{ "revw  $dst, $src" %}
6207 
6208   ins_encode %{
6209     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
6210   %}
6211 
6212   ins_pipe(ialu_reg);
6213 %}
6214 
6215 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
6216   match(Set dst (ReverseBytesL src));
6217 
6218   ins_cost(INSN_COST);
6219   format %{ "rev  $dst, $src" %}
6220 
6221   ins_encode %{
6222     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
6223   %}
6224 
6225   ins_pipe(ialu_reg);
6226 %}
6227 
6228 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
6229   match(Set dst (ReverseBytesUS src));
6230 
6231   ins_cost(INSN_COST);
6232   format %{ "rev16w  $dst, $src" %}
6233 
6234   ins_encode %{
6235     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
6236   %}
6237 
6238   ins_pipe(ialu_reg);
6239 %}
6240 
6241 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
6242   match(Set dst (ReverseBytesS src));
6243 
6244   ins_cost(INSN_COST);
6245   format %{ "rev16w  $dst, $src\n\t"
6246             "sbfmw $dst, $dst, #0, #15" %}
6247 
6248   ins_encode %{
6249     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
6250     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
6251   %}
6252 
6253   ins_pipe(ialu_reg);
6254 %}
6255 
6256 // ============================================================================
6257 // Zero Count Instructions
6258 
6259 instruct countLeadingZerosI(iRegINoSp dst, iRegI src) %{
6260   match(Set dst (CountLeadingZerosI src));
6261 
6262   ins_cost(INSN_COST);
6263   format %{ "clzw  $dst, $src" %}
6264   ins_encode %{
6265     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
6266   %}
6267 
6268   ins_pipe(ialu_reg);
6269 %}
6270 
6271 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
6272   match(Set dst (CountLeadingZerosL src));
6273 
6274   ins_cost(INSN_COST);
6275   format %{ "clz   $dst, $src" %}
6276   ins_encode %{
6277     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
6278   %}
6279 
6280   ins_pipe(ialu_reg);
6281 %}
6282 
6283 instruct countTrailingZerosI(iRegINoSp dst, iRegI src) %{
6284   match(Set dst (CountTrailingZerosI src));
6285 
6286   ins_cost(INSN_COST * 2);
6287   format %{ "rbitw  $dst, $src\n\t"
6288             "clzw   $dst, $dst" %}
6289   ins_encode %{
6290     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
6291     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
6292   %}
6293 
6294   ins_pipe(ialu_reg);
6295 %}
6296 
6297 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
6298   match(Set dst (CountTrailingZerosL src));
6299 
6300   ins_cost(INSN_COST * 2);
6301   format %{ "rbit   $dst, $src\n\t"
6302             "clz    $dst, $dst" %}
6303   ins_encode %{
6304     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
6305     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
6306   %}
6307 
6308   ins_pipe(ialu_reg);
6309 %}
6310 
6311 // ============================================================================
6312 // MemBar Instruction
6313 
6314 instruct load_fence() %{
6315   match(LoadFence);
6316   ins_cost(VOLATILE_REF_COST);
6317 
6318   format %{ "load_fence" %}
6319 
6320   ins_encode %{
6321     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
6322   %}
6323   ins_pipe(pipe_serial);
6324 %}
6325 
6326 instruct unnecessary_membar_acquire() %{
6327   predicate(! UseBarriersForVolatile && preceded_by_ordered_load(n));
6328   match(MemBarAcquire);
6329   ins_cost(0);
6330 
6331   format %{ "membar_acquire (elided)" %}
6332 
6333   ins_encode %{
6334     __ block_comment("membar_acquire (elided)");
6335   %}
6336 
6337   ins_pipe(pipe_class_empty);
6338 %}
6339 
6340 instruct membar_acquire() %{
6341   match(MemBarAcquire);
6342   ins_cost(VOLATILE_REF_COST);
6343 
6344   format %{ "membar_acquire" %}
6345 
6346   ins_encode %{
6347     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
6348   %}
6349 
6350   ins_pipe(pipe_serial);
6351 %}
6352 
6353 
6354 instruct membar_acquire_lock() %{
6355   match(MemBarAcquireLock);
6356   ins_cost(VOLATILE_REF_COST);
6357 
6358   format %{ "membar_acquire_lock" %}
6359 
6360   ins_encode %{
6361     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
6362   %}
6363 
6364   ins_pipe(pipe_serial);
6365 %}
6366 
6367 instruct store_fence() %{
6368   match(StoreFence);
6369   ins_cost(VOLATILE_REF_COST);
6370 
6371   format %{ "store_fence" %}
6372 
6373   ins_encode %{
6374     __ membar(Assembler::LoadStore|Assembler::StoreStore);
6375   %}
6376   ins_pipe(pipe_serial);
6377 %}
6378 
6379 instruct membar_release() %{
6380   match(MemBarRelease);
6381   ins_cost(VOLATILE_REF_COST);
6382 
6383   format %{ "membar_release" %}
6384 
6385   ins_encode %{
6386     __ membar(Assembler::LoadStore|Assembler::StoreStore);
6387   %}
6388   ins_pipe(pipe_serial);
6389 %}
6390 
6391 instruct membar_storestore() %{
6392   match(MemBarStoreStore);
6393   ins_cost(VOLATILE_REF_COST);
6394 
6395   format %{ "MEMBAR-store-store" %}
6396 
6397   ins_encode %{
6398     __ membar(Assembler::StoreStore);
6399   %}
6400   ins_pipe(pipe_serial);
6401 %}
6402 
6403 instruct membar_release_lock() %{
6404   match(MemBarReleaseLock);
6405   ins_cost(VOLATILE_REF_COST);
6406 
6407   format %{ "membar_release_lock" %}
6408 
6409   ins_encode %{
6410     __ membar(Assembler::LoadStore|Assembler::StoreStore);
6411   %}
6412 
6413   ins_pipe(pipe_serial);
6414 %}
6415 
6416 instruct membar_volatile() %{
6417   match(MemBarVolatile);
6418   ins_cost(VOLATILE_REF_COST*100);
6419 
6420   format %{ "membar_volatile" %}
6421 
6422   ins_encode %{
6423     __ membar(Assembler::StoreLoad);
6424   %}
6425 
6426   ins_pipe(pipe_serial);
6427 %}
6428 
6429 // ============================================================================
6430 // Cast/Convert Instructions
6431 
6432 instruct castX2P(iRegPNoSp dst, iRegL src) %{
6433   match(Set dst (CastX2P src));
6434 
6435   ins_cost(INSN_COST);
6436   format %{ "mov $dst, $src\t# long -> ptr" %}
6437 
6438   ins_encode %{
6439     if ($dst$$reg != $src$$reg) {
6440       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
6441     }
6442   %}
6443 
6444   ins_pipe(ialu_reg);
6445 %}
6446 
6447 instruct castP2X(iRegLNoSp dst, iRegP src) %{
6448   match(Set dst (CastP2X src));
6449 
6450   ins_cost(INSN_COST);
6451   format %{ "mov $dst, $src\t# ptr -> long" %}
6452 
6453   ins_encode %{
6454     if ($dst$$reg != $src$$reg) {
6455       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
6456     }
6457   %}
6458 
6459   ins_pipe(ialu_reg);
6460 %}
6461 
6462 // Convert oop into int for vectors alignment masking
6463 instruct convP2I(iRegINoSp dst, iRegP src) %{
6464   match(Set dst (ConvL2I (CastP2X src)));
6465 
6466   ins_cost(INSN_COST);
6467   format %{ "movw $dst, $src\t# ptr -> int" %}
6468   ins_encode %{
6469     __ movw($dst$$Register, $src$$Register);
6470   %}
6471 
6472   ins_pipe(ialu_reg);
6473 %}
6474 
6475 // Convert compressed oop into int for vectors alignment masking
6476 // in case of 32bit oops (heap < 4Gb).
6477 instruct convN2I(iRegINoSp dst, iRegN src)
6478 %{
6479   predicate(Universe::narrow_oop_shift() == 0);
6480   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6481 
6482   ins_cost(INSN_COST);
6483   format %{ "mov dst, $src\t# compressed ptr -> int" %}
6484   ins_encode %{
6485     __ movw($dst$$Register, $src$$Register);
6486   %}
6487 
6488   ins_pipe(ialu_reg);
6489 %}
6490 
6491 
6492 // Convert oop pointer into compressed form
6493 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
6494   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6495   match(Set dst (EncodeP src));
6496   effect(KILL cr);
6497   ins_cost(INSN_COST * 3);
6498   format %{ "encode_heap_oop $dst, $src" %}
6499   ins_encode %{
6500     Register s = $src$$Register;
6501     Register d = $dst$$Register;
6502     __ encode_heap_oop(d, s);
6503   %}
6504   ins_pipe(ialu_reg);
6505 %}
6506 
6507 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
6508   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6509   match(Set dst (EncodeP src));
6510   ins_cost(INSN_COST * 3);
6511   format %{ "encode_heap_oop_not_null $dst, $src" %}
6512   ins_encode %{
6513     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6514   %}
6515   ins_pipe(ialu_reg);
6516 %}
6517 
6518 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
6519   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6520             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6521   match(Set dst (DecodeN src));
6522   ins_cost(INSN_COST * 3);
6523   format %{ "decode_heap_oop $dst, $src" %}
6524   ins_encode %{
6525     Register s = $src$$Register;
6526     Register d = $dst$$Register;
6527     __ decode_heap_oop(d, s);
6528   %}
6529   ins_pipe(ialu_reg);
6530 %}
6531 
6532 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
6533   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6534             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6535   match(Set dst (DecodeN src));
6536   ins_cost(INSN_COST * 3);
6537   format %{ "decode_heap_oop_not_null $dst, $src" %}
6538   ins_encode %{
6539     Register s = $src$$Register;
6540     Register d = $dst$$Register;
6541     __ decode_heap_oop_not_null(d, s);
6542   %}
6543   ins_pipe(ialu_reg);
6544 %}
6545 
6546 // n.b. AArch64 implementations of encode_klass_not_null and
6547 // decode_klass_not_null do not modify the flags register so, unlike
6548 // Intel, we don't kill CR as a side effect here
6549 
6550 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
6551   match(Set dst (EncodePKlass src));
6552 
6553   ins_cost(INSN_COST * 3);
6554   format %{ "encode_klass_not_null $dst,$src" %}
6555 
6556   ins_encode %{
6557     Register src_reg = as_Register($src$$reg);
6558     Register dst_reg = as_Register($dst$$reg);
6559     __ encode_klass_not_null(dst_reg, src_reg);
6560   %}
6561 
6562    ins_pipe(ialu_reg);
6563 %}
6564 
6565 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
6566   match(Set dst (DecodeNKlass src));
6567 
6568   ins_cost(INSN_COST * 3);
6569   format %{ "decode_klass_not_null $dst,$src" %}
6570 
6571   ins_encode %{
6572     Register src_reg = as_Register($src$$reg);
6573     Register dst_reg = as_Register($dst$$reg);
6574     if (dst_reg != src_reg) {
6575       __ decode_klass_not_null(dst_reg, src_reg);
6576     } else {
6577       __ decode_klass_not_null(dst_reg);
6578     }
6579   %}
6580 
6581    ins_pipe(ialu_reg);
6582 %}
6583 
6584 instruct checkCastPP(iRegPNoSp dst)
6585 %{
6586   match(Set dst (CheckCastPP dst));
6587 
6588   size(0);
6589   format %{ "# checkcastPP of $dst" %}
6590   ins_encode(/* empty encoding */);
6591   ins_pipe(pipe_class_empty);
6592 %}
6593 
6594 instruct castPP(iRegPNoSp dst)
6595 %{
6596   match(Set dst (CastPP dst));
6597 
6598   size(0);
6599   format %{ "# castPP of $dst" %}
6600   ins_encode(/* empty encoding */);
6601   ins_pipe(pipe_class_empty);
6602 %}
6603 
6604 instruct castII(iRegI dst)
6605 %{
6606   match(Set dst (CastII dst));
6607 
6608   size(0);
6609   format %{ "# castII of $dst" %}
6610   ins_encode(/* empty encoding */);
6611   ins_cost(0);
6612   ins_pipe(pipe_class_empty);
6613 %}
6614 
6615 // ============================================================================
6616 // Atomic operation instructions
6617 //
6618 // Intel and SPARC both implement Ideal Node LoadPLocked and
6619 // Store{PIL}Conditional instructions using a normal load for the
6620 // LoadPLocked and a CAS for the Store{PIL}Conditional.
6621 //
6622 // The ideal code appears only to use LoadPLocked/StorePLocked as a
6623 // pair to lock object allocations from Eden space when not using
6624 // TLABs.
6625 //
6626 // There does not appear to be a Load{IL}Locked Ideal Node and the
6627 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
6628 // and to use StoreIConditional only for 32-bit and StoreLConditional
6629 // only for 64-bit.
6630 //
6631 // We implement LoadPLocked and StorePLocked instructions using,
6632 // respectively the AArch64 hw load-exclusive and store-conditional
6633 // instructions. Whereas we must implement each of
6634 // Store{IL}Conditional using a CAS which employs a pair of
6635 // instructions comprising a load-exclusive followed by a
6636 // store-conditional.
6637 
6638 
6639 // Locked-load (linked load) of the current heap-top
6640 // used when updating the eden heap top
6641 // implemented using ldaxr on AArch64
6642 
6643 instruct loadPLocked(iRegPNoSp dst, indirect mem)
6644 %{
6645   match(Set dst (LoadPLocked mem));
6646 
6647   ins_cost(VOLATILE_REF_COST);
6648 
6649   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
6650 
6651   ins_encode(aarch64_enc_ldaxr(dst, mem));
6652 
6653   ins_pipe(pipe_serial);
6654 %}
6655 
6656 // Conditional-store of the updated heap-top.
6657 // Used during allocation of the shared heap.
6658 // Sets flag (EQ) on success.
6659 // implemented using stlxr on AArch64.
6660 
6661 instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
6662 %{
6663   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
6664 
6665   ins_cost(VOLATILE_REF_COST);
6666 
6667  // TODO
6668  // do we need to do a store-conditional release or can we just use a
6669  // plain store-conditional?
6670 
6671   format %{
6672     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
6673     "cmpw rscratch1, zr\t# EQ on successful write"
6674   %}
6675 
6676   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
6677 
6678   ins_pipe(pipe_serial);
6679 %}
6680 
6681 // this has to be implemented as a CAS
6682 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
6683 %{
6684   match(Set cr (StoreLConditional mem (Binary oldval newval)));
6685 
6686   ins_cost(VOLATILE_REF_COST);
6687 
6688   format %{
6689     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
6690     "cmpw rscratch1, zr\t# EQ on successful write"
6691   %}
6692 
6693   ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval));
6694 
6695   ins_pipe(pipe_slow);
6696 %}
6697 
6698 // this has to be implemented as a CAS
6699 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
6700 %{
6701   match(Set cr (StoreIConditional mem (Binary oldval newval)));
6702 
6703   ins_cost(VOLATILE_REF_COST);
6704 
6705   format %{
6706     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
6707     "cmpw rscratch1, zr\t# EQ on successful write"
6708   %}
6709 
6710   ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval));
6711 
6712   ins_pipe(pipe_slow);
6713 %}
6714 
6715 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
6716 // can't match them
6717 
6718 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
6719 
6720   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
6721 
6722   effect(KILL cr);
6723 
6724  format %{
6725     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
6726     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
6727  %}
6728 
6729  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
6730             aarch64_enc_cset_eq(res));
6731 
6732   ins_pipe(pipe_slow);
6733 %}
6734 
6735 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
6736 
6737   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
6738 
6739   effect(KILL cr);
6740 
6741  format %{
6742     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
6743     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
6744  %}
6745 
6746  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
6747             aarch64_enc_cset_eq(res));
6748 
6749   ins_pipe(pipe_slow);
6750 %}
6751 
6752 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
6753 
6754   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
6755 
6756   effect(KILL cr);
6757 
6758  format %{
6759     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
6760     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
6761  %}
6762 
6763  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
6764             aarch64_enc_cset_eq(res));
6765 
6766   ins_pipe(pipe_slow);
6767 %}
6768 
6769 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
6770 
6771   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
6772 
6773   effect(KILL cr);
6774 
6775  format %{
6776     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
6777     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
6778  %}
6779 
6780  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
6781             aarch64_enc_cset_eq(res));
6782 
6783   ins_pipe(pipe_slow);
6784 %}
6785 
6786 
6787 instruct get_and_setI(indirect mem, iRegINoSp newv, iRegI prev) %{
6788   match(Set prev (GetAndSetI mem newv));
6789   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
6790   ins_encode %{
6791     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
6792   %}
6793   ins_pipe(pipe_serial);
6794 %}
6795 
6796 instruct get_and_setL(indirect mem, iRegLNoSp newv, iRegL prev) %{
6797   match(Set prev (GetAndSetL mem newv));
6798   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
6799   ins_encode %{
6800     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
6801   %}
6802   ins_pipe(pipe_serial);
6803 %}
6804 
6805 instruct get_and_setN(indirect mem, iRegNNoSp newv, iRegI prev) %{
6806   match(Set prev (GetAndSetN mem newv));
6807   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
6808   ins_encode %{
6809     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
6810   %}
6811   ins_pipe(pipe_serial);
6812 %}
6813 
6814 instruct get_and_setP(indirect mem, iRegPNoSp newv, iRegP prev) %{
6815   match(Set prev (GetAndSetP mem newv));
6816   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
6817   ins_encode %{
6818     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
6819   %}
6820   ins_pipe(pipe_serial);
6821 %}
6822 
6823 
6824 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
6825   match(Set newval (GetAndAddL mem incr));
6826   ins_cost(INSN_COST * 10);
6827   format %{ "get_and_addL $newval, [$mem], $incr" %}
6828   ins_encode %{
6829     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
6830   %}
6831   ins_pipe(pipe_serial);
6832 %}
6833 
6834 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
6835   predicate(n->as_LoadStore()->result_not_used());
6836   match(Set dummy (GetAndAddL mem incr));
6837   ins_cost(INSN_COST * 9);
6838   format %{ "get_and_addL [$mem], $incr" %}
6839   ins_encode %{
6840     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
6841   %}
6842   ins_pipe(pipe_serial);
6843 %}
6844 
6845 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
6846   match(Set newval (GetAndAddL mem incr));
6847   ins_cost(INSN_COST * 10);
6848   format %{ "get_and_addL $newval, [$mem], $incr" %}
6849   ins_encode %{
6850     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
6851   %}
6852   ins_pipe(pipe_serial);
6853 %}
6854 
6855 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
6856   predicate(n->as_LoadStore()->result_not_used());
6857   match(Set dummy (GetAndAddL mem incr));
6858   ins_cost(INSN_COST * 9);
6859   format %{ "get_and_addL [$mem], $incr" %}
6860   ins_encode %{
6861     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
6862   %}
6863   ins_pipe(pipe_serial);
6864 %}
6865 
6866 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
6867   match(Set newval (GetAndAddI mem incr));
6868   ins_cost(INSN_COST * 10);
6869   format %{ "get_and_addI $newval, [$mem], $incr" %}
6870   ins_encode %{
6871     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
6872   %}
6873   ins_pipe(pipe_serial);
6874 %}
6875 
6876 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
6877   predicate(n->as_LoadStore()->result_not_used());
6878   match(Set dummy (GetAndAddI mem incr));
6879   ins_cost(INSN_COST * 9);
6880   format %{ "get_and_addI [$mem], $incr" %}
6881   ins_encode %{
6882     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
6883   %}
6884   ins_pipe(pipe_serial);
6885 %}
6886 
6887 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
6888   match(Set newval (GetAndAddI mem incr));
6889   ins_cost(INSN_COST * 10);
6890   format %{ "get_and_addI $newval, [$mem], $incr" %}
6891   ins_encode %{
6892     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
6893   %}
6894   ins_pipe(pipe_serial);
6895 %}
6896 
6897 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
6898   predicate(n->as_LoadStore()->result_not_used());
6899   match(Set dummy (GetAndAddI mem incr));
6900   ins_cost(INSN_COST * 9);
6901   format %{ "get_and_addI [$mem], $incr" %}
6902   ins_encode %{
6903     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
6904   %}
6905   ins_pipe(pipe_serial);
6906 %}
6907 
6908 // Manifest a CmpL result in an integer register.
6909 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
6910 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
6911 %{
6912   match(Set dst (CmpL3 src1 src2));
6913   effect(KILL flags);
6914 
6915   ins_cost(INSN_COST * 6);
6916   format %{
6917       "cmp $src1, $src2"
6918       "csetw $dst, ne"
6919       "cnegw $dst, lt"
6920   %}
6921   // format %{ "CmpL3 $dst, $src1, $src2" %}
6922   ins_encode %{
6923     __ cmp($src1$$Register, $src2$$Register);
6924     __ csetw($dst$$Register, Assembler::NE);
6925     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
6926   %}
6927 
6928   ins_pipe(pipe_class_default);
6929 %}
6930 
6931 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
6932 %{
6933   match(Set dst (CmpL3 src1 src2));
6934   effect(KILL flags);
6935 
6936   ins_cost(INSN_COST * 6);
6937   format %{
6938       "cmp $src1, $src2"
6939       "csetw $dst, ne"
6940       "cnegw $dst, lt"
6941   %}
6942   ins_encode %{
6943     int32_t con = (int32_t)$src2$$constant;
6944      if (con < 0) {
6945       __ adds(zr, $src1$$Register, -con);
6946     } else {
6947       __ subs(zr, $src1$$Register, con);
6948     }
6949     __ csetw($dst$$Register, Assembler::NE);
6950     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
6951   %}
6952 
6953   ins_pipe(pipe_class_default);
6954 %}
6955 
6956 // ============================================================================
6957 // Conditional Move Instructions
6958 
6959 // n.b. we have identical rules for both a signed compare op (cmpOp)
6960 // and an unsigned compare op (cmpOpU). it would be nice if we could
6961 // define an op class which merged both inputs and use it to type the
6962 // argument to a single rule. unfortunatelyt his fails because the
6963 // opclass does not live up to the COND_INTER interface of its
6964 // component operands. When the generic code tries to negate the
6965 // operand it ends up running the generci Machoper::negate method
6966 // which throws a ShouldNotHappen. So, we have to provide two flavours
6967 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
6968 
6969 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegI src1, iRegI src2) %{
6970   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
6971 
6972   ins_cost(INSN_COST * 2);
6973   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
6974 
6975   ins_encode %{
6976     __ cselw(as_Register($dst$$reg),
6977              as_Register($src2$$reg),
6978              as_Register($src1$$reg),
6979              (Assembler::Condition)$cmp$$cmpcode);
6980   %}
6981 
6982   ins_pipe(icond_reg_reg);
6983 %}
6984 
6985 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegI src1, iRegI src2) %{
6986   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
6987 
6988   ins_cost(INSN_COST * 2);
6989   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
6990 
6991   ins_encode %{
6992     __ cselw(as_Register($dst$$reg),
6993              as_Register($src2$$reg),
6994              as_Register($src1$$reg),
6995              (Assembler::Condition)$cmp$$cmpcode);
6996   %}
6997 
6998   ins_pipe(icond_reg_reg);
6999 %}
7000 
7001 // special cases where one arg is zero
7002 
7003 // n.b. this is selected in preference to the rule above because it
7004 // avoids loading constant 0 into a source register
7005 
7006 // TODO
7007 // we ought only to be able to cull one of these variants as the ideal
7008 // transforms ought always to order the zero consistently (to left/right?)
7009 
7010 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegI src) %{
7011   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
7012 
7013   ins_cost(INSN_COST * 2);
7014   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
7015 
7016   ins_encode %{
7017     __ cselw(as_Register($dst$$reg),
7018              as_Register($src$$reg),
7019              zr,
7020              (Assembler::Condition)$cmp$$cmpcode);
7021   %}
7022 
7023   ins_pipe(icond_reg);
7024 %}
7025 
7026 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegI src) %{
7027   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
7028 
7029   ins_cost(INSN_COST * 2);
7030   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
7031 
7032   ins_encode %{
7033     __ cselw(as_Register($dst$$reg),
7034              as_Register($src$$reg),
7035              zr,
7036              (Assembler::Condition)$cmp$$cmpcode);
7037   %}
7038 
7039   ins_pipe(icond_reg);
7040 %}
7041 
7042 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegI src, immI0 zero) %{
7043   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
7044 
7045   ins_cost(INSN_COST * 2);
7046   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
7047 
7048   ins_encode %{
7049     __ cselw(as_Register($dst$$reg),
7050              zr,
7051              as_Register($src$$reg),
7052              (Assembler::Condition)$cmp$$cmpcode);
7053   %}
7054 
7055   ins_pipe(icond_reg);
7056 %}
7057 
7058 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegI src, immI0 zero) %{
7059   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
7060 
7061   ins_cost(INSN_COST * 2);
7062   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
7063 
7064   ins_encode %{
7065     __ cselw(as_Register($dst$$reg),
7066              zr,
7067              as_Register($src$$reg),
7068              (Assembler::Condition)$cmp$$cmpcode);
7069   %}
7070 
7071   ins_pipe(icond_reg);
7072 %}
7073 
7074 // special case for creating a boolean 0 or 1
7075 
7076 // n.b. this is selected in preference to the rule above because it
7077 // avoids loading constants 0 and 1 into a source register
7078 
7079 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
7080   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
7081 
7082   ins_cost(INSN_COST * 2);
7083   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
7084 
7085   ins_encode %{
7086     // equivalently
7087     // cset(as_Register($dst$$reg),
7088     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
7089     __ csincw(as_Register($dst$$reg),
7090              zr,
7091              zr,
7092              (Assembler::Condition)$cmp$$cmpcode);
7093   %}
7094 
7095   ins_pipe(icond_none);
7096 %}
7097 
7098 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
7099   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
7100 
7101   ins_cost(INSN_COST * 2);
7102   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
7103 
7104   ins_encode %{
7105     // equivalently
7106     // cset(as_Register($dst$$reg),
7107     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
7108     __ csincw(as_Register($dst$$reg),
7109              zr,
7110              zr,
7111              (Assembler::Condition)$cmp$$cmpcode);
7112   %}
7113 
7114   ins_pipe(icond_none);
7115 %}
7116 
7117 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
7118   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
7119 
7120   ins_cost(INSN_COST * 2);
7121   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
7122 
7123   ins_encode %{
7124     __ csel(as_Register($dst$$reg),
7125             as_Register($src2$$reg),
7126             as_Register($src1$$reg),
7127             (Assembler::Condition)$cmp$$cmpcode);
7128   %}
7129 
7130   ins_pipe(icond_reg_reg);
7131 %}
7132 
7133 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
7134   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
7135 
7136   ins_cost(INSN_COST * 2);
7137   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
7138 
7139   ins_encode %{
7140     __ csel(as_Register($dst$$reg),
7141             as_Register($src2$$reg),
7142             as_Register($src1$$reg),
7143             (Assembler::Condition)$cmp$$cmpcode);
7144   %}
7145 
7146   ins_pipe(icond_reg_reg);
7147 %}
7148 
7149 // special cases where one arg is zero
7150 
7151 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
7152   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
7153 
7154   ins_cost(INSN_COST * 2);
7155   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
7156 
7157   ins_encode %{
7158     __ csel(as_Register($dst$$reg),
7159             zr,
7160             as_Register($src$$reg),
7161             (Assembler::Condition)$cmp$$cmpcode);
7162   %}
7163 
7164   ins_pipe(icond_reg);
7165 %}
7166 
7167 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
7168   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
7169 
7170   ins_cost(INSN_COST * 2);
7171   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
7172 
7173   ins_encode %{
7174     __ csel(as_Register($dst$$reg),
7175             zr,
7176             as_Register($src$$reg),
7177             (Assembler::Condition)$cmp$$cmpcode);
7178   %}
7179 
7180   ins_pipe(icond_reg);
7181 %}
7182 
7183 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
7184   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
7185 
7186   ins_cost(INSN_COST * 2);
7187   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
7188 
7189   ins_encode %{
7190     __ csel(as_Register($dst$$reg),
7191             as_Register($src$$reg),
7192             zr,
7193             (Assembler::Condition)$cmp$$cmpcode);
7194   %}
7195 
7196   ins_pipe(icond_reg);
7197 %}
7198 
7199 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
7200   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
7201 
7202   ins_cost(INSN_COST * 2);
7203   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
7204 
7205   ins_encode %{
7206     __ csel(as_Register($dst$$reg),
7207             as_Register($src$$reg),
7208             zr,
7209             (Assembler::Condition)$cmp$$cmpcode);
7210   %}
7211 
7212   ins_pipe(icond_reg);
7213 %}
7214 
7215 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
7216   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
7217 
7218   ins_cost(INSN_COST * 2);
7219   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
7220 
7221   ins_encode %{
7222     __ csel(as_Register($dst$$reg),
7223             as_Register($src2$$reg),
7224             as_Register($src1$$reg),
7225             (Assembler::Condition)$cmp$$cmpcode);
7226   %}
7227 
7228   ins_pipe(icond_reg_reg);
7229 %}
7230 
7231 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
7232   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
7233 
7234   ins_cost(INSN_COST * 2);
7235   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
7236 
7237   ins_encode %{
7238     __ csel(as_Register($dst$$reg),
7239             as_Register($src2$$reg),
7240             as_Register($src1$$reg),
7241             (Assembler::Condition)$cmp$$cmpcode);
7242   %}
7243 
7244   ins_pipe(icond_reg_reg);
7245 %}
7246 
7247 // special cases where one arg is zero
7248 
7249 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
7250   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
7251 
7252   ins_cost(INSN_COST * 2);
7253   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
7254 
7255   ins_encode %{
7256     __ csel(as_Register($dst$$reg),
7257             zr,
7258             as_Register($src$$reg),
7259             (Assembler::Condition)$cmp$$cmpcode);
7260   %}
7261 
7262   ins_pipe(icond_reg);
7263 %}
7264 
7265 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
7266   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
7267 
7268   ins_cost(INSN_COST * 2);
7269   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
7270 
7271   ins_encode %{
7272     __ csel(as_Register($dst$$reg),
7273             zr,
7274             as_Register($src$$reg),
7275             (Assembler::Condition)$cmp$$cmpcode);
7276   %}
7277 
7278   ins_pipe(icond_reg);
7279 %}
7280 
7281 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
7282   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
7283 
7284   ins_cost(INSN_COST * 2);
7285   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
7286 
7287   ins_encode %{
7288     __ csel(as_Register($dst$$reg),
7289             as_Register($src$$reg),
7290             zr,
7291             (Assembler::Condition)$cmp$$cmpcode);
7292   %}
7293 
7294   ins_pipe(icond_reg);
7295 %}
7296 
7297 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
7298   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
7299 
7300   ins_cost(INSN_COST * 2);
7301   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
7302 
7303   ins_encode %{
7304     __ csel(as_Register($dst$$reg),
7305             as_Register($src$$reg),
7306             zr,
7307             (Assembler::Condition)$cmp$$cmpcode);
7308   %}
7309 
7310   ins_pipe(icond_reg);
7311 %}
7312 
7313 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
7314   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
7315 
7316   ins_cost(INSN_COST * 2);
7317   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
7318 
7319   ins_encode %{
7320     __ cselw(as_Register($dst$$reg),
7321              as_Register($src2$$reg),
7322              as_Register($src1$$reg),
7323              (Assembler::Condition)$cmp$$cmpcode);
7324   %}
7325 
7326   ins_pipe(icond_reg_reg);
7327 %}
7328 
7329 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
7330   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
7331 
7332   ins_cost(INSN_COST * 2);
7333   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
7334 
7335   ins_encode %{
7336     __ cselw(as_Register($dst$$reg),
7337              as_Register($src2$$reg),
7338              as_Register($src1$$reg),
7339              (Assembler::Condition)$cmp$$cmpcode);
7340   %}
7341 
7342   ins_pipe(icond_reg_reg);
7343 %}
7344 
7345 // special cases where one arg is zero
7346 
7347 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
7348   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
7349 
7350   ins_cost(INSN_COST * 2);
7351   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
7352 
7353   ins_encode %{
7354     __ cselw(as_Register($dst$$reg),
7355              zr,
7356              as_Register($src$$reg),
7357              (Assembler::Condition)$cmp$$cmpcode);
7358   %}
7359 
7360   ins_pipe(icond_reg);
7361 %}
7362 
7363 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
7364   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
7365 
7366   ins_cost(INSN_COST * 2);
7367   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
7368 
7369   ins_encode %{
7370     __ cselw(as_Register($dst$$reg),
7371              zr,
7372              as_Register($src$$reg),
7373              (Assembler::Condition)$cmp$$cmpcode);
7374   %}
7375 
7376   ins_pipe(icond_reg);
7377 %}
7378 
7379 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
7380   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
7381 
7382   ins_cost(INSN_COST * 2);
7383   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
7384 
7385   ins_encode %{
7386     __ cselw(as_Register($dst$$reg),
7387              as_Register($src$$reg),
7388              zr,
7389              (Assembler::Condition)$cmp$$cmpcode);
7390   %}
7391 
7392   ins_pipe(icond_reg);
7393 %}
7394 
7395 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
7396   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
7397 
7398   ins_cost(INSN_COST * 2);
7399   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
7400 
7401   ins_encode %{
7402     __ cselw(as_Register($dst$$reg),
7403              as_Register($src$$reg),
7404              zr,
7405              (Assembler::Condition)$cmp$$cmpcode);
7406   %}
7407 
7408   ins_pipe(icond_reg);
7409 %}
7410 
7411 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
7412 %{
7413   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
7414 
7415   ins_cost(INSN_COST * 3);
7416 
7417   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
7418   ins_encode %{
7419     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
7420     __ fcsels(as_FloatRegister($dst$$reg),
7421               as_FloatRegister($src2$$reg),
7422               as_FloatRegister($src1$$reg),
7423               cond);
7424   %}
7425 
7426   ins_pipe(pipe_class_default);
7427 %}
7428 
7429 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
7430 %{
7431   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
7432 
7433   ins_cost(INSN_COST * 3);
7434 
7435   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
7436   ins_encode %{
7437     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
7438     __ fcsels(as_FloatRegister($dst$$reg),
7439               as_FloatRegister($src2$$reg),
7440               as_FloatRegister($src1$$reg),
7441               cond);
7442   %}
7443 
7444   ins_pipe(pipe_class_default);
7445 %}
7446 
7447 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
7448 %{
7449   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
7450 
7451   ins_cost(INSN_COST * 3);
7452 
7453   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
7454   ins_encode %{
7455     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
7456     __ fcseld(as_FloatRegister($dst$$reg),
7457               as_FloatRegister($src2$$reg),
7458               as_FloatRegister($src1$$reg),
7459               cond);
7460   %}
7461 
7462   ins_pipe(pipe_class_default);
7463 %}
7464 
7465 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
7466 %{
7467   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
7468 
7469   ins_cost(INSN_COST * 3);
7470 
7471   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
7472   ins_encode %{
7473     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
7474     __ fcseld(as_FloatRegister($dst$$reg),
7475               as_FloatRegister($src2$$reg),
7476               as_FloatRegister($src1$$reg),
7477               cond);
7478   %}
7479 
7480   ins_pipe(pipe_class_default);
7481 %}
7482 
7483 // ============================================================================
7484 // Arithmetic Instructions
7485 //
7486 
7487 // Integer Addition
7488 
7489 // TODO
7490 // these currently employ operations which do not set CR and hence are
7491 // not flagged as killing CR but we would like to isolate the cases
7492 // where we want to set flags from those where we don't. need to work
7493 // out how to do that.
7494 
7495 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7496   match(Set dst (AddI src1 src2));
7497 
7498   ins_cost(INSN_COST);
7499   format %{ "addw  $dst, $src1, $src2" %}
7500 
7501   ins_encode %{
7502     __ addw(as_Register($dst$$reg),
7503             as_Register($src1$$reg),
7504             as_Register($src2$$reg));
7505   %}
7506 
7507   ins_pipe(ialu_reg_reg);
7508 %}
7509 
7510 instruct addI_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2) %{
7511   match(Set dst (AddI src1 src2));
7512 
7513   ins_cost(INSN_COST);
7514   format %{ "addw $dst, $src1, $src2" %}
7515 
7516   // use opcode to indicate that this is an add not a sub
7517   opcode(0x0);
7518 
7519   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
7520 
7521   ins_pipe(ialu_reg_imm);
7522 %}
7523 
7524 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
7525   match(Set dst (AddI (ConvL2I src1) src2));
7526 
7527   ins_cost(INSN_COST);
7528   format %{ "addw $dst, $src1, $src2" %}
7529 
7530   // use opcode to indicate that this is an add not a sub
7531   opcode(0x0);
7532 
7533   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
7534 
7535   ins_pipe(ialu_reg_imm);
7536 %}
7537 
7538 // Pointer Addition
7539 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
7540   match(Set dst (AddP src1 src2));
7541 
7542   ins_cost(INSN_COST);
7543   format %{ "add $dst, $src1, $src2\t# ptr" %}
7544 
7545   ins_encode %{
7546     __ add(as_Register($dst$$reg),
7547            as_Register($src1$$reg),
7548            as_Register($src2$$reg));
7549   %}
7550 
7551   ins_pipe(ialu_reg_reg);
7552 %}
7553 
7554 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
7555   match(Set dst (AddP src1 (ConvI2L src2)));
7556 
7557   ins_cost(1.9 * INSN_COST);
7558   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
7559 
7560   ins_encode %{
7561     __ add(as_Register($dst$$reg),
7562            as_Register($src1$$reg),
7563            as_Register($src2$$reg), ext::sxtw);
7564   %}
7565 
7566   ins_pipe(ialu_reg_reg);
7567 %}
7568 
7569 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
7570   match(Set dst (AddP src1 (LShiftL src2 scale)));
7571 
7572   ins_cost(1.9 * INSN_COST);
7573   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
7574 
7575   ins_encode %{
7576     __ lea(as_Register($dst$$reg),
7577            Address(as_Register($src1$$reg), as_Register($src2$$reg),
7578                    Address::lsl($scale$$constant)));
7579   %}
7580 
7581   ins_pipe(ialu_reg_reg_shift);
7582 %}
7583 
7584 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
7585   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
7586 
7587   ins_cost(1.9 * INSN_COST);
7588   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
7589 
7590   ins_encode %{
7591     __ lea(as_Register($dst$$reg),
7592            Address(as_Register($src1$$reg), as_Register($src2$$reg),
7593                    Address::sxtw($scale$$constant)));
7594   %}
7595 
7596   ins_pipe(ialu_reg_reg_shift);
7597 %}
7598 
7599 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
7600   match(Set dst (LShiftL (ConvI2L src) scale));
7601 
7602   ins_cost(INSN_COST);
7603   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
7604 
7605   ins_encode %{
7606     __ sbfiz(as_Register($dst$$reg),
7607           as_Register($src$$reg),
7608           $scale$$constant & 63, MIN(32, (-$scale$$constant) & 63));
7609   %}
7610 
7611   ins_pipe(ialu_reg_shift);
7612 %}
7613 
7614 // Pointer Immediate Addition
7615 // n.b. this needs to be more expensive than using an indirect memory
7616 // operand
7617 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
7618   match(Set dst (AddP src1 src2));
7619 
7620   ins_cost(INSN_COST);
7621   format %{ "add $dst, $src1, $src2\t# ptr" %}
7622 
7623   // use opcode to indicate that this is an add not a sub
7624   opcode(0x0);
7625 
7626   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
7627 
7628   ins_pipe(ialu_reg_imm);
7629 %}
7630 
7631 // Long Addition
7632 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
7633 
7634   match(Set dst (AddL src1 src2));
7635 
7636   ins_cost(INSN_COST);
7637   format %{ "add  $dst, $src1, $src2" %}
7638 
7639   ins_encode %{
7640     __ add(as_Register($dst$$reg),
7641            as_Register($src1$$reg),
7642            as_Register($src2$$reg));
7643   %}
7644 
7645   ins_pipe(ialu_reg_reg);
7646 %}
7647 
7648 // No constant pool entries requiredLong Immediate Addition.
7649 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
7650   match(Set dst (AddL src1 src2));
7651 
7652   ins_cost(INSN_COST);
7653   format %{ "add $dst, $src1, $src2" %}
7654 
7655   // use opcode to indicate that this is an add not a sub
7656   opcode(0x0);
7657 
7658   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
7659 
7660   ins_pipe(ialu_reg_imm);
7661 %}
7662 
7663 // Integer Subtraction
7664 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7665   match(Set dst (SubI src1 src2));
7666 
7667   ins_cost(INSN_COST);
7668   format %{ "subw  $dst, $src1, $src2" %}
7669 
7670   ins_encode %{
7671     __ subw(as_Register($dst$$reg),
7672             as_Register($src1$$reg),
7673             as_Register($src2$$reg));
7674   %}
7675 
7676   ins_pipe(ialu_reg_reg);
7677 %}
7678 
7679 // Immediate Subtraction
7680 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
7681   match(Set dst (SubI src1 src2));
7682 
7683   ins_cost(INSN_COST);
7684   format %{ "subw $dst, $src1, $src2" %}
7685 
7686   // use opcode to indicate that this is a sub not an add
7687   opcode(0x1);
7688 
7689   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
7690 
7691   ins_pipe(ialu_reg_imm);
7692 %}
7693 
7694 // Long Subtraction
7695 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
7696 
7697   match(Set dst (SubL src1 src2));
7698 
7699   ins_cost(INSN_COST);
7700   format %{ "sub  $dst, $src1, $src2" %}
7701 
7702   ins_encode %{
7703     __ sub(as_Register($dst$$reg),
7704            as_Register($src1$$reg),
7705            as_Register($src2$$reg));
7706   %}
7707 
7708   ins_pipe(ialu_reg_reg);
7709 %}
7710 
7711 // No constant pool entries requiredLong Immediate Subtraction.
7712 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
7713   match(Set dst (SubL src1 src2));
7714 
7715   ins_cost(INSN_COST);
7716   format %{ "sub$dst, $src1, $src2" %}
7717 
7718   // use opcode to indicate that this is a sub not an add
7719   opcode(0x1);
7720 
7721   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
7722 
7723   ins_pipe(ialu_reg_imm);
7724 %}
7725 
7726 // Integer Negation (special case for sub)
7727 
7728 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
7729   match(Set dst (SubI zero src));
7730 
7731   ins_cost(INSN_COST);
7732   format %{ "negw $dst, $src\t# int" %}
7733 
7734   ins_encode %{
7735     __ negw(as_Register($dst$$reg),
7736             as_Register($src$$reg));
7737   %}
7738 
7739   ins_pipe(ialu_reg);
7740 %}
7741 
7742 // Long Negation
7743 
7744 instruct negL_reg(iRegLNoSp dst, iRegIorL2I src, immL0 zero, rFlagsReg cr) %{
7745   match(Set dst (SubL zero src));
7746 
7747   ins_cost(INSN_COST);
7748   format %{ "neg $dst, $src\t# long" %}
7749 
7750   ins_encode %{
7751     __ neg(as_Register($dst$$reg),
7752            as_Register($src$$reg));
7753   %}
7754 
7755   ins_pipe(ialu_reg);
7756 %}
7757 
7758 // Integer Multiply
7759 
7760 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7761   match(Set dst (MulI src1 src2));
7762 
7763   ins_cost(INSN_COST * 3);
7764   format %{ "mulw  $dst, $src1, $src2" %}
7765 
7766   ins_encode %{
7767     __ mulw(as_Register($dst$$reg),
7768             as_Register($src1$$reg),
7769             as_Register($src2$$reg));
7770   %}
7771 
7772   ins_pipe(imul_reg_reg);
7773 %}
7774 
7775 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7776   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
7777 
7778   ins_cost(INSN_COST * 3);
7779   format %{ "smull  $dst, $src1, $src2" %}
7780 
7781   ins_encode %{
7782     __ smull(as_Register($dst$$reg),
7783              as_Register($src1$$reg),
7784              as_Register($src2$$reg));
7785   %}
7786 
7787   ins_pipe(imul_reg_reg);
7788 %}
7789 
7790 // Long Multiply
7791 
7792 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
7793   match(Set dst (MulL src1 src2));
7794 
7795   ins_cost(INSN_COST * 5);
7796   format %{ "mul  $dst, $src1, $src2" %}
7797 
7798   ins_encode %{
7799     __ mul(as_Register($dst$$reg),
7800            as_Register($src1$$reg),
7801            as_Register($src2$$reg));
7802   %}
7803 
7804   ins_pipe(lmul_reg_reg);
7805 %}
7806 
7807 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
7808 %{
7809   match(Set dst (MulHiL src1 src2));
7810 
7811   ins_cost(INSN_COST * 7);
7812   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
7813 
7814   ins_encode %{
7815     __ smulh(as_Register($dst$$reg),
7816              as_Register($src1$$reg),
7817              as_Register($src2$$reg));
7818   %}
7819 
7820   ins_pipe(lmul_reg_reg);
7821 %}
7822 
7823 // Combined Integer Multiply & Add/Sub
7824 
7825 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
7826   match(Set dst (AddI src3 (MulI src1 src2)));
7827 
7828   ins_cost(INSN_COST * 3);
7829   format %{ "madd  $dst, $src1, $src2, $src3" %}
7830 
7831   ins_encode %{
7832     __ maddw(as_Register($dst$$reg),
7833              as_Register($src1$$reg),
7834              as_Register($src2$$reg),
7835              as_Register($src3$$reg));
7836   %}
7837 
7838   ins_pipe(imac_reg_reg);
7839 %}
7840 
7841 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
7842   match(Set dst (SubI src3 (MulI src1 src2)));
7843 
7844   ins_cost(INSN_COST * 3);
7845   format %{ "msub  $dst, $src1, $src2, $src3" %}
7846 
7847   ins_encode %{
7848     __ msubw(as_Register($dst$$reg),
7849              as_Register($src1$$reg),
7850              as_Register($src2$$reg),
7851              as_Register($src3$$reg));
7852   %}
7853 
7854   ins_pipe(imac_reg_reg);
7855 %}
7856 
7857 // Combined Long Multiply & Add/Sub
7858 
7859 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
7860   match(Set dst (AddL src3 (MulL src1 src2)));
7861 
7862   ins_cost(INSN_COST * 5);
7863   format %{ "madd  $dst, $src1, $src2, $src3" %}
7864 
7865   ins_encode %{
7866     __ madd(as_Register($dst$$reg),
7867             as_Register($src1$$reg),
7868             as_Register($src2$$reg),
7869             as_Register($src3$$reg));
7870   %}
7871 
7872   ins_pipe(lmac_reg_reg);
7873 %}
7874 
7875 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
7876   match(Set dst (SubL src3 (MulL src1 src2)));
7877 
7878   ins_cost(INSN_COST * 5);
7879   format %{ "msub  $dst, $src1, $src2, $src3" %}
7880 
7881   ins_encode %{
7882     __ msub(as_Register($dst$$reg),
7883             as_Register($src1$$reg),
7884             as_Register($src2$$reg),
7885             as_Register($src3$$reg));
7886   %}
7887 
7888   ins_pipe(lmac_reg_reg);
7889 %}
7890 
7891 // Integer Divide
7892 
7893 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7894   match(Set dst (DivI src1 src2));
7895 
7896   ins_cost(INSN_COST * 19);
7897   format %{ "sdivw  $dst, $src1, $src2" %}
7898 
7899   ins_encode(aarch64_enc_divw(dst, src1, src2));
7900   ins_pipe(idiv_reg_reg);
7901 %}
7902 
7903 instruct signExtract(iRegINoSp dst, iRegI src1, immI_31 div1, immI_31 div2) %{
7904   match(Set dst (URShiftI (RShiftI src1 div1) div2));
7905   ins_cost(INSN_COST);
7906   format %{ "lsrw $dst, $src1, $div1" %}
7907   ins_encode %{
7908     __ lsrw(as_Register($dst$$reg), as_Register($src1$$reg), 31);
7909   %}
7910   ins_pipe(ialu_reg_shift);
7911 %}
7912 
7913 instruct div2Round(iRegINoSp dst, iRegI src, immI_31 div1, immI_31 div2) %{
7914   match(Set dst (AddI src (URShiftI (RShiftI src div1) div2)));
7915   ins_cost(INSN_COST);
7916   format %{ "addw $dst, $src, LSR $div1" %}
7917 
7918   ins_encode %{
7919     __ addw(as_Register($dst$$reg),
7920               as_Register($src$$reg),
7921               as_Register($src$$reg),
7922               Assembler::LSR, 31);
7923   %}
7924   ins_pipe(ialu_reg);
7925 %}
7926 
7927 // Long Divide
7928 
7929 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
7930   match(Set dst (DivL src1 src2));
7931 
7932   ins_cost(INSN_COST * 35);
7933   format %{ "sdiv   $dst, $src1, $src2" %}
7934 
7935   ins_encode(aarch64_enc_div(dst, src1, src2));
7936   ins_pipe(ldiv_reg_reg);
7937 %}
7938 
7939 instruct signExtractL(iRegLNoSp dst, iRegL src1, immL_63 div1, immL_63 div2) %{
7940   match(Set dst (URShiftL (RShiftL src1 div1) div2));
7941   ins_cost(INSN_COST);
7942   format %{ "lsr $dst, $src1, $div1" %}
7943   ins_encode %{
7944     __ lsr(as_Register($dst$$reg), as_Register($src1$$reg), 63);
7945   %}
7946   ins_pipe(ialu_reg_shift);
7947 %}
7948 
7949 instruct div2RoundL(iRegLNoSp dst, iRegL src, immL_63 div1, immL_63 div2) %{
7950   match(Set dst (AddL src (URShiftL (RShiftL src div1) div2)));
7951   ins_cost(INSN_COST);
7952   format %{ "add $dst, $src, $div1" %}
7953 
7954   ins_encode %{
7955     __ add(as_Register($dst$$reg),
7956               as_Register($src$$reg),
7957               as_Register($src$$reg),
7958               Assembler::LSR, 63);
7959   %}
7960   ins_pipe(ialu_reg);
7961 %}
7962 
7963 // Integer Remainder
7964 
7965 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7966   match(Set dst (ModI src1 src2));
7967 
7968   ins_cost(INSN_COST * 22);
7969   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
7970             "msubw($dst, rscratch1, $src2, $src1" %}
7971 
7972   ins_encode(aarch64_enc_modw(dst, src1, src2));
7973   ins_pipe(idiv_reg_reg);
7974 %}
7975 
7976 // Long Remainder
7977 
7978 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
7979   match(Set dst (ModL src1 src2));
7980 
7981   ins_cost(INSN_COST * 38);
7982   format %{ "sdiv   rscratch1, $src1, $src2\n"
7983             "msub($dst, rscratch1, $src2, $src1" %}
7984 
7985   ins_encode(aarch64_enc_mod(dst, src1, src2));
7986   ins_pipe(ldiv_reg_reg);
7987 %}
7988 
7989 // Integer Shifts
7990 
7991 // Shift Left Register
7992 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7993   match(Set dst (LShiftI src1 src2));
7994 
7995   ins_cost(INSN_COST * 2);
7996   format %{ "lslvw  $dst, $src1, $src2" %}
7997 
7998   ins_encode %{
7999     __ lslvw(as_Register($dst$$reg),
8000              as_Register($src1$$reg),
8001              as_Register($src2$$reg));
8002   %}
8003 
8004   ins_pipe(ialu_reg_reg_vshift);
8005 %}
8006 
8007 // Shift Left Immediate
8008 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
8009   match(Set dst (LShiftI src1 src2));
8010 
8011   ins_cost(INSN_COST);
8012   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
8013 
8014   ins_encode %{
8015     __ lslw(as_Register($dst$$reg),
8016             as_Register($src1$$reg),
8017             $src2$$constant & 0x1f);
8018   %}
8019 
8020   ins_pipe(ialu_reg_shift);
8021 %}
8022 
8023 // Shift Right Logical Register
8024 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8025   match(Set dst (URShiftI src1 src2));
8026 
8027   ins_cost(INSN_COST * 2);
8028   format %{ "lsrvw  $dst, $src1, $src2" %}
8029 
8030   ins_encode %{
8031     __ lsrvw(as_Register($dst$$reg),
8032              as_Register($src1$$reg),
8033              as_Register($src2$$reg));
8034   %}
8035 
8036   ins_pipe(ialu_reg_reg_vshift);
8037 %}
8038 
8039 // Shift Right Logical Immediate
8040 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
8041   match(Set dst (URShiftI src1 src2));
8042 
8043   ins_cost(INSN_COST);
8044   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
8045 
8046   ins_encode %{
8047     __ lsrw(as_Register($dst$$reg),
8048             as_Register($src1$$reg),
8049             $src2$$constant & 0x1f);
8050   %}
8051 
8052   ins_pipe(ialu_reg_shift);
8053 %}
8054 
8055 // Shift Right Arithmetic Register
8056 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8057   match(Set dst (RShiftI src1 src2));
8058 
8059   ins_cost(INSN_COST * 2);
8060   format %{ "asrvw  $dst, $src1, $src2" %}
8061 
8062   ins_encode %{
8063     __ asrvw(as_Register($dst$$reg),
8064              as_Register($src1$$reg),
8065              as_Register($src2$$reg));
8066   %}
8067 
8068   ins_pipe(ialu_reg_reg_vshift);
8069 %}
8070 
8071 // Shift Right Arithmetic Immediate
8072 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
8073   match(Set dst (RShiftI src1 src2));
8074 
8075   ins_cost(INSN_COST);
8076   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
8077 
8078   ins_encode %{
8079     __ asrw(as_Register($dst$$reg),
8080             as_Register($src1$$reg),
8081             $src2$$constant & 0x1f);
8082   %}
8083 
8084   ins_pipe(ialu_reg_shift);
8085 %}
8086 
8087 // Combined Int Mask and Right Shift (using UBFM)
8088 // TODO
8089 
8090 // Long Shifts
8091 
8092 // Shift Left Register
8093 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
8094   match(Set dst (LShiftL src1 src2));
8095 
8096   ins_cost(INSN_COST * 2);
8097   format %{ "lslv  $dst, $src1, $src2" %}
8098 
8099   ins_encode %{
8100     __ lslv(as_Register($dst$$reg),
8101             as_Register($src1$$reg),
8102             as_Register($src2$$reg));
8103   %}
8104 
8105   ins_pipe(ialu_reg_reg_vshift);
8106 %}
8107 
8108 // Shift Left Immediate
8109 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
8110   match(Set dst (LShiftL src1 src2));
8111 
8112   ins_cost(INSN_COST);
8113   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
8114 
8115   ins_encode %{
8116     __ lsl(as_Register($dst$$reg),
8117             as_Register($src1$$reg),
8118             $src2$$constant & 0x3f);
8119   %}
8120 
8121   ins_pipe(ialu_reg_shift);
8122 %}
8123 
8124 // Shift Right Logical Register
8125 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
8126   match(Set dst (URShiftL src1 src2));
8127 
8128   ins_cost(INSN_COST * 2);
8129   format %{ "lsrv  $dst, $src1, $src2" %}
8130 
8131   ins_encode %{
8132     __ lsrv(as_Register($dst$$reg),
8133             as_Register($src1$$reg),
8134             as_Register($src2$$reg));
8135   %}
8136 
8137   ins_pipe(ialu_reg_reg_vshift);
8138 %}
8139 
8140 // Shift Right Logical Immediate
8141 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
8142   match(Set dst (URShiftL src1 src2));
8143 
8144   ins_cost(INSN_COST);
8145   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
8146 
8147   ins_encode %{
8148     __ lsr(as_Register($dst$$reg),
8149            as_Register($src1$$reg),
8150            $src2$$constant & 0x3f);
8151   %}
8152 
8153   ins_pipe(ialu_reg_shift);
8154 %}
8155 
8156 // A special-case pattern for card table stores.
8157 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
8158   match(Set dst (URShiftL (CastP2X src1) src2));
8159 
8160   ins_cost(INSN_COST);
8161   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
8162 
8163   ins_encode %{
8164     __ lsr(as_Register($dst$$reg),
8165            as_Register($src1$$reg),
8166            $src2$$constant & 0x3f);
8167   %}
8168 
8169   ins_pipe(ialu_reg_shift);
8170 %}
8171 
8172 // Shift Right Arithmetic Register
8173 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
8174   match(Set dst (RShiftL src1 src2));
8175 
8176   ins_cost(INSN_COST * 2);
8177   format %{ "asrv  $dst, $src1, $src2" %}
8178 
8179   ins_encode %{
8180     __ asrv(as_Register($dst$$reg),
8181             as_Register($src1$$reg),
8182             as_Register($src2$$reg));
8183   %}
8184 
8185   ins_pipe(ialu_reg_reg_vshift);
8186 %}
8187 
8188 // Shift Right Arithmetic Immediate
8189 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
8190   match(Set dst (RShiftL src1 src2));
8191 
8192   ins_cost(INSN_COST);
8193   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
8194 
8195   ins_encode %{
8196     __ asr(as_Register($dst$$reg),
8197            as_Register($src1$$reg),
8198            $src2$$constant & 0x3f);
8199   %}
8200 
8201   ins_pipe(ialu_reg_shift);
8202 %}
8203 
8204 // BEGIN This section of the file is automatically generated. Do not edit --------------
8205 
8206 instruct regL_not_reg(iRegLNoSp dst,
8207                          iRegL src1, immL_M1 m1,
8208                          rFlagsReg cr) %{
8209   match(Set dst (XorL src1 m1));
8210   ins_cost(INSN_COST);
8211   format %{ "eon  $dst, $src1, zr" %}
8212 
8213   ins_encode %{
8214     __ eon(as_Register($dst$$reg),
8215               as_Register($src1$$reg),
8216               zr,
8217               Assembler::LSL, 0);
8218   %}
8219 
8220   ins_pipe(ialu_reg);
8221 %}
8222 instruct regI_not_reg(iRegINoSp dst,
8223                          iRegI src1, immI_M1 m1,
8224                          rFlagsReg cr) %{
8225   match(Set dst (XorI src1 m1));
8226   ins_cost(INSN_COST);
8227   format %{ "eonw  $dst, $src1, zr" %}
8228 
8229   ins_encode %{
8230     __ eonw(as_Register($dst$$reg),
8231               as_Register($src1$$reg),
8232               zr,
8233               Assembler::LSL, 0);
8234   %}
8235 
8236   ins_pipe(ialu_reg);
8237 %}
8238 
8239 instruct AndI_reg_not_reg(iRegINoSp dst,
8240                          iRegI src1, iRegI src2, immI_M1 m1,
8241                          rFlagsReg cr) %{
8242   match(Set dst (AndI src1 (XorI src2 m1)));
8243   ins_cost(INSN_COST);
8244   format %{ "bic  $dst, $src1, $src2" %}
8245 
8246   ins_encode %{
8247     __ bic(as_Register($dst$$reg),
8248               as_Register($src1$$reg),
8249               as_Register($src2$$reg),
8250               Assembler::LSL, 0);
8251   %}
8252 
8253   ins_pipe(ialu_reg_reg);
8254 %}
8255 
8256 instruct AndL_reg_not_reg(iRegLNoSp dst,
8257                          iRegL src1, iRegL src2, immL_M1 m1,
8258                          rFlagsReg cr) %{
8259   match(Set dst (AndL src1 (XorL src2 m1)));
8260   ins_cost(INSN_COST);
8261   format %{ "bic  $dst, $src1, $src2" %}
8262 
8263   ins_encode %{
8264     __ bic(as_Register($dst$$reg),
8265               as_Register($src1$$reg),
8266               as_Register($src2$$reg),
8267               Assembler::LSL, 0);
8268   %}
8269 
8270   ins_pipe(ialu_reg_reg);
8271 %}
8272 
8273 instruct OrI_reg_not_reg(iRegINoSp dst,
8274                          iRegI src1, iRegI src2, immI_M1 m1,
8275                          rFlagsReg cr) %{
8276   match(Set dst (OrI src1 (XorI src2 m1)));
8277   ins_cost(INSN_COST);
8278   format %{ "orn  $dst, $src1, $src2" %}
8279 
8280   ins_encode %{
8281     __ orn(as_Register($dst$$reg),
8282               as_Register($src1$$reg),
8283               as_Register($src2$$reg),
8284               Assembler::LSL, 0);
8285   %}
8286 
8287   ins_pipe(ialu_reg_reg);
8288 %}
8289 
8290 instruct OrL_reg_not_reg(iRegLNoSp dst,
8291                          iRegL src1, iRegL src2, immL_M1 m1,
8292                          rFlagsReg cr) %{
8293   match(Set dst (OrL src1 (XorL src2 m1)));
8294   ins_cost(INSN_COST);
8295   format %{ "orn  $dst, $src1, $src2" %}
8296 
8297   ins_encode %{
8298     __ orn(as_Register($dst$$reg),
8299               as_Register($src1$$reg),
8300               as_Register($src2$$reg),
8301               Assembler::LSL, 0);
8302   %}
8303 
8304   ins_pipe(ialu_reg_reg);
8305 %}
8306 
8307 instruct XorI_reg_not_reg(iRegINoSp dst,
8308                          iRegI src1, iRegI src2, immI_M1 m1,
8309                          rFlagsReg cr) %{
8310   match(Set dst (XorI m1 (XorI src2 src1)));
8311   ins_cost(INSN_COST);
8312   format %{ "eon  $dst, $src1, $src2" %}
8313 
8314   ins_encode %{
8315     __ eon(as_Register($dst$$reg),
8316               as_Register($src1$$reg),
8317               as_Register($src2$$reg),
8318               Assembler::LSL, 0);
8319   %}
8320 
8321   ins_pipe(ialu_reg_reg);
8322 %}
8323 
8324 instruct XorL_reg_not_reg(iRegLNoSp dst,
8325                          iRegL src1, iRegL src2, immL_M1 m1,
8326                          rFlagsReg cr) %{
8327   match(Set dst (XorL m1 (XorL src2 src1)));
8328   ins_cost(INSN_COST);
8329   format %{ "eon  $dst, $src1, $src2" %}
8330 
8331   ins_encode %{
8332     __ eon(as_Register($dst$$reg),
8333               as_Register($src1$$reg),
8334               as_Register($src2$$reg),
8335               Assembler::LSL, 0);
8336   %}
8337 
8338   ins_pipe(ialu_reg_reg);
8339 %}
8340 
8341 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
8342                          iRegI src1, iRegI src2,
8343                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8344   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
8345   ins_cost(1.9 * INSN_COST);
8346   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
8347 
8348   ins_encode %{
8349     __ bicw(as_Register($dst$$reg),
8350               as_Register($src1$$reg),
8351               as_Register($src2$$reg),
8352               Assembler::LSR,
8353               $src3$$constant & 0x3f);
8354   %}
8355 
8356   ins_pipe(ialu_reg_reg_shift);
8357 %}
8358 
8359 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
8360                          iRegL src1, iRegL src2,
8361                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8362   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
8363   ins_cost(1.9 * INSN_COST);
8364   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
8365 
8366   ins_encode %{
8367     __ bic(as_Register($dst$$reg),
8368               as_Register($src1$$reg),
8369               as_Register($src2$$reg),
8370               Assembler::LSR,
8371               $src3$$constant & 0x3f);
8372   %}
8373 
8374   ins_pipe(ialu_reg_reg_shift);
8375 %}
8376 
8377 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
8378                          iRegI src1, iRegI src2,
8379                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8380   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
8381   ins_cost(1.9 * INSN_COST);
8382   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
8383 
8384   ins_encode %{
8385     __ bicw(as_Register($dst$$reg),
8386               as_Register($src1$$reg),
8387               as_Register($src2$$reg),
8388               Assembler::ASR,
8389               $src3$$constant & 0x3f);
8390   %}
8391 
8392   ins_pipe(ialu_reg_reg_shift);
8393 %}
8394 
8395 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
8396                          iRegL src1, iRegL src2,
8397                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8398   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
8399   ins_cost(1.9 * INSN_COST);
8400   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
8401 
8402   ins_encode %{
8403     __ bic(as_Register($dst$$reg),
8404               as_Register($src1$$reg),
8405               as_Register($src2$$reg),
8406               Assembler::ASR,
8407               $src3$$constant & 0x3f);
8408   %}
8409 
8410   ins_pipe(ialu_reg_reg_shift);
8411 %}
8412 
8413 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
8414                          iRegI src1, iRegI src2,
8415                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8416   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
8417   ins_cost(1.9 * INSN_COST);
8418   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
8419 
8420   ins_encode %{
8421     __ bicw(as_Register($dst$$reg),
8422               as_Register($src1$$reg),
8423               as_Register($src2$$reg),
8424               Assembler::LSL,
8425               $src3$$constant & 0x3f);
8426   %}
8427 
8428   ins_pipe(ialu_reg_reg_shift);
8429 %}
8430 
8431 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
8432                          iRegL src1, iRegL src2,
8433                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8434   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
8435   ins_cost(1.9 * INSN_COST);
8436   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
8437 
8438   ins_encode %{
8439     __ bic(as_Register($dst$$reg),
8440               as_Register($src1$$reg),
8441               as_Register($src2$$reg),
8442               Assembler::LSL,
8443               $src3$$constant & 0x3f);
8444   %}
8445 
8446   ins_pipe(ialu_reg_reg_shift);
8447 %}
8448 
8449 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
8450                          iRegI src1, iRegI src2,
8451                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8452   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
8453   ins_cost(1.9 * INSN_COST);
8454   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
8455 
8456   ins_encode %{
8457     __ eonw(as_Register($dst$$reg),
8458               as_Register($src1$$reg),
8459               as_Register($src2$$reg),
8460               Assembler::LSR,
8461               $src3$$constant & 0x3f);
8462   %}
8463 
8464   ins_pipe(ialu_reg_reg_shift);
8465 %}
8466 
8467 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
8468                          iRegL src1, iRegL src2,
8469                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8470   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
8471   ins_cost(1.9 * INSN_COST);
8472   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
8473 
8474   ins_encode %{
8475     __ eon(as_Register($dst$$reg),
8476               as_Register($src1$$reg),
8477               as_Register($src2$$reg),
8478               Assembler::LSR,
8479               $src3$$constant & 0x3f);
8480   %}
8481 
8482   ins_pipe(ialu_reg_reg_shift);
8483 %}
8484 
8485 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
8486                          iRegI src1, iRegI src2,
8487                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8488   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
8489   ins_cost(1.9 * INSN_COST);
8490   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
8491 
8492   ins_encode %{
8493     __ eonw(as_Register($dst$$reg),
8494               as_Register($src1$$reg),
8495               as_Register($src2$$reg),
8496               Assembler::ASR,
8497               $src3$$constant & 0x3f);
8498   %}
8499 
8500   ins_pipe(ialu_reg_reg_shift);
8501 %}
8502 
8503 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
8504                          iRegL src1, iRegL src2,
8505                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8506   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
8507   ins_cost(1.9 * INSN_COST);
8508   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
8509 
8510   ins_encode %{
8511     __ eon(as_Register($dst$$reg),
8512               as_Register($src1$$reg),
8513               as_Register($src2$$reg),
8514               Assembler::ASR,
8515               $src3$$constant & 0x3f);
8516   %}
8517 
8518   ins_pipe(ialu_reg_reg_shift);
8519 %}
8520 
8521 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
8522                          iRegI src1, iRegI src2,
8523                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8524   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
8525   ins_cost(1.9 * INSN_COST);
8526   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
8527 
8528   ins_encode %{
8529     __ eonw(as_Register($dst$$reg),
8530               as_Register($src1$$reg),
8531               as_Register($src2$$reg),
8532               Assembler::LSL,
8533               $src3$$constant & 0x3f);
8534   %}
8535 
8536   ins_pipe(ialu_reg_reg_shift);
8537 %}
8538 
8539 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
8540                          iRegL src1, iRegL src2,
8541                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8542   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
8543   ins_cost(1.9 * INSN_COST);
8544   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
8545 
8546   ins_encode %{
8547     __ eon(as_Register($dst$$reg),
8548               as_Register($src1$$reg),
8549               as_Register($src2$$reg),
8550               Assembler::LSL,
8551               $src3$$constant & 0x3f);
8552   %}
8553 
8554   ins_pipe(ialu_reg_reg_shift);
8555 %}
8556 
8557 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
8558                          iRegI src1, iRegI src2,
8559                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8560   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
8561   ins_cost(1.9 * INSN_COST);
8562   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
8563 
8564   ins_encode %{
8565     __ ornw(as_Register($dst$$reg),
8566               as_Register($src1$$reg),
8567               as_Register($src2$$reg),
8568               Assembler::LSR,
8569               $src3$$constant & 0x3f);
8570   %}
8571 
8572   ins_pipe(ialu_reg_reg_shift);
8573 %}
8574 
8575 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
8576                          iRegL src1, iRegL src2,
8577                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8578   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
8579   ins_cost(1.9 * INSN_COST);
8580   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
8581 
8582   ins_encode %{
8583     __ orn(as_Register($dst$$reg),
8584               as_Register($src1$$reg),
8585               as_Register($src2$$reg),
8586               Assembler::LSR,
8587               $src3$$constant & 0x3f);
8588   %}
8589 
8590   ins_pipe(ialu_reg_reg_shift);
8591 %}
8592 
8593 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
8594                          iRegI src1, iRegI src2,
8595                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8596   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
8597   ins_cost(1.9 * INSN_COST);
8598   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
8599 
8600   ins_encode %{
8601     __ ornw(as_Register($dst$$reg),
8602               as_Register($src1$$reg),
8603               as_Register($src2$$reg),
8604               Assembler::ASR,
8605               $src3$$constant & 0x3f);
8606   %}
8607 
8608   ins_pipe(ialu_reg_reg_shift);
8609 %}
8610 
8611 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
8612                          iRegL src1, iRegL src2,
8613                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8614   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
8615   ins_cost(1.9 * INSN_COST);
8616   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
8617 
8618   ins_encode %{
8619     __ orn(as_Register($dst$$reg),
8620               as_Register($src1$$reg),
8621               as_Register($src2$$reg),
8622               Assembler::ASR,
8623               $src3$$constant & 0x3f);
8624   %}
8625 
8626   ins_pipe(ialu_reg_reg_shift);
8627 %}
8628 
8629 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
8630                          iRegI src1, iRegI src2,
8631                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8632   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
8633   ins_cost(1.9 * INSN_COST);
8634   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
8635 
8636   ins_encode %{
8637     __ ornw(as_Register($dst$$reg),
8638               as_Register($src1$$reg),
8639               as_Register($src2$$reg),
8640               Assembler::LSL,
8641               $src3$$constant & 0x3f);
8642   %}
8643 
8644   ins_pipe(ialu_reg_reg_shift);
8645 %}
8646 
8647 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
8648                          iRegL src1, iRegL src2,
8649                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8650   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
8651   ins_cost(1.9 * INSN_COST);
8652   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
8653 
8654   ins_encode %{
8655     __ orn(as_Register($dst$$reg),
8656               as_Register($src1$$reg),
8657               as_Register($src2$$reg),
8658               Assembler::LSL,
8659               $src3$$constant & 0x3f);
8660   %}
8661 
8662   ins_pipe(ialu_reg_reg_shift);
8663 %}
8664 
8665 instruct AndI_reg_URShift_reg(iRegINoSp dst,
8666                          iRegI src1, iRegI src2,
8667                          immI src3, rFlagsReg cr) %{
8668   match(Set dst (AndI src1 (URShiftI src2 src3)));
8669 
8670   ins_cost(1.9 * INSN_COST);
8671   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
8672 
8673   ins_encode %{
8674     __ andw(as_Register($dst$$reg),
8675               as_Register($src1$$reg),
8676               as_Register($src2$$reg),
8677               Assembler::LSR,
8678               $src3$$constant & 0x3f);
8679   %}
8680 
8681   ins_pipe(ialu_reg_reg_shift);
8682 %}
8683 
8684 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
8685                          iRegL src1, iRegL src2,
8686                          immI src3, rFlagsReg cr) %{
8687   match(Set dst (AndL src1 (URShiftL src2 src3)));
8688 
8689   ins_cost(1.9 * INSN_COST);
8690   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
8691 
8692   ins_encode %{
8693     __ andr(as_Register($dst$$reg),
8694               as_Register($src1$$reg),
8695               as_Register($src2$$reg),
8696               Assembler::LSR,
8697               $src3$$constant & 0x3f);
8698   %}
8699 
8700   ins_pipe(ialu_reg_reg_shift);
8701 %}
8702 
8703 instruct AndI_reg_RShift_reg(iRegINoSp dst,
8704                          iRegI src1, iRegI src2,
8705                          immI src3, rFlagsReg cr) %{
8706   match(Set dst (AndI src1 (RShiftI src2 src3)));
8707 
8708   ins_cost(1.9 * INSN_COST);
8709   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
8710 
8711   ins_encode %{
8712     __ andw(as_Register($dst$$reg),
8713               as_Register($src1$$reg),
8714               as_Register($src2$$reg),
8715               Assembler::ASR,
8716               $src3$$constant & 0x3f);
8717   %}
8718 
8719   ins_pipe(ialu_reg_reg_shift);
8720 %}
8721 
8722 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
8723                          iRegL src1, iRegL src2,
8724                          immI src3, rFlagsReg cr) %{
8725   match(Set dst (AndL src1 (RShiftL src2 src3)));
8726 
8727   ins_cost(1.9 * INSN_COST);
8728   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
8729 
8730   ins_encode %{
8731     __ andr(as_Register($dst$$reg),
8732               as_Register($src1$$reg),
8733               as_Register($src2$$reg),
8734               Assembler::ASR,
8735               $src3$$constant & 0x3f);
8736   %}
8737 
8738   ins_pipe(ialu_reg_reg_shift);
8739 %}
8740 
8741 instruct AndI_reg_LShift_reg(iRegINoSp dst,
8742                          iRegI src1, iRegI src2,
8743                          immI src3, rFlagsReg cr) %{
8744   match(Set dst (AndI src1 (LShiftI src2 src3)));
8745 
8746   ins_cost(1.9 * INSN_COST);
8747   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
8748 
8749   ins_encode %{
8750     __ andw(as_Register($dst$$reg),
8751               as_Register($src1$$reg),
8752               as_Register($src2$$reg),
8753               Assembler::LSL,
8754               $src3$$constant & 0x3f);
8755   %}
8756 
8757   ins_pipe(ialu_reg_reg_shift);
8758 %}
8759 
8760 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
8761                          iRegL src1, iRegL src2,
8762                          immI src3, rFlagsReg cr) %{
8763   match(Set dst (AndL src1 (LShiftL src2 src3)));
8764 
8765   ins_cost(1.9 * INSN_COST);
8766   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
8767 
8768   ins_encode %{
8769     __ andr(as_Register($dst$$reg),
8770               as_Register($src1$$reg),
8771               as_Register($src2$$reg),
8772               Assembler::LSL,
8773               $src3$$constant & 0x3f);
8774   %}
8775 
8776   ins_pipe(ialu_reg_reg_shift);
8777 %}
8778 
8779 instruct XorI_reg_URShift_reg(iRegINoSp dst,
8780                          iRegI src1, iRegI src2,
8781                          immI src3, rFlagsReg cr) %{
8782   match(Set dst (XorI src1 (URShiftI src2 src3)));
8783 
8784   ins_cost(1.9 * INSN_COST);
8785   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
8786 
8787   ins_encode %{
8788     __ eorw(as_Register($dst$$reg),
8789               as_Register($src1$$reg),
8790               as_Register($src2$$reg),
8791               Assembler::LSR,
8792               $src3$$constant & 0x3f);
8793   %}
8794 
8795   ins_pipe(ialu_reg_reg_shift);
8796 %}
8797 
8798 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
8799                          iRegL src1, iRegL src2,
8800                          immI src3, rFlagsReg cr) %{
8801   match(Set dst (XorL src1 (URShiftL src2 src3)));
8802 
8803   ins_cost(1.9 * INSN_COST);
8804   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
8805 
8806   ins_encode %{
8807     __ eor(as_Register($dst$$reg),
8808               as_Register($src1$$reg),
8809               as_Register($src2$$reg),
8810               Assembler::LSR,
8811               $src3$$constant & 0x3f);
8812   %}
8813 
8814   ins_pipe(ialu_reg_reg_shift);
8815 %}
8816 
8817 instruct XorI_reg_RShift_reg(iRegINoSp dst,
8818                          iRegI src1, iRegI src2,
8819                          immI src3, rFlagsReg cr) %{
8820   match(Set dst (XorI src1 (RShiftI src2 src3)));
8821 
8822   ins_cost(1.9 * INSN_COST);
8823   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
8824 
8825   ins_encode %{
8826     __ eorw(as_Register($dst$$reg),
8827               as_Register($src1$$reg),
8828               as_Register($src2$$reg),
8829               Assembler::ASR,
8830               $src3$$constant & 0x3f);
8831   %}
8832 
8833   ins_pipe(ialu_reg_reg_shift);
8834 %}
8835 
8836 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
8837                          iRegL src1, iRegL src2,
8838                          immI src3, rFlagsReg cr) %{
8839   match(Set dst (XorL src1 (RShiftL src2 src3)));
8840 
8841   ins_cost(1.9 * INSN_COST);
8842   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
8843 
8844   ins_encode %{
8845     __ eor(as_Register($dst$$reg),
8846               as_Register($src1$$reg),
8847               as_Register($src2$$reg),
8848               Assembler::ASR,
8849               $src3$$constant & 0x3f);
8850   %}
8851 
8852   ins_pipe(ialu_reg_reg_shift);
8853 %}
8854 
8855 instruct XorI_reg_LShift_reg(iRegINoSp dst,
8856                          iRegI src1, iRegI src2,
8857                          immI src3, rFlagsReg cr) %{
8858   match(Set dst (XorI src1 (LShiftI src2 src3)));
8859 
8860   ins_cost(1.9 * INSN_COST);
8861   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
8862 
8863   ins_encode %{
8864     __ eorw(as_Register($dst$$reg),
8865               as_Register($src1$$reg),
8866               as_Register($src2$$reg),
8867               Assembler::LSL,
8868               $src3$$constant & 0x3f);
8869   %}
8870 
8871   ins_pipe(ialu_reg_reg_shift);
8872 %}
8873 
8874 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
8875                          iRegL src1, iRegL src2,
8876                          immI src3, rFlagsReg cr) %{
8877   match(Set dst (XorL src1 (LShiftL src2 src3)));
8878 
8879   ins_cost(1.9 * INSN_COST);
8880   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
8881 
8882   ins_encode %{
8883     __ eor(as_Register($dst$$reg),
8884               as_Register($src1$$reg),
8885               as_Register($src2$$reg),
8886               Assembler::LSL,
8887               $src3$$constant & 0x3f);
8888   %}
8889 
8890   ins_pipe(ialu_reg_reg_shift);
8891 %}
8892 
8893 instruct OrI_reg_URShift_reg(iRegINoSp dst,
8894                          iRegI src1, iRegI src2,
8895                          immI src3, rFlagsReg cr) %{
8896   match(Set dst (OrI src1 (URShiftI src2 src3)));
8897 
8898   ins_cost(1.9 * INSN_COST);
8899   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
8900 
8901   ins_encode %{
8902     __ orrw(as_Register($dst$$reg),
8903               as_Register($src1$$reg),
8904               as_Register($src2$$reg),
8905               Assembler::LSR,
8906               $src3$$constant & 0x3f);
8907   %}
8908 
8909   ins_pipe(ialu_reg_reg_shift);
8910 %}
8911 
8912 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
8913                          iRegL src1, iRegL src2,
8914                          immI src3, rFlagsReg cr) %{
8915   match(Set dst (OrL src1 (URShiftL src2 src3)));
8916 
8917   ins_cost(1.9 * INSN_COST);
8918   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
8919 
8920   ins_encode %{
8921     __ orr(as_Register($dst$$reg),
8922               as_Register($src1$$reg),
8923               as_Register($src2$$reg),
8924               Assembler::LSR,
8925               $src3$$constant & 0x3f);
8926   %}
8927 
8928   ins_pipe(ialu_reg_reg_shift);
8929 %}
8930 
8931 instruct OrI_reg_RShift_reg(iRegINoSp dst,
8932                          iRegI src1, iRegI src2,
8933                          immI src3, rFlagsReg cr) %{
8934   match(Set dst (OrI src1 (RShiftI src2 src3)));
8935 
8936   ins_cost(1.9 * INSN_COST);
8937   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
8938 
8939   ins_encode %{
8940     __ orrw(as_Register($dst$$reg),
8941               as_Register($src1$$reg),
8942               as_Register($src2$$reg),
8943               Assembler::ASR,
8944               $src3$$constant & 0x3f);
8945   %}
8946 
8947   ins_pipe(ialu_reg_reg_shift);
8948 %}
8949 
8950 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
8951                          iRegL src1, iRegL src2,
8952                          immI src3, rFlagsReg cr) %{
8953   match(Set dst (OrL src1 (RShiftL src2 src3)));
8954 
8955   ins_cost(1.9 * INSN_COST);
8956   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
8957 
8958   ins_encode %{
8959     __ orr(as_Register($dst$$reg),
8960               as_Register($src1$$reg),
8961               as_Register($src2$$reg),
8962               Assembler::ASR,
8963               $src3$$constant & 0x3f);
8964   %}
8965 
8966   ins_pipe(ialu_reg_reg_shift);
8967 %}
8968 
8969 instruct OrI_reg_LShift_reg(iRegINoSp dst,
8970                          iRegI src1, iRegI src2,
8971                          immI src3, rFlagsReg cr) %{
8972   match(Set dst (OrI src1 (LShiftI src2 src3)));
8973 
8974   ins_cost(1.9 * INSN_COST);
8975   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
8976 
8977   ins_encode %{
8978     __ orrw(as_Register($dst$$reg),
8979               as_Register($src1$$reg),
8980               as_Register($src2$$reg),
8981               Assembler::LSL,
8982               $src3$$constant & 0x3f);
8983   %}
8984 
8985   ins_pipe(ialu_reg_reg_shift);
8986 %}
8987 
8988 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
8989                          iRegL src1, iRegL src2,
8990                          immI src3, rFlagsReg cr) %{
8991   match(Set dst (OrL src1 (LShiftL src2 src3)));
8992 
8993   ins_cost(1.9 * INSN_COST);
8994   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
8995 
8996   ins_encode %{
8997     __ orr(as_Register($dst$$reg),
8998               as_Register($src1$$reg),
8999               as_Register($src2$$reg),
9000               Assembler::LSL,
9001               $src3$$constant & 0x3f);
9002   %}
9003 
9004   ins_pipe(ialu_reg_reg_shift);
9005 %}
9006 
9007 instruct AddI_reg_URShift_reg(iRegINoSp dst,
9008                          iRegI src1, iRegI src2,
9009                          immI src3, rFlagsReg cr) %{
9010   match(Set dst (AddI src1 (URShiftI src2 src3)));
9011 
9012   ins_cost(1.9 * INSN_COST);
9013   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
9014 
9015   ins_encode %{
9016     __ addw(as_Register($dst$$reg),
9017               as_Register($src1$$reg),
9018               as_Register($src2$$reg),
9019               Assembler::LSR,
9020               $src3$$constant & 0x3f);
9021   %}
9022 
9023   ins_pipe(ialu_reg_reg_shift);
9024 %}
9025 
9026 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
9027                          iRegL src1, iRegL src2,
9028                          immI src3, rFlagsReg cr) %{
9029   match(Set dst (AddL src1 (URShiftL src2 src3)));
9030 
9031   ins_cost(1.9 * INSN_COST);
9032   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
9033 
9034   ins_encode %{
9035     __ add(as_Register($dst$$reg),
9036               as_Register($src1$$reg),
9037               as_Register($src2$$reg),
9038               Assembler::LSR,
9039               $src3$$constant & 0x3f);
9040   %}
9041 
9042   ins_pipe(ialu_reg_reg_shift);
9043 %}
9044 
9045 instruct AddI_reg_RShift_reg(iRegINoSp dst,
9046                          iRegI src1, iRegI src2,
9047                          immI src3, rFlagsReg cr) %{
9048   match(Set dst (AddI src1 (RShiftI src2 src3)));
9049 
9050   ins_cost(1.9 * INSN_COST);
9051   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
9052 
9053   ins_encode %{
9054     __ addw(as_Register($dst$$reg),
9055               as_Register($src1$$reg),
9056               as_Register($src2$$reg),
9057               Assembler::ASR,
9058               $src3$$constant & 0x3f);
9059   %}
9060 
9061   ins_pipe(ialu_reg_reg_shift);
9062 %}
9063 
9064 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
9065                          iRegL src1, iRegL src2,
9066                          immI src3, rFlagsReg cr) %{
9067   match(Set dst (AddL src1 (RShiftL src2 src3)));
9068 
9069   ins_cost(1.9 * INSN_COST);
9070   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
9071 
9072   ins_encode %{
9073     __ add(as_Register($dst$$reg),
9074               as_Register($src1$$reg),
9075               as_Register($src2$$reg),
9076               Assembler::ASR,
9077               $src3$$constant & 0x3f);
9078   %}
9079 
9080   ins_pipe(ialu_reg_reg_shift);
9081 %}
9082 
9083 instruct AddI_reg_LShift_reg(iRegINoSp dst,
9084                          iRegI src1, iRegI src2,
9085                          immI src3, rFlagsReg cr) %{
9086   match(Set dst (AddI src1 (LShiftI src2 src3)));
9087 
9088   ins_cost(1.9 * INSN_COST);
9089   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
9090 
9091   ins_encode %{
9092     __ addw(as_Register($dst$$reg),
9093               as_Register($src1$$reg),
9094               as_Register($src2$$reg),
9095               Assembler::LSL,
9096               $src3$$constant & 0x3f);
9097   %}
9098 
9099   ins_pipe(ialu_reg_reg_shift);
9100 %}
9101 
9102 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
9103                          iRegL src1, iRegL src2,
9104                          immI src3, rFlagsReg cr) %{
9105   match(Set dst (AddL src1 (LShiftL src2 src3)));
9106 
9107   ins_cost(1.9 * INSN_COST);
9108   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
9109 
9110   ins_encode %{
9111     __ add(as_Register($dst$$reg),
9112               as_Register($src1$$reg),
9113               as_Register($src2$$reg),
9114               Assembler::LSL,
9115               $src3$$constant & 0x3f);
9116   %}
9117 
9118   ins_pipe(ialu_reg_reg_shift);
9119 %}
9120 
9121 instruct SubI_reg_URShift_reg(iRegINoSp dst,
9122                          iRegI src1, iRegI src2,
9123                          immI src3, rFlagsReg cr) %{
9124   match(Set dst (SubI src1 (URShiftI src2 src3)));
9125 
9126   ins_cost(1.9 * INSN_COST);
9127   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
9128 
9129   ins_encode %{
9130     __ subw(as_Register($dst$$reg),
9131               as_Register($src1$$reg),
9132               as_Register($src2$$reg),
9133               Assembler::LSR,
9134               $src3$$constant & 0x3f);
9135   %}
9136 
9137   ins_pipe(ialu_reg_reg_shift);
9138 %}
9139 
9140 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
9141                          iRegL src1, iRegL src2,
9142                          immI src3, rFlagsReg cr) %{
9143   match(Set dst (SubL src1 (URShiftL src2 src3)));
9144 
9145   ins_cost(1.9 * INSN_COST);
9146   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
9147 
9148   ins_encode %{
9149     __ sub(as_Register($dst$$reg),
9150               as_Register($src1$$reg),
9151               as_Register($src2$$reg),
9152               Assembler::LSR,
9153               $src3$$constant & 0x3f);
9154   %}
9155 
9156   ins_pipe(ialu_reg_reg_shift);
9157 %}
9158 
9159 instruct SubI_reg_RShift_reg(iRegINoSp dst,
9160                          iRegI src1, iRegI src2,
9161                          immI src3, rFlagsReg cr) %{
9162   match(Set dst (SubI src1 (RShiftI src2 src3)));
9163 
9164   ins_cost(1.9 * INSN_COST);
9165   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
9166 
9167   ins_encode %{
9168     __ subw(as_Register($dst$$reg),
9169               as_Register($src1$$reg),
9170               as_Register($src2$$reg),
9171               Assembler::ASR,
9172               $src3$$constant & 0x3f);
9173   %}
9174 
9175   ins_pipe(ialu_reg_reg_shift);
9176 %}
9177 
9178 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
9179                          iRegL src1, iRegL src2,
9180                          immI src3, rFlagsReg cr) %{
9181   match(Set dst (SubL src1 (RShiftL src2 src3)));
9182 
9183   ins_cost(1.9 * INSN_COST);
9184   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
9185 
9186   ins_encode %{
9187     __ sub(as_Register($dst$$reg),
9188               as_Register($src1$$reg),
9189               as_Register($src2$$reg),
9190               Assembler::ASR,
9191               $src3$$constant & 0x3f);
9192   %}
9193 
9194   ins_pipe(ialu_reg_reg_shift);
9195 %}
9196 
9197 instruct SubI_reg_LShift_reg(iRegINoSp dst,
9198                          iRegI src1, iRegI src2,
9199                          immI src3, rFlagsReg cr) %{
9200   match(Set dst (SubI src1 (LShiftI src2 src3)));
9201 
9202   ins_cost(1.9 * INSN_COST);
9203   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
9204 
9205   ins_encode %{
9206     __ subw(as_Register($dst$$reg),
9207               as_Register($src1$$reg),
9208               as_Register($src2$$reg),
9209               Assembler::LSL,
9210               $src3$$constant & 0x3f);
9211   %}
9212 
9213   ins_pipe(ialu_reg_reg_shift);
9214 %}
9215 
9216 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
9217                          iRegL src1, iRegL src2,
9218                          immI src3, rFlagsReg cr) %{
9219   match(Set dst (SubL src1 (LShiftL src2 src3)));
9220 
9221   ins_cost(1.9 * INSN_COST);
9222   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
9223 
9224   ins_encode %{
9225     __ sub(as_Register($dst$$reg),
9226               as_Register($src1$$reg),
9227               as_Register($src2$$reg),
9228               Assembler::LSL,
9229               $src3$$constant & 0x3f);
9230   %}
9231 
9232   ins_pipe(ialu_reg_reg_shift);
9233 %}
9234 
9235 
9236 
9237 // Shift Left followed by Shift Right.
9238 // This idiom is used by the compiler for the i2b bytecode etc.
9239 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
9240 %{
9241   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
9242   // Make sure we are not going to exceed what sbfm can do.
9243   predicate((unsigned int)n->in(2)->get_int() <= 63
9244             && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
9245 
9246   ins_cost(INSN_COST * 2);
9247   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
9248   ins_encode %{
9249     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
9250     int s = 63 - lshift;
9251     int r = (rshift - lshift) & 63;
9252     __ sbfm(as_Register($dst$$reg),
9253             as_Register($src$$reg),
9254             r, s);
9255   %}
9256 
9257   ins_pipe(ialu_reg_shift);
9258 %}
9259 
9260 // Shift Left followed by Shift Right.
9261 // This idiom is used by the compiler for the i2b bytecode etc.
9262 instruct sbfmwI(iRegINoSp dst, iRegI src, immI lshift_count, immI rshift_count)
9263 %{
9264   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
9265   // Make sure we are not going to exceed what sbfmw can do.
9266   predicate((unsigned int)n->in(2)->get_int() <= 31
9267             && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
9268 
9269   ins_cost(INSN_COST * 2);
9270   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
9271   ins_encode %{
9272     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
9273     int s = 31 - lshift;
9274     int r = (rshift - lshift) & 31;
9275     __ sbfmw(as_Register($dst$$reg),
9276             as_Register($src$$reg),
9277             r, s);
9278   %}
9279 
9280   ins_pipe(ialu_reg_shift);
9281 %}
9282 
9283 // Shift Left followed by Shift Right.
9284 // This idiom is used by the compiler for the i2b bytecode etc.
9285 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
9286 %{
9287   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
9288   // Make sure we are not going to exceed what ubfm can do.
9289   predicate((unsigned int)n->in(2)->get_int() <= 63
9290             && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
9291 
9292   ins_cost(INSN_COST * 2);
9293   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
9294   ins_encode %{
9295     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
9296     int s = 63 - lshift;
9297     int r = (rshift - lshift) & 63;
9298     __ ubfm(as_Register($dst$$reg),
9299             as_Register($src$$reg),
9300             r, s);
9301   %}
9302 
9303   ins_pipe(ialu_reg_shift);
9304 %}
9305 
9306 // Shift Left followed by Shift Right.
9307 // This idiom is used by the compiler for the i2b bytecode etc.
9308 instruct ubfmwI(iRegINoSp dst, iRegI src, immI lshift_count, immI rshift_count)
9309 %{
9310   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
9311   // Make sure we are not going to exceed what ubfmw can do.
9312   predicate((unsigned int)n->in(2)->get_int() <= 31
9313             && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
9314 
9315   ins_cost(INSN_COST * 2);
9316   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
9317   ins_encode %{
9318     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
9319     int s = 31 - lshift;
9320     int r = (rshift - lshift) & 31;
9321     __ ubfmw(as_Register($dst$$reg),
9322             as_Register($src$$reg),
9323             r, s);
9324   %}
9325 
9326   ins_pipe(ialu_reg_shift);
9327 %}
9328 // Bitfield extract with shift & mask
9329 
9330 instruct ubfxwI(iRegINoSp dst, iRegI src, immI rshift, immI_bitmask mask)
9331 %{
9332   match(Set dst (AndI (URShiftI src rshift) mask));
9333 
9334   ins_cost(INSN_COST);
9335   format %{ "ubfxw $dst, $src, $mask" %}
9336   ins_encode %{
9337     int rshift = $rshift$$constant;
9338     long mask = $mask$$constant;
9339     int width = exact_log2(mask+1);
9340     __ ubfxw(as_Register($dst$$reg),
9341             as_Register($src$$reg), rshift, width);
9342   %}
9343   ins_pipe(ialu_reg_shift);
9344 %}
9345 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
9346 %{
9347   match(Set dst (AndL (URShiftL src rshift) mask));
9348 
9349   ins_cost(INSN_COST);
9350   format %{ "ubfx $dst, $src, $mask" %}
9351   ins_encode %{
9352     int rshift = $rshift$$constant;
9353     long mask = $mask$$constant;
9354     int width = exact_log2(mask+1);
9355     __ ubfx(as_Register($dst$$reg),
9356             as_Register($src$$reg), rshift, width);
9357   %}
9358   ins_pipe(ialu_reg_shift);
9359 %}
9360 
9361 // We can use ubfx when extending an And with a mask when we know mask
9362 // is positive.  We know that because immI_bitmask guarantees it.
9363 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
9364 %{
9365   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
9366 
9367   ins_cost(INSN_COST * 2);
9368   format %{ "ubfx $dst, $src, $mask" %}
9369   ins_encode %{
9370     int rshift = $rshift$$constant;
9371     long mask = $mask$$constant;
9372     int width = exact_log2(mask+1);
9373     __ ubfx(as_Register($dst$$reg),
9374             as_Register($src$$reg), rshift, width);
9375   %}
9376   ins_pipe(ialu_reg_shift);
9377 %}
9378 
9379 // Rotations
9380 
9381 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
9382 %{
9383   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
9384   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
9385 
9386   ins_cost(INSN_COST);
9387   format %{ "extr $dst, $src1, $src2, #$rshift" %}
9388 
9389   ins_encode %{
9390     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
9391             $rshift$$constant & 63);
9392   %}
9393   ins_pipe(ialu_reg_reg_extr);
9394 %}
9395 
9396 instruct extrOrI(iRegINoSp dst, iRegI src1, iRegI src2, immI lshift, immI rshift, rFlagsReg cr)
9397 %{
9398   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
9399   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
9400 
9401   ins_cost(INSN_COST);
9402   format %{ "extr $dst, $src1, $src2, #$rshift" %}
9403 
9404   ins_encode %{
9405     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
9406             $rshift$$constant & 31);
9407   %}
9408   ins_pipe(ialu_reg_reg_extr);
9409 %}
9410 
9411 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
9412 %{
9413   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
9414   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
9415 
9416   ins_cost(INSN_COST);
9417   format %{ "extr $dst, $src1, $src2, #$rshift" %}
9418 
9419   ins_encode %{
9420     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
9421             $rshift$$constant & 63);
9422   %}
9423   ins_pipe(ialu_reg_reg_extr);
9424 %}
9425 
9426 instruct extrAddI(iRegINoSp dst, iRegI src1, iRegI src2, immI lshift, immI rshift, rFlagsReg cr)
9427 %{
9428   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
9429   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
9430 
9431   ins_cost(INSN_COST);
9432   format %{ "extr $dst, $src1, $src2, #$rshift" %}
9433 
9434   ins_encode %{
9435     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
9436             $rshift$$constant & 31);
9437   %}
9438   ins_pipe(ialu_reg_reg_extr);
9439 %}
9440 
9441 
9442 // rol expander
9443 
9444 instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
9445 %{
9446   effect(DEF dst, USE src, USE shift);
9447 
9448   format %{ "rol    $dst, $src, $shift" %}
9449   ins_cost(INSN_COST * 3);
9450   ins_encode %{
9451     __ subw(rscratch1, zr, as_Register($shift$$reg));
9452     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
9453             rscratch1);
9454     %}
9455   ins_pipe(ialu_reg_reg_vshift);
9456 %}
9457 
9458 // rol expander
9459 
9460 instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
9461 %{
9462   effect(DEF dst, USE src, USE shift);
9463 
9464   format %{ "rol    $dst, $src, $shift" %}
9465   ins_cost(INSN_COST * 3);
9466   ins_encode %{
9467     __ subw(rscratch1, zr, as_Register($shift$$reg));
9468     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
9469             rscratch1);
9470     %}
9471   ins_pipe(ialu_reg_reg_vshift);
9472 %}
9473 
9474 instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
9475 %{
9476   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
9477 
9478   expand %{
9479     rolL_rReg(dst, src, shift, cr);
9480   %}
9481 %}
9482 
9483 instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
9484 %{
9485   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
9486 
9487   expand %{
9488     rolL_rReg(dst, src, shift, cr);
9489   %}
9490 %}
9491 
9492 instruct rolI_rReg_Var_C_32(iRegLNoSp dst, iRegL src, iRegI shift, immI_32 c_32, rFlagsReg cr)
9493 %{
9494   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
9495 
9496   expand %{
9497     rolL_rReg(dst, src, shift, cr);
9498   %}
9499 %}
9500 
9501 instruct rolI_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
9502 %{
9503   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
9504 
9505   expand %{
9506     rolL_rReg(dst, src, shift, cr);
9507   %}
9508 %}
9509 
9510 // ror expander
9511 
9512 instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
9513 %{
9514   effect(DEF dst, USE src, USE shift);
9515 
9516   format %{ "ror    $dst, $src, $shift" %}
9517   ins_cost(INSN_COST);
9518   ins_encode %{
9519     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
9520             as_Register($shift$$reg));
9521     %}
9522   ins_pipe(ialu_reg_reg_vshift);
9523 %}
9524 
9525 // ror expander
9526 
9527 instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
9528 %{
9529   effect(DEF dst, USE src, USE shift);
9530 
9531   format %{ "ror    $dst, $src, $shift" %}
9532   ins_cost(INSN_COST);
9533   ins_encode %{
9534     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
9535             as_Register($shift$$reg));
9536     %}
9537   ins_pipe(ialu_reg_reg_vshift);
9538 %}
9539 
9540 instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
9541 %{
9542   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
9543 
9544   expand %{
9545     rorL_rReg(dst, src, shift, cr);
9546   %}
9547 %}
9548 
9549 instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
9550 %{
9551   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
9552 
9553   expand %{
9554     rorL_rReg(dst, src, shift, cr);
9555   %}
9556 %}
9557 
9558 instruct rorI_rReg_Var_C_32(iRegLNoSp dst, iRegL src, iRegI shift, immI_32 c_32, rFlagsReg cr)
9559 %{
9560   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
9561 
9562   expand %{
9563     rorL_rReg(dst, src, shift, cr);
9564   %}
9565 %}
9566 
9567 instruct rorI_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
9568 %{
9569   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
9570 
9571   expand %{
9572     rorL_rReg(dst, src, shift, cr);
9573   %}
9574 %}
9575 
9576 // Add/subtract (extended)
9577 
9578 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
9579 %{
9580   match(Set dst (AddL src1 (ConvI2L src2)));
9581   ins_cost(INSN_COST);
9582   format %{ "add  $dst, $src1, sxtw $src2" %}
9583 
9584    ins_encode %{
9585      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9586             as_Register($src2$$reg), ext::sxtw);
9587    %}
9588   ins_pipe(ialu_reg_reg);
9589 %};
9590 
9591 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
9592 %{
9593   match(Set dst (SubL src1 (ConvI2L src2)));
9594   ins_cost(INSN_COST);
9595   format %{ "sub  $dst, $src1, sxtw $src2" %}
9596 
9597    ins_encode %{
9598      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
9599             as_Register($src2$$reg), ext::sxtw);
9600    %}
9601   ins_pipe(ialu_reg_reg);
9602 %};
9603 
9604 
9605 instruct AddExtI_sxth(iRegINoSp dst, iRegI src1, iRegI src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
9606 %{
9607   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
9608   ins_cost(INSN_COST);
9609   format %{ "add  $dst, $src1, sxth $src2" %}
9610 
9611    ins_encode %{
9612      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9613             as_Register($src2$$reg), ext::sxth);
9614    %}
9615   ins_pipe(ialu_reg_reg);
9616 %}
9617 
9618 instruct AddExtI_sxtb(iRegINoSp dst, iRegI src1, iRegI src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
9619 %{
9620   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
9621   ins_cost(INSN_COST);
9622   format %{ "add  $dst, $src1, sxtb $src2" %}
9623 
9624    ins_encode %{
9625      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9626             as_Register($src2$$reg), ext::sxtb);
9627    %}
9628   ins_pipe(ialu_reg_reg);
9629 %}
9630 
9631 instruct AddExtI_uxtb(iRegINoSp dst, iRegI src1, iRegI src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
9632 %{
9633   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
9634   ins_cost(INSN_COST);
9635   format %{ "add  $dst, $src1, uxtb $src2" %}
9636 
9637    ins_encode %{
9638      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9639             as_Register($src2$$reg), ext::uxtb);
9640    %}
9641   ins_pipe(ialu_reg_reg);
9642 %}
9643 
9644 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
9645 %{
9646   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
9647   ins_cost(INSN_COST);
9648   format %{ "add  $dst, $src1, sxth $src2" %}
9649 
9650    ins_encode %{
9651      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9652             as_Register($src2$$reg), ext::sxth);
9653    %}
9654   ins_pipe(ialu_reg_reg);
9655 %}
9656 
9657 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
9658 %{
9659   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
9660   ins_cost(INSN_COST);
9661   format %{ "add  $dst, $src1, sxtw $src2" %}
9662 
9663    ins_encode %{
9664      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9665             as_Register($src2$$reg), ext::sxtw);
9666    %}
9667   ins_pipe(ialu_reg_reg);
9668 %}
9669 
9670 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
9671 %{
9672   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
9673   ins_cost(INSN_COST);
9674   format %{ "add  $dst, $src1, sxtb $src2" %}
9675 
9676    ins_encode %{
9677      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9678             as_Register($src2$$reg), ext::sxtb);
9679    %}
9680   ins_pipe(ialu_reg_reg);
9681 %}
9682 
9683 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
9684 %{
9685   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
9686   ins_cost(INSN_COST);
9687   format %{ "add  $dst, $src1, uxtb $src2" %}
9688 
9689    ins_encode %{
9690      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9691             as_Register($src2$$reg), ext::uxtb);
9692    %}
9693   ins_pipe(ialu_reg_reg);
9694 %}
9695 
9696 
9697 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegI src1, iRegI src2, immI_255 mask, rFlagsReg cr)
9698 %{
9699   match(Set dst (AddI src1 (AndI src2 mask)));
9700   ins_cost(INSN_COST);
9701   format %{ "addw  $dst, $src1, $src2, uxtb" %}
9702 
9703    ins_encode %{
9704      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
9705             as_Register($src2$$reg), ext::uxtb);
9706    %}
9707   ins_pipe(ialu_reg_reg);
9708 %}
9709 
9710 instruct AddExtI_uxth_and(iRegINoSp dst, iRegI src1, iRegI src2, immI_65535 mask, rFlagsReg cr)
9711 %{
9712   match(Set dst (AddI src1 (AndI src2 mask)));
9713   ins_cost(INSN_COST);
9714   format %{ "addw  $dst, $src1, $src2, uxth" %}
9715 
9716    ins_encode %{
9717      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
9718             as_Register($src2$$reg), ext::uxth);
9719    %}
9720   ins_pipe(ialu_reg_reg);
9721 %}
9722 
9723 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
9724 %{
9725   match(Set dst (AddL src1 (AndL src2 mask)));
9726   ins_cost(INSN_COST);
9727   format %{ "add  $dst, $src1, $src2, uxtb" %}
9728 
9729    ins_encode %{
9730      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9731             as_Register($src2$$reg), ext::uxtb);
9732    %}
9733   ins_pipe(ialu_reg_reg);
9734 %}
9735 
9736 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
9737 %{
9738   match(Set dst (AddL src1 (AndL src2 mask)));
9739   ins_cost(INSN_COST);
9740   format %{ "add  $dst, $src1, $src2, uxth" %}
9741 
9742    ins_encode %{
9743      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9744             as_Register($src2$$reg), ext::uxth);
9745    %}
9746   ins_pipe(ialu_reg_reg);
9747 %}
9748 
9749 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
9750 %{
9751   match(Set dst (AddL src1 (AndL src2 mask)));
9752   ins_cost(INSN_COST);
9753   format %{ "add  $dst, $src1, $src2, uxtw" %}
9754 
9755    ins_encode %{
9756      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9757             as_Register($src2$$reg), ext::uxtw);
9758    %}
9759   ins_pipe(ialu_reg_reg);
9760 %}
9761 
9762 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegI src1, iRegI src2, immI_255 mask, rFlagsReg cr)
9763 %{
9764   match(Set dst (SubI src1 (AndI src2 mask)));
9765   ins_cost(INSN_COST);
9766   format %{ "subw  $dst, $src1, $src2, uxtb" %}
9767 
9768    ins_encode %{
9769      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
9770             as_Register($src2$$reg), ext::uxtb);
9771    %}
9772   ins_pipe(ialu_reg_reg);
9773 %}
9774 
9775 instruct SubExtI_uxth_and(iRegINoSp dst, iRegI src1, iRegI src2, immI_65535 mask, rFlagsReg cr)
9776 %{
9777   match(Set dst (SubI src1 (AndI src2 mask)));
9778   ins_cost(INSN_COST);
9779   format %{ "subw  $dst, $src1, $src2, uxth" %}
9780 
9781    ins_encode %{
9782      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
9783             as_Register($src2$$reg), ext::uxth);
9784    %}
9785   ins_pipe(ialu_reg_reg);
9786 %}
9787 
9788 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
9789 %{
9790   match(Set dst (SubL src1 (AndL src2 mask)));
9791   ins_cost(INSN_COST);
9792   format %{ "sub  $dst, $src1, $src2, uxtb" %}
9793 
9794    ins_encode %{
9795      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
9796             as_Register($src2$$reg), ext::uxtb);
9797    %}
9798   ins_pipe(ialu_reg_reg);
9799 %}
9800 
9801 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
9802 %{
9803   match(Set dst (SubL src1 (AndL src2 mask)));
9804   ins_cost(INSN_COST);
9805   format %{ "sub  $dst, $src1, $src2, uxth" %}
9806 
9807    ins_encode %{
9808      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
9809             as_Register($src2$$reg), ext::uxth);
9810    %}
9811   ins_pipe(ialu_reg_reg);
9812 %}
9813 
9814 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
9815 %{
9816   match(Set dst (SubL src1 (AndL src2 mask)));
9817   ins_cost(INSN_COST);
9818   format %{ "sub  $dst, $src1, $src2, uxtw" %}
9819 
9820    ins_encode %{
9821      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
9822             as_Register($src2$$reg), ext::uxtw);
9823    %}
9824   ins_pipe(ialu_reg_reg);
9825 %}
9826 
9827 // END This section of the file is automatically generated. Do not edit --------------
9828 
9829 // ============================================================================
9830 // Floating Point Arithmetic Instructions
9831 
9832 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
9833   match(Set dst (AddF src1 src2));
9834 
9835   ins_cost(INSN_COST * 5);
9836   format %{ "fadds   $dst, $src1, $src2" %}
9837 
9838   ins_encode %{
9839     __ fadds(as_FloatRegister($dst$$reg),
9840              as_FloatRegister($src1$$reg),
9841              as_FloatRegister($src2$$reg));
9842   %}
9843 
9844   ins_pipe(pipe_class_default);
9845 %}
9846 
9847 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
9848   match(Set dst (AddD src1 src2));
9849 
9850   ins_cost(INSN_COST * 5);
9851   format %{ "faddd   $dst, $src1, $src2" %}
9852 
9853   ins_encode %{
9854     __ faddd(as_FloatRegister($dst$$reg),
9855              as_FloatRegister($src1$$reg),
9856              as_FloatRegister($src2$$reg));
9857   %}
9858 
9859   ins_pipe(pipe_class_default);
9860 %}
9861 
9862 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
9863   match(Set dst (SubF src1 src2));
9864 
9865   ins_cost(INSN_COST * 5);
9866   format %{ "fsubs   $dst, $src1, $src2" %}
9867 
9868   ins_encode %{
9869     __ fsubs(as_FloatRegister($dst$$reg),
9870              as_FloatRegister($src1$$reg),
9871              as_FloatRegister($src2$$reg));
9872   %}
9873 
9874   ins_pipe(pipe_class_default);
9875 %}
9876 
9877 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
9878   match(Set dst (SubD src1 src2));
9879 
9880   ins_cost(INSN_COST * 5);
9881   format %{ "fsubd   $dst, $src1, $src2" %}
9882 
9883   ins_encode %{
9884     __ fsubd(as_FloatRegister($dst$$reg),
9885              as_FloatRegister($src1$$reg),
9886              as_FloatRegister($src2$$reg));
9887   %}
9888 
9889   ins_pipe(pipe_class_default);
9890 %}
9891 
9892 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
9893   match(Set dst (MulF src1 src2));
9894 
9895   ins_cost(INSN_COST * 6);
9896   format %{ "fmuls   $dst, $src1, $src2" %}
9897 
9898   ins_encode %{
9899     __ fmuls(as_FloatRegister($dst$$reg),
9900              as_FloatRegister($src1$$reg),
9901              as_FloatRegister($src2$$reg));
9902   %}
9903 
9904   ins_pipe(pipe_class_default);
9905 %}
9906 
9907 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
9908   match(Set dst (MulD src1 src2));
9909 
9910   ins_cost(INSN_COST * 6);
9911   format %{ "fmuld   $dst, $src1, $src2" %}
9912 
9913   ins_encode %{
9914     __ fmuld(as_FloatRegister($dst$$reg),
9915              as_FloatRegister($src1$$reg),
9916              as_FloatRegister($src2$$reg));
9917   %}
9918 
9919   ins_pipe(pipe_class_default);
9920 %}
9921 
9922 // We cannot use these fused mul w add/sub ops because they don't
9923 // produce the same result as the equivalent separated ops
9924 // (essentially they don't round the intermediate result). that's a
9925 // shame. leaving them here in case we can idenitfy cases where it is
9926 // legitimate to use them
9927 
9928 
9929 // instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
9930 //   match(Set dst (AddF (MulF src1 src2) src3));
9931 
9932 //   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
9933 
9934 //   ins_encode %{
9935 //     __ fmadds(as_FloatRegister($dst$$reg),
9936 //              as_FloatRegister($src1$$reg),
9937 //              as_FloatRegister($src2$$reg),
9938 //              as_FloatRegister($src3$$reg));
9939 //   %}
9940 
9941 //   ins_pipe(pipe_class_default);
9942 // %}
9943 
9944 // instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
9945 //   match(Set dst (AddD (MulD src1 src2) src3));
9946 
9947 //   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
9948 
9949 //   ins_encode %{
9950 //     __ fmaddd(as_FloatRegister($dst$$reg),
9951 //              as_FloatRegister($src1$$reg),
9952 //              as_FloatRegister($src2$$reg),
9953 //              as_FloatRegister($src3$$reg));
9954 //   %}
9955 
9956 //   ins_pipe(pipe_class_default);
9957 // %}
9958 
9959 // instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
9960 //   match(Set dst (AddF (MulF (NegF src1) src2) src3));
9961 //   match(Set dst (AddF (NegF (MulF src1 src2)) src3));
9962 
9963 //   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
9964 
9965 //   ins_encode %{
9966 //     __ fmsubs(as_FloatRegister($dst$$reg),
9967 //               as_FloatRegister($src1$$reg),
9968 //               as_FloatRegister($src2$$reg),
9969 //              as_FloatRegister($src3$$reg));
9970 //   %}
9971 
9972 //   ins_pipe(pipe_class_default);
9973 // %}
9974 
9975 // instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
9976 //   match(Set dst (AddD (MulD (NegD src1) src2) src3));
9977 //   match(Set dst (AddD (NegD (MulD src1 src2)) src3));
9978 
9979 //   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
9980 
9981 //   ins_encode %{
9982 //     __ fmsubd(as_FloatRegister($dst$$reg),
9983 //               as_FloatRegister($src1$$reg),
9984 //               as_FloatRegister($src2$$reg),
9985 //               as_FloatRegister($src3$$reg));
9986 //   %}
9987 
9988 //   ins_pipe(pipe_class_default);
9989 // %}
9990 
9991 // instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
9992 //   match(Set dst (SubF (MulF (NegF src1) src2) src3));
9993 //   match(Set dst (SubF (NegF (MulF src1 src2)) src3));
9994 
9995 //   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
9996 
9997 //   ins_encode %{
9998 //     __ fnmadds(as_FloatRegister($dst$$reg),
9999 //                as_FloatRegister($src1$$reg),
10000 //                as_FloatRegister($src2$$reg),
10001 //                as_FloatRegister($src3$$reg));
10002 //   %}
10003 
10004 //   ins_pipe(pipe_class_default);
10005 // %}
10006 
10007 // instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
10008 //   match(Set dst (SubD (MulD (NegD src1) src2) src3));
10009 //   match(Set dst (SubD (NegD (MulD src1 src2)) src3));
10010 
10011 //   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
10012 
10013 //   ins_encode %{
10014 //     __ fnmaddd(as_FloatRegister($dst$$reg),
10015 //                as_FloatRegister($src1$$reg),
10016 //                as_FloatRegister($src2$$reg),
10017 //                as_FloatRegister($src3$$reg));
10018 //   %}
10019 
10020 //   ins_pipe(pipe_class_default);
10021 // %}
10022 
10023 // instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
10024 //   match(Set dst (SubF (MulF src1 src2) src3));
10025 
10026 //   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
10027 
10028 //   ins_encode %{
10029 //     __ fnmsubs(as_FloatRegister($dst$$reg),
10030 //                as_FloatRegister($src1$$reg),
10031 //                as_FloatRegister($src2$$reg),
10032 //                as_FloatRegister($src3$$reg));
10033 //   %}
10034 
10035 //   ins_pipe(pipe_class_default);
10036 // %}
10037 
10038 // instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
10039 //   match(Set dst (SubD (MulD src1 src2) src3));
10040 
10041 //   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
10042 
10043 //   ins_encode %{
10044 //   // n.b. insn name should be fnmsubd
10045 //     __ fnmsub(as_FloatRegister($dst$$reg),
10046 //                as_FloatRegister($src1$$reg),
10047 //                as_FloatRegister($src2$$reg),
10048 //                as_FloatRegister($src3$$reg));
10049 //   %}
10050 
10051 //   ins_pipe(pipe_class_default);
10052 // %}
10053 
10054 
10055 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
10056   match(Set dst (DivF src1  src2));
10057 
10058   ins_cost(INSN_COST * 18);
10059   format %{ "fdivs   $dst, $src1, $src2" %}
10060 
10061   ins_encode %{
10062     __ fdivs(as_FloatRegister($dst$$reg),
10063              as_FloatRegister($src1$$reg),
10064              as_FloatRegister($src2$$reg));
10065   %}
10066 
10067   ins_pipe(pipe_class_default);
10068 %}
10069 
10070 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
10071   match(Set dst (DivD src1  src2));
10072 
10073   ins_cost(INSN_COST * 32);
10074   format %{ "fdivd   $dst, $src1, $src2" %}
10075 
10076   ins_encode %{
10077     __ fdivd(as_FloatRegister($dst$$reg),
10078              as_FloatRegister($src1$$reg),
10079              as_FloatRegister($src2$$reg));
10080   %}
10081 
10082   ins_pipe(pipe_class_default);
10083 %}
10084 
10085 instruct negF_reg_reg(vRegF dst, vRegF src) %{
10086   match(Set dst (NegF src));
10087 
10088   ins_cost(INSN_COST * 3);
10089   format %{ "fneg   $dst, $src" %}
10090 
10091   ins_encode %{
10092     __ fnegs(as_FloatRegister($dst$$reg),
10093              as_FloatRegister($src$$reg));
10094   %}
10095 
10096   ins_pipe(pipe_class_default);
10097 %}
10098 
10099 instruct negD_reg_reg(vRegD dst, vRegD src) %{
10100   match(Set dst (NegD src));
10101 
10102   ins_cost(INSN_COST * 3);
10103   format %{ "fnegd   $dst, $src" %}
10104 
10105   ins_encode %{
10106     __ fnegd(as_FloatRegister($dst$$reg),
10107              as_FloatRegister($src$$reg));
10108   %}
10109 
10110   ins_pipe(pipe_class_default);
10111 %}
10112 
10113 instruct absF_reg(vRegF dst, vRegF src) %{
10114   match(Set dst (AbsF src));
10115 
10116   ins_cost(INSN_COST * 3);
10117   format %{ "fabss   $dst, $src" %}
10118   ins_encode %{
10119     __ fabss(as_FloatRegister($dst$$reg),
10120              as_FloatRegister($src$$reg));
10121   %}
10122 
10123   ins_pipe(pipe_class_default);
10124 %}
10125 
10126 instruct absD_reg(vRegD dst, vRegD src) %{
10127   match(Set dst (AbsD src));
10128 
10129   ins_cost(INSN_COST * 3);
10130   format %{ "fabsd   $dst, $src" %}
10131   ins_encode %{
10132     __ fabsd(as_FloatRegister($dst$$reg),
10133              as_FloatRegister($src$$reg));
10134   %}
10135 
10136   ins_pipe(pipe_class_default);
10137 %}
10138 
10139 instruct sqrtD_reg(vRegD dst, vRegD src) %{
10140   match(Set dst (SqrtD src));
10141 
10142   ins_cost(INSN_COST * 50);
10143   format %{ "fsqrtd  $dst, $src" %}
10144   ins_encode %{
10145     __ fsqrtd(as_FloatRegister($dst$$reg),
10146              as_FloatRegister($src$$reg));
10147   %}
10148 
10149   ins_pipe(pipe_class_default);
10150 %}
10151 
10152 instruct sqrtF_reg(vRegF dst, vRegF src) %{
10153   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
10154 
10155   ins_cost(INSN_COST * 50);
10156   format %{ "fsqrts  $dst, $src" %}
10157   ins_encode %{
10158     __ fsqrts(as_FloatRegister($dst$$reg),
10159              as_FloatRegister($src$$reg));
10160   %}
10161 
10162   ins_pipe(pipe_class_default);
10163 %}
10164 
10165 // ============================================================================
10166 // Logical Instructions
10167 
10168 // Integer Logical Instructions
10169 
10170 // And Instructions
10171 
10172 
10173 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
10174   match(Set dst (AndI src1 src2));
10175 
10176   format %{ "andw  $dst, $src1, $src2\t# int" %}
10177 
10178   ins_cost(INSN_COST);
10179   ins_encode %{
10180     __ andw(as_Register($dst$$reg),
10181             as_Register($src1$$reg),
10182             as_Register($src2$$reg));
10183   %}
10184 
10185   ins_pipe(ialu_reg_reg);
10186 %}
10187 
10188 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
10189   match(Set dst (AndI src1 src2));
10190 
10191   format %{ "andsw  $dst, $src1, $src2\t# int" %}
10192 
10193   ins_cost(INSN_COST);
10194   ins_encode %{
10195     __ andw(as_Register($dst$$reg),
10196             as_Register($src1$$reg),
10197             (unsigned long)($src2$$constant));
10198   %}
10199 
10200   ins_pipe(ialu_reg_imm);
10201 %}
10202 
10203 // Or Instructions
10204 
10205 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10206   match(Set dst (OrI src1 src2));
10207 
10208   format %{ "orrw  $dst, $src1, $src2\t# int" %}
10209 
10210   ins_cost(INSN_COST);
10211   ins_encode %{
10212     __ orrw(as_Register($dst$$reg),
10213             as_Register($src1$$reg),
10214             as_Register($src2$$reg));
10215   %}
10216 
10217   ins_pipe(ialu_reg_reg);
10218 %}
10219 
10220 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
10221   match(Set dst (OrI src1 src2));
10222 
10223   format %{ "orrw  $dst, $src1, $src2\t# int" %}
10224 
10225   ins_cost(INSN_COST);
10226   ins_encode %{
10227     __ orrw(as_Register($dst$$reg),
10228             as_Register($src1$$reg),
10229             (unsigned long)($src2$$constant));
10230   %}
10231 
10232   ins_pipe(ialu_reg_imm);
10233 %}
10234 
10235 // Xor Instructions
10236 
10237 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10238   match(Set dst (XorI src1 src2));
10239 
10240   format %{ "eorw  $dst, $src1, $src2\t# int" %}
10241 
10242   ins_cost(INSN_COST);
10243   ins_encode %{
10244     __ eorw(as_Register($dst$$reg),
10245             as_Register($src1$$reg),
10246             as_Register($src2$$reg));
10247   %}
10248 
10249   ins_pipe(ialu_reg_reg);
10250 %}
10251 
10252 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
10253   match(Set dst (XorI src1 src2));
10254 
10255   format %{ "eorw  $dst, $src1, $src2\t# int" %}
10256 
10257   ins_cost(INSN_COST);
10258   ins_encode %{
10259     __ eorw(as_Register($dst$$reg),
10260             as_Register($src1$$reg),
10261             (unsigned long)($src2$$constant));
10262   %}
10263 
10264   ins_pipe(ialu_reg_imm);
10265 %}
10266 
10267 // Long Logical Instructions
10268 // TODO
10269 
10270 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
10271   match(Set dst (AndL src1 src2));
10272 
10273   format %{ "and  $dst, $src1, $src2\t# int" %}
10274 
10275   ins_cost(INSN_COST);
10276   ins_encode %{
10277     __ andr(as_Register($dst$$reg),
10278             as_Register($src1$$reg),
10279             as_Register($src2$$reg));
10280   %}
10281 
10282   ins_pipe(ialu_reg_reg);
10283 %}
10284 
10285 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
10286   match(Set dst (AndL src1 src2));
10287 
10288   format %{ "and  $dst, $src1, $src2\t# int" %}
10289 
10290   ins_cost(INSN_COST);
10291   ins_encode %{
10292     __ andr(as_Register($dst$$reg),
10293             as_Register($src1$$reg),
10294             (unsigned long)($src2$$constant));
10295   %}
10296 
10297   ins_pipe(ialu_reg_imm);
10298 %}
10299 
10300 // Or Instructions
10301 
10302 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10303   match(Set dst (OrL src1 src2));
10304 
10305   format %{ "orr  $dst, $src1, $src2\t# int" %}
10306 
10307   ins_cost(INSN_COST);
10308   ins_encode %{
10309     __ orr(as_Register($dst$$reg),
10310            as_Register($src1$$reg),
10311            as_Register($src2$$reg));
10312   %}
10313 
10314   ins_pipe(ialu_reg_reg);
10315 %}
10316 
10317 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
10318   match(Set dst (OrL src1 src2));
10319 
10320   format %{ "orr  $dst, $src1, $src2\t# int" %}
10321 
10322   ins_cost(INSN_COST);
10323   ins_encode %{
10324     __ orr(as_Register($dst$$reg),
10325            as_Register($src1$$reg),
10326            (unsigned long)($src2$$constant));
10327   %}
10328 
10329   ins_pipe(ialu_reg_imm);
10330 %}
10331 
10332 // Xor Instructions
10333 
10334 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10335   match(Set dst (XorL src1 src2));
10336 
10337   format %{ "eor  $dst, $src1, $src2\t# int" %}
10338 
10339   ins_cost(INSN_COST);
10340   ins_encode %{
10341     __ eor(as_Register($dst$$reg),
10342            as_Register($src1$$reg),
10343            as_Register($src2$$reg));
10344   %}
10345 
10346   ins_pipe(ialu_reg_reg);
10347 %}
10348 
10349 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
10350   match(Set dst (XorL src1 src2));
10351 
10352   ins_cost(INSN_COST);
10353   format %{ "eor  $dst, $src1, $src2\t# int" %}
10354 
10355   ins_encode %{
10356     __ eor(as_Register($dst$$reg),
10357            as_Register($src1$$reg),
10358            (unsigned long)($src2$$constant));
10359   %}
10360 
10361   ins_pipe(ialu_reg_imm);
10362 %}
10363 
10364 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
10365 %{
10366   match(Set dst (ConvI2L src));
10367 
10368   ins_cost(INSN_COST);
10369   format %{ "sxtw  $dst, $src\t# i2l" %}
10370   ins_encode %{
10371     __ sbfm($dst$$Register, $src$$Register, 0, 31);
10372   %}
10373   ins_pipe(ialu_reg_shift);
10374 %}
10375 
10376 // this pattern occurs in bigmath arithmetic
10377 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegI src, immL_32bits mask)
10378 %{
10379   match(Set dst (AndL (ConvI2L src) mask));
10380 
10381   ins_cost(INSN_COST);
10382   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
10383   ins_encode %{
10384     __ ubfm($dst$$Register, $src$$Register, 0, 31);
10385   %}
10386 
10387   ins_pipe(ialu_reg_shift);
10388 %}
10389 
10390 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
10391   match(Set dst (ConvL2I src));
10392 
10393   ins_cost(INSN_COST);
10394   format %{ "movw  $dst, $src \t// l2i" %}
10395 
10396   ins_encode %{
10397     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
10398   %}
10399 
10400   ins_pipe(ialu_reg);
10401 %}
10402 
10403 instruct convI2B(iRegINoSp dst, iRegI src, rFlagsReg cr)
10404 %{
10405   match(Set dst (Conv2B src));
10406   effect(KILL cr);
10407 
10408   format %{
10409     "cmpw $src, zr\n\t"
10410     "cset $dst, ne"
10411   %}
10412 
10413   ins_encode %{
10414     __ cmpw(as_Register($src$$reg), zr);
10415     __ cset(as_Register($dst$$reg), Assembler::NE);
10416   %}
10417 
10418   ins_pipe(ialu_reg);
10419 %}
10420 
10421 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
10422 %{
10423   match(Set dst (Conv2B src));
10424   effect(KILL cr);
10425 
10426   format %{
10427     "cmp  $src, zr\n\t"
10428     "cset $dst, ne"
10429   %}
10430 
10431   ins_encode %{
10432     __ cmp(as_Register($src$$reg), zr);
10433     __ cset(as_Register($dst$$reg), Assembler::NE);
10434   %}
10435 
10436   ins_pipe(ialu_reg);
10437 %}
10438 
10439 instruct convD2F_reg(vRegF dst, vRegD src) %{
10440   match(Set dst (ConvD2F src));
10441 
10442   ins_cost(INSN_COST * 5);
10443   format %{ "fcvtd  $dst, $src \t// d2f" %}
10444 
10445   ins_encode %{
10446     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
10447   %}
10448 
10449   ins_pipe(pipe_class_default);
10450 %}
10451 
10452 instruct convF2D_reg(vRegD dst, vRegF src) %{
10453   match(Set dst (ConvF2D src));
10454 
10455   ins_cost(INSN_COST * 5);
10456   format %{ "fcvts  $dst, $src \t// f2d" %}
10457 
10458   ins_encode %{
10459     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
10460   %}
10461 
10462   ins_pipe(pipe_class_default);
10463 %}
10464 
10465 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
10466   match(Set dst (ConvF2I src));
10467 
10468   ins_cost(INSN_COST * 5);
10469   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
10470 
10471   ins_encode %{
10472     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
10473   %}
10474 
10475   ins_pipe(pipe_class_default);
10476 %}
10477 
10478 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
10479   match(Set dst (ConvF2L src));
10480 
10481   ins_cost(INSN_COST * 5);
10482   format %{ "fcvtzs  $dst, $src \t// f2l" %}
10483 
10484   ins_encode %{
10485     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
10486   %}
10487 
10488   ins_pipe(pipe_class_default);
10489 %}
10490 
10491 instruct convI2F_reg_reg(vRegF dst, iRegI src) %{
10492   match(Set dst (ConvI2F src));
10493 
10494   ins_cost(INSN_COST * 5);
10495   format %{ "scvtfws  $dst, $src \t// i2f" %}
10496 
10497   ins_encode %{
10498     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
10499   %}
10500 
10501   ins_pipe(pipe_class_default);
10502 %}
10503 
10504 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
10505   match(Set dst (ConvL2F src));
10506 
10507   ins_cost(INSN_COST * 5);
10508   format %{ "scvtfs  $dst, $src \t// l2f" %}
10509 
10510   ins_encode %{
10511     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
10512   %}
10513 
10514   ins_pipe(pipe_class_default);
10515 %}
10516 
10517 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
10518   match(Set dst (ConvD2I src));
10519 
10520   ins_cost(INSN_COST * 5);
10521   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
10522 
10523   ins_encode %{
10524     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
10525   %}
10526 
10527   ins_pipe(pipe_class_default);
10528 %}
10529 
10530 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
10531   match(Set dst (ConvD2L src));
10532 
10533   ins_cost(INSN_COST * 5);
10534   format %{ "fcvtzd  $dst, $src \t// d2l" %}
10535 
10536   ins_encode %{
10537     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
10538   %}
10539 
10540   ins_pipe(pipe_class_default);
10541 %}
10542 
10543 instruct convI2D_reg_reg(vRegD dst, iRegI src) %{
10544   match(Set dst (ConvI2D src));
10545 
10546   ins_cost(INSN_COST * 5);
10547   format %{ "scvtfwd  $dst, $src \t// i2d" %}
10548 
10549   ins_encode %{
10550     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
10551   %}
10552 
10553   ins_pipe(pipe_class_default);
10554 %}
10555 
10556 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
10557   match(Set dst (ConvL2D src));
10558 
10559   ins_cost(INSN_COST * 5);
10560   format %{ "scvtfd  $dst, $src \t// l2d" %}
10561 
10562   ins_encode %{
10563     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
10564   %}
10565 
10566   ins_pipe(pipe_class_default);
10567 %}
10568 
10569 // stack <-> reg and reg <-> reg shuffles with no conversion
10570 
10571 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
10572 
10573   match(Set dst (MoveF2I src));
10574 
10575   effect(DEF dst, USE src);
10576 
10577   ins_cost(4 * INSN_COST);
10578 
10579   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
10580 
10581   ins_encode %{
10582     __ ldrw($dst$$Register, Address(sp, $src$$disp));
10583   %}
10584 
10585   ins_pipe(iload_reg_reg);
10586 
10587 %}
10588 
10589 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
10590 
10591   match(Set dst (MoveI2F src));
10592 
10593   effect(DEF dst, USE src);
10594 
10595   ins_cost(4 * INSN_COST);
10596 
10597   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
10598 
10599   ins_encode %{
10600     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
10601   %}
10602 
10603   ins_pipe(pipe_class_memory);
10604 
10605 %}
10606 
10607 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
10608 
10609   match(Set dst (MoveD2L src));
10610 
10611   effect(DEF dst, USE src);
10612 
10613   ins_cost(4 * INSN_COST);
10614 
10615   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
10616 
10617   ins_encode %{
10618     __ ldr($dst$$Register, Address(sp, $src$$disp));
10619   %}
10620 
10621   ins_pipe(iload_reg_reg);
10622 
10623 %}
10624 
10625 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
10626 
10627   match(Set dst (MoveL2D src));
10628 
10629   effect(DEF dst, USE src);
10630 
10631   ins_cost(4 * INSN_COST);
10632 
10633   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
10634 
10635   ins_encode %{
10636     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
10637   %}
10638 
10639   ins_pipe(pipe_class_memory);
10640 
10641 %}
10642 
10643 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
10644 
10645   match(Set dst (MoveF2I src));
10646 
10647   effect(DEF dst, USE src);
10648 
10649   ins_cost(INSN_COST);
10650 
10651   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
10652 
10653   ins_encode %{
10654     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
10655   %}
10656 
10657   ins_pipe(pipe_class_memory);
10658 
10659 %}
10660 
10661 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
10662 
10663   match(Set dst (MoveI2F src));
10664 
10665   effect(DEF dst, USE src);
10666 
10667   ins_cost(INSN_COST);
10668 
10669   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
10670 
10671   ins_encode %{
10672     __ strw($src$$Register, Address(sp, $dst$$disp));
10673   %}
10674 
10675   ins_pipe(istore_reg_reg);
10676 
10677 %}
10678 
10679 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
10680 
10681   match(Set dst (MoveD2L src));
10682 
10683   effect(DEF dst, USE src);
10684 
10685   ins_cost(INSN_COST);
10686 
10687   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
10688 
10689   ins_encode %{
10690     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
10691   %}
10692 
10693   ins_pipe(pipe_class_memory);
10694 
10695 %}
10696 
10697 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
10698 
10699   match(Set dst (MoveL2D src));
10700 
10701   effect(DEF dst, USE src);
10702 
10703   ins_cost(INSN_COST);
10704 
10705   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
10706 
10707   ins_encode %{
10708     __ str($src$$Register, Address(sp, $dst$$disp));
10709   %}
10710 
10711   ins_pipe(istore_reg_reg);
10712 
10713 %}
10714 
10715 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
10716 
10717   match(Set dst (MoveF2I src));
10718 
10719   effect(DEF dst, USE src);
10720 
10721   ins_cost(INSN_COST);
10722 
10723   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
10724 
10725   ins_encode %{
10726     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
10727   %}
10728 
10729   ins_pipe(pipe_class_memory);
10730 
10731 %}
10732 
10733 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
10734 
10735   match(Set dst (MoveI2F src));
10736 
10737   effect(DEF dst, USE src);
10738 
10739   ins_cost(INSN_COST);
10740 
10741   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
10742 
10743   ins_encode %{
10744     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
10745   %}
10746 
10747   ins_pipe(pipe_class_memory);
10748 
10749 %}
10750 
10751 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
10752 
10753   match(Set dst (MoveD2L src));
10754 
10755   effect(DEF dst, USE src);
10756 
10757   ins_cost(INSN_COST);
10758 
10759   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
10760 
10761   ins_encode %{
10762     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
10763   %}
10764 
10765   ins_pipe(pipe_class_memory);
10766 
10767 %}
10768 
10769 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
10770 
10771   match(Set dst (MoveL2D src));
10772 
10773   effect(DEF dst, USE src);
10774 
10775   ins_cost(INSN_COST);
10776 
10777   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
10778 
10779   ins_encode %{
10780     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
10781   %}
10782 
10783   ins_pipe(pipe_class_memory);
10784 
10785 %}
10786 
10787 // ============================================================================
10788 // clearing of an array
10789 
10790 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
10791 %{
10792   match(Set dummy (ClearArray cnt base));
10793   effect(USE_KILL cnt, USE_KILL base);
10794 
10795   ins_cost(4 * INSN_COST);
10796   format %{ "ClearArray $cnt, $base" %}
10797 
10798   ins_encode(aarch64_enc_clear_array_reg_reg(cnt, base));
10799 
10800   ins_pipe(pipe_class_memory);
10801 %}
10802 
10803 // ============================================================================
10804 // Overflow Math Instructions
10805 
10806 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
10807 %{
10808   match(Set cr (OverflowAddI op1 op2));
10809 
10810   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
10811   ins_cost(INSN_COST);
10812   ins_encode %{
10813     __ cmnw($op1$$Register, $op2$$Register);
10814   %}
10815 
10816   ins_pipe(icmp_reg_reg);
10817 %}
10818 
10819 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegI op1, immIAddSub op2)
10820 %{
10821   match(Set cr (OverflowAddI op1 op2));
10822 
10823   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
10824   ins_cost(INSN_COST);
10825   ins_encode %{
10826     __ cmnw($op1$$Register, $op2$$constant);
10827   %}
10828 
10829   ins_pipe(icmp_reg_imm);
10830 %}
10831 
10832 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
10833 %{
10834   match(Set cr (OverflowAddL op1 op2));
10835 
10836   format %{ "cmn   $op1, $op2\t# overflow check long" %}
10837   ins_cost(INSN_COST);
10838   ins_encode %{
10839     __ cmn($op1$$Register, $op2$$Register);
10840   %}
10841 
10842   ins_pipe(icmp_reg_reg);
10843 %}
10844 
10845 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
10846 %{
10847   match(Set cr (OverflowAddL op1 op2));
10848 
10849   format %{ "cmn   $op1, $op2\t# overflow check long" %}
10850   ins_cost(INSN_COST);
10851   ins_encode %{
10852     __ cmn($op1$$Register, $op2$$constant);
10853   %}
10854 
10855   ins_pipe(icmp_reg_imm);
10856 %}
10857 
10858 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
10859 %{
10860   match(Set cr (OverflowSubI op1 op2));
10861 
10862   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
10863   ins_cost(INSN_COST);
10864   ins_encode %{
10865     __ cmpw($op1$$Register, $op2$$Register);
10866   %}
10867 
10868   ins_pipe(icmp_reg_reg);
10869 %}
10870 
10871 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegI op1, immIAddSub op2)
10872 %{
10873   match(Set cr (OverflowSubI op1 op2));
10874 
10875   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
10876   ins_cost(INSN_COST);
10877   ins_encode %{
10878     __ cmpw($op1$$Register, $op2$$constant);
10879   %}
10880 
10881   ins_pipe(icmp_reg_imm);
10882 %}
10883 
10884 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
10885 %{
10886   match(Set cr (OverflowSubL op1 op2));
10887 
10888   format %{ "cmp   $op1, $op2\t# overflow check long" %}
10889   ins_cost(INSN_COST);
10890   ins_encode %{
10891     __ cmp($op1$$Register, $op2$$Register);
10892   %}
10893 
10894   ins_pipe(icmp_reg_reg);
10895 %}
10896 
10897 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
10898 %{
10899   match(Set cr (OverflowSubL op1 op2));
10900 
10901   format %{ "cmp   $op1, $op2\t# overflow check long" %}
10902   ins_cost(INSN_COST);
10903   ins_encode %{
10904     __ cmp($op1$$Register, $op2$$constant);
10905   %}
10906 
10907   ins_pipe(icmp_reg_imm);
10908 %}
10909 
10910 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegI op1)
10911 %{
10912   match(Set cr (OverflowSubI zero op1));
10913 
10914   format %{ "cmpw  zr, $op1\t# overflow check int" %}
10915   ins_cost(INSN_COST);
10916   ins_encode %{
10917     __ cmpw(zr, $op1$$Register);
10918   %}
10919 
10920   ins_pipe(icmp_reg_imm);
10921 %}
10922 
10923 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
10924 %{
10925   match(Set cr (OverflowSubL zero op1));
10926 
10927   format %{ "cmp   zr, $op1\t# overflow check long" %}
10928   ins_cost(INSN_COST);
10929   ins_encode %{
10930     __ cmp(zr, $op1$$Register);
10931   %}
10932 
10933   ins_pipe(icmp_reg_imm);
10934 %}
10935 
10936 instruct overflowMulI_reg(rFlagsReg cr, iRegI op1, iRegI op2)
10937 %{
10938   match(Set cr (OverflowMulI op1 op2));
10939 
10940   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
10941             "cmp   rscratch1, rscratch1, sxtw\n\t"
10942             "movw  rscratch1, #0x80000000\n\t"
10943             "cselw rscratch1, rscratch1, zr, NE\n\t"
10944             "cmpw  rscratch1, #1" %}
10945   ins_cost(5 * INSN_COST);
10946   ins_encode %{
10947     __ smull(rscratch1, $op1$$Register, $op2$$Register);
10948     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
10949     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
10950     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
10951     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
10952   %}
10953 
10954   ins_pipe(pipe_slow);
10955 %}
10956 
10957 instruct overflowMulI_reg_branch(cmpOp cmp, iRegI op1, iRegI op2, label labl, rFlagsReg cr)
10958 %{
10959   match(If cmp (OverflowMulI op1 op2));
10960   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
10961             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
10962   effect(USE labl, KILL cr);
10963 
10964   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
10965             "cmp   rscratch1, rscratch1, sxtw\n\t"
10966             "b$cmp   $labl" %}
10967   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
10968   ins_encode %{
10969     Label* L = $labl$$label;
10970     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10971     __ smull(rscratch1, $op1$$Register, $op2$$Register);
10972     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
10973     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
10974   %}
10975 
10976   ins_pipe(pipe_serial);
10977 %}
10978 
10979 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
10980 %{
10981   match(Set cr (OverflowMulL op1 op2));
10982 
10983   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
10984             "smulh rscratch2, $op1, $op2\n\t"
10985             "cmp   rscratch2, rscratch1, ASR #31\n\t"
10986             "movw  rscratch1, #0x80000000\n\t"
10987             "cselw rscratch1, rscratch1, zr, NE\n\t"
10988             "cmpw  rscratch1, #1" %}
10989   ins_cost(6 * INSN_COST);
10990   ins_encode %{
10991     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
10992     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
10993     __ cmp(rscratch2, rscratch1, Assembler::ASR, 31);    // Top is pure sign ext
10994     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
10995     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
10996     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
10997   %}
10998 
10999   ins_pipe(pipe_slow);
11000 %}
11001 
11002 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
11003 %{
11004   match(If cmp (OverflowMulL op1 op2));
11005   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
11006             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
11007   effect(USE labl, KILL cr);
11008 
11009   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
11010             "smulh rscratch2, $op1, $op2\n\t"
11011             "cmp   rscratch2, rscratch1, ASR #31\n\t"
11012             "b$cmp $labl" %}
11013   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
11014   ins_encode %{
11015     Label* L = $labl$$label;
11016     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
11017     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
11018     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
11019     __ cmp(rscratch2, rscratch1, Assembler::ASR, 31);    // Top is pure sign ext
11020     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
11021   %}
11022 
11023   ins_pipe(pipe_serial);
11024 %}
11025 
11026 // ============================================================================
11027 // Compare Instructions
11028 
11029 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
11030 %{
11031   match(Set cr (CmpI op1 op2));
11032 
11033   effect(DEF cr, USE op1, USE op2);
11034 
11035   ins_cost(INSN_COST);
11036   format %{ "cmpw  $op1, $op2" %}
11037 
11038   ins_encode(aarch64_enc_cmpw(op1, op2));
11039 
11040   ins_pipe(icmp_reg_reg);
11041 %}
11042 
11043 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
11044 %{
11045   match(Set cr (CmpI op1 zero));
11046 
11047   effect(DEF cr, USE op1);
11048 
11049   ins_cost(INSN_COST);
11050   format %{ "cmpw $op1, 0" %}
11051 
11052   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
11053 
11054   ins_pipe(icmp_reg_imm);
11055 %}
11056 
11057 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
11058 %{
11059   match(Set cr (CmpI op1 op2));
11060 
11061   effect(DEF cr, USE op1);
11062 
11063   ins_cost(INSN_COST);
11064   format %{ "cmpw  $op1, $op2" %}
11065 
11066   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
11067 
11068   ins_pipe(icmp_reg_imm);
11069 %}
11070 
11071 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
11072 %{
11073   match(Set cr (CmpI op1 op2));
11074 
11075   effect(DEF cr, USE op1);
11076 
11077   ins_cost(INSN_COST * 2);
11078   format %{ "cmpw  $op1, $op2" %}
11079 
11080   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
11081 
11082   ins_pipe(icmp_reg_imm);
11083 %}
11084 
11085 // Unsigned compare Instructions; really, same as signed compare
11086 // except it should only be used to feed an If or a CMovI which takes a
11087 // cmpOpU.
11088 
11089 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
11090 %{
11091   match(Set cr (CmpU op1 op2));
11092 
11093   effect(DEF cr, USE op1, USE op2);
11094 
11095   ins_cost(INSN_COST);
11096   format %{ "cmpw  $op1, $op2\t# unsigned" %}
11097 
11098   ins_encode(aarch64_enc_cmpw(op1, op2));
11099 
11100   ins_pipe(icmp_reg_reg);
11101 %}
11102 
11103 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
11104 %{
11105   match(Set cr (CmpU op1 zero));
11106 
11107   effect(DEF cr, USE op1);
11108 
11109   ins_cost(INSN_COST);
11110   format %{ "cmpw $op1, #0\t# unsigned" %}
11111 
11112   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
11113 
11114   ins_pipe(icmp_reg_imm);
11115 %}
11116 
11117 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
11118 %{
11119   match(Set cr (CmpU op1 op2));
11120 
11121   effect(DEF cr, USE op1);
11122 
11123   ins_cost(INSN_COST);
11124   format %{ "cmpw  $op1, $op2\t# unsigned" %}
11125 
11126   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
11127 
11128   ins_pipe(icmp_reg_imm);
11129 %}
11130 
11131 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
11132 %{
11133   match(Set cr (CmpU op1 op2));
11134 
11135   effect(DEF cr, USE op1);
11136 
11137   ins_cost(INSN_COST * 2);
11138   format %{ "cmpw  $op1, $op2\t# unsigned" %}
11139 
11140   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
11141 
11142   ins_pipe(icmp_reg_imm);
11143 %}
11144 
11145 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
11146 %{
11147   match(Set cr (CmpL op1 op2));
11148 
11149   effect(DEF cr, USE op1, USE op2);
11150 
11151   ins_cost(INSN_COST);
11152   format %{ "cmp  $op1, $op2" %}
11153 
11154   ins_encode(aarch64_enc_cmp(op1, op2));
11155 
11156   ins_pipe(icmp_reg_reg);
11157 %}
11158 
11159 instruct compL_reg_immI0(rFlagsReg cr, iRegL op1, immI0 zero)
11160 %{
11161   match(Set cr (CmpL op1 zero));
11162 
11163   effect(DEF cr, USE op1);
11164 
11165   ins_cost(INSN_COST);
11166   format %{ "tst  $op1" %}
11167 
11168   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
11169 
11170   ins_pipe(icmp_reg_imm);
11171 %}
11172 
11173 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
11174 %{
11175   match(Set cr (CmpL op1 op2));
11176 
11177   effect(DEF cr, USE op1);
11178 
11179   ins_cost(INSN_COST);
11180   format %{ "cmp  $op1, $op2" %}
11181 
11182   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
11183 
11184   ins_pipe(icmp_reg_imm);
11185 %}
11186 
11187 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
11188 %{
11189   match(Set cr (CmpL op1 op2));
11190 
11191   effect(DEF cr, USE op1);
11192 
11193   ins_cost(INSN_COST * 2);
11194   format %{ "cmp  $op1, $op2" %}
11195 
11196   ins_encode(aarch64_enc_cmp_imm(op1, op2));
11197 
11198   ins_pipe(icmp_reg_imm);
11199 %}
11200 
11201 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
11202 %{
11203   match(Set cr (CmpP op1 op2));
11204 
11205   effect(DEF cr, USE op1, USE op2);
11206 
11207   ins_cost(INSN_COST);
11208   format %{ "cmp  $op1, $op2\t // ptr" %}
11209 
11210   ins_encode(aarch64_enc_cmpp(op1, op2));
11211 
11212   ins_pipe(icmp_reg_reg);
11213 %}
11214 
11215 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
11216 %{
11217   match(Set cr (CmpN op1 op2));
11218 
11219   effect(DEF cr, USE op1, USE op2);
11220 
11221   ins_cost(INSN_COST);
11222   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
11223 
11224   ins_encode(aarch64_enc_cmpn(op1, op2));
11225 
11226   ins_pipe(icmp_reg_reg);
11227 %}
11228 
11229 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
11230 %{
11231   match(Set cr (CmpP op1 zero));
11232 
11233   effect(DEF cr, USE op1, USE zero);
11234 
11235   ins_cost(INSN_COST);
11236   format %{ "cmp  $op1, 0\t // ptr" %}
11237 
11238   ins_encode(aarch64_enc_testp(op1));
11239 
11240   ins_pipe(icmp_reg_imm);
11241 %}
11242 
11243 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
11244 %{
11245   match(Set cr (CmpN op1 zero));
11246 
11247   effect(DEF cr, USE op1, USE zero);
11248 
11249   ins_cost(INSN_COST);
11250   format %{ "cmp  $op1, 0\t // compressed ptr" %}
11251 
11252   ins_encode(aarch64_enc_testn(op1));
11253 
11254   ins_pipe(icmp_reg_imm);
11255 %}
11256 
11257 // FP comparisons
11258 //
11259 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
11260 // using normal cmpOp. See declaration of rFlagsReg for details.
11261 
11262 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
11263 %{
11264   match(Set cr (CmpF src1 src2));
11265 
11266   ins_cost(3 * INSN_COST);
11267   format %{ "fcmps $src1, $src2" %}
11268 
11269   ins_encode %{
11270     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
11271   %}
11272 
11273   ins_pipe(pipe_class_compare);
11274 %}
11275 
11276 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
11277 %{
11278   match(Set cr (CmpF src1 src2));
11279 
11280   ins_cost(3 * INSN_COST);
11281   format %{ "fcmps $src1, 0.0" %}
11282 
11283   ins_encode %{
11284     __ fcmps(as_FloatRegister($src1$$reg), 0.0D);
11285   %}
11286 
11287   ins_pipe(pipe_class_compare);
11288 %}
11289 // FROM HERE
11290 
11291 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
11292 %{
11293   match(Set cr (CmpD src1 src2));
11294 
11295   ins_cost(3 * INSN_COST);
11296   format %{ "fcmpd $src1, $src2" %}
11297 
11298   ins_encode %{
11299     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
11300   %}
11301 
11302   ins_pipe(pipe_class_compare);
11303 %}
11304 
11305 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
11306 %{
11307   match(Set cr (CmpD src1 src2));
11308 
11309   ins_cost(3 * INSN_COST);
11310   format %{ "fcmpd $src1, 0.0" %}
11311 
11312   ins_encode %{
11313     __ fcmpd(as_FloatRegister($src1$$reg), 0.0D);
11314   %}
11315 
11316   ins_pipe(pipe_class_compare);
11317 %}
11318 
11319 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
11320 %{
11321   match(Set dst (CmpF3 src1 src2));
11322   effect(KILL cr);
11323 
11324   ins_cost(5 * INSN_COST);
11325   format %{ "fcmps $src1, $src2\n\t"
11326             "csinvw($dst, zr, zr, eq\n\t"
11327             "csnegw($dst, $dst, $dst, lt)"
11328   %}
11329 
11330   ins_encode %{
11331     Label done;
11332     FloatRegister s1 = as_FloatRegister($src1$$reg);
11333     FloatRegister s2 = as_FloatRegister($src2$$reg);
11334     Register d = as_Register($dst$$reg);
11335     __ fcmps(s1, s2);
11336     // installs 0 if EQ else -1
11337     __ csinvw(d, zr, zr, Assembler::EQ);
11338     // keeps -1 if less or unordered else installs 1
11339     __ csnegw(d, d, d, Assembler::LT);
11340     __ bind(done);
11341   %}
11342 
11343   ins_pipe(pipe_class_default);
11344 
11345 %}
11346 
11347 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
11348 %{
11349   match(Set dst (CmpD3 src1 src2));
11350   effect(KILL cr);
11351 
11352   ins_cost(5 * INSN_COST);
11353   format %{ "fcmpd $src1, $src2\n\t"
11354             "csinvw($dst, zr, zr, eq\n\t"
11355             "csnegw($dst, $dst, $dst, lt)"
11356   %}
11357 
11358   ins_encode %{
11359     Label done;
11360     FloatRegister s1 = as_FloatRegister($src1$$reg);
11361     FloatRegister s2 = as_FloatRegister($src2$$reg);
11362     Register d = as_Register($dst$$reg);
11363     __ fcmpd(s1, s2);
11364     // installs 0 if EQ else -1
11365     __ csinvw(d, zr, zr, Assembler::EQ);
11366     // keeps -1 if less or unordered else installs 1
11367     __ csnegw(d, d, d, Assembler::LT);
11368     __ bind(done);
11369   %}
11370   ins_pipe(pipe_class_default);
11371 
11372 %}
11373 
11374 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
11375 %{
11376   match(Set dst (CmpF3 src1 zero));
11377   effect(KILL cr);
11378 
11379   ins_cost(5 * INSN_COST);
11380   format %{ "fcmps $src1, 0.0\n\t"
11381             "csinvw($dst, zr, zr, eq\n\t"
11382             "csnegw($dst, $dst, $dst, lt)"
11383   %}
11384 
11385   ins_encode %{
11386     Label done;
11387     FloatRegister s1 = as_FloatRegister($src1$$reg);
11388     Register d = as_Register($dst$$reg);
11389     __ fcmps(s1, 0.0D);
11390     // installs 0 if EQ else -1
11391     __ csinvw(d, zr, zr, Assembler::EQ);
11392     // keeps -1 if less or unordered else installs 1
11393     __ csnegw(d, d, d, Assembler::LT);
11394     __ bind(done);
11395   %}
11396 
11397   ins_pipe(pipe_class_default);
11398 
11399 %}
11400 
11401 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
11402 %{
11403   match(Set dst (CmpD3 src1 zero));
11404   effect(KILL cr);
11405 
11406   ins_cost(5 * INSN_COST);
11407   format %{ "fcmpd $src1, 0.0\n\t"
11408             "csinvw($dst, zr, zr, eq\n\t"
11409             "csnegw($dst, $dst, $dst, lt)"
11410   %}
11411 
11412   ins_encode %{
11413     Label done;
11414     FloatRegister s1 = as_FloatRegister($src1$$reg);
11415     Register d = as_Register($dst$$reg);
11416     __ fcmpd(s1, 0.0D);
11417     // installs 0 if EQ else -1
11418     __ csinvw(d, zr, zr, Assembler::EQ);
11419     // keeps -1 if less or unordered else installs 1
11420     __ csnegw(d, d, d, Assembler::LT);
11421     __ bind(done);
11422   %}
11423   ins_pipe(pipe_class_default);
11424 
11425 %}
11426 
11427 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegI p, iRegI q, rFlagsReg cr)
11428 %{
11429   match(Set dst (CmpLTMask p q));
11430   effect(KILL cr);
11431 
11432   ins_cost(3 * INSN_COST);
11433 
11434   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
11435             "csetw $dst, lt\n\t"
11436             "subw $dst, zr, $dst"
11437   %}
11438 
11439   ins_encode %{
11440     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
11441     __ csetw(as_Register($dst$$reg), Assembler::LT);
11442     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
11443   %}
11444 
11445   ins_pipe(ialu_reg_reg);
11446 %}
11447 
11448 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegI src, immI0 zero, rFlagsReg cr)
11449 %{
11450   match(Set dst (CmpLTMask src zero));
11451   effect(KILL cr);
11452 
11453   ins_cost(INSN_COST);
11454 
11455   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
11456 
11457   ins_encode %{
11458     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
11459   %}
11460 
11461   ins_pipe(ialu_reg_shift);
11462 %}
11463 
11464 // ============================================================================
11465 // Max and Min
11466 
11467 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
11468 %{
11469   match(Set dst (MinI src1 src2));
11470 
11471   effect(DEF dst, USE src1, USE src2, KILL cr);
11472   size(8);
11473 
11474   ins_cost(INSN_COST * 3);
11475   format %{
11476     "cmpw $src1 $src2\t signed int\n\t"
11477     "cselw $dst, $src1, $src2 lt\t"
11478   %}
11479 
11480   ins_encode %{
11481     __ cmpw(as_Register($src1$$reg),
11482             as_Register($src2$$reg));
11483     __ cselw(as_Register($dst$$reg),
11484              as_Register($src1$$reg),
11485              as_Register($src2$$reg),
11486              Assembler::LT);
11487   %}
11488 
11489   ins_pipe(ialu_reg_reg);
11490 %}
11491 // FROM HERE
11492 
11493 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
11494 %{
11495   match(Set dst (MaxI src1 src2));
11496 
11497   effect(DEF dst, USE src1, USE src2, KILL cr);
11498   size(8);
11499 
11500   ins_cost(INSN_COST * 3);
11501   format %{
11502     "cmpw $src1 $src2\t signed int\n\t"
11503     "cselw $dst, $src1, $src2 gt\t"
11504   %}
11505 
11506   ins_encode %{
11507     __ cmpw(as_Register($src1$$reg),
11508             as_Register($src2$$reg));
11509     __ cselw(as_Register($dst$$reg),
11510              as_Register($src1$$reg),
11511              as_Register($src2$$reg),
11512              Assembler::GT);
11513   %}
11514 
11515   ins_pipe(ialu_reg_reg);
11516 %}
11517 
11518 // ============================================================================
11519 // Branch Instructions
11520 
11521 // Direct Branch.
11522 instruct branch(label lbl)
11523 %{
11524   match(Goto);
11525 
11526   effect(USE lbl);
11527 
11528   ins_cost(BRANCH_COST);
11529   format %{ "b  $lbl" %}
11530 
11531   ins_encode(aarch64_enc_b(lbl));
11532 
11533   ins_pipe(pipe_branch);
11534 %}
11535 
11536 // Conditional Near Branch
11537 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
11538 %{
11539   // Same match rule as `branchConFar'.
11540   match(If cmp cr);
11541 
11542   effect(USE lbl);
11543 
11544   ins_cost(BRANCH_COST);
11545   // If set to 1 this indicates that the current instruction is a
11546   // short variant of a long branch. This avoids using this
11547   // instruction in first-pass matching. It will then only be used in
11548   // the `Shorten_branches' pass.
11549   // ins_short_branch(1);
11550   format %{ "b$cmp  $lbl" %}
11551 
11552   ins_encode(aarch64_enc_br_con(cmp, lbl));
11553 
11554   ins_pipe(pipe_branch_cond);
11555 %}
11556 
11557 // Conditional Near Branch Unsigned
11558 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
11559 %{
11560   // Same match rule as `branchConFar'.
11561   match(If cmp cr);
11562 
11563   effect(USE lbl);
11564 
11565   ins_cost(BRANCH_COST);
11566   // If set to 1 this indicates that the current instruction is a
11567   // short variant of a long branch. This avoids using this
11568   // instruction in first-pass matching. It will then only be used in
11569   // the `Shorten_branches' pass.
11570   // ins_short_branch(1);
11571   format %{ "b$cmp  $lbl\t# unsigned" %}
11572 
11573   ins_encode(aarch64_enc_br_conU(cmp, lbl));
11574 
11575   ins_pipe(pipe_branch_cond);
11576 %}
11577 
11578 // Make use of CBZ and CBNZ.  These instructions, as well as being
11579 // shorter than (cmp; branch), have the additional benefit of not
11580 // killing the flags.
11581 
11582 instruct cmpI_imm0_branch(cmpOp cmp, iRegI op1, immI0 op2, label labl, rFlagsReg cr) %{
11583   match(If cmp (CmpI op1 op2));
11584   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
11585             || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
11586   effect(USE labl);
11587 
11588   ins_cost(BRANCH_COST);
11589   format %{ "cbw$cmp   $op1, $labl" %}
11590   ins_encode %{
11591     Label* L = $labl$$label;
11592     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
11593     if (cond == Assembler::EQ)
11594       __ cbzw($op1$$Register, *L);
11595     else
11596       __ cbnzw($op1$$Register, *L);
11597   %}
11598   ins_pipe(pipe_cmp_branch);
11599 %}
11600 
11601 instruct cmpL_imm0_branch(cmpOp cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
11602   match(If cmp (CmpL op1 op2));
11603   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
11604             || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
11605   effect(USE labl);
11606 
11607   ins_cost(BRANCH_COST);
11608   format %{ "cb$cmp   $op1, $labl" %}
11609   ins_encode %{
11610     Label* L = $labl$$label;
11611     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
11612     if (cond == Assembler::EQ)
11613       __ cbz($op1$$Register, *L);
11614     else
11615       __ cbnz($op1$$Register, *L);
11616   %}
11617   ins_pipe(pipe_cmp_branch);
11618 %}
11619 
11620 instruct cmpP_imm0_branch(cmpOp cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
11621   match(If cmp (CmpP op1 op2));
11622   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
11623             || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
11624   effect(USE labl);
11625 
11626   ins_cost(BRANCH_COST);
11627   format %{ "cb$cmp   $op1, $labl" %}
11628   ins_encode %{
11629     Label* L = $labl$$label;
11630     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
11631     if (cond == Assembler::EQ)
11632       __ cbz($op1$$Register, *L);
11633     else
11634       __ cbnz($op1$$Register, *L);
11635   %}
11636   ins_pipe(pipe_cmp_branch);
11637 %}
11638 
11639 // Conditional Far Branch
11640 // Conditional Far Branch Unsigned
11641 // TODO: fixme
11642 
11643 // counted loop end branch near
11644 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
11645 %{
11646   match(CountedLoopEnd cmp cr);
11647 
11648   effect(USE lbl);
11649 
11650   ins_cost(BRANCH_COST);
11651   // short variant.
11652   // ins_short_branch(1);
11653   format %{ "b$cmp $lbl \t// counted loop end" %}
11654 
11655   ins_encode(aarch64_enc_br_con(cmp, lbl));
11656 
11657   ins_pipe(pipe_branch);
11658 %}
11659 
11660 // counted loop end branch near Unsigned
11661 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
11662 %{
11663   match(CountedLoopEnd cmp cr);
11664 
11665   effect(USE lbl);
11666 
11667   ins_cost(BRANCH_COST);
11668   // short variant.
11669   // ins_short_branch(1);
11670   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
11671 
11672   ins_encode(aarch64_enc_br_conU(cmp, lbl));
11673 
11674   ins_pipe(pipe_branch);
11675 %}
11676 
11677 // counted loop end branch far
11678 // counted loop end branch far unsigned
11679 // TODO: fixme
11680 
11681 // ============================================================================
11682 // inlined locking and unlocking
11683 
11684 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
11685 %{
11686   match(Set cr (FastLock object box));
11687   effect(TEMP tmp, TEMP tmp2);
11688 
11689   // TODO
11690   // identify correct cost
11691   ins_cost(5 * INSN_COST);
11692   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
11693 
11694   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
11695 
11696   ins_pipe(pipe_serial);
11697 %}
11698 
11699 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
11700 %{
11701   match(Set cr (FastUnlock object box));
11702   effect(TEMP tmp, TEMP tmp2);
11703 
11704   ins_cost(5 * INSN_COST);
11705   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
11706 
11707   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
11708 
11709   ins_pipe(pipe_serial);
11710 %}
11711 
11712 
11713 // ============================================================================
11714 // Safepoint Instructions
11715 
11716 // TODO
11717 // provide a near and far version of this code
11718 
11719 instruct safePoint(iRegP poll)
11720 %{
11721   match(SafePoint poll);
11722 
11723   format %{
11724     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
11725   %}
11726   ins_encode %{
11727     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
11728   %}
11729   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
11730 %}
11731 
11732 
11733 // ============================================================================
11734 // Procedure Call/Return Instructions
11735 
11736 // Call Java Static Instruction
11737 
11738 instruct CallStaticJavaDirect(method meth)
11739 %{
11740   match(CallStaticJava);
11741 
11742   effect(USE meth);
11743 
11744   predicate(!((CallStaticJavaNode*)n)->is_method_handle_invoke());
11745 
11746   ins_cost(CALL_COST);
11747 
11748   format %{ "call,static $meth \t// ==> " %}
11749 
11750   ins_encode( aarch64_enc_java_static_call(meth),
11751               aarch64_enc_call_epilog );
11752 
11753   ins_pipe(pipe_class_call);
11754 %}
11755 
11756 // TO HERE
11757 
11758 // Call Java Static Instruction (method handle version)
11759 
11760 instruct CallStaticJavaDirectHandle(method meth, iRegP_FP reg_mh_save)
11761 %{
11762   match(CallStaticJava);
11763 
11764   effect(USE meth);
11765 
11766   predicate(((CallStaticJavaNode*)n)->is_method_handle_invoke());
11767 
11768   ins_cost(CALL_COST);
11769 
11770   format %{ "call,static $meth \t// (methodhandle) ==> " %}
11771 
11772   ins_encode( aarch64_enc_java_handle_call(meth),
11773               aarch64_enc_call_epilog );
11774 
11775   ins_pipe(pipe_class_call);
11776 %}
11777 
11778 // Call Java Dynamic Instruction
11779 instruct CallDynamicJavaDirect(method meth)
11780 %{
11781   match(CallDynamicJava);
11782 
11783   effect(USE meth);
11784 
11785   ins_cost(CALL_COST);
11786 
11787   format %{ "CALL,dynamic $meth \t// ==> " %}
11788 
11789   ins_encode( aarch64_enc_java_dynamic_call(meth),
11790                aarch64_enc_call_epilog );
11791 
11792   ins_pipe(pipe_class_call);
11793 %}
11794 
11795 // Call Runtime Instruction
11796 
11797 instruct CallRuntimeDirect(method meth)
11798 %{
11799   match(CallRuntime);
11800 
11801   effect(USE meth);
11802 
11803   ins_cost(CALL_COST);
11804 
11805   format %{ "CALL, runtime $meth" %}
11806 
11807   ins_encode( aarch64_enc_java_to_runtime(meth) );
11808 
11809   ins_pipe(pipe_class_call);
11810 %}
11811 
11812 // Call Runtime Instruction
11813 
11814 instruct CallLeafDirect(method meth)
11815 %{
11816   match(CallLeaf);
11817 
11818   effect(USE meth);
11819 
11820   ins_cost(CALL_COST);
11821 
11822   format %{ "CALL, runtime leaf $meth" %}
11823 
11824   ins_encode( aarch64_enc_java_to_runtime(meth) );
11825 
11826   ins_pipe(pipe_class_call);
11827 %}
11828 
11829 // Call Runtime Instruction
11830 
11831 instruct CallLeafNoFPDirect(method meth)
11832 %{
11833   match(CallLeafNoFP);
11834 
11835   effect(USE meth);
11836 
11837   ins_cost(CALL_COST);
11838 
11839   format %{ "CALL, runtime leaf nofp $meth" %}
11840 
11841   ins_encode( aarch64_enc_java_to_runtime(meth) );
11842 
11843   ins_pipe(pipe_class_call);
11844 %}
11845 
11846 // Tail Call; Jump from runtime stub to Java code.
11847 // Also known as an 'interprocedural jump'.
11848 // Target of jump will eventually return to caller.
11849 // TailJump below removes the return address.
11850 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop)
11851 %{
11852   match(TailCall jump_target method_oop);
11853 
11854   ins_cost(CALL_COST);
11855 
11856   format %{ "br $jump_target\t# $method_oop holds method oop" %}
11857 
11858   ins_encode(aarch64_enc_tail_call(jump_target));
11859 
11860   ins_pipe(pipe_class_call);
11861 %}
11862 
11863 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
11864 %{
11865   match(TailJump jump_target ex_oop);
11866 
11867   ins_cost(CALL_COST);
11868 
11869   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
11870 
11871   ins_encode(aarch64_enc_tail_jmp(jump_target));
11872 
11873   ins_pipe(pipe_class_call);
11874 %}
11875 
11876 // Create exception oop: created by stack-crawling runtime code.
11877 // Created exception is now available to this handler, and is setup
11878 // just prior to jumping to this handler. No code emitted.
11879 // TODO check
11880 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
11881 instruct CreateException(iRegP_R0 ex_oop)
11882 %{
11883   match(Set ex_oop (CreateEx));
11884 
11885   format %{ " -- \t// exception oop; no code emitted" %}
11886 
11887   size(0);
11888 
11889   ins_encode( /*empty*/ );
11890 
11891   ins_pipe(pipe_class_empty);
11892 %}
11893 
11894 // Rethrow exception: The exception oop will come in the first
11895 // argument position. Then JUMP (not call) to the rethrow stub code.
11896 instruct RethrowException() %{
11897   match(Rethrow);
11898   ins_cost(CALL_COST);
11899 
11900   format %{ "b rethrow_stub" %}
11901 
11902   ins_encode( aarch64_enc_rethrow() );
11903 
11904   ins_pipe(pipe_class_call);
11905 %}
11906 
11907 
11908 // Return Instruction
11909 // epilog node loads ret address into lr as part of frame pop
11910 instruct Ret()
11911 %{
11912   match(Return);
11913 
11914   format %{ "ret\t// return register" %}
11915 
11916   ins_encode( aarch64_enc_ret() );
11917 
11918   ins_pipe(pipe_branch);
11919 %}
11920 
11921 // Die now.
11922 instruct ShouldNotReachHere() %{
11923   match(Halt);
11924 
11925   ins_cost(CALL_COST);
11926   format %{ "ShouldNotReachHere" %}
11927 
11928   ins_encode %{
11929     // TODO
11930     // implement proper trap call here
11931     __ brk(999);
11932   %}
11933 
11934   ins_pipe(pipe_class_default);
11935 %}
11936 
11937 // ============================================================================
11938 // Partial Subtype Check
11939 //
11940 // superklass array for an instance of the superklass.  Set a hidden
11941 // internal cache on a hit (cache is checked with exposed code in
11942 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11943 // encoding ALSO sets flags.
11944 
11945 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
11946 %{
11947   match(Set result (PartialSubtypeCheck sub super));
11948   effect(KILL cr, KILL temp);
11949 
11950   ins_cost(1100);  // slightly larger than the next version
11951   format %{ "partialSubtypeCheck $result, $sub, $super" %}
11952 
11953   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
11954 
11955   opcode(0x1); // Force zero of result reg on hit
11956 
11957   ins_pipe(pipe_class_memory);
11958 %}
11959 
11960 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
11961 %{
11962   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11963   effect(KILL temp, KILL result);
11964 
11965   ins_cost(1100);  // slightly larger than the next version
11966   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
11967 
11968   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
11969 
11970   opcode(0x0); // Don't zero result reg on hit
11971 
11972   ins_pipe(pipe_class_memory);
11973 %}
11974 
11975 instruct string_compare(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
11976                         iRegI_R0 result, iRegP_R10 tmp1, rFlagsReg cr)
11977 %{
11978   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11979   effect(KILL tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11980 
11981   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
11982   ins_encode %{
11983     __ string_compare($str1$$Register, $str2$$Register,
11984                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11985                       $tmp1$$Register);
11986   %}
11987   ins_pipe(pipe_class_memory);
11988 %}
11989 
11990 instruct string_indexof(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
11991        iRegI_R0 result, iRegI tmp1, iRegI tmp2, iRegI tmp3, iRegI tmp4, rFlagsReg cr)
11992 %{
11993   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11994   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
11995          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
11996   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result" %}
11997 
11998   ins_encode %{
11999     __ string_indexof($str1$$Register, $str2$$Register,
12000                       $cnt1$$Register, $cnt2$$Register,
12001                       $tmp1$$Register, $tmp2$$Register,
12002                       $tmp3$$Register, $tmp4$$Register,
12003                       -1, $result$$Register);
12004   %}
12005   ins_pipe(pipe_class_memory);
12006 %}
12007 
12008 instruct string_indexof_con(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
12009                  immI_le_4 int_cnt2, iRegI_R0 result, iRegI tmp1, iRegI tmp2,
12010                  iRegI tmp3, iRegI tmp4, rFlagsReg cr)
12011 %{
12012   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12013   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
12014          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
12015   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result" %}
12016 
12017   ins_encode %{
12018     int icnt2 = (int)$int_cnt2$$constant;
12019     __ string_indexof($str1$$Register, $str2$$Register,
12020                       $cnt1$$Register, zr,
12021                       $tmp1$$Register, $tmp2$$Register,
12022                       $tmp3$$Register, $tmp4$$Register,
12023                       icnt2, $result$$Register);
12024   %}
12025   ins_pipe(pipe_class_memory);
12026 %}
12027 
12028 instruct string_equals(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
12029                         iRegI_R0 result, iRegP_R10 tmp, rFlagsReg cr)
12030 %{
12031   match(Set result (StrEquals (Binary str1 str2) cnt));
12032   effect(KILL tmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
12033 
12034   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp" %}
12035   ins_encode %{
12036     __ string_equals($str1$$Register, $str2$$Register,
12037                       $cnt$$Register, $result$$Register,
12038                       $tmp$$Register);
12039   %}
12040   ins_pipe(pipe_class_memory);
12041 %}
12042 
12043 instruct array_equals(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
12044                       iRegP_R10 tmp, rFlagsReg cr)
12045 %{
12046   match(Set result (AryEq ary1 ary2));
12047   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, KILL cr);
12048 
12049   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
12050   ins_encode %{
12051     __ char_arrays_equals($ary1$$Register, $ary2$$Register,
12052                           $result$$Register, $tmp$$Register);
12053   %}
12054   ins_pipe(pipe_class_memory);
12055 %}
12056 
12057 // encode char[] to byte[] in ISO_8859_1
12058 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
12059                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
12060                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
12061                           iRegI_R0 result, rFlagsReg cr)
12062 %{
12063   match(Set result (EncodeISOArray src (Binary dst len)));
12064   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
12065          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
12066 
12067   format %{ "Encode array $src,$dst,$len -> $result" %}
12068   ins_encode %{
12069     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12070          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
12071          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
12072   %}
12073   ins_pipe( pipe_class_memory );
12074 %}
12075 
12076 // ============================================================================
12077 // This name is KNOWN by the ADLC and cannot be changed.
12078 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12079 // for this guy.
12080 instruct tlsLoadP(thread_RegP dst)
12081 %{
12082   match(Set dst (ThreadLocal));
12083 
12084   ins_cost(0);
12085 
12086   format %{ " -- \t// $dst=Thread::current(), empty" %}
12087 
12088   size(0);
12089 
12090   ins_encode( /*empty*/ );
12091 
12092   ins_pipe(pipe_class_empty);
12093 %}
12094 
12095 
12096 
12097 //----------PEEPHOLE RULES-----------------------------------------------------
12098 // These must follow all instruction definitions as they use the names
12099 // defined in the instructions definitions.
12100 //
12101 // peepmatch ( root_instr_name [preceding_instruction]* );
12102 //
12103 // peepconstraint %{
12104 // (instruction_number.operand_name relational_op instruction_number.operand_name
12105 //  [, ...] );
12106 // // instruction numbers are zero-based using left to right order in peepmatch
12107 //
12108 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12109 // // provide an instruction_number.operand_name for each operand that appears
12110 // // in the replacement instruction's match rule
12111 //
12112 // ---------VM FLAGS---------------------------------------------------------
12113 //
12114 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12115 //
12116 // Each peephole rule is given an identifying number starting with zero and
12117 // increasing by one in the order seen by the parser.  An individual peephole
12118 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12119 // on the command-line.
12120 //
12121 // ---------CURRENT LIMITATIONS----------------------------------------------
12122 //
12123 // Only match adjacent instructions in same basic block
12124 // Only equality constraints
12125 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12126 // Only one replacement instruction
12127 //
12128 // ---------EXAMPLE----------------------------------------------------------
12129 //
12130 // // pertinent parts of existing instructions in architecture description
12131 // instruct movI(iRegINoSp dst, iRegI src)
12132 // %{
12133 //   match(Set dst (CopyI src));
12134 // %}
12135 //
12136 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
12137 // %{
12138 //   match(Set dst (AddI dst src));
12139 //   effect(KILL cr);
12140 // %}
12141 //
12142 // // Change (inc mov) to lea
12143 // peephole %{
12144 //   // increment preceeded by register-register move
12145 //   peepmatch ( incI_iReg movI );
12146 //   // require that the destination register of the increment
12147 //   // match the destination register of the move
12148 //   peepconstraint ( 0.dst == 1.dst );
12149 //   // construct a replacement instruction that sets
12150 //   // the destination to ( move's source register + one )
12151 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
12152 // %}
12153 //
12154 
12155 // Implementation no longer uses movX instructions since
12156 // machine-independent system no longer uses CopyX nodes.
12157 //
12158 // peephole
12159 // %{
12160 //   peepmatch (incI_iReg movI);
12161 //   peepconstraint (0.dst == 1.dst);
12162 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
12163 // %}
12164 
12165 // peephole
12166 // %{
12167 //   peepmatch (decI_iReg movI);
12168 //   peepconstraint (0.dst == 1.dst);
12169 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
12170 // %}
12171 
12172 // peephole
12173 // %{
12174 //   peepmatch (addI_iReg_imm movI);
12175 //   peepconstraint (0.dst == 1.dst);
12176 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
12177 // %}
12178 
12179 // peephole
12180 // %{
12181 //   peepmatch (incL_iReg movL);
12182 //   peepconstraint (0.dst == 1.dst);
12183 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
12184 // %}
12185 
12186 // peephole
12187 // %{
12188 //   peepmatch (decL_iReg movL);
12189 //   peepconstraint (0.dst == 1.dst);
12190 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
12191 // %}
12192 
12193 // peephole
12194 // %{
12195 //   peepmatch (addL_iReg_imm movL);
12196 //   peepconstraint (0.dst == 1.dst);
12197 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
12198 // %}
12199 
12200 // peephole
12201 // %{
12202 //   peepmatch (addP_iReg_imm movP);
12203 //   peepconstraint (0.dst == 1.dst);
12204 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
12205 // %}
12206 
12207 // // Change load of spilled value to only a spill
12208 // instruct storeI(memory mem, iRegI src)
12209 // %{
12210 //   match(Set mem (StoreI mem src));
12211 // %}
12212 //
12213 // instruct loadI(iRegINoSp dst, memory mem)
12214 // %{
12215 //   match(Set dst (LoadI mem));
12216 // %}
12217 //
12218 
12219 //----------SMARTSPILL RULES---------------------------------------------------
12220 // These must follow all instruction definitions as they use the names
12221 // defined in the instructions definitions.
12222 
12223 // Local Variables:
12224 // mode: c++
12225 // End: