1 //
   2 // Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 // AArch64 Architecture Description File
  27 
  28 //----------REGISTER DEFINITION BLOCK------------------------------------------
  29 // This information is used by the matcher and the register allocator to
  30 // describe individual registers and classes of registers within the target
  31 // archtecture.
  32 
  33 register %{
  34 //----------Architecture Description Register Definitions----------------------
  35 // General Registers
  36 // "reg_def"  name ( register save type, C convention save type,
  37 //                   ideal register type, encoding );
  38 // Register Save Types:
  39 //
  40 // NS  = No-Save:       The register allocator assumes that these registers
  41 //                      can be used without saving upon entry to the method, &
  42 //                      that they do not need to be saved at call sites.
  43 //
  44 // SOC = Save-On-Call:  The register allocator assumes that these registers
  45 //                      can be used without saving upon entry to the method,
  46 //                      but that they must be saved at call sites.
  47 //
  48 // SOE = Save-On-Entry: The register allocator assumes that these registers
  49 //                      must be saved before using them upon entry to the
  50 //                      method, but they do not need to be saved at call
  51 //                      sites.
  52 //
  53 // AS  = Always-Save:   The register allocator assumes that these registers
  54 //                      must be saved before using them upon entry to the
  55 //                      method, & that they must be saved at call sites.
  56 //
  57 // Ideal Register Type is used to determine how to save & restore a
  58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  60 //
  61 // The encoding number is the actual bit-pattern placed into the opcodes.
  62 
  63 // We must define the 64 bit int registers in two 32 bit halves, the
  64 // real lower register and a virtual upper half register. upper halves
  65 // are used by the register allocator but are not actually supplied as
  66 // operands to memory ops.
  67 //
  68 // follow the C1 compiler in making registers
  69 //
  70 //   r0-r7,r10-r26 volatile (caller save)
  71 //   r27-r32 system (no save, no allocate)
  72 //   r8-r9 invisible to the allocator (so we can use them as scratch regs)
  73 //
  74 // as regards Java usage. we don't use any callee save registers
  75 // because this makes it difficult to de-optimise a frame (see comment
  76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
  77 //
  78 
  79 // General Registers
  80 
  81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
  82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
  83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
  84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
  85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
  86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
  87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
  88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
  89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
  90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
  91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
  92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
  93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
  94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
  95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
  96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
  97 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  98 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  99 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
 100 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 101 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
 102 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
 103 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
 104 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
 105 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
 106 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
 107 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
 108 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
 109 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
 110 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
 111 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
 112 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
 113 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()        );
 114 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
 115 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
 116 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
 117 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
 118 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
 119 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
 120 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
 121 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
 122 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
 123 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
 124 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
 125 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
 126 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
 127 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
 128 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
 129 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
 130 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
 131 reg_def R27     (  NS, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
 132 reg_def R27_H   (  NS, SOE, Op_RegI, 27, r27->as_VMReg()->next());
 133 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
 134 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
 135 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
 136 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
 137 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
 138 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
 139 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
 140 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
 141 
 142 // ----------------------------
 143 // Float/Double Registers
 144 // ----------------------------
 145 
 146 // Double Registers
 147 
 148 // The rules of ADL require that double registers be defined in pairs.
 149 // Each pair must be two 32-bit values, but not necessarily a pair of
 150 // single float registers. In each pair, ADLC-assigned register numbers
 151 // must be adjacent, with the lower number even. Finally, when the
 152 // CPU stores such a register pair to memory, the word associated with
 153 // the lower ADLC-assigned number must be stored to the lower address.
 154 
 155 // AArch64 has 32 floating-point registers. Each can store a vector of
 156 // single or double precision floating-point values up to 8 * 32
 157 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
 158 // use the first float or double element of the vector.
 159 
 160 // for Java use float registers v0-v15 are always save on call whereas
 161 // the platform ABI treats v8-v15 as callee save). float registers
 162 // v16-v31 are SOC as per the platform spec
 163 
 164   reg_def V0   ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()         );
 165   reg_def V0_H ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next() );
 166   reg_def V1   ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()         );
 167   reg_def V1_H ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next() );
 168   reg_def V2   ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()         );
 169   reg_def V2_H ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next() );
 170   reg_def V3   ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()         );
 171   reg_def V3_H ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next() );
 172   reg_def V4   ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()         );
 173   reg_def V4_H ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next() );
 174   reg_def V5   ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()         );
 175   reg_def V5_H ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next() );
 176   reg_def V6   ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()         );
 177   reg_def V6_H ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next() );
 178   reg_def V7   ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()         );
 179   reg_def V7_H ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next() );
 180   reg_def V8   ( SOC, SOE, Op_RegF,  8, v8->as_VMReg()         );
 181   reg_def V8_H ( SOC, SOE, Op_RegF,  8, v8->as_VMReg()->next() );
 182   reg_def V9   ( SOC, SOE, Op_RegF,  9, v9->as_VMReg()         );
 183   reg_def V9_H ( SOC, SOE, Op_RegF,  9, v9->as_VMReg()->next() );
 184   reg_def V10  ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()        );
 185   reg_def V10_H( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next());
 186   reg_def V11  ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()        );
 187   reg_def V11_H( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next());
 188   reg_def V12  ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()        );
 189   reg_def V12_H( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next());
 190   reg_def V13  ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()        );
 191   reg_def V13_H( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next());
 192   reg_def V14  ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()        );
 193   reg_def V14_H( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next());
 194   reg_def V15  ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()        );
 195   reg_def V15_H( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next());
 196   reg_def V16  ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()        );
 197   reg_def V16_H( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next());
 198   reg_def V17  ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()        );
 199   reg_def V17_H( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next());
 200   reg_def V18  ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()        );
 201   reg_def V18_H( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next());
 202   reg_def V19  ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()        );
 203   reg_def V19_H( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next());
 204   reg_def V20  ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()        );
 205   reg_def V20_H( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next());
 206   reg_def V21  ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()        );
 207   reg_def V21_H( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next());
 208   reg_def V22  ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()        );
 209   reg_def V22_H( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next());
 210   reg_def V23  ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()        );
 211   reg_def V23_H( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next());
 212   reg_def V24  ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()        );
 213   reg_def V24_H( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next());
 214   reg_def V25  ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()        );
 215   reg_def V25_H( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next());
 216   reg_def V26  ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()        );
 217   reg_def V26_H( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next());
 218   reg_def V27  ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()        );
 219   reg_def V27_H( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next());
 220   reg_def V28  ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()        );
 221   reg_def V28_H( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next());
 222   reg_def V29  ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()        );
 223   reg_def V29_H( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next());
 224   reg_def V30  ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()        );
 225   reg_def V30_H( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next());
 226   reg_def V31  ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()        );
 227   reg_def V31_H( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next());
 228 
 229 // ----------------------------
 230 // Special Registers
 231 // ----------------------------
 232 
 233 // the AArch64 CSPR status flag register is not directly acessible as
 234 // instruction operand. the FPSR status flag register is a system
 235 // register which can be written/read using MSR/MRS but again does not
 236 // appear as an operand (a code identifying the FSPR occurs as an
 237 // immediate value in the instruction).
 238 
 239 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
 240 
 241 
 242 // Specify priority of register selection within phases of register
 243 // allocation.  Highest priority is first.  A useful heuristic is to
 244 // give registers a low priority when they are required by machine
 245 // instructions, like EAX and EDX on I486, and choose no-save registers
 246 // before save-on-call, & save-on-call before save-on-entry.  Registers
 247 // which participate in fixed calling sequences should come last.
 248 // Registers which are used as pairs must fall on an even boundary.
 249 
 250 alloc_class chunk0(
 251     // volatiles
 252     R10, R10_H,
 253     R11, R11_H,
 254     R12, R12_H,
 255     R13, R13_H,
 256     R14, R14_H,
 257     R15, R15_H,
 258     R16, R16_H,
 259     R17, R17_H,
 260     R18, R18_H,
 261 
 262     // arg registers
 263     R0, R0_H,
 264     R1, R1_H,
 265     R2, R2_H,
 266     R3, R3_H,
 267     R4, R4_H,
 268     R5, R5_H,
 269     R6, R6_H,
 270     R7, R7_H,
 271 
 272     // non-volatiles
 273     R19, R19_H,
 274     R20, R20_H,
 275     R21, R21_H,
 276     R22, R22_H,
 277     R23, R23_H,
 278     R24, R24_H,
 279     R25, R25_H,
 280     R26, R26_H,
 281 
 282     // non-allocatable registers
 283 
 284     R27, R27_H, // heapbase
 285     R28, R28_H, // thread
 286     R29, R29_H, // fp
 287     R30, R30_H, // lr
 288     R31, R31_H, // sp
 289 );
 290 
 291 alloc_class chunk1(
 292 
 293     // no save
 294     V16, V16_H,
 295     V17, V17_H,
 296     V18, V18_H,
 297     V19, V19_H,
 298     V20, V20_H,
 299     V21, V21_H,
 300     V22, V22_H,
 301     V23, V23_H,
 302     V24, V24_H,
 303     V25, V25_H,
 304     V26, V26_H,
 305     V27, V27_H,
 306     V28, V28_H,
 307     V29, V29_H,
 308     V30, V30_H,
 309     V31, V31_H,
 310 
 311     // arg registers
 312     V0, V0_H,
 313     V1, V1_H,
 314     V2, V2_H,
 315     V3, V3_H,
 316     V4, V4_H,
 317     V5, V5_H,
 318     V6, V6_H,
 319     V7, V7_H,
 320 
 321     // non-volatiles
 322     V8, V8_H,
 323     V9, V9_H,
 324     V10, V10_H,
 325     V11, V11_H,
 326     V12, V12_H,
 327     V13, V13_H,
 328     V14, V14_H,
 329     V15, V15_H,
 330 );
 331 
 332 alloc_class chunk2(RFLAGS);
 333 
 334 //----------Architecture Description Register Classes--------------------------
 335 // Several register classes are automatically defined based upon information in
 336 // this architecture description.
 337 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 338 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 339 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 340 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 341 //
 342 
 343 // Class for all 32 bit integer registers -- excludes SP which will
 344 // never be used as an integer register
 345 reg_class any_reg32(
 346     R0,
 347     R1,
 348     R2,
 349     R3,
 350     R4,
 351     R5,
 352     R6,
 353     R7,
 354     R10,
 355     R11,
 356     R12,
 357     R13,
 358     R14,
 359     R15,
 360     R16,
 361     R17,
 362     R18,
 363     R19,
 364     R20,
 365     R21,
 366     R22,
 367     R23,
 368     R24,
 369     R25,
 370     R26,
 371     R27,
 372     R28,
 373     R29,
 374     R30
 375 );
 376 
 377 // Singleton class for R0 int register
 378 reg_class int_r0_reg(R0);
 379 
 380 // Singleton class for R2 int register
 381 reg_class int_r2_reg(R2);
 382 
 383 // Singleton class for R3 int register
 384 reg_class int_r3_reg(R3);
 385 
 386 // Singleton class for R4 int register
 387 reg_class int_r4_reg(R4);
 388 
 389 // Class for all long integer registers (including RSP)
 390 reg_class any_reg(
 391     R0, R0_H,
 392     R1, R1_H,
 393     R2, R2_H,
 394     R3, R3_H,
 395     R4, R4_H,
 396     R5, R5_H,
 397     R6, R6_H,
 398     R7, R7_H,
 399     R10, R10_H,
 400     R11, R11_H,
 401     R12, R12_H,
 402     R13, R13_H,
 403     R14, R14_H,
 404     R15, R15_H,
 405     R16, R16_H,
 406     R17, R17_H,
 407     R18, R18_H,
 408     R19, R19_H,
 409     R20, R20_H,
 410     R21, R21_H,
 411     R22, R22_H,
 412     R23, R23_H,
 413     R24, R24_H,
 414     R25, R25_H,
 415     R26, R26_H,
 416     R27, R27_H,
 417     R28, R28_H,
 418     R29, R29_H,
 419     R30, R30_H,
 420     R31, R31_H
 421 );
 422 
 423 // Class for all non-special integer registers
 424 reg_class no_special_reg32(
 425     R0,
 426     R1,
 427     R2,
 428     R3,
 429     R4,
 430     R5,
 431     R6,
 432     R7,
 433     R10,
 434     R11,
 435     R12,                        // rmethod
 436     R13,
 437     R14,
 438     R15,
 439     R16,
 440     R17,
 441     R18,
 442     R19,
 443     R20,
 444     R21,
 445     R22,
 446     R23,
 447     R24,
 448     R25,
 449     R26
 450  /* R27, */                     // heapbase
 451  /* R28, */                     // thread
 452  /* R29, */                     // fp
 453  /* R30, */                     // lr
 454  /* R31 */                      // sp
 455 );
 456 
 457 // Class for all non-special long integer registers
 458 reg_class no_special_reg(
 459     R0, R0_H,
 460     R1, R1_H,
 461     R2, R2_H,
 462     R3, R3_H,
 463     R4, R4_H,
 464     R5, R5_H,
 465     R6, R6_H,
 466     R7, R7_H,
 467     R10, R10_H,
 468     R11, R11_H,
 469     R12, R12_H,                 // rmethod
 470     R13, R13_H,
 471     R14, R14_H,
 472     R15, R15_H,
 473     R16, R16_H,
 474     R17, R17_H,
 475     R18, R18_H,
 476     R19, R19_H,
 477     R20, R20_H,
 478     R21, R21_H,
 479     R22, R22_H,
 480     R23, R23_H,
 481     R24, R24_H,
 482     R25, R25_H,
 483     R26, R26_H,
 484  /* R27, R27_H, */              // heapbase
 485  /* R28, R28_H, */              // thread
 486  /* R29, R29_H, */              // fp
 487  /* R30, R30_H, */              // lr
 488  /* R31, R31_H */               // sp
 489 );
 490 
 491 // Class for 64 bit register r0
 492 reg_class r0_reg(
 493     R0, R0_H
 494 );
 495 
 496 // Class for 64 bit register r1
 497 reg_class r1_reg(
 498     R1, R1_H
 499 );
 500 
 501 // Class for 64 bit register r2
 502 reg_class r2_reg(
 503     R2, R2_H
 504 );
 505 
 506 // Class for 64 bit register r3
 507 reg_class r3_reg(
 508     R3, R3_H
 509 );
 510 
 511 // Class for 64 bit register r4
 512 reg_class r4_reg(
 513     R4, R4_H
 514 );
 515 
 516 // Class for 64 bit register r5
 517 reg_class r5_reg(
 518     R5, R5_H
 519 );
 520 
 521 // Class for 64 bit register r10
 522 reg_class r10_reg(
 523     R10, R10_H
 524 );
 525 
 526 // Class for 64 bit register r11
 527 reg_class r11_reg(
 528     R11, R11_H
 529 );
 530 
 531 // Class for method register
 532 reg_class method_reg(
 533     R12, R12_H
 534 );
 535 
 536 // Class for heapbase register
 537 reg_class heapbase_reg(
 538     R27, R27_H
 539 );
 540 
 541 // Class for thread register
 542 reg_class thread_reg(
 543     R28, R28_H
 544 );
 545 
 546 // Class for frame pointer register
 547 reg_class fp_reg(
 548     R29, R29_H
 549 );
 550 
 551 // Class for link register
 552 reg_class lr_reg(
 553     R30, R30_H
 554 );
 555 
 556 // Class for long sp register
 557 reg_class sp_reg(
 558   R31, R31_H
 559 );
 560 
 561 // Class for all pointer registers
 562 reg_class ptr_reg(
 563     R0, R0_H,
 564     R1, R1_H,
 565     R2, R2_H,
 566     R3, R3_H,
 567     R4, R4_H,
 568     R5, R5_H,
 569     R6, R6_H,
 570     R7, R7_H,
 571     R10, R10_H,
 572     R11, R11_H,
 573     R12, R12_H,
 574     R13, R13_H,
 575     R14, R14_H,
 576     R15, R15_H,
 577     R16, R16_H,
 578     R17, R17_H,
 579     R18, R18_H,
 580     R19, R19_H,
 581     R20, R20_H,
 582     R21, R21_H,
 583     R22, R22_H,
 584     R23, R23_H,
 585     R24, R24_H,
 586     R25, R25_H,
 587     R26, R26_H,
 588     R27, R27_H,
 589     R28, R28_H,
 590     R29, R29_H,
 591     R30, R30_H,
 592     R31, R31_H
 593 );
 594 
 595 // Class for all non_special pointer registers
 596 reg_class no_special_ptr_reg(
 597     R0, R0_H,
 598     R1, R1_H,
 599     R2, R2_H,
 600     R3, R3_H,
 601     R4, R4_H,
 602     R5, R5_H,
 603     R6, R6_H,
 604     R7, R7_H,
 605     R10, R10_H,
 606     R11, R11_H,
 607     R12, R12_H,
 608     R13, R13_H,
 609     R14, R14_H,
 610     R15, R15_H,
 611     R16, R16_H,
 612     R17, R17_H,
 613     R18, R18_H,
 614     R19, R19_H,
 615     R20, R20_H,
 616     R21, R21_H,
 617     R22, R22_H,
 618     R23, R23_H,
 619     R24, R24_H,
 620     R25, R25_H,
 621     R26, R26_H,
 622  /* R27, R27_H, */              // heapbase
 623  /* R28, R28_H, */              // thread
 624  /* R29, R29_H, */              // fp
 625  /* R30, R30_H, */              // lr
 626  /* R31, R31_H */               // sp
 627 );
 628 
 629 // Class for all float registers
 630 reg_class float_reg(
 631     V0,
 632     V1,
 633     V2,
 634     V3,
 635     V4,
 636     V5,
 637     V6,
 638     V7,
 639     V8,
 640     V9,
 641     V10,
 642     V11,
 643     V12,
 644     V13,
 645     V14,
 646     V15,
 647     V16,
 648     V17,
 649     V18,
 650     V19,
 651     V20,
 652     V21,
 653     V22,
 654     V23,
 655     V24,
 656     V25,
 657     V26,
 658     V27,
 659     V28,
 660     V29,
 661     V30,
 662     V31
 663 );
 664 
 665 // Double precision float registers have virtual `high halves' that
 666 // are needed by the allocator.
 667 // Class for all double registers
 668 reg_class double_reg(
 669     V0, V0_H,
 670     V1, V1_H,
 671     V2, V2_H,
 672     V3, V3_H,
 673     V4, V4_H,
 674     V5, V5_H,
 675     V6, V6_H,
 676     V7, V7_H,
 677     V8, V8_H,
 678     V9, V9_H,
 679     V10, V10_H,
 680     V11, V11_H,
 681     V12, V12_H,
 682     V13, V13_H,
 683     V14, V14_H,
 684     V15, V15_H,
 685     V16, V16_H,
 686     V17, V17_H,
 687     V18, V18_H,
 688     V19, V19_H,
 689     V20, V20_H,
 690     V21, V21_H,
 691     V22, V22_H,
 692     V23, V23_H,
 693     V24, V24_H,
 694     V25, V25_H,
 695     V26, V26_H,
 696     V27, V27_H,
 697     V28, V28_H,
 698     V29, V29_H,
 699     V30, V30_H,
 700     V31, V31_H
 701 );
 702 
 703 // Class for 128 bit register v0
 704 reg_class v0_reg(
 705     V0, V0_H
 706 );
 707 
 708 // Class for 128 bit register v1
 709 reg_class v1_reg(
 710     V1, V1_H
 711 );
 712 
 713 // Class for 128 bit register v2
 714 reg_class v2_reg(
 715     V2, V2_H
 716 );
 717 
 718 // Class for 128 bit register v3
 719 reg_class v3_reg(
 720     V3, V3_H
 721 );
 722 
 723 // Singleton class for condition codes
 724 reg_class int_flags(RFLAGS);
 725 
 726 %}
 727 
 728 //----------DEFINITION BLOCK---------------------------------------------------
 729 // Define name --> value mappings to inform the ADLC of an integer valued name
 730 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 731 // Format:
 732 //        int_def  <name>         ( <int_value>, <expression>);
 733 // Generated Code in ad_<arch>.hpp
 734 //        #define  <name>   (<expression>)
 735 //        // value == <int_value>
 736 // Generated code in ad_<arch>.cpp adlc_verification()
 737 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 738 //
 739 
 740 // we follow the ppc-aix port in using a simple cost model which ranks
 741 // register operations as cheap, memory ops as more expensive and
 742 // branches as most expensive. the first two have a low as well as a
 743 // normal cost. huge cost appears to be a way of saying don't do
 744 // something
 745 
 746 definitions %{
 747   // The default cost (of a register move instruction).
 748   int_def INSN_COST            (    100,     100);
 749   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 750   int_def CALL_COST            (    200,     2 * INSN_COST);
 751   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 752 %}
 753 
 754 
 755 //----------SOURCE BLOCK-------------------------------------------------------
 756 // This is a block of C++ code which provides values, functions, and
 757 // definitions necessary in the rest of the architecture description
 758 
 759 source_hpp %{
 760 
 761 class CallStubImpl {
 762 
 763   //--------------------------------------------------------------
 764   //---<  Used for optimization in Compile::shorten_branches  >---
 765   //--------------------------------------------------------------
 766 
 767  public:
 768   // Size of call trampoline stub.
 769   static uint size_call_trampoline() {
 770     return 0; // no call trampolines on this platform
 771   }
 772 
 773   // number of relocations needed by a call trampoline stub
 774   static uint reloc_call_trampoline() {
 775     return 0; // no call trampolines on this platform
 776   }
 777 };
 778 
 779 class HandlerImpl {
 780 
 781  public:
 782 
 783   static int emit_exception_handler(CodeBuffer &cbuf);
 784   static int emit_deopt_handler(CodeBuffer& cbuf);
 785 
 786   static uint size_exception_handler() {
 787     return MacroAssembler::far_branch_size();
 788   }
 789 
 790   static uint size_deopt_handler() {
 791     // count one adr and one far branch instruction
 792     return 4 * NativeInstruction::instruction_size;
 793   }
 794 };
 795 
 796   bool preceded_by_ordered_load(const Node *barrier);
 797 
 798   // Use barrier instructions rather than load acquire / store
 799   // release.
 800   const bool UseBarriersForVolatile = true;
 801 %}
 802 
 803 source %{
 804 
 805   // AArch64 has load acquire and store release instructions which we
 806   // use for ordered memory accesses, e.g. for volatiles.  The ideal
 807   // graph generator also inserts memory barriers around volatile
 808   // accesses, and we don't want to generate both barriers and acq/rel
 809   // instructions.  So, when we emit a MemBarAcquire we look back in
 810   // the ideal graph for an ordered load and only emit the barrier if
 811   // we don't find one.
 812 
 813 bool preceded_by_ordered_load(const Node *barrier) {
 814   Node *x = barrier->lookup(TypeFunc::Parms);
 815 
 816   if (! x)
 817     return false;
 818 
 819   if (x->is_DecodeNarrowPtr())
 820     x = x->in(1);
 821 
 822   if (x->is_Load())
 823     return ! x->as_Load()->is_unordered();
 824 
 825   return false;
 826 }
 827 
 828 #define __ _masm.
 829 
 830 // advance declarations for helper functions to convert register
 831 // indices to register objects
 832 
 833 // the ad file has to provide implementations of certain methods
 834 // expected by the generic code
 835 //
 836 // REQUIRED FUNCTIONALITY
 837 
 838 //=============================================================================
 839 
 840 // !!!!! Special hack to get all types of calls to specify the byte offset
 841 //       from the start of the call to the point where the return address
 842 //       will point.
 843 
 844 int MachCallStaticJavaNode::ret_addr_offset()
 845 {
 846   // call should be a simple bl
 847   // unless this is a method handle invoke in which case it is
 848   // mov(rfp, sp), bl, mov(sp, rfp)
 849   int off = 4;
 850   if (_method_handle_invoke) {
 851     off += 4;
 852   }
 853   return off;
 854 }
 855 
 856 int MachCallDynamicJavaNode::ret_addr_offset()
 857 {
 858   return 16; // movz, movk, movk, bl
 859 }
 860 
 861 int MachCallRuntimeNode::ret_addr_offset() {
 862   // for generated stubs the call will be
 863   //   far_call(addr)
 864   // for real runtime callouts it will be six instructions
 865   // see aarch64_enc_java_to_runtime
 866   //   adr(rscratch2, retaddr)
 867   //   lea(rscratch1, RuntimeAddress(addr)
 868   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 869   //   blrt rscratch1
 870   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 871   if (cb) {
 872     return MacroAssembler::far_branch_size();
 873   } else {
 874     return 6 * NativeInstruction::instruction_size;
 875   }
 876 }
 877 
 878 // Indicate if the safepoint node needs the polling page as an input
 879 
 880 // the shared code plants the oop data at the start of the generated
 881 // code for the safepoint node and that needs ot be at the load
 882 // instruction itself. so we cannot plant a mov of the safepoint poll
 883 // address followed by a load. setting this to true means the mov is
 884 // scheduled as a prior instruction. that's better for scheduling
 885 // anyway.
 886 
 887 bool SafePointNode::needs_polling_address_input()
 888 {
 889   return true;
 890 }
 891 
 892 //=============================================================================
 893 
 894 #ifndef PRODUCT
 895 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 896   st->print("BREAKPOINT");
 897 }
 898 #endif
 899 
 900 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 901   MacroAssembler _masm(&cbuf);
 902   __ brk(0);
 903 }
 904 
 905 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 906   return MachNode::size(ra_);
 907 }
 908 
 909 //=============================================================================
 910 
 911 #ifndef PRODUCT
 912   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 913     st->print("nop \t# %d bytes pad for loops and calls", _count);
 914   }
 915 #endif
 916 
 917   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 918     MacroAssembler _masm(&cbuf);
 919     for (int i = 0; i < _count; i++) {
 920       __ nop();
 921     }
 922   }
 923 
 924   uint MachNopNode::size(PhaseRegAlloc*) const {
 925     return _count * NativeInstruction::instruction_size;
 926   }
 927 
 928 //=============================================================================
 929 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 930 
 931 int Compile::ConstantTable::calculate_table_base_offset() const {
 932   return 0;  // absolute addressing, no offset
 933 }
 934 
 935 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 936 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 937   ShouldNotReachHere();
 938 }
 939 
 940 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 941   // Empty encoding
 942 }
 943 
 944 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 945   return 0;
 946 }
 947 
 948 #ifndef PRODUCT
 949 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 950   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 951 }
 952 #endif
 953 
 954 #ifndef PRODUCT
 955 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 956   Compile* C = ra_->C;
 957 
 958   int framesize = C->frame_slots() << LogBytesPerInt;
 959 
 960   if (C->need_stack_bang(framesize))
 961     st->print("# stack bang size=%d\n\t", framesize);
 962 
 963   if (framesize == 0) {
 964     // Is this even possible?
 965     st->print("stp  lr, rfp, [sp, #%d]!", -(2 * wordSize));
 966   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 967     st->print("sub  sp, sp, #%d\n\t", framesize);
 968     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 969   } else {
 970     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 971     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 972     st->print("sub  sp, sp, rscratch1");
 973   }
 974 }
 975 #endif
 976 
 977 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 978   Compile* C = ra_->C;
 979   MacroAssembler _masm(&cbuf);
 980 
 981   // n.b. frame size includes space for return pc and rfp
 982   const long framesize = C->frame_size_in_bytes();
 983   assert(framesize%(2*wordSize) == 0, "must preserve 2*wordSize alignment");
 984 
 985   // insert a nop at the start of the prolog so we can patch in a
 986   // branch if we need to invalidate the method later
 987   __ nop();
 988 
 989   int bangsize = C->bang_size_in_bytes();
 990   if (C->need_stack_bang(bangsize) && UseStackBanging)
 991     __ generate_stack_overflow_check(bangsize);
 992 
 993   __ build_frame(framesize);
 994 
 995   if (NotifySimulator) {
 996     __ notify(Assembler::method_entry);
 997   }
 998 
 999   if (VerifyStackAtCalls) {
1000     Unimplemented();
1001   }
1002 
1003   C->set_frame_complete(cbuf.insts_size());
1004 
1005   if (C->has_mach_constant_base_node()) {
1006     // NOTE: We set the table base offset here because users might be
1007     // emitted before MachConstantBaseNode.
1008     Compile::ConstantTable& constant_table = C->constant_table();
1009     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1010   }
1011 }
1012 
1013 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1014 {
1015   return MachNode::size(ra_); // too many variables; just compute it
1016                               // the hard way
1017 }
1018 
1019 int MachPrologNode::reloc() const
1020 {
1021   return 0;
1022 }
1023 
1024 //=============================================================================
1025 
1026 #ifndef PRODUCT
1027 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1028   Compile* C = ra_->C;
1029   int framesize = C->frame_slots() << LogBytesPerInt;
1030 
1031   st->print("# pop frame %d\n\t",framesize);
1032 
1033   if (framesize == 0) {
1034     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1035   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1036     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1037     st->print("add  sp, sp, #%d\n\t", framesize);
1038   } else {
1039     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1040     st->print("add  sp, sp, rscratch1\n\t");
1041     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1042   }
1043 
1044   if (do_polling() && C->is_method_compilation()) {
1045     st->print("# touch polling page\n\t");
1046     st->print("mov  rscratch1, #0x%lx\n\t", p2i(os::get_polling_page()));
1047     st->print("ldr zr, [rscratch1]");
1048   }
1049 }
1050 #endif
1051 
1052 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1053   Compile* C = ra_->C;
1054   MacroAssembler _masm(&cbuf);
1055   int framesize = C->frame_slots() << LogBytesPerInt;
1056 
1057   __ remove_frame(framesize);
1058 
1059   if (NotifySimulator) {
1060     __ notify(Assembler::method_reentry);
1061   }
1062 
1063   if (do_polling() && C->is_method_compilation()) {
1064     __ read_polling_page(rscratch1, os::get_polling_page(), relocInfo::poll_return_type);
1065   }
1066 }
1067 
1068 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1069   // Variable size. Determine dynamically.
1070   return MachNode::size(ra_);
1071 }
1072 
1073 int MachEpilogNode::reloc() const {
1074   // Return number of relocatable values contained in this instruction.
1075   return 1; // 1 for polling page.
1076 }
1077 
1078 const Pipeline * MachEpilogNode::pipeline() const {
1079   return MachNode::pipeline_class();
1080 }
1081 
1082 // This method seems to be obsolete. It is declared in machnode.hpp
1083 // and defined in all *.ad files, but it is never called. Should we
1084 // get rid of it?
1085 int MachEpilogNode::safepoint_offset() const {
1086   assert(do_polling(), "no return for this epilog node");
1087   return 4;
1088 }
1089 
1090 //=============================================================================
1091 
1092 // Figure out which register class each belongs in: rc_int, rc_float or
1093 // rc_stack.
1094 enum RC { rc_bad, rc_int, rc_float, rc_stack };
1095 
1096 static enum RC rc_class(OptoReg::Name reg) {
1097 
1098   if (reg == OptoReg::Bad) {
1099     return rc_bad;
1100   }
1101 
1102   // we have 30 int registers * 2 halves
1103   // (rscratch1 and rscratch2 are omitted)
1104 
1105   if (reg < 60) {
1106     return rc_int;
1107   }
1108 
1109   // we have 32 float register * 2 halves
1110   if (reg < 60 + 64) {
1111     return rc_float;
1112   }
1113 
1114   // Between float regs & stack is the flags regs.
1115   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1116 
1117   return rc_stack;
1118 }
1119 
1120 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1121   Compile* C = ra_->C;
1122 
1123   // Get registers to move.
1124   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1125   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1126   OptoReg::Name dst_hi = ra_->get_reg_second(this);
1127   OptoReg::Name dst_lo = ra_->get_reg_first(this);
1128 
1129   enum RC src_hi_rc = rc_class(src_hi);
1130   enum RC src_lo_rc = rc_class(src_lo);
1131   enum RC dst_hi_rc = rc_class(dst_hi);
1132   enum RC dst_lo_rc = rc_class(dst_lo);
1133 
1134   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1135 
1136   if (src_hi != OptoReg::Bad) {
1137     assert((src_lo&1)==0 && src_lo+1==src_hi &&
1138            (dst_lo&1)==0 && dst_lo+1==dst_hi,
1139            "expected aligned-adjacent pairs");
1140   }
1141 
1142   if (src_lo == dst_lo && src_hi == dst_hi) {
1143     return 0;            // Self copy, no move.
1144   }
1145 
1146   switch (src_lo_rc) {
1147   case rc_int:
1148     if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
1149       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1150           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1151           // 64 bit
1152         if (cbuf) {
1153           MacroAssembler _masm(cbuf);
1154           __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1155                  as_Register(Matcher::_regEncode[src_lo]));
1156         } else if (st) {
1157           st->print("mov  %s, %s\t# shuffle",
1158                     Matcher::regName[dst_lo],
1159                     Matcher::regName[src_lo]);
1160         }
1161       } else {
1162         // 32 bit
1163         if (cbuf) {
1164           MacroAssembler _masm(cbuf);
1165           __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1166                   as_Register(Matcher::_regEncode[src_lo]));
1167         } else if (st) {
1168           st->print("movw  %s, %s\t# shuffle",
1169                     Matcher::regName[dst_lo],
1170                     Matcher::regName[src_lo]);
1171         }
1172       }
1173     } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1174       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1175           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1176           // 64 bit
1177         if (cbuf) {
1178           MacroAssembler _masm(cbuf);
1179           __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1180                    as_Register(Matcher::_regEncode[src_lo]));
1181         } else if (st) {
1182           st->print("fmovd  %s, %s\t# shuffle",
1183                     Matcher::regName[dst_lo],
1184                     Matcher::regName[src_lo]);
1185         }
1186       } else {
1187         // 32 bit
1188         if (cbuf) {
1189           MacroAssembler _masm(cbuf);
1190           __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1191                    as_Register(Matcher::_regEncode[src_lo]));
1192         } else if (st) {
1193           st->print("fmovs  %s, %s\t# shuffle",
1194                     Matcher::regName[dst_lo],
1195                     Matcher::regName[src_lo]);
1196         }
1197       }
1198     } else {                    // gpr --> stack spill
1199       assert(dst_lo_rc == rc_stack, "spill to bad register class");
1200       int dst_offset = ra_->reg2offset(dst_lo);
1201       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1202           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1203           // 64 bit
1204         if (cbuf) {
1205           MacroAssembler _masm(cbuf);
1206           __ str(as_Register(Matcher::_regEncode[src_lo]),
1207                  Address(sp, dst_offset));
1208         } else if (st) {
1209           st->print("str  %s, [sp, #%d]\t# spill",
1210                     Matcher::regName[src_lo],
1211                     dst_offset);
1212         }
1213       } else {
1214         // 32 bit
1215         if (cbuf) {
1216           MacroAssembler _masm(cbuf);
1217           __ strw(as_Register(Matcher::_regEncode[src_lo]),
1218                  Address(sp, dst_offset));
1219         } else if (st) {
1220           st->print("strw  %s, [sp, #%d]\t# spill",
1221                     Matcher::regName[src_lo],
1222                     dst_offset);
1223         }
1224       }
1225     }
1226     return 4;
1227   case rc_float:
1228     if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
1229       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1230           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1231           // 64 bit
1232         if (cbuf) {
1233           MacroAssembler _masm(cbuf);
1234           __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
1235                    as_FloatRegister(Matcher::_regEncode[src_lo]));
1236         } else if (st) {
1237           st->print("fmovd  %s, %s\t# shuffle",
1238                     Matcher::regName[dst_lo],
1239                     Matcher::regName[src_lo]);
1240         }
1241       } else {
1242         // 32 bit
1243         if (cbuf) {
1244           MacroAssembler _masm(cbuf);
1245           __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
1246                    as_FloatRegister(Matcher::_regEncode[src_lo]));
1247         } else if (st) {
1248           st->print("fmovs  %s, %s\t# shuffle",
1249                     Matcher::regName[dst_lo],
1250                     Matcher::regName[src_lo]);
1251         }
1252       }
1253     } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
1254       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1255           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1256           // 64 bit
1257         if (cbuf) {
1258           MacroAssembler _masm(cbuf);
1259           __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1260                    as_FloatRegister(Matcher::_regEncode[src_lo]));
1261         } else if (st) {
1262           st->print("fmovd  %s, %s\t# shuffle",
1263                     Matcher::regName[dst_lo],
1264                     Matcher::regName[src_lo]);
1265         }
1266       } else {
1267         // 32 bit
1268         if (cbuf) {
1269           MacroAssembler _masm(cbuf);
1270           __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1271                    as_FloatRegister(Matcher::_regEncode[src_lo]));
1272         } else if (st) {
1273           st->print("fmovs  %s, %s\t# shuffle",
1274                     Matcher::regName[dst_lo],
1275                     Matcher::regName[src_lo]);
1276         }
1277       }
1278     } else {                    // fpr --> stack spill
1279       assert(dst_lo_rc == rc_stack, "spill to bad register class");
1280       int dst_offset = ra_->reg2offset(dst_lo);
1281       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1282           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1283           // 64 bit
1284         if (cbuf) {
1285           MacroAssembler _masm(cbuf);
1286           __ strd(as_FloatRegister(Matcher::_regEncode[src_lo]),
1287                  Address(sp, dst_offset));
1288         } else if (st) {
1289           st->print("strd  %s, [sp, #%d]\t# spill",
1290                     Matcher::regName[src_lo],
1291                     dst_offset);
1292         }
1293       } else {
1294         // 32 bit
1295         if (cbuf) {
1296           MacroAssembler _masm(cbuf);
1297           __ strs(as_FloatRegister(Matcher::_regEncode[src_lo]),
1298                  Address(sp, dst_offset));
1299         } else if (st) {
1300           st->print("strs  %s, [sp, #%d]\t# spill",
1301                     Matcher::regName[src_lo],
1302                     dst_offset);
1303         }
1304       }
1305     }
1306     return 4;
1307   case rc_stack:
1308     int src_offset = ra_->reg2offset(src_lo);
1309     if (dst_lo_rc == rc_int) {  // stack --> gpr load
1310       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1311           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1312           // 64 bit
1313         if (cbuf) {
1314           MacroAssembler _masm(cbuf);
1315           __ ldr(as_Register(Matcher::_regEncode[dst_lo]),
1316                  Address(sp, src_offset));
1317         } else if (st) {
1318           st->print("ldr  %s, [sp, %d]\t# restore",
1319                     Matcher::regName[dst_lo],
1320                     src_offset);
1321         }
1322       } else {
1323         // 32 bit
1324         if (cbuf) {
1325           MacroAssembler _masm(cbuf);
1326           __ ldrw(as_Register(Matcher::_regEncode[dst_lo]),
1327                   Address(sp, src_offset));
1328         } else if (st) {
1329           st->print("ldr  %s, [sp, %d]\t# restore",
1330                     Matcher::regName[dst_lo],
1331                    src_offset);
1332         }
1333       }
1334       return 4;
1335     } else if (dst_lo_rc == rc_float) { // stack --> fpr load
1336       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1337           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1338           // 64 bit
1339         if (cbuf) {
1340           MacroAssembler _masm(cbuf);
1341           __ ldrd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1342                  Address(sp, src_offset));
1343         } else if (st) {
1344           st->print("ldrd  %s, [sp, %d]\t# restore",
1345                     Matcher::regName[dst_lo],
1346                     src_offset);
1347         }
1348       } else {
1349         // 32 bit
1350         if (cbuf) {
1351           MacroAssembler _masm(cbuf);
1352           __ ldrs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1353                   Address(sp, src_offset));
1354         } else if (st) {
1355           st->print("ldrs  %s, [sp, %d]\t# restore",
1356                     Matcher::regName[dst_lo],
1357                    src_offset);
1358         }
1359       }
1360       return 4;
1361     } else {                    // stack --> stack copy
1362       assert(dst_lo_rc == rc_stack, "spill to bad register class");
1363       int dst_offset = ra_->reg2offset(dst_lo);
1364       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1365           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1366           // 64 bit
1367         if (cbuf) {
1368           MacroAssembler _masm(cbuf);
1369           __ ldr(rscratch1, Address(sp, src_offset));
1370           __ str(rscratch1, Address(sp, dst_offset));
1371         } else if (st) {
1372           st->print("ldr  rscratch1, [sp, %d]\t# mem-mem spill",
1373                     src_offset);
1374           st->print("\n\t");
1375           st->print("str  rscratch1, [sp, %d]",
1376                     dst_offset);
1377         }
1378       } else {
1379         // 32 bit
1380         if (cbuf) {
1381           MacroAssembler _masm(cbuf);
1382           __ ldrw(rscratch1, Address(sp, src_offset));
1383           __ strw(rscratch1, Address(sp, dst_offset));
1384         } else if (st) {
1385           st->print("ldrw  rscratch1, [sp, %d]\t# mem-mem spill",
1386                     src_offset);
1387           st->print("\n\t");
1388           st->print("strw  rscratch1, [sp, %d]",
1389                     dst_offset);
1390         }
1391       }
1392       return 8;
1393     }
1394   }
1395 
1396   assert(false," bad rc_class for spill ");
1397   Unimplemented();
1398   return 0;
1399 
1400 }
1401 
1402 #ifndef PRODUCT
1403 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1404   if (!ra_)
1405     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
1406   else
1407     implementation(NULL, ra_, false, st);
1408 }
1409 #endif
1410 
1411 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1412   implementation(&cbuf, ra_, false, NULL);
1413 }
1414 
1415 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1416   return implementation(NULL, ra_, true, NULL);
1417 }
1418 
1419 //=============================================================================
1420 
1421 #ifndef PRODUCT
1422 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1423   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1424   int reg = ra_->get_reg_first(this);
1425   st->print("add %s, rsp, #%d]\t# box lock",
1426             Matcher::regName[reg], offset);
1427 }
1428 #endif
1429 
1430 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1431   MacroAssembler _masm(&cbuf);
1432 
1433   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1434   int reg    = ra_->get_encode(this);
1435 
1436   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
1437     __ add(as_Register(reg), sp, offset);
1438   } else {
1439     ShouldNotReachHere();
1440   }
1441 }
1442 
1443 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
1444   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
1445   return 4;
1446 }
1447 
1448 //=============================================================================
1449 
1450 #ifndef PRODUCT
1451 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1452 {
1453   st->print_cr("# MachUEPNode");
1454   if (UseCompressedClassPointers) {
1455     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1456     if (Universe::narrow_klass_shift() != 0) {
1457       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1458     }
1459   } else {
1460    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1461   }
1462   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
1463   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
1464 }
1465 #endif
1466 
1467 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1468 {
1469   // This is the unverified entry point.
1470   MacroAssembler _masm(&cbuf);
1471 
1472   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
1473   Label skip;
1474   // TODO
1475   // can we avoid this skip and still use a reloc?
1476   __ br(Assembler::EQ, skip);
1477   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1478   __ bind(skip);
1479 }
1480 
1481 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1482 {
1483   return MachNode::size(ra_);
1484 }
1485 
1486 // REQUIRED EMIT CODE
1487 
1488 //=============================================================================
1489 
1490 // Emit exception handler code.
1491 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
1492 {
1493   // mov rscratch1 #exception_blob_entry_point
1494   // br rscratch1
1495   // Note that the code buffer's insts_mark is always relative to insts.
1496   // That's why we must use the macroassembler to generate a handler.
1497   MacroAssembler _masm(&cbuf);
1498   address base =
1499   __ start_a_stub(size_exception_handler());
1500   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1501   int offset = __ offset();
1502   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
1503   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1504   __ end_a_stub();
1505   return offset;
1506 }
1507 
1508 // Emit deopt handler code.
1509 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
1510 {
1511   // Note that the code buffer's insts_mark is always relative to insts.
1512   // That's why we must use the macroassembler to generate a handler.
1513   MacroAssembler _masm(&cbuf);
1514   address base =
1515   __ start_a_stub(size_deopt_handler());
1516   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1517   int offset = __ offset();
1518 
1519   __ adr(lr, __ pc());
1520   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
1521 
1522   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1523   __ end_a_stub();
1524   return offset;
1525 }
1526 
1527 // REQUIRED MATCHER CODE
1528 
1529 //=============================================================================
1530 
1531 const bool Matcher::match_rule_supported(int opcode) {
1532 
1533   // TODO
1534   // identify extra cases that we might want to provide match rules for
1535   // e.g. Op_StrEquals and other intrinsics
1536   if (!has_match_rule(opcode)) {
1537     return false;
1538   }
1539 
1540   return true;  // Per default match rules are supported.
1541 }
1542 
1543 int Matcher::regnum_to_fpu_offset(int regnum)
1544 {
1545   Unimplemented();
1546   return 0;
1547 }
1548 
1549 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset)
1550 {
1551   Unimplemented();
1552   return false;
1553 }
1554 
1555 const bool Matcher::isSimpleConstant64(jlong value) {
1556   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1557   // Probably always true, even if a temp register is required.
1558   return true;
1559 }
1560 
1561 // true just means we have fast l2f conversion
1562 const bool Matcher::convL2FSupported(void) {
1563   return true;
1564 }
1565 
1566 // Vector width in bytes.
1567 const int Matcher::vector_width_in_bytes(BasicType bt) {
1568   // TODO fixme
1569   return 0;
1570 }
1571 
1572 // Limits on vector size (number of elements) loaded into vector.
1573 const int Matcher::max_vector_size(const BasicType bt) {
1574   return vector_width_in_bytes(bt)/type2aelembytes(bt);
1575 }
1576 const int Matcher::min_vector_size(const BasicType bt) {
1577   int max_size = max_vector_size(bt);
1578   // Min size which can be loaded into vector is 4 bytes.
1579   int size = (type2aelembytes(bt) == 1) ? 4 : 2;
1580   return MIN2(size,max_size);
1581 }
1582 
1583 // Vector ideal reg.
1584 const int Matcher::vector_ideal_reg(int len) {
1585   // TODO fixme
1586   return Op_RegD;
1587 }
1588 
1589 // Only lowest bits of xmm reg are used for vector shift count.
1590 const int Matcher::vector_shift_count_ideal_reg(int size) {
1591   // TODO fixme
1592   return Op_RegL;
1593 }
1594 
1595 // AES support not yet implemented
1596 const bool Matcher::pass_original_key_for_aes() {
1597   return false;
1598 }
1599 
1600 // x86 supports misaligned vectors store/load.
1601 const bool Matcher::misaligned_vectors_ok() {
1602   // TODO fixme
1603   // return !AlignVector; // can be changed by flag
1604   return false;
1605 }
1606 
1607 // false => size gets scaled to BytesPerLong, ok.
1608 const bool Matcher::init_array_count_is_in_bytes = false;
1609 
1610 // Threshold size for cleararray.
1611 const int Matcher::init_array_short_size = 18 * BytesPerLong;
1612 
1613 // Use conditional move (CMOVL)
1614 const int Matcher::long_cmove_cost() {
1615   // long cmoves are no more expensive than int cmoves
1616   return 0;
1617 }
1618 
1619 const int Matcher::float_cmove_cost() {
1620   // float cmoves are no more expensive than int cmoves
1621   return 0;
1622 }
1623 
1624 // Does the CPU require late expand (see block.cpp for description of late expand)?
1625 const bool Matcher::require_postalloc_expand = false;
1626 
1627 // Should the Matcher clone shifts on addressing modes, expecting them
1628 // to be subsumed into complex addressing expressions or compute them
1629 // into registers?  True for Intel but false for most RISCs
1630 const bool Matcher::clone_shift_expressions = false;
1631 
1632 // Do we need to mask the count passed to shift instructions or does
1633 // the cpu only look at the lower 5/6 bits anyway?
1634 const bool Matcher::need_masked_shift_count = false;
1635 
1636 // This affects two different things:
1637 //  - how Decode nodes are matched
1638 //  - how ImplicitNullCheck opportunities are recognized
1639 // If true, the matcher will try to remove all Decodes and match them
1640 // (as operands) into nodes. NullChecks are not prepared to deal with
1641 // Decodes by final_graph_reshaping().
1642 // If false, final_graph_reshaping() forces the decode behind the Cmp
1643 // for a NullCheck. The matcher matches the Decode node into a register.
1644 // Implicit_null_check optimization moves the Decode along with the
1645 // memory operation back up before the NullCheck.
1646 bool Matcher::narrow_oop_use_complex_address() {
1647   return Universe::narrow_oop_shift() == 0;
1648 }
1649 
1650 bool Matcher::narrow_klass_use_complex_address() {
1651 // TODO
1652 // decide whether we need to set this to true
1653   return false;
1654 }
1655 
1656 // Is it better to copy float constants, or load them directly from
1657 // memory?  Intel can load a float constant from a direct address,
1658 // requiring no extra registers.  Most RISCs will have to materialize
1659 // an address into a register first, so they would do better to copy
1660 // the constant from stack.
1661 const bool Matcher::rematerialize_float_constants = false;
1662 
1663 // If CPU can load and store mis-aligned doubles directly then no
1664 // fixup is needed.  Else we split the double into 2 integer pieces
1665 // and move it piece-by-piece.  Only happens when passing doubles into
1666 // C code as the Java calling convention forces doubles to be aligned.
1667 const bool Matcher::misaligned_doubles_ok = true;
1668 
1669 // No-op on amd64
1670 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
1671   Unimplemented();
1672 }
1673 
1674 // Advertise here if the CPU requires explicit rounding operations to
1675 // implement the UseStrictFP mode.
1676 const bool Matcher::strict_fp_requires_explicit_rounding = false;
1677 
1678 // Are floats converted to double when stored to stack during
1679 // deoptimization?
1680 bool Matcher::float_in_double() { return true; }
1681 
1682 // Do ints take an entire long register or just half?
1683 // The relevant question is how the int is callee-saved:
1684 // the whole long is written but de-opt'ing will have to extract
1685 // the relevant 32 bits.
1686 const bool Matcher::int_in_long = true;
1687 
1688 // Return whether or not this register is ever used as an argument.
1689 // This function is used on startup to build the trampoline stubs in
1690 // generateOptoStub.  Registers not mentioned will be killed by the VM
1691 // call in the trampoline, and arguments in those registers not be
1692 // available to the callee.
1693 bool Matcher::can_be_java_arg(int reg)
1694 {
1695   return
1696     reg ==  R0_num || reg == R0_H_num ||
1697     reg ==  R1_num || reg == R1_H_num ||
1698     reg ==  R2_num || reg == R2_H_num ||
1699     reg ==  R3_num || reg == R3_H_num ||
1700     reg ==  R4_num || reg == R4_H_num ||
1701     reg ==  R5_num || reg == R5_H_num ||
1702     reg ==  R6_num || reg == R6_H_num ||
1703     reg ==  R7_num || reg == R7_H_num ||
1704     reg ==  V0_num || reg == V0_H_num ||
1705     reg ==  V1_num || reg == V1_H_num ||
1706     reg ==  V2_num || reg == V2_H_num ||
1707     reg ==  V3_num || reg == V3_H_num ||
1708     reg ==  V4_num || reg == V4_H_num ||
1709     reg ==  V5_num || reg == V5_H_num ||
1710     reg ==  V6_num || reg == V6_H_num ||
1711     reg ==  V7_num || reg == V7_H_num;
1712 }
1713 
1714 bool Matcher::is_spillable_arg(int reg)
1715 {
1716   return can_be_java_arg(reg);
1717 }
1718 
1719 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
1720   return false;
1721 }
1722 
1723 RegMask Matcher::divI_proj_mask() {
1724   ShouldNotReachHere();
1725   return RegMask();
1726 }
1727 
1728 // Register for MODI projection of divmodI.
1729 RegMask Matcher::modI_proj_mask() {
1730   ShouldNotReachHere();
1731   return RegMask();
1732 }
1733 
1734 // Register for DIVL projection of divmodL.
1735 RegMask Matcher::divL_proj_mask() {
1736   ShouldNotReachHere();
1737   return RegMask();
1738 }
1739 
1740 // Register for MODL projection of divmodL.
1741 RegMask Matcher::modL_proj_mask() {
1742   ShouldNotReachHere();
1743   return RegMask();
1744 }
1745 
1746 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1747   return RegMask();
1748 }
1749 
1750 // helper for encoding java_to_runtime calls on sim
1751 //
1752 // this is needed to compute the extra arguments required when
1753 // planting a call to the simulator blrt instruction. the TypeFunc
1754 // can be queried to identify the counts for integral, and floating
1755 // arguments and the return type
1756 
1757 static void getCallInfo(const TypeFunc *tf, int &gpcnt, int &fpcnt, int &rtype)
1758 {
1759   int gps = 0;
1760   int fps = 0;
1761   const TypeTuple *domain = tf->domain();
1762   int max = domain->cnt();
1763   for (int i = TypeFunc::Parms; i < max; i++) {
1764     const Type *t = domain->field_at(i);
1765     switch(t->basic_type()) {
1766     case T_FLOAT:
1767     case T_DOUBLE:
1768       fps++;
1769     default:
1770       gps++;
1771     }
1772   }
1773   gpcnt = gps;
1774   fpcnt = fps;
1775   BasicType rt = tf->return_type();
1776   switch (rt) {
1777   case T_VOID:
1778     rtype = MacroAssembler::ret_type_void;
1779     break;
1780   default:
1781     rtype = MacroAssembler::ret_type_integral;
1782     break;
1783   case T_FLOAT:
1784     rtype = MacroAssembler::ret_type_float;
1785     break;
1786   case T_DOUBLE:
1787     rtype = MacroAssembler::ret_type_double;
1788     break;
1789   }
1790 }
1791 
1792 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
1793   MacroAssembler _masm(&cbuf);                                          \
1794   {                                                                     \
1795     guarantee(INDEX == -1, "mode not permitted for volatile");          \
1796     guarantee(DISP == 0, "mode not permitted for volatile");            \
1797     guarantee(SCALE == 0, "mode not permitted for volatile");           \
1798     __ INSN(REG, as_Register(BASE));                                    \
1799   }
1800 
1801 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
1802 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
1803 
1804   // Used for all non-volatile memory accesses.  The use of
1805   // $mem->opcode() to discover whether this pattern uses sign-extended
1806   // offsets is something of a kludge.
1807   static void loadStore(MacroAssembler masm, mem_insn insn,
1808                          Register reg, int opcode,
1809                          Register base, int index, int size, int disp)
1810   {
1811     Address::extend scale;
1812 
1813     // Hooboy, this is fugly.  We need a way to communicate to the
1814     // encoder that the index needs to be sign extended, so we have to
1815     // enumerate all the cases.
1816     switch (opcode) {
1817     case INDINDEXSCALEDOFFSETI2L:
1818     case INDINDEXSCALEDI2L:
1819     case INDINDEXSCALEDOFFSETI2LN:
1820     case INDINDEXSCALEDI2LN:
1821       scale = Address::sxtw(size);
1822       break;
1823     default:
1824       scale = Address::lsl(size);
1825     }
1826 
1827     if (index == -1) {
1828       (masm.*insn)(reg, Address(base, disp));
1829     } else {
1830       if (disp == 0) {
1831         (masm.*insn)(reg, Address(base, as_Register(index), scale));
1832       } else {
1833         masm.lea(rscratch1, Address(base, disp));
1834         (masm.*insn)(reg, Address(rscratch1, as_Register(index), scale));
1835       }
1836     }
1837   }
1838 
1839   static void loadStore(MacroAssembler masm, mem_float_insn insn,
1840                          FloatRegister reg, int opcode,
1841                          Register base, int index, int size, int disp)
1842   {
1843     Address::extend scale;
1844 
1845     switch (opcode) {
1846     case INDINDEXSCALEDOFFSETI2L:
1847     case INDINDEXSCALEDI2L:
1848     case INDINDEXSCALEDOFFSETI2LN:
1849     case INDINDEXSCALEDI2LN:
1850       scale = Address::sxtw(size);
1851       break;
1852     default:
1853       scale = Address::lsl(size);
1854     }
1855 
1856      if (index == -1) {
1857       (masm.*insn)(reg, Address(base, disp));
1858     } else {
1859       if (disp == 0) {
1860         (masm.*insn)(reg, Address(base, as_Register(index), scale));
1861       } else {
1862         masm.lea(rscratch1, Address(base, disp));
1863         (masm.*insn)(reg, Address(rscratch1, as_Register(index), scale));
1864       }
1865     }
1866   }
1867 
1868 %}
1869 
1870 
1871 
1872 //----------ENCODING BLOCK-----------------------------------------------------
1873 // This block specifies the encoding classes used by the compiler to
1874 // output byte streams.  Encoding classes are parameterized macros
1875 // used by Machine Instruction Nodes in order to generate the bit
1876 // encoding of the instruction.  Operands specify their base encoding
1877 // interface with the interface keyword.  There are currently
1878 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1879 // COND_INTER.  REG_INTER causes an operand to generate a function
1880 // which returns its register number when queried.  CONST_INTER causes
1881 // an operand to generate a function which returns the value of the
1882 // constant when queried.  MEMORY_INTER causes an operand to generate
1883 // four functions which return the Base Register, the Index Register,
1884 // the Scale Value, and the Offset Value of the operand when queried.
1885 // COND_INTER causes an operand to generate six functions which return
1886 // the encoding code (ie - encoding bits for the instruction)
1887 // associated with each basic boolean condition for a conditional
1888 // instruction.
1889 //
1890 // Instructions specify two basic values for encoding.  Again, a
1891 // function is available to check if the constant displacement is an
1892 // oop. They use the ins_encode keyword to specify their encoding
1893 // classes (which must be a sequence of enc_class names, and their
1894 // parameters, specified in the encoding block), and they use the
1895 // opcode keyword to specify, in order, their primary, secondary, and
1896 // tertiary opcode.  Only the opcode sections which a particular
1897 // instruction needs for encoding need to be specified.
1898 encode %{
1899   // Build emit functions for each basic byte or larger field in the
1900   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1901   // from C++ code in the enc_class source block.  Emit functions will
1902   // live in the main source block for now.  In future, we can
1903   // generalize this by adding a syntax that specifies the sizes of
1904   // fields in an order, so that the adlc can build the emit functions
1905   // automagically
1906 
1907   // catch all for unimplemented encodings
1908   enc_class enc_unimplemented %{
1909     MacroAssembler _masm(&cbuf);
1910     __ unimplemented("C2 catch all");
1911   %}
1912 
1913   // BEGIN Non-volatile memory access
1914 
1915   enc_class aarch64_enc_ldrsbw(iRegI dst, memory mem) %{
1916     Register dst_reg = as_Register($dst$$reg);
1917     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
1918                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1919   %}
1920 
1921   enc_class aarch64_enc_ldrsb(iRegI dst, memory mem) %{
1922     Register dst_reg = as_Register($dst$$reg);
1923     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
1924                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1925   %}
1926 
1927   enc_class aarch64_enc_ldrb(iRegI dst, memory mem) %{
1928     Register dst_reg = as_Register($dst$$reg);
1929     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
1930                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1931   %}
1932 
1933   enc_class aarch64_enc_ldrb(iRegL dst, memory mem) %{
1934     Register dst_reg = as_Register($dst$$reg);
1935     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
1936                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1937   %}
1938 
1939   enc_class aarch64_enc_ldrshw(iRegI dst, memory mem) %{
1940     Register dst_reg = as_Register($dst$$reg);
1941     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
1942                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1943   %}
1944 
1945   enc_class aarch64_enc_ldrsh(iRegI dst, memory mem) %{
1946     Register dst_reg = as_Register($dst$$reg);
1947     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
1948                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1949   %}
1950 
1951   enc_class aarch64_enc_ldrh(iRegI dst, memory mem) %{
1952     Register dst_reg = as_Register($dst$$reg);
1953     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
1954                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1955   %}
1956 
1957   enc_class aarch64_enc_ldrh(iRegL dst, memory mem) %{
1958     Register dst_reg = as_Register($dst$$reg);
1959     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
1960                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1961   %}
1962 
1963   enc_class aarch64_enc_ldrw(iRegI dst, memory mem) %{
1964     Register dst_reg = as_Register($dst$$reg);
1965     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
1966                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1967   %}
1968 
1969   enc_class aarch64_enc_ldrw(iRegL dst, memory mem) %{
1970     Register dst_reg = as_Register($dst$$reg);
1971     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
1972                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1973   %}
1974 
1975   enc_class aarch64_enc_ldrsw(iRegL dst, memory mem) %{
1976     Register dst_reg = as_Register($dst$$reg);
1977     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
1978                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1979   %}
1980 
1981   enc_class aarch64_enc_ldr(iRegL dst, memory mem) %{
1982     Register dst_reg = as_Register($dst$$reg);
1983     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
1984                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1985   %}
1986 
1987   enc_class aarch64_enc_ldrs(vRegF dst, memory mem) %{
1988     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
1989     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
1990                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1991   %}
1992 
1993   enc_class aarch64_enc_ldrd(vRegD dst, memory mem) %{
1994     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
1995     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
1996                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1997   %}
1998 
1999   enc_class aarch64_enc_strb(iRegI src, memory mem) %{
2000     Register src_reg = as_Register($src$$reg);
2001     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2002                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2003   %}
2004 
2005   enc_class aarch64_enc_strb0(memory mem) %{
2006     MacroAssembler _masm(&cbuf);
2007     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2008                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2009   %}
2010 
2011   enc_class aarch64_enc_strh(iRegI src, memory mem) %{
2012     Register src_reg = as_Register($src$$reg);
2013     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2014                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2015   %}
2016 
2017   enc_class aarch64_enc_strh0(memory mem) %{
2018     MacroAssembler _masm(&cbuf);
2019     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2020                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2021   %}
2022 
2023   enc_class aarch64_enc_strw(iRegI src, memory mem) %{
2024     Register src_reg = as_Register($src$$reg);
2025     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2026                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2027   %}
2028 
2029   enc_class aarch64_enc_strw0(memory mem) %{
2030     MacroAssembler _masm(&cbuf);
2031     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2032                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2033   %}
2034 
2035   enc_class aarch64_enc_str(iRegL src, memory mem) %{
2036     Register src_reg = as_Register($src$$reg);
2037     // we sometimes get asked to store the stack pointer into the
2038     // current thread -- we cannot do that directly on AArch64
2039     if (src_reg == r31_sp) {
2040       MacroAssembler _masm(&cbuf);
2041       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2042       __ mov(rscratch2, sp);
2043       src_reg = rscratch2;
2044     }
2045     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2046                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2047   %}
2048 
2049   enc_class aarch64_enc_str0(memory mem) %{
2050     MacroAssembler _masm(&cbuf);
2051     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2052                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2053   %}
2054 
2055   enc_class aarch64_enc_strs(vRegF src, memory mem) %{
2056     FloatRegister src_reg = as_FloatRegister($src$$reg);
2057     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2058                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2059   %}
2060 
2061   enc_class aarch64_enc_strd(vRegD src, memory mem) %{
2062     FloatRegister src_reg = as_FloatRegister($src$$reg);
2063     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2064                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2065   %}
2066 
2067   // END Non-volatile memory access
2068 
2069   // volatile loads and stores
2070 
2071   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
2072     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2073                  rscratch1, stlrb);
2074   %}
2075 
2076   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
2077     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2078                  rscratch1, stlrh);
2079   %}
2080 
2081   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
2082     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2083                  rscratch1, stlrw);
2084   %}
2085 
2086 
2087   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
2088     Register dst_reg = as_Register($dst$$reg);
2089     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2090              rscratch1, ldarb);
2091     __ sxtbw(dst_reg, dst_reg);
2092   %}
2093 
2094   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
2095     Register dst_reg = as_Register($dst$$reg);
2096     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2097              rscratch1, ldarb);
2098     __ sxtb(dst_reg, dst_reg);
2099   %}
2100 
2101   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
2102     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2103              rscratch1, ldarb);
2104   %}
2105 
2106   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
2107     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2108              rscratch1, ldarb);
2109   %}
2110 
2111   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
2112     Register dst_reg = as_Register($dst$$reg);
2113     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2114              rscratch1, ldarh);
2115     __ sxthw(dst_reg, dst_reg);
2116   %}
2117 
2118   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
2119     Register dst_reg = as_Register($dst$$reg);
2120     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2121              rscratch1, ldarh);
2122     __ sxth(dst_reg, dst_reg);
2123   %}
2124 
2125   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
2126     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2127              rscratch1, ldarh);
2128   %}
2129 
2130   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
2131     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2132              rscratch1, ldarh);
2133   %}
2134 
2135   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
2136     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2137              rscratch1, ldarw);
2138   %}
2139 
2140   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
2141     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2142              rscratch1, ldarw);
2143   %}
2144 
2145   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
2146     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2147              rscratch1, ldar);
2148   %}
2149 
2150   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
2151     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2152              rscratch1, ldarw);
2153     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
2154   %}
2155 
2156   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
2157     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2158              rscratch1, ldar);
2159     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
2160   %}
2161 
2162   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
2163     Register src_reg = as_Register($src$$reg);
2164     // we sometimes get asked to store the stack pointer into the
2165     // current thread -- we cannot do that directly on AArch64
2166     if (src_reg == r31_sp) {
2167         MacroAssembler _masm(&cbuf);
2168       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2169       __ mov(rscratch2, sp);
2170       src_reg = rscratch2;
2171     }
2172     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2173                  rscratch1, stlr);
2174   %}
2175 
2176   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
2177     {
2178       MacroAssembler _masm(&cbuf);
2179       FloatRegister src_reg = as_FloatRegister($src$$reg);
2180       __ fmovs(rscratch2, src_reg);
2181     }
2182     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2183                  rscratch1, stlrw);
2184   %}
2185 
2186   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
2187     {
2188       MacroAssembler _masm(&cbuf);
2189       FloatRegister src_reg = as_FloatRegister($src$$reg);
2190       __ fmovd(rscratch2, src_reg);
2191     }
2192     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2193                  rscratch1, stlr);
2194   %}
2195 
2196   // synchronized read/update encodings
2197 
2198   enc_class aarch64_enc_ldaxr(iRegL dst, memory mem) %{
2199     MacroAssembler _masm(&cbuf);
2200     Register dst_reg = as_Register($dst$$reg);
2201     Register base = as_Register($mem$$base);
2202     int index = $mem$$index;
2203     int scale = $mem$$scale;
2204     int disp = $mem$$disp;
2205     if (index == -1) {
2206        if (disp != 0) {
2207         __ lea(rscratch1, Address(base, disp));
2208         __ ldaxr(dst_reg, rscratch1);
2209       } else {
2210         // TODO
2211         // should we ever get anything other than this case?
2212         __ ldaxr(dst_reg, base);
2213       }
2214     } else {
2215       Register index_reg = as_Register(index);
2216       if (disp == 0) {
2217         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
2218         __ ldaxr(dst_reg, rscratch1);
2219       } else {
2220         __ lea(rscratch1, Address(base, disp));
2221         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
2222         __ ldaxr(dst_reg, rscratch1);
2223       }
2224     }
2225   %}
2226 
2227   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory mem) %{
2228     MacroAssembler _masm(&cbuf);
2229     Register src_reg = as_Register($src$$reg);
2230     Register base = as_Register($mem$$base);
2231     int index = $mem$$index;
2232     int scale = $mem$$scale;
2233     int disp = $mem$$disp;
2234     if (index == -1) {
2235        if (disp != 0) {
2236         __ lea(rscratch2, Address(base, disp));
2237         __ stlxr(rscratch1, src_reg, rscratch2);
2238       } else {
2239         // TODO
2240         // should we ever get anything other than this case?
2241         __ stlxr(rscratch1, src_reg, base);
2242       }
2243     } else {
2244       Register index_reg = as_Register(index);
2245       if (disp == 0) {
2246         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
2247         __ stlxr(rscratch1, src_reg, rscratch2);
2248       } else {
2249         __ lea(rscratch2, Address(base, disp));
2250         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
2251         __ stlxr(rscratch1, src_reg, rscratch2);
2252       }
2253     }
2254     __ cmpw(rscratch1, zr);
2255   %}
2256 
2257   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2258     MacroAssembler _masm(&cbuf);
2259     Register old_reg = as_Register($oldval$$reg);
2260     Register new_reg = as_Register($newval$$reg);
2261     Register base = as_Register($mem$$base);
2262     Register addr_reg;
2263     int index = $mem$$index;
2264     int scale = $mem$$scale;
2265     int disp = $mem$$disp;
2266     if (index == -1) {
2267        if (disp != 0) {
2268         __ lea(rscratch2, Address(base, disp));
2269         addr_reg = rscratch2;
2270       } else {
2271         // TODO
2272         // should we ever get anything other than this case?
2273         addr_reg = base;
2274       }
2275     } else {
2276       Register index_reg = as_Register(index);
2277       if (disp == 0) {
2278         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
2279         addr_reg = rscratch2;
2280       } else {
2281         __ lea(rscratch2, Address(base, disp));
2282         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
2283         addr_reg = rscratch2;
2284       }
2285     }
2286     Label retry_load, done;
2287     __ bind(retry_load);
2288     __ ldxr(rscratch1, addr_reg);
2289     __ cmp(rscratch1, old_reg);
2290     __ br(Assembler::NE, done);
2291     __ stlxr(rscratch1, new_reg, addr_reg);
2292     __ cbnzw(rscratch1, retry_load);
2293     __ bind(done);
2294   %}
2295 
2296   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2297     MacroAssembler _masm(&cbuf);
2298     Register old_reg = as_Register($oldval$$reg);
2299     Register new_reg = as_Register($newval$$reg);
2300     Register base = as_Register($mem$$base);
2301     Register addr_reg;
2302     int index = $mem$$index;
2303     int scale = $mem$$scale;
2304     int disp = $mem$$disp;
2305     if (index == -1) {
2306        if (disp != 0) {
2307         __ lea(rscratch2, Address(base, disp));
2308         addr_reg = rscratch2;
2309       } else {
2310         // TODO
2311         // should we ever get anything other than this case?
2312         addr_reg = base;
2313       }
2314     } else {
2315       Register index_reg = as_Register(index);
2316       if (disp == 0) {
2317         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
2318         addr_reg = rscratch2;
2319       } else {
2320         __ lea(rscratch2, Address(base, disp));
2321         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
2322         addr_reg = rscratch2;
2323       }
2324     }
2325     Label retry_load, done;
2326     __ bind(retry_load);
2327     __ ldxrw(rscratch1, addr_reg);
2328     __ cmpw(rscratch1, old_reg);
2329     __ br(Assembler::NE, done);
2330     __ stlxrw(rscratch1, new_reg, addr_reg);
2331     __ cbnzw(rscratch1, retry_load);
2332     __ bind(done);
2333   %}
2334 
2335   // auxiliary used for CompareAndSwapX to set result register
2336   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
2337     MacroAssembler _masm(&cbuf);
2338     Register res_reg = as_Register($res$$reg);
2339     __ cset(res_reg, Assembler::EQ);
2340   %}
2341 
2342   // prefetch encodings
2343 
2344   enc_class aarch64_enc_prefetchw(memory mem) %{
2345     MacroAssembler _masm(&cbuf);
2346     Register base = as_Register($mem$$base);
2347     int index = $mem$$index;
2348     int scale = $mem$$scale;
2349     int disp = $mem$$disp;
2350     if (index == -1) {
2351       __ prfm(Address(base, disp), PSTL1KEEP);
2352       __ nop();
2353     } else {
2354       Register index_reg = as_Register(index);
2355       if (disp == 0) {
2356         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
2357       } else {
2358         __ lea(rscratch1, Address(base, disp));
2359         __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
2360       }
2361     }
2362   %}
2363 
2364   enc_class aarch64_enc_clear_array_reg_reg(iRegL_R11 cnt, iRegP_R10 base) %{
2365     MacroAssembler _masm(&cbuf);
2366     Register cnt_reg = as_Register($cnt$$reg);
2367     Register base_reg = as_Register($base$$reg);
2368     // base is word aligned
2369     // cnt is count of words
2370 
2371     Label loop;
2372     Label entry;
2373 
2374 //  Algorithm:
2375 //
2376 //    scratch1 = cnt & 7;
2377 //    cnt -= scratch1;
2378 //    p += scratch1;
2379 //    switch (scratch1) {
2380 //      do {
2381 //        cnt -= 8;
2382 //          p[-8] = 0;
2383 //        case 7:
2384 //          p[-7] = 0;
2385 //        case 6:
2386 //          p[-6] = 0;
2387 //          // ...
2388 //        case 1:
2389 //          p[-1] = 0;
2390 //        case 0:
2391 //          p += 8;
2392 //      } while (cnt);
2393 //    }
2394 
2395     const int unroll = 8; // Number of str(zr) instructions we'll unroll
2396 
2397     __ andr(rscratch1, cnt_reg, unroll - 1);  // tmp1 = cnt % unroll
2398     __ sub(cnt_reg, cnt_reg, rscratch1);      // cnt -= unroll
2399     // base_reg always points to the end of the region we're about to zero
2400     __ add(base_reg, base_reg, rscratch1, Assembler::LSL, exact_log2(wordSize));
2401     __ adr(rscratch2, entry);
2402     __ sub(rscratch2, rscratch2, rscratch1, Assembler::LSL, 2);
2403     __ br(rscratch2);
2404     __ bind(loop);
2405     __ sub(cnt_reg, cnt_reg, unroll);
2406     for (int i = -unroll; i < 0; i++)
2407       __ str(zr, Address(base_reg, i * wordSize));
2408     __ bind(entry);
2409     __ add(base_reg, base_reg, unroll * wordSize);
2410     __ cbnz(cnt_reg, loop);
2411   %}
2412 
2413   /// mov envcodings
2414 
2415   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
2416     MacroAssembler _masm(&cbuf);
2417     u_int32_t con = (u_int32_t)$src$$constant;
2418     Register dst_reg = as_Register($dst$$reg);
2419     if (con == 0) {
2420       __ movw(dst_reg, zr);
2421     } else {
2422       __ movw(dst_reg, con);
2423     }
2424   %}
2425 
2426   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
2427     MacroAssembler _masm(&cbuf);
2428     Register dst_reg = as_Register($dst$$reg);
2429     u_int64_t con = (u_int64_t)$src$$constant;
2430     if (con == 0) {
2431       __ mov(dst_reg, zr);
2432     } else {
2433       __ mov(dst_reg, con);
2434     }
2435   %}
2436 
2437   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
2438     MacroAssembler _masm(&cbuf);
2439     Register dst_reg = as_Register($dst$$reg);
2440     address con = (address)$src$$constant;
2441     if (con == NULL || con == (address)1) {
2442       ShouldNotReachHere();
2443     } else {
2444       relocInfo::relocType rtype = $src->constant_reloc();
2445       if (rtype == relocInfo::oop_type) {
2446         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
2447       } else if (rtype == relocInfo::metadata_type) {
2448         __ mov_metadata(dst_reg, (Metadata*)con);
2449       } else {
2450         assert(rtype == relocInfo::none, "unexpected reloc type");
2451         if (con < (address)(uintptr_t)os::vm_page_size()) {
2452           __ mov(dst_reg, con);
2453         } else {
2454           unsigned long offset;
2455           __ adrp(dst_reg, con, offset);
2456           __ add(dst_reg, dst_reg, offset);
2457         }
2458       }
2459     }
2460   %}
2461 
2462   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
2463     MacroAssembler _masm(&cbuf);
2464     Register dst_reg = as_Register($dst$$reg);
2465     __ mov(dst_reg, zr);
2466   %}
2467 
2468   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
2469     MacroAssembler _masm(&cbuf);
2470     Register dst_reg = as_Register($dst$$reg);
2471     __ mov(dst_reg, (u_int64_t)1);
2472   %}
2473 
2474   enc_class aarch64_enc_mov_poll_page(iRegP dst, immPollPage src) %{
2475     MacroAssembler _masm(&cbuf);
2476     address page = (address)$src$$constant;
2477     Register dst_reg = as_Register($dst$$reg);
2478     unsigned long off;
2479     __ adrp(dst_reg, Address(page, relocInfo::poll_type), off);
2480     assert(off == 0, "assumed offset == 0");
2481   %}
2482 
2483   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
2484     MacroAssembler _masm(&cbuf);
2485     address page = (address)$src$$constant;
2486     Register dst_reg = as_Register($dst$$reg);
2487     unsigned long off;
2488     __ adrp(dst_reg, ExternalAddress(page), off);
2489     assert(off == 0, "assumed offset == 0");
2490   %}
2491 
2492   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
2493     MacroAssembler _masm(&cbuf);
2494     Register dst_reg = as_Register($dst$$reg);
2495     address con = (address)$src$$constant;
2496     if (con == NULL) {
2497       ShouldNotReachHere();
2498     } else {
2499       relocInfo::relocType rtype = $src->constant_reloc();
2500       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
2501       __ set_narrow_oop(dst_reg, (jobject)con);
2502     }
2503   %}
2504 
2505   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
2506     MacroAssembler _masm(&cbuf);
2507     Register dst_reg = as_Register($dst$$reg);
2508     __ mov(dst_reg, zr);
2509   %}
2510 
2511   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
2512     MacroAssembler _masm(&cbuf);
2513     Register dst_reg = as_Register($dst$$reg);
2514     address con = (address)$src$$constant;
2515     if (con == NULL) {
2516       ShouldNotReachHere();
2517     } else {
2518       relocInfo::relocType rtype = $src->constant_reloc();
2519       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
2520       __ set_narrow_klass(dst_reg, (Klass *)con);
2521     }
2522   %}
2523 
2524   // arithmetic encodings
2525 
2526   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
2527     MacroAssembler _masm(&cbuf);
2528     Register dst_reg = as_Register($dst$$reg);
2529     Register src_reg = as_Register($src1$$reg);
2530     int32_t con = (int32_t)$src2$$constant;
2531     // add has primary == 0, subtract has primary == 1
2532     if ($primary) { con = -con; }
2533     if (con < 0) {
2534       __ subw(dst_reg, src_reg, -con);
2535     } else {
2536       __ addw(dst_reg, src_reg, con);
2537     }
2538   %}
2539 
2540   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
2541     MacroAssembler _masm(&cbuf);
2542     Register dst_reg = as_Register($dst$$reg);
2543     Register src_reg = as_Register($src1$$reg);
2544     int32_t con = (int32_t)$src2$$constant;
2545     // add has primary == 0, subtract has primary == 1
2546     if ($primary) { con = -con; }
2547     if (con < 0) {
2548       __ sub(dst_reg, src_reg, -con);
2549     } else {
2550       __ add(dst_reg, src_reg, con);
2551     }
2552   %}
2553 
2554   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
2555     MacroAssembler _masm(&cbuf);
2556    Register dst_reg = as_Register($dst$$reg);
2557    Register src1_reg = as_Register($src1$$reg);
2558    Register src2_reg = as_Register($src2$$reg);
2559     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
2560   %}
2561 
2562   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
2563     MacroAssembler _masm(&cbuf);
2564    Register dst_reg = as_Register($dst$$reg);
2565    Register src1_reg = as_Register($src1$$reg);
2566    Register src2_reg = as_Register($src2$$reg);
2567     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
2568   %}
2569 
2570   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
2571     MacroAssembler _masm(&cbuf);
2572    Register dst_reg = as_Register($dst$$reg);
2573    Register src1_reg = as_Register($src1$$reg);
2574    Register src2_reg = as_Register($src2$$reg);
2575     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
2576   %}
2577 
2578   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
2579     MacroAssembler _masm(&cbuf);
2580    Register dst_reg = as_Register($dst$$reg);
2581    Register src1_reg = as_Register($src1$$reg);
2582    Register src2_reg = as_Register($src2$$reg);
2583     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
2584   %}
2585 
2586   // compare instruction encodings
2587 
2588   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
2589     MacroAssembler _masm(&cbuf);
2590     Register reg1 = as_Register($src1$$reg);
2591     Register reg2 = as_Register($src2$$reg);
2592     __ cmpw(reg1, reg2);
2593   %}
2594 
2595   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
2596     MacroAssembler _masm(&cbuf);
2597     Register reg = as_Register($src1$$reg);
2598     int32_t val = $src2$$constant;
2599     if (val >= 0) {
2600       __ subsw(zr, reg, val);
2601     } else {
2602       __ addsw(zr, reg, -val);
2603     }
2604   %}
2605 
2606   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
2607     MacroAssembler _masm(&cbuf);
2608     Register reg1 = as_Register($src1$$reg);
2609     u_int32_t val = (u_int32_t)$src2$$constant;
2610     __ movw(rscratch1, val);
2611     __ cmpw(reg1, rscratch1);
2612   %}
2613 
2614   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
2615     MacroAssembler _masm(&cbuf);
2616     Register reg1 = as_Register($src1$$reg);
2617     Register reg2 = as_Register($src2$$reg);
2618     __ cmp(reg1, reg2);
2619   %}
2620 
2621   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
2622     MacroAssembler _masm(&cbuf);
2623     Register reg = as_Register($src1$$reg);
2624     int64_t val = $src2$$constant;
2625     if (val >= 0) {
2626       __ subs(zr, reg, val);
2627     } else if (val != -val) {
2628       __ adds(zr, reg, -val);
2629     } else {
2630     // aargh, Long.MIN_VALUE is a special case
2631       __ orr(rscratch1, zr, (u_int64_t)val);
2632       __ subs(zr, reg, rscratch1);
2633     }
2634   %}
2635 
2636   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
2637     MacroAssembler _masm(&cbuf);
2638     Register reg1 = as_Register($src1$$reg);
2639     u_int64_t val = (u_int64_t)$src2$$constant;
2640     __ mov(rscratch1, val);
2641     __ cmp(reg1, rscratch1);
2642   %}
2643 
2644   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
2645     MacroAssembler _masm(&cbuf);
2646     Register reg1 = as_Register($src1$$reg);
2647     Register reg2 = as_Register($src2$$reg);
2648     __ cmp(reg1, reg2);
2649   %}
2650 
2651   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
2652     MacroAssembler _masm(&cbuf);
2653     Register reg1 = as_Register($src1$$reg);
2654     Register reg2 = as_Register($src2$$reg);
2655     __ cmpw(reg1, reg2);
2656   %}
2657 
2658   enc_class aarch64_enc_testp(iRegP src) %{
2659     MacroAssembler _masm(&cbuf);
2660     Register reg = as_Register($src$$reg);
2661     __ cmp(reg, zr);
2662   %}
2663 
2664   enc_class aarch64_enc_testn(iRegN src) %{
2665     MacroAssembler _masm(&cbuf);
2666     Register reg = as_Register($src$$reg);
2667     __ cmpw(reg, zr);
2668   %}
2669 
2670   enc_class aarch64_enc_b(label lbl) %{
2671     MacroAssembler _masm(&cbuf);
2672     Label *L = $lbl$$label;
2673     __ b(*L);
2674   %}
2675 
2676   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
2677     MacroAssembler _masm(&cbuf);
2678     Label *L = $lbl$$label;
2679     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
2680   %}
2681 
2682   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
2683     MacroAssembler _masm(&cbuf);
2684     Label *L = $lbl$$label;
2685     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
2686   %}
2687 
2688   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
2689   %{
2690      Register sub_reg = as_Register($sub$$reg);
2691      Register super_reg = as_Register($super$$reg);
2692      Register temp_reg = as_Register($temp$$reg);
2693      Register result_reg = as_Register($result$$reg);
2694 
2695      Label miss;
2696      MacroAssembler _masm(&cbuf);
2697      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
2698                                      NULL, &miss,
2699                                      /*set_cond_codes:*/ true);
2700      if ($primary) {
2701        __ mov(result_reg, zr);
2702      }
2703      __ bind(miss);
2704   %}
2705 
2706   enc_class aarch64_enc_java_static_call(method meth) %{
2707     MacroAssembler _masm(&cbuf);
2708 
2709     address addr = (address)$meth$$method;
2710     if (!_method) {
2711       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
2712       __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
2713     } else if (_optimized_virtual) {
2714       __ trampoline_call(Address(addr, relocInfo::opt_virtual_call_type), &cbuf);
2715     } else {
2716       __ trampoline_call(Address(addr, relocInfo::static_call_type), &cbuf);
2717     }
2718 
2719     if (_method) {
2720       // Emit stub for static call
2721       CompiledStaticCall::emit_to_interp_stub(cbuf);
2722     }
2723   %}
2724 
2725   enc_class aarch64_enc_java_handle_call(method meth) %{
2726     MacroAssembler _masm(&cbuf);
2727     relocInfo::relocType reloc;
2728 
2729     // RFP is preserved across all calls, even compiled calls.
2730     // Use it to preserve SP.
2731     __ mov(rfp, sp);
2732 
2733     const int start_offset = __ offset();
2734     address addr = (address)$meth$$method;
2735     if (!_method) {
2736       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
2737       __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
2738     } else if (_optimized_virtual) {
2739       __ trampoline_call(Address(addr, relocInfo::opt_virtual_call_type), &cbuf);
2740     } else {
2741       __ trampoline_call(Address(addr, relocInfo::static_call_type), &cbuf);
2742     }
2743 
2744     if (_method) {
2745       // Emit stub for static call
2746       CompiledStaticCall::emit_to_interp_stub(cbuf);
2747     }
2748 
2749     // now restore sp
2750     __ mov(sp, rfp);
2751   %}
2752 
2753   enc_class aarch64_enc_java_dynamic_call(method meth) %{
2754     MacroAssembler _masm(&cbuf);
2755     __ ic_call((address)$meth$$method);
2756   %}
2757 
2758   enc_class aarch64_enc_call_epilog() %{
2759     MacroAssembler _masm(&cbuf);
2760     if (VerifyStackAtCalls) {
2761       // Check that stack depth is unchanged: find majik cookie on stack
2762       __ call_Unimplemented();
2763     }
2764   %}
2765 
2766   enc_class aarch64_enc_java_to_runtime(method meth) %{
2767     MacroAssembler _masm(&cbuf);
2768 
2769     // some calls to generated routines (arraycopy code) are scheduled
2770     // by C2 as runtime calls. if so we can call them using a br (they
2771     // will be in a reachable segment) otherwise we have to use a blrt
2772     // which loads the absolute address into a register.
2773     address entry = (address)$meth$$method;
2774     CodeBlob *cb = CodeCache::find_blob(entry);
2775     if (cb) {
2776       __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
2777     } else {
2778       int gpcnt;
2779       int fpcnt;
2780       int rtype;
2781       getCallInfo(tf(), gpcnt, fpcnt, rtype);
2782       Label retaddr;
2783       __ adr(rscratch2, retaddr);
2784       __ lea(rscratch1, RuntimeAddress(entry));
2785       // Leave a breadcrumb for JavaThread::pd_last_frame().
2786       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
2787       __ blrt(rscratch1, gpcnt, fpcnt, rtype);
2788       __ bind(retaddr);
2789       __ add(sp, sp, 2 * wordSize);
2790     }
2791   %}
2792 
2793   enc_class aarch64_enc_rethrow() %{
2794     MacroAssembler _masm(&cbuf);
2795     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
2796   %}
2797 
2798   enc_class aarch64_enc_ret() %{
2799     MacroAssembler _masm(&cbuf);
2800     __ ret(lr);
2801   %}
2802 
2803   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
2804     MacroAssembler _masm(&cbuf);
2805     Register target_reg = as_Register($jump_target$$reg);
2806     __ br(target_reg);
2807   %}
2808 
2809   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
2810     MacroAssembler _masm(&cbuf);
2811     Register target_reg = as_Register($jump_target$$reg);
2812     // exception oop should be in r0
2813     // ret addr has been popped into lr
2814     // callee expects it in r3
2815     __ mov(r3, lr);
2816     __ br(target_reg);
2817   %}
2818 
2819   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
2820     MacroAssembler _masm(&cbuf);
2821     Register oop = as_Register($object$$reg);
2822     Register box = as_Register($box$$reg);
2823     Register disp_hdr = as_Register($tmp$$reg);
2824     Register tmp = as_Register($tmp2$$reg);
2825     Label cont;
2826     Label object_has_monitor;
2827     Label cas_failed;
2828 
2829     assert_different_registers(oop, box, tmp, disp_hdr);
2830 
2831     // Load markOop from object into displaced_header.
2832     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
2833 
2834     // Always do locking in runtime.
2835     if (EmitSync & 0x01) {
2836       __ cmp(oop, zr);
2837       return;
2838     }
2839 
2840     if (UseBiasedLocking) {
2841       __ biased_locking_enter(disp_hdr, oop, box, tmp, true, cont);
2842     }
2843 
2844     // Handle existing monitor
2845     if (EmitSync & 0x02) {
2846       // we can use AArch64's bit test and branch here but
2847       // markoopDesc does not define a bit index just the bit value
2848       // so assert in case the bit pos changes
2849 #     define __monitor_value_log2 1
2850       assert(markOopDesc::monitor_value == (1 << __monitor_value_log2), "incorrect bit position");
2851       __ tbnz(disp_hdr, __monitor_value_log2, object_has_monitor);
2852 #     undef __monitor_value_log2
2853     }
2854 
2855     // Set displaced_header to be (markOop of object | UNLOCK_VALUE).
2856     __ orr(disp_hdr, disp_hdr, markOopDesc::unlocked_value);
2857 
2858     // Load Compare Value application register.
2859 
2860     // Initialize the box. (Must happen before we update the object mark!)
2861     __ str(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
2862 
2863     // Compare object markOop with mark and if equal exchange scratch1
2864     // with object markOop.
2865     // Note that this is simply a CAS: it does not generate any
2866     // barriers.  These are separately generated by
2867     // membar_acquire_lock().
2868     {
2869       Label retry_load;
2870       __ bind(retry_load);
2871       __ ldxr(tmp, oop);
2872       __ cmp(tmp, disp_hdr);
2873       __ br(Assembler::NE, cas_failed);
2874       // use stlxr to ensure update is immediately visible
2875       __ stlxr(tmp, box, oop);
2876       __ cbzw(tmp, cont);
2877       __ b(retry_load);
2878     }
2879 
2880     // Formerly:
2881     // __ cmpxchgptr(/*oldv=*/disp_hdr,
2882     //               /*newv=*/box,
2883     //               /*addr=*/oop,
2884     //               /*tmp=*/tmp,
2885     //               cont,
2886     //               /*fail*/NULL);
2887 
2888     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2889 
2890     // If the compare-and-exchange succeeded, then we found an unlocked
2891     // object, will have now locked it will continue at label cont
2892 
2893     __ bind(cas_failed);
2894     // We did not see an unlocked object so try the fast recursive case.
2895 
2896     // Check if the owner is self by comparing the value in the
2897     // markOop of object (disp_hdr) with the stack pointer.
2898     __ mov(rscratch1, sp);
2899     __ sub(disp_hdr, disp_hdr, rscratch1);
2900     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place));
2901     // If condition is true we are cont and hence we can store 0 as the
2902     // displaced header in the box, which indicates that it is a recursive lock.
2903     __ ands(tmp/*==0?*/, disp_hdr, tmp);
2904     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
2905 
2906     // Handle existing monitor.
2907     if ((EmitSync & 0x02) == 0) {
2908       __ b(cont);
2909 
2910       __ bind(object_has_monitor);
2911       // The object's monitor m is unlocked iff m->owner == NULL,
2912       // otherwise m->owner may contain a thread or a stack address.
2913       //
2914       // Try to CAS m->owner from NULL to current thread.
2915       __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
2916       __ mov(disp_hdr, zr);
2917 
2918       {
2919         Label retry_load, fail;
2920         __ bind(retry_load);
2921         __ ldxr(rscratch1, tmp);
2922         __ cmp(disp_hdr, rscratch1);
2923         __ br(Assembler::NE, fail);
2924         // use stlxr to ensure update is immediately visible
2925         __ stlxr(rscratch1, rthread, tmp);
2926         __ cbnzw(rscratch1, retry_load);
2927         __ bind(fail);
2928       }
2929 
2930       // Label next;
2931       // __ cmpxchgptr(/*oldv=*/disp_hdr,
2932       //               /*newv=*/rthread,
2933       //               /*addr=*/tmp,
2934       //               /*tmp=*/rscratch1,
2935       //               /*succeed*/next,
2936       //               /*fail*/NULL);
2937       // __ bind(next);
2938 
2939       // store a non-null value into the box.
2940       __ str(box, Address(box, BasicLock::displaced_header_offset_in_bytes()));
2941 
2942       // PPC port checks the following invariants
2943       // #ifdef ASSERT
2944       // bne(flag, cont);
2945       // We have acquired the monitor, check some invariants.
2946       // addw(/*monitor=*/tmp, tmp, -ObjectMonitor::owner_offset_in_bytes());
2947       // Invariant 1: _recursions should be 0.
2948       // assert(ObjectMonitor::recursions_size_in_bytes() == 8, "unexpected size");
2949       // assert_mem8_is_zero(ObjectMonitor::recursions_offset_in_bytes(), tmp,
2950       //                        "monitor->_recursions should be 0", -1);
2951       // Invariant 2: OwnerIsThread shouldn't be 0.
2952       // assert(ObjectMonitor::OwnerIsThread_size_in_bytes() == 4, "unexpected size");
2953       //assert_mem4_isnot_zero(ObjectMonitor::OwnerIsThread_offset_in_bytes(), tmp,
2954       //                           "monitor->OwnerIsThread shouldn't be 0", -1);
2955       // #endif
2956     }
2957 
2958     __ bind(cont);
2959     // flag == EQ indicates success
2960     // flag == NE indicates failure
2961 
2962   %}
2963 
2964   // TODO
2965   // reimplement this with custom cmpxchgptr code
2966   // which avoids some of the unnecessary branching
2967   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
2968     MacroAssembler _masm(&cbuf);
2969     Register oop = as_Register($object$$reg);
2970     Register box = as_Register($box$$reg);
2971     Register disp_hdr = as_Register($tmp$$reg);
2972     Register tmp = as_Register($tmp2$$reg);
2973     Label cont;
2974     Label object_has_monitor;
2975     Label cas_failed;
2976 
2977     assert_different_registers(oop, box, tmp, disp_hdr);
2978 
2979     // Always do locking in runtime.
2980     if (EmitSync & 0x01) {
2981       __ cmp(oop, zr); // Oop can't be 0 here => always false.
2982       return;
2983     }
2984 
2985     if (UseBiasedLocking) {
2986       __ biased_locking_exit(oop, tmp, cont);
2987     }
2988 
2989     // Find the lock address and load the displaced header from the stack.
2990     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
2991 
2992     // If the displaced header is 0, we have a recursive unlock.
2993     __ cmp(disp_hdr, zr);
2994     __ br(Assembler::EQ, cont);
2995 
2996 
2997     // Handle existing monitor.
2998     if ((EmitSync & 0x02) == 0) {
2999       __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3000       __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3001     }
3002 
3003     // Check if it is still a light weight lock, this is is true if we
3004     // see the stack address of the basicLock in the markOop of the
3005     // object.
3006 
3007       {
3008         Label retry_load;
3009         __ bind(retry_load);
3010         __ ldxr(tmp, oop);
3011         __ cmp(box, tmp);
3012         __ br(Assembler::NE, cas_failed);
3013         // use stlxr to ensure update is immediately visible
3014         __ stlxr(tmp, disp_hdr, oop);
3015         __ cbzw(tmp, cont);
3016         __ b(retry_load);
3017       }
3018 
3019     // __ cmpxchgptr(/*compare_value=*/box,
3020     //               /*exchange_value=*/disp_hdr,
3021     //               /*where=*/oop,
3022     //               /*result=*/tmp,
3023     //               cont,
3024     //               /*cas_failed*/NULL);
3025     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3026 
3027     __ bind(cas_failed);
3028 
3029     // Handle existing monitor.
3030     if ((EmitSync & 0x02) == 0) {
3031       __ b(cont);
3032 
3033       __ bind(object_has_monitor);
3034       __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor
3035       __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3036       __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3037       __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3038       __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3039       __ cmp(rscratch1, zr);
3040       __ br(Assembler::NE, cont);
3041 
3042       __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3043       __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3044       __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3045       __ cmp(rscratch1, zr);
3046       __ cbnz(rscratch1, cont);
3047       // need a release store here
3048       __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3049       __ stlr(rscratch1, tmp); // rscratch1 is zero
3050     }
3051 
3052     __ bind(cont);
3053     // flag == EQ indicates success
3054     // flag == NE indicates failure
3055   %}
3056 
3057 %}
3058 
3059 //----------FRAME--------------------------------------------------------------
3060 // Definition of frame structure and management information.
3061 //
3062 //  S T A C K   L A Y O U T    Allocators stack-slot number
3063 //                             |   (to get allocators register number
3064 //  G  Owned by    |        |  v    add OptoReg::stack0())
3065 //  r   CALLER     |        |
3066 //  o     |        +--------+      pad to even-align allocators stack-slot
3067 //  w     V        |  pad0  |        numbers; owned by CALLER
3068 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
3069 //  h     ^        |   in   |  5
3070 //        |        |  args  |  4   Holes in incoming args owned by SELF
3071 //  |     |        |        |  3
3072 //  |     |        +--------+
3073 //  V     |        | old out|      Empty on Intel, window on Sparc
3074 //        |    old |preserve|      Must be even aligned.
3075 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
3076 //        |        |   in   |  3   area for Intel ret address
3077 //     Owned by    |preserve|      Empty on Sparc.
3078 //       SELF      +--------+
3079 //        |        |  pad2  |  2   pad to align old SP
3080 //        |        +--------+  1
3081 //        |        | locks  |  0
3082 //        |        +--------+----> OptoReg::stack0(), even aligned
3083 //        |        |  pad1  | 11   pad to align new SP
3084 //        |        +--------+
3085 //        |        |        | 10
3086 //        |        | spills |  9   spills
3087 //        V        |        |  8   (pad0 slot for callee)
3088 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
3089 //        ^        |  out   |  7
3090 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
3091 //     Owned by    +--------+
3092 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
3093 //        |    new |preserve|      Must be even-aligned.
3094 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
3095 //        |        |        |
3096 //
3097 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
3098 //         known from SELF's arguments and the Java calling convention.
3099 //         Region 6-7 is determined per call site.
3100 // Note 2: If the calling convention leaves holes in the incoming argument
3101 //         area, those holes are owned by SELF.  Holes in the outgoing area
3102 //         are owned by the CALLEE.  Holes should not be nessecary in the
3103 //         incoming area, as the Java calling convention is completely under
3104 //         the control of the AD file.  Doubles can be sorted and packed to
3105 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
3106 //         varargs C calling conventions.
3107 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
3108 //         even aligned with pad0 as needed.
3109 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
3110 //           (the latter is true on Intel but is it false on AArch64?)
3111 //         region 6-11 is even aligned; it may be padded out more so that
3112 //         the region from SP to FP meets the minimum stack alignment.
3113 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3114 //         alignment.  Region 11, pad1, may be dynamically extended so that
3115 //         SP meets the minimum alignment.
3116 
3117 frame %{
3118   // What direction does stack grow in (assumed to be same for C & Java)
3119   stack_direction(TOWARDS_LOW);
3120 
3121   // These three registers define part of the calling convention
3122   // between compiled code and the interpreter.
3123 
3124   // Inline Cache Register or methodOop for I2C.
3125   inline_cache_reg(R12);
3126 
3127   // Method Oop Register when calling interpreter.
3128   interpreter_method_oop_reg(R12);
3129 
3130   // Number of stack slots consumed by locking an object
3131   sync_stack_slots(2);
3132 
3133   // Compiled code's Frame Pointer
3134   frame_pointer(R31);
3135 
3136   // Interpreter stores its frame pointer in a register which is
3137   // stored to the stack by I2CAdaptors.
3138   // I2CAdaptors convert from interpreted java to compiled java.
3139   interpreter_frame_pointer(R29);
3140 
3141   // Stack alignment requirement
3142   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3143 
3144   // Number of stack slots between incoming argument block and the start of
3145   // a new frame.  The PROLOG must add this many slots to the stack.  The
3146   // EPILOG must remove this many slots. aarch64 needs two slots for
3147   // return address and fp.
3148   // TODO think this is correct but check
3149   in_preserve_stack_slots(4);
3150 
3151   // Number of outgoing stack slots killed above the out_preserve_stack_slots
3152   // for calls to C.  Supports the var-args backing area for register parms.
3153   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3154 
3155   // The after-PROLOG location of the return address.  Location of
3156   // return address specifies a type (REG or STACK) and a number
3157   // representing the register number (i.e. - use a register name) or
3158   // stack slot.
3159   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3160   // Otherwise, it is above the locks and verification slot and alignment word
3161   // TODO this may well be correct but need to check why that - 2 is there
3162   // ppc port uses 0 but we definitely need to allow for fixed_slots
3163   // which folds in the space used for monitors
3164   return_addr(STACK - 2 +
3165               round_to((Compile::current()->in_preserve_stack_slots() +
3166                         Compile::current()->fixed_slots()),
3167                        stack_alignment_in_slots()));
3168 
3169   // Body of function which returns an integer array locating
3170   // arguments either in registers or in stack slots.  Passed an array
3171   // of ideal registers called "sig" and a "length" count.  Stack-slot
3172   // offsets are based on outgoing arguments, i.e. a CALLER setting up
3173   // arguments for a CALLEE.  Incoming stack arguments are
3174   // automatically biased by the preserve_stack_slots field above.
3175 
3176   calling_convention
3177   %{
3178     // No difference between ingoing/outgoing just pass false
3179     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3180   %}
3181 
3182   c_calling_convention
3183   %{
3184     // This is obviously always outgoing
3185     (void) SharedRuntime::c_calling_convention(sig_bt, regs, NULL, length);
3186   %}
3187 
3188   // Location of compiled Java return values.  Same as C for now.
3189   return_value
3190   %{
3191     // TODO do we allow ideal_reg == Op_RegN???
3192     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3193            "only return normal values");
3194 
3195     static const int lo[Op_RegL + 1] = { // enum name
3196       0,                                 // Op_Node
3197       0,                                 // Op_Set
3198       R0_num,                            // Op_RegN
3199       R0_num,                            // Op_RegI
3200       R0_num,                            // Op_RegP
3201       V0_num,                            // Op_RegF
3202       V0_num,                            // Op_RegD
3203       R0_num                             // Op_RegL
3204     };
3205 
3206     static const int hi[Op_RegL + 1] = { // enum name
3207       0,                                 // Op_Node
3208       0,                                 // Op_Set
3209       OptoReg::Bad,                       // Op_RegN
3210       OptoReg::Bad,                      // Op_RegI
3211       R0_H_num,                          // Op_RegP
3212       OptoReg::Bad,                      // Op_RegF
3213       V0_H_num,                          // Op_RegD
3214       R0_H_num                           // Op_RegL
3215     };
3216 
3217     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3218   %}
3219 %}
3220 
3221 //----------ATTRIBUTES---------------------------------------------------------
3222 //----------Operand Attributes-------------------------------------------------
3223 op_attrib op_cost(1);        // Required cost attribute
3224 
3225 //----------Instruction Attributes---------------------------------------------
3226 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3227 ins_attrib ins_size(32);        // Required size attribute (in bits)
3228 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3229                                 // a non-matching short branch variant
3230                                 // of some long branch?
3231 ins_attrib ins_alignment(4);    // Required alignment attribute (must
3232                                 // be a power of 2) specifies the
3233                                 // alignment that some part of the
3234                                 // instruction (not necessarily the
3235                                 // start) requires.  If > 1, a
3236                                 // compute_padding() function must be
3237                                 // provided for the instruction
3238 
3239 //----------OPERANDS-----------------------------------------------------------
3240 // Operand definitions must precede instruction definitions for correct parsing
3241 // in the ADLC because operands constitute user defined types which are used in
3242 // instruction definitions.
3243 
3244 //----------Simple Operands----------------------------------------------------
3245 
3246 // Integer operands 32 bit
3247 // 32 bit immediate
3248 operand immI()
3249 %{
3250   match(ConI);
3251 
3252   op_cost(0);
3253   format %{ %}
3254   interface(CONST_INTER);
3255 %}
3256 
3257 // 32 bit zero
3258 operand immI0()
3259 %{
3260   predicate(n->get_int() == 0);
3261   match(ConI);
3262 
3263   op_cost(0);
3264   format %{ %}
3265   interface(CONST_INTER);
3266 %}
3267 
3268 // 32 bit unit increment
3269 operand immI_1()
3270 %{
3271   predicate(n->get_int() == 1);
3272   match(ConI);
3273 
3274   op_cost(0);
3275   format %{ %}
3276   interface(CONST_INTER);
3277 %}
3278 
3279 // 32 bit unit decrement
3280 operand immI_M1()
3281 %{
3282   predicate(n->get_int() == -1);
3283   match(ConI);
3284 
3285   op_cost(0);
3286   format %{ %}
3287   interface(CONST_INTER);
3288 %}
3289 
3290 operand immI_le_4()
3291 %{
3292   predicate(n->get_int() <= 4);
3293   match(ConI);
3294 
3295   op_cost(0);
3296   format %{ %}
3297   interface(CONST_INTER);
3298 %}
3299 
3300 operand immI_31()
3301 %{
3302   predicate(n->get_int() == 31);
3303   match(ConI);
3304 
3305   op_cost(0);
3306   format %{ %}
3307   interface(CONST_INTER);
3308 %}
3309 
3310 operand immI_8()
3311 %{
3312   predicate(n->get_int() == 8);
3313   match(ConI);
3314 
3315   op_cost(0);
3316   format %{ %}
3317   interface(CONST_INTER);
3318 %}
3319 
3320 operand immI_16()
3321 %{
3322   predicate(n->get_int() == 16);
3323   match(ConI);
3324 
3325   op_cost(0);
3326   format %{ %}
3327   interface(CONST_INTER);
3328 %}
3329 
3330 operand immI_24()
3331 %{
3332   predicate(n->get_int() == 24);
3333   match(ConI);
3334 
3335   op_cost(0);
3336   format %{ %}
3337   interface(CONST_INTER);
3338 %}
3339 
3340 operand immI_32()
3341 %{
3342   predicate(n->get_int() == 32);
3343   match(ConI);
3344 
3345   op_cost(0);
3346   format %{ %}
3347   interface(CONST_INTER);
3348 %}
3349 
3350 operand immI_48()
3351 %{
3352   predicate(n->get_int() == 48);
3353   match(ConI);
3354 
3355   op_cost(0);
3356   format %{ %}
3357   interface(CONST_INTER);
3358 %}
3359 
3360 operand immI_56()
3361 %{
3362   predicate(n->get_int() == 56);
3363   match(ConI);
3364 
3365   op_cost(0);
3366   format %{ %}
3367   interface(CONST_INTER);
3368 %}
3369 
3370 operand immI_64()
3371 %{
3372   predicate(n->get_int() == 64);
3373   match(ConI);
3374 
3375   op_cost(0);
3376   format %{ %}
3377   interface(CONST_INTER);
3378 %}
3379 
3380 operand immI_255()
3381 %{
3382   predicate(n->get_int() == 255);
3383   match(ConI);
3384 
3385   op_cost(0);
3386   format %{ %}
3387   interface(CONST_INTER);
3388 %}
3389 
3390 operand immI_65535()
3391 %{
3392   predicate(n->get_int() == 65535);
3393   match(ConI);
3394 
3395   op_cost(0);
3396   format %{ %}
3397   interface(CONST_INTER);
3398 %}
3399 
3400 operand immL_63()
3401 %{
3402   predicate(n->get_int() == 63);
3403   match(ConI);
3404 
3405   op_cost(0);
3406   format %{ %}
3407   interface(CONST_INTER);
3408 %}
3409 
3410 operand immL_255()
3411 %{
3412   predicate(n->get_int() == 255);
3413   match(ConI);
3414 
3415   op_cost(0);
3416   format %{ %}
3417   interface(CONST_INTER);
3418 %}
3419 
3420 operand immL_65535()
3421 %{
3422   predicate(n->get_long() == 65535L);
3423   match(ConL);
3424 
3425   op_cost(0);
3426   format %{ %}
3427   interface(CONST_INTER);
3428 %}
3429 
3430 operand immL_4294967295()
3431 %{
3432   predicate(n->get_long() == 4294967295L);
3433   match(ConL);
3434 
3435   op_cost(0);
3436   format %{ %}
3437   interface(CONST_INTER);
3438 %}
3439 
3440 operand immL_bitmask()
3441 %{
3442   predicate(((n->get_long() & 0xc000000000000000l) == 0)
3443             && is_power_of_2(n->get_long() + 1));
3444   match(ConL);
3445 
3446   op_cost(0);
3447   format %{ %}
3448   interface(CONST_INTER);
3449 %}
3450 
3451 operand immI_bitmask()
3452 %{
3453   predicate(((n->get_int() & 0xc0000000) == 0)
3454             && is_power_of_2(n->get_int() + 1));
3455   match(ConI);
3456 
3457   op_cost(0);
3458   format %{ %}
3459   interface(CONST_INTER);
3460 %}
3461 
3462 // Scale values for scaled offset addressing modes (up to long but not quad)
3463 operand immIScale()
3464 %{
3465   predicate(0 <= n->get_int() && (n->get_int() <= 3));
3466   match(ConI);
3467 
3468   op_cost(0);
3469   format %{ %}
3470   interface(CONST_INTER);
3471 %}
3472 
3473 // 26 bit signed offset -- for pc-relative branches
3474 operand immI26()
3475 %{
3476   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
3477   match(ConI);
3478 
3479   op_cost(0);
3480   format %{ %}
3481   interface(CONST_INTER);
3482 %}
3483 
3484 // 19 bit signed offset -- for pc-relative loads
3485 operand immI19()
3486 %{
3487   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
3488   match(ConI);
3489 
3490   op_cost(0);
3491   format %{ %}
3492   interface(CONST_INTER);
3493 %}
3494 
3495 // 12 bit unsigned offset -- for base plus immediate loads
3496 operand immIU12()
3497 %{
3498   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
3499   match(ConI);
3500 
3501   op_cost(0);
3502   format %{ %}
3503   interface(CONST_INTER);
3504 %}
3505 
3506 operand immLU12()
3507 %{
3508   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
3509   match(ConL);
3510 
3511   op_cost(0);
3512   format %{ %}
3513   interface(CONST_INTER);
3514 %}
3515 
3516 // Offset for scaled or unscaled immediate loads and stores
3517 operand immIOffset()
3518 %{
3519   predicate(Address::offset_ok_for_immed(n->get_int()));
3520   match(ConI);
3521 
3522   op_cost(0);
3523   format %{ %}
3524   interface(CONST_INTER);
3525 %}
3526 
3527 operand immLoffset()
3528 %{
3529   predicate(Address::offset_ok_for_immed(n->get_long()));
3530   match(ConL);
3531 
3532   op_cost(0);
3533   format %{ %}
3534   interface(CONST_INTER);
3535 %}
3536 
3537 // 32 bit integer valid for add sub immediate
3538 operand immIAddSub()
3539 %{
3540   predicate(Assembler::operand_valid_for_add_sub_immediate((long)n->get_int()));
3541   match(ConI);
3542   op_cost(0);
3543   format %{ %}
3544   interface(CONST_INTER);
3545 %}
3546 
3547 // 32 bit unsigned integer valid for logical immediate
3548 // TODO -- check this is right when e.g the mask is 0x80000000
3549 operand immILog()
3550 %{
3551   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (unsigned long)n->get_int()));
3552   match(ConI);
3553 
3554   op_cost(0);
3555   format %{ %}
3556   interface(CONST_INTER);
3557 %}
3558 
3559 // Integer operands 64 bit
3560 // 64 bit immediate
3561 operand immL()
3562 %{
3563   match(ConL);
3564 
3565   op_cost(0);
3566   format %{ %}
3567   interface(CONST_INTER);
3568 %}
3569 
3570 // 64 bit zero
3571 operand immL0()
3572 %{
3573   predicate(n->get_long() == 0);
3574   match(ConL);
3575 
3576   op_cost(0);
3577   format %{ %}
3578   interface(CONST_INTER);
3579 %}
3580 
3581 // 64 bit unit increment
3582 operand immL_1()
3583 %{
3584   predicate(n->get_long() == 1);
3585   match(ConL);
3586 
3587   op_cost(0);
3588   format %{ %}
3589   interface(CONST_INTER);
3590 %}
3591 
3592 // 64 bit unit decrement
3593 operand immL_M1()
3594 %{
3595   predicate(n->get_long() == -1);
3596   match(ConL);
3597 
3598   op_cost(0);
3599   format %{ %}
3600   interface(CONST_INTER);
3601 %}
3602 
3603 // 32 bit offset of pc in thread anchor
3604 
3605 operand immL_pc_off()
3606 %{
3607   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
3608                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
3609   match(ConL);
3610 
3611   op_cost(0);
3612   format %{ %}
3613   interface(CONST_INTER);
3614 %}
3615 
3616 // 64 bit integer valid for add sub immediate
3617 operand immLAddSub()
3618 %{
3619   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
3620   match(ConL);
3621   op_cost(0);
3622   format %{ %}
3623   interface(CONST_INTER);
3624 %}
3625 
3626 // 64 bit integer valid for logical immediate
3627 operand immLLog()
3628 %{
3629   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (unsigned long)n->get_long()));
3630   match(ConL);
3631   op_cost(0);
3632   format %{ %}
3633   interface(CONST_INTER);
3634 %}
3635 
3636 // Long Immediate: low 32-bit mask
3637 operand immL_32bits()
3638 %{
3639   predicate(n->get_long() == 0xFFFFFFFFL);
3640   match(ConL);
3641   op_cost(0);
3642   format %{ %}
3643   interface(CONST_INTER);
3644 %}
3645 
3646 // Pointer operands
3647 // Pointer Immediate
3648 operand immP()
3649 %{
3650   match(ConP);
3651 
3652   op_cost(0);
3653   format %{ %}
3654   interface(CONST_INTER);
3655 %}
3656 
3657 // NULL Pointer Immediate
3658 operand immP0()
3659 %{
3660   predicate(n->get_ptr() == 0);
3661   match(ConP);
3662 
3663   op_cost(0);
3664   format %{ %}
3665   interface(CONST_INTER);
3666 %}
3667 
3668 // Pointer Immediate One
3669 // this is used in object initialization (initial object header)
3670 operand immP_1()
3671 %{
3672   predicate(n->get_ptr() == 1);
3673   match(ConP);
3674 
3675   op_cost(0);
3676   format %{ %}
3677   interface(CONST_INTER);
3678 %}
3679 
3680 // Polling Page Pointer Immediate
3681 operand immPollPage()
3682 %{
3683   predicate((address)n->get_ptr() == os::get_polling_page());
3684   match(ConP);
3685 
3686   op_cost(0);
3687   format %{ %}
3688   interface(CONST_INTER);
3689 %}
3690 
3691 // Card Table Byte Map Base
3692 operand immByteMapBase()
3693 %{
3694   // Get base of card map
3695   predicate((jbyte*)n->get_ptr() ==
3696         ((CardTableModRefBS*)(Universe::heap()->barrier_set()))->byte_map_base);
3697   match(ConP);
3698 
3699   op_cost(0);
3700   format %{ %}
3701   interface(CONST_INTER);
3702 %}
3703 
3704 // Pointer Immediate Minus One
3705 // this is used when we want to write the current PC to the thread anchor
3706 operand immP_M1()
3707 %{
3708   predicate(n->get_ptr() == -1);
3709   match(ConP);
3710 
3711   op_cost(0);
3712   format %{ %}
3713   interface(CONST_INTER);
3714 %}
3715 
3716 // Pointer Immediate Minus Two
3717 // this is used when we want to write the current PC to the thread anchor
3718 operand immP_M2()
3719 %{
3720   predicate(n->get_ptr() == -2);
3721   match(ConP);
3722 
3723   op_cost(0);
3724   format %{ %}
3725   interface(CONST_INTER);
3726 %}
3727 
3728 // Float and Double operands
3729 // Double Immediate
3730 operand immD()
3731 %{
3732   match(ConD);
3733   op_cost(0);
3734   format %{ %}
3735   interface(CONST_INTER);
3736 %}
3737 
3738 // Double Immediate: +0.0d
3739 operand immD0()
3740 %{
3741   predicate(jlong_cast(n->getd()) == 0);
3742   match(ConD);
3743 
3744   op_cost(0);
3745   format %{ %}
3746   interface(CONST_INTER);
3747 %}
3748 
3749 // constant 'double +0.0'.
3750 operand immDPacked()
3751 %{
3752   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
3753   match(ConD);
3754   op_cost(0);
3755   format %{ %}
3756   interface(CONST_INTER);
3757 %}
3758 
3759 // Float Immediate
3760 operand immF()
3761 %{
3762   match(ConF);
3763   op_cost(0);
3764   format %{ %}
3765   interface(CONST_INTER);
3766 %}
3767 
3768 // Float Immediate: +0.0f.
3769 operand immF0()
3770 %{
3771   predicate(jint_cast(n->getf()) == 0);
3772   match(ConF);
3773 
3774   op_cost(0);
3775   format %{ %}
3776   interface(CONST_INTER);
3777 %}
3778 
3779 //
3780 operand immFPacked()
3781 %{
3782   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
3783   match(ConF);
3784   op_cost(0);
3785   format %{ %}
3786   interface(CONST_INTER);
3787 %}
3788 
3789 // Narrow pointer operands
3790 // Narrow Pointer Immediate
3791 operand immN()
3792 %{
3793   match(ConN);
3794 
3795   op_cost(0);
3796   format %{ %}
3797   interface(CONST_INTER);
3798 %}
3799 
3800 // Narrow NULL Pointer Immediate
3801 operand immN0()
3802 %{
3803   predicate(n->get_narrowcon() == 0);
3804   match(ConN);
3805 
3806   op_cost(0);
3807   format %{ %}
3808   interface(CONST_INTER);
3809 %}
3810 
3811 operand immNKlass()
3812 %{
3813   match(ConNKlass);
3814 
3815   op_cost(0);
3816   format %{ %}
3817   interface(CONST_INTER);
3818 %}
3819 
3820 // Integer 32 bit Register Operands
3821 // Integer 32 bitRegister (excludes SP)
3822 operand iRegI()
3823 %{
3824   constraint(ALLOC_IN_RC(any_reg32));
3825   match(RegI);
3826   match(iRegINoSp);
3827   op_cost(0);
3828   format %{ %}
3829   interface(REG_INTER);
3830 %}
3831 
3832 // Integer 32 bit Register not Special
3833 operand iRegINoSp()
3834 %{
3835   constraint(ALLOC_IN_RC(no_special_reg32));
3836   match(RegI);
3837   op_cost(0);
3838   format %{ %}
3839   interface(REG_INTER);
3840 %}
3841 
3842 // Integer 64 bit Register Operands
3843 // Integer 64 bit Register (includes SP)
3844 operand iRegL()
3845 %{
3846   constraint(ALLOC_IN_RC(any_reg));
3847   match(RegL);
3848   match(iRegLNoSp);
3849   op_cost(0);
3850   format %{ %}
3851   interface(REG_INTER);
3852 %}
3853 
3854 // Integer 64 bit Register not Special
3855 operand iRegLNoSp()
3856 %{
3857   constraint(ALLOC_IN_RC(no_special_reg));
3858   match(RegL);
3859   format %{ %}
3860   interface(REG_INTER);
3861 %}
3862 
3863 // Pointer Register Operands
3864 // Pointer Register
3865 operand iRegP()
3866 %{
3867   constraint(ALLOC_IN_RC(ptr_reg));
3868   match(RegP);
3869   match(iRegPNoSp);
3870   match(iRegP_R0);
3871   //match(iRegP_R2);
3872   //match(iRegP_R4);
3873   //match(iRegP_R5);
3874   match(thread_RegP);
3875   op_cost(0);
3876   format %{ %}
3877   interface(REG_INTER);
3878 %}
3879 
3880 // Pointer 64 bit Register not Special
3881 operand iRegPNoSp()
3882 %{
3883   constraint(ALLOC_IN_RC(no_special_ptr_reg));
3884   match(RegP);
3885   // match(iRegP);
3886   // match(iRegP_R0);
3887   // match(iRegP_R2);
3888   // match(iRegP_R4);
3889   // match(iRegP_R5);
3890   // match(thread_RegP);
3891   op_cost(0);
3892   format %{ %}
3893   interface(REG_INTER);
3894 %}
3895 
3896 // Pointer 64 bit Register R0 only
3897 operand iRegP_R0()
3898 %{
3899   constraint(ALLOC_IN_RC(r0_reg));
3900   match(RegP);
3901   // match(iRegP);
3902   match(iRegPNoSp);
3903   op_cost(0);
3904   format %{ %}
3905   interface(REG_INTER);
3906 %}
3907 
3908 // Pointer 64 bit Register R1 only
3909 operand iRegP_R1()
3910 %{
3911   constraint(ALLOC_IN_RC(r1_reg));
3912   match(RegP);
3913   // match(iRegP);
3914   match(iRegPNoSp);
3915   op_cost(0);
3916   format %{ %}
3917   interface(REG_INTER);
3918 %}
3919 
3920 // Pointer 64 bit Register R2 only
3921 operand iRegP_R2()
3922 %{
3923   constraint(ALLOC_IN_RC(r2_reg));
3924   match(RegP);
3925   // match(iRegP);
3926   match(iRegPNoSp);
3927   op_cost(0);
3928   format %{ %}
3929   interface(REG_INTER);
3930 %}
3931 
3932 // Pointer 64 bit Register R3 only
3933 operand iRegP_R3()
3934 %{
3935   constraint(ALLOC_IN_RC(r3_reg));
3936   match(RegP);
3937   // match(iRegP);
3938   match(iRegPNoSp);
3939   op_cost(0);
3940   format %{ %}
3941   interface(REG_INTER);
3942 %}
3943 
3944 // Pointer 64 bit Register R4 only
3945 operand iRegP_R4()
3946 %{
3947   constraint(ALLOC_IN_RC(r4_reg));
3948   match(RegP);
3949   // match(iRegP);
3950   match(iRegPNoSp);
3951   op_cost(0);
3952   format %{ %}
3953   interface(REG_INTER);
3954 %}
3955 
3956 // Pointer 64 bit Register R5 only
3957 operand iRegP_R5()
3958 %{
3959   constraint(ALLOC_IN_RC(r5_reg));
3960   match(RegP);
3961   // match(iRegP);
3962   match(iRegPNoSp);
3963   op_cost(0);
3964   format %{ %}
3965   interface(REG_INTER);
3966 %}
3967 
3968 // Pointer 64 bit Register R10 only
3969 operand iRegP_R10()
3970 %{
3971   constraint(ALLOC_IN_RC(r10_reg));
3972   match(RegP);
3973   // match(iRegP);
3974   match(iRegPNoSp);
3975   op_cost(0);
3976   format %{ %}
3977   interface(REG_INTER);
3978 %}
3979 
3980 // Long 64 bit Register R11 only
3981 operand iRegL_R11()
3982 %{
3983   constraint(ALLOC_IN_RC(r11_reg));
3984   match(RegL);
3985   match(iRegLNoSp);
3986   op_cost(0);
3987   format %{ %}
3988   interface(REG_INTER);
3989 %}
3990 
3991 // Pointer 64 bit Register FP only
3992 operand iRegP_FP()
3993 %{
3994   constraint(ALLOC_IN_RC(fp_reg));
3995   match(RegP);
3996   // match(iRegP);
3997   op_cost(0);
3998   format %{ %}
3999   interface(REG_INTER);
4000 %}
4001 
4002 // Register R0 only
4003 operand iRegI_R0()
4004 %{
4005   constraint(ALLOC_IN_RC(int_r0_reg));
4006   match(RegI);
4007   match(iRegINoSp);
4008   op_cost(0);
4009   format %{ %}
4010   interface(REG_INTER);
4011 %}
4012 
4013 // Register R2 only
4014 operand iRegI_R2()
4015 %{
4016   constraint(ALLOC_IN_RC(int_r2_reg));
4017   match(RegI);
4018   match(iRegINoSp);
4019   op_cost(0);
4020   format %{ %}
4021   interface(REG_INTER);
4022 %}
4023 
4024 // Register R3 only
4025 operand iRegI_R3()
4026 %{
4027   constraint(ALLOC_IN_RC(int_r3_reg));
4028   match(RegI);
4029   match(iRegINoSp);
4030   op_cost(0);
4031   format %{ %}
4032   interface(REG_INTER);
4033 %}
4034 
4035 
4036 // Register R2 only
4037 operand iRegI_R4()
4038 %{
4039   constraint(ALLOC_IN_RC(int_r4_reg));
4040   match(RegI);
4041   match(iRegINoSp);
4042   op_cost(0);
4043   format %{ %}
4044   interface(REG_INTER);
4045 %}
4046 
4047 
4048 // Pointer Register Operands
4049 // Narrow Pointer Register
4050 operand iRegN()
4051 %{
4052   constraint(ALLOC_IN_RC(any_reg32));
4053   match(RegN);
4054   match(iRegNNoSp);
4055   op_cost(0);
4056   format %{ %}
4057   interface(REG_INTER);
4058 %}
4059 
4060 // Integer 64 bit Register not Special
4061 operand iRegNNoSp()
4062 %{
4063   constraint(ALLOC_IN_RC(no_special_reg32));
4064   match(RegN);
4065   op_cost(0);
4066   format %{ %}
4067   interface(REG_INTER);
4068 %}
4069 
4070 // heap base register -- used for encoding immN0
4071 
4072 operand iRegIHeapbase()
4073 %{
4074   constraint(ALLOC_IN_RC(heapbase_reg));
4075   match(RegI);
4076   op_cost(0);
4077   format %{ %}
4078   interface(REG_INTER);
4079 %}
4080 
4081 // Float Register
4082 // Float register operands
4083 operand vRegF()
4084 %{
4085   constraint(ALLOC_IN_RC(float_reg));
4086   match(RegF);
4087 
4088   op_cost(0);
4089   format %{ %}
4090   interface(REG_INTER);
4091 %}
4092 
4093 // Double Register
4094 // Double register operands
4095 operand vRegD()
4096 %{
4097   constraint(ALLOC_IN_RC(double_reg));
4098   match(RegD);
4099 
4100   op_cost(0);
4101   format %{ %}
4102   interface(REG_INTER);
4103 %}
4104 
4105 operand vRegD_V0()
4106 %{
4107   constraint(ALLOC_IN_RC(v0_reg));
4108   match(RegD);
4109   op_cost(0);
4110   format %{ %}
4111   interface(REG_INTER);
4112 %}
4113 
4114 operand vRegD_V1()
4115 %{
4116   constraint(ALLOC_IN_RC(v1_reg));
4117   match(RegD);
4118   op_cost(0);
4119   format %{ %}
4120   interface(REG_INTER);
4121 %}
4122 
4123 operand vRegD_V2()
4124 %{
4125   constraint(ALLOC_IN_RC(v2_reg));
4126   match(RegD);
4127   op_cost(0);
4128   format %{ %}
4129   interface(REG_INTER);
4130 %}
4131 
4132 operand vRegD_V3()
4133 %{
4134   constraint(ALLOC_IN_RC(v3_reg));
4135   match(RegD);
4136   op_cost(0);
4137   format %{ %}
4138   interface(REG_INTER);
4139 %}
4140 
4141 // Flags register, used as output of signed compare instructions
4142 
4143 // note that on AArch64 we also use this register as the output for
4144 // for floating point compare instructions (CmpF CmpD). this ensures
4145 // that ordered inequality tests use GT, GE, LT or LE none of which
4146 // pass through cases where the result is unordered i.e. one or both
4147 // inputs to the compare is a NaN. this means that the ideal code can
4148 // replace e.g. a GT with an LE and not end up capturing the NaN case
4149 // (where the comparison should always fail). EQ and NE tests are
4150 // always generated in ideal code so that unordered folds into the NE
4151 // case, matching the behaviour of AArch64 NE.
4152 //
4153 // This differs from x86 where the outputs of FP compares use a
4154 // special FP flags registers and where compares based on this
4155 // register are distinguished into ordered inequalities (cmpOpUCF) and
4156 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
4157 // to explicitly handle the unordered case in branches. x86 also has
4158 // to include extra CMoveX rules to accept a cmpOpUCF input.
4159 
4160 operand rFlagsReg()
4161 %{
4162   constraint(ALLOC_IN_RC(int_flags));
4163   match(RegFlags);
4164 
4165   op_cost(0);
4166   format %{ "RFLAGS" %}
4167   interface(REG_INTER);
4168 %}
4169 
4170 // Flags register, used as output of unsigned compare instructions
4171 operand rFlagsRegU()
4172 %{
4173   constraint(ALLOC_IN_RC(int_flags));
4174   match(RegFlags);
4175 
4176   op_cost(0);
4177   format %{ "RFLAGSU" %}
4178   interface(REG_INTER);
4179 %}
4180 
4181 // Special Registers
4182 
4183 // Method Register
4184 operand inline_cache_RegP(iRegP reg)
4185 %{
4186   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
4187   match(reg);
4188   match(iRegPNoSp);
4189   op_cost(0);
4190   format %{ %}
4191   interface(REG_INTER);
4192 %}
4193 
4194 operand interpreter_method_oop_RegP(iRegP reg)
4195 %{
4196   constraint(ALLOC_IN_RC(method_reg)); // interpreter_method_oop_reg
4197   match(reg);
4198   match(iRegPNoSp);
4199   op_cost(0);
4200   format %{ %}
4201   interface(REG_INTER);
4202 %}
4203 
4204 // Thread Register
4205 operand thread_RegP(iRegP reg)
4206 %{
4207   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
4208   match(reg);
4209   op_cost(0);
4210   format %{ %}
4211   interface(REG_INTER);
4212 %}
4213 
4214 operand lr_RegP(iRegP reg)
4215 %{
4216   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
4217   match(reg);
4218   op_cost(0);
4219   format %{ %}
4220   interface(REG_INTER);
4221 %}
4222 
4223 //----------Memory Operands----------------------------------------------------
4224 
4225 operand indirect(iRegP reg)
4226 %{
4227   constraint(ALLOC_IN_RC(ptr_reg));
4228   match(reg);
4229   op_cost(0);
4230   format %{ "[$reg]" %}
4231   interface(MEMORY_INTER) %{
4232     base($reg);
4233     index(0xffffffff);
4234     scale(0x0);
4235     disp(0x0);
4236   %}
4237 %}
4238 
4239 operand indIndexScaledOffsetI(iRegP reg, iRegL lreg, immIScale scale, immIU12 off)
4240 %{
4241   constraint(ALLOC_IN_RC(ptr_reg));
4242   match(AddP (AddP reg (LShiftL lreg scale)) off);
4243   op_cost(INSN_COST);
4244   format %{ "$reg, $lreg lsl($scale), $off" %}
4245   interface(MEMORY_INTER) %{
4246     base($reg);
4247     index($lreg);
4248     scale($scale);
4249     disp($off);
4250   %}
4251 %}
4252 
4253 operand indIndexScaledOffsetL(iRegP reg, iRegL lreg, immIScale scale, immLU12 off)
4254 %{
4255   constraint(ALLOC_IN_RC(ptr_reg));
4256   match(AddP (AddP reg (LShiftL lreg scale)) off);
4257   op_cost(INSN_COST);
4258   format %{ "$reg, $lreg lsl($scale), $off" %}
4259   interface(MEMORY_INTER) %{
4260     base($reg);
4261     index($lreg);
4262     scale($scale);
4263     disp($off);
4264   %}
4265 %}
4266 
4267 operand indIndexScaledOffsetI2L(iRegP reg, iRegI ireg, immIScale scale, immLU12 off)
4268 %{
4269   constraint(ALLOC_IN_RC(ptr_reg));
4270   match(AddP (AddP reg (LShiftL (ConvI2L ireg) scale)) off);
4271   op_cost(INSN_COST);
4272   format %{ "$reg, $ireg sxtw($scale), $off I2L" %}
4273   interface(MEMORY_INTER) %{
4274     base($reg);
4275     index($ireg);
4276     scale($scale);
4277     disp($off);
4278   %}
4279 %}
4280 
4281 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
4282 %{
4283   constraint(ALLOC_IN_RC(ptr_reg));
4284   match(AddP reg (LShiftL (ConvI2L ireg) scale));
4285   op_cost(0);
4286   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
4287   interface(MEMORY_INTER) %{
4288     base($reg);
4289     index($ireg);
4290     scale($scale);
4291     disp(0x0);
4292   %}
4293 %}
4294 
4295 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
4296 %{
4297   constraint(ALLOC_IN_RC(ptr_reg));
4298   match(AddP reg (LShiftL lreg scale));
4299   op_cost(0);
4300   format %{ "$reg, $lreg lsl($scale)" %}
4301   interface(MEMORY_INTER) %{
4302     base($reg);
4303     index($lreg);
4304     scale($scale);
4305     disp(0x0);
4306   %}
4307 %}
4308 
4309 operand indIndex(iRegP reg, iRegL lreg)
4310 %{
4311   constraint(ALLOC_IN_RC(ptr_reg));
4312   match(AddP reg lreg);
4313   op_cost(0);
4314   format %{ "$reg, $lreg" %}
4315   interface(MEMORY_INTER) %{
4316     base($reg);
4317     index($lreg);
4318     scale(0x0);
4319     disp(0x0);
4320   %}
4321 %}
4322 
4323 operand indOffI(iRegP reg, immIOffset off)
4324 %{
4325   constraint(ALLOC_IN_RC(ptr_reg));
4326   match(AddP reg off);
4327   op_cost(INSN_COST);
4328   format %{ "[$reg, $off]" %}
4329   interface(MEMORY_INTER) %{
4330     base($reg);
4331     index(0xffffffff);
4332     scale(0x0);
4333     disp($off);
4334   %}
4335 %}
4336 
4337 operand indOffL(iRegP reg, immLoffset off)
4338 %{
4339   constraint(ALLOC_IN_RC(ptr_reg));
4340   match(AddP reg off);
4341   op_cost(0);
4342   format %{ "[$reg, $off]" %}
4343   interface(MEMORY_INTER) %{
4344     base($reg);
4345     index(0xffffffff);
4346     scale(0x0);
4347     disp($off);
4348   %}
4349 %}
4350 
4351 
4352 operand indirectN(iRegN reg)
4353 %{
4354   predicate(Universe::narrow_oop_shift() == 0);
4355   constraint(ALLOC_IN_RC(ptr_reg));
4356   match(DecodeN reg);
4357   op_cost(0);
4358   format %{ "[$reg]\t# narrow" %}
4359   interface(MEMORY_INTER) %{
4360     base($reg);
4361     index(0xffffffff);
4362     scale(0x0);
4363     disp(0x0);
4364   %}
4365 %}
4366 
4367 operand indIndexScaledOffsetIN(iRegN reg, iRegL lreg, immIScale scale, immIU12 off)
4368 %{
4369   predicate(Universe::narrow_oop_shift() == 0);
4370   constraint(ALLOC_IN_RC(ptr_reg));
4371   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4372   op_cost(0);
4373   format %{ "$reg, $lreg lsl($scale), $off\t# narrow" %}
4374   interface(MEMORY_INTER) %{
4375     base($reg);
4376     index($lreg);
4377     scale($scale);
4378     disp($off);
4379   %}
4380 %}
4381 
4382 operand indIndexScaledOffsetLN(iRegN reg, iRegL lreg, immIScale scale, immLU12 off)
4383 %{
4384   predicate(Universe::narrow_oop_shift() == 0);
4385   constraint(ALLOC_IN_RC(ptr_reg));
4386   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4387   op_cost(INSN_COST);
4388   format %{ "$reg, $lreg lsl($scale), $off\t# narrow" %}
4389   interface(MEMORY_INTER) %{
4390     base($reg);
4391     index($lreg);
4392     scale($scale);
4393     disp($off);
4394   %}
4395 %}
4396 
4397 operand indIndexScaledOffsetI2LN(iRegN reg, iRegI ireg, immIScale scale, immLU12 off)
4398 %{
4399   predicate(Universe::narrow_oop_shift() == 0);
4400   constraint(ALLOC_IN_RC(ptr_reg));
4401   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale)) off);
4402   op_cost(INSN_COST);
4403   format %{ "$reg, $ireg sxtw($scale), $off I2L\t# narrow" %}
4404   interface(MEMORY_INTER) %{
4405     base($reg);
4406     index($ireg);
4407     scale($scale);
4408     disp($off);
4409   %}
4410 %}
4411 
4412 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
4413 %{
4414   predicate(Universe::narrow_oop_shift() == 0);
4415   constraint(ALLOC_IN_RC(ptr_reg));
4416   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
4417   op_cost(0);
4418   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
4419   interface(MEMORY_INTER) %{
4420     base($reg);
4421     index($ireg);
4422     scale($scale);
4423     disp(0x0);
4424   %}
4425 %}
4426 
4427 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
4428 %{
4429   predicate(Universe::narrow_oop_shift() == 0);
4430   constraint(ALLOC_IN_RC(ptr_reg));
4431   match(AddP (DecodeN reg) (LShiftL lreg scale));
4432   op_cost(0);
4433   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
4434   interface(MEMORY_INTER) %{
4435     base($reg);
4436     index($lreg);
4437     scale($scale);
4438     disp(0x0);
4439   %}
4440 %}
4441 
4442 operand indIndexN(iRegN reg, iRegL lreg)
4443 %{
4444   predicate(Universe::narrow_oop_shift() == 0);
4445   constraint(ALLOC_IN_RC(ptr_reg));
4446   match(AddP (DecodeN reg) lreg);
4447   op_cost(0);
4448   format %{ "$reg, $lreg\t# narrow" %}
4449   interface(MEMORY_INTER) %{
4450     base($reg);
4451     index($lreg);
4452     scale(0x0);
4453     disp(0x0);
4454   %}
4455 %}
4456 
4457 operand indOffIN(iRegN reg, immIOffset off)
4458 %{
4459   predicate(Universe::narrow_oop_shift() == 0);
4460   constraint(ALLOC_IN_RC(ptr_reg));
4461   match(AddP (DecodeN reg) off);
4462   op_cost(0);
4463   format %{ "[$reg, $off]\t# narrow" %}
4464   interface(MEMORY_INTER) %{
4465     base($reg);
4466     index(0xffffffff);
4467     scale(0x0);
4468     disp($off);
4469   %}
4470 %}
4471 
4472 operand indOffLN(iRegN reg, immLoffset off)
4473 %{
4474   predicate(Universe::narrow_oop_shift() == 0);
4475   constraint(ALLOC_IN_RC(ptr_reg));
4476   match(AddP (DecodeN reg) off);
4477   op_cost(0);
4478   format %{ "[$reg, $off]\t# narrow" %}
4479   interface(MEMORY_INTER) %{
4480     base($reg);
4481     index(0xffffffff);
4482     scale(0x0);
4483     disp($off);
4484   %}
4485 %}
4486 
4487 
4488 
4489 // AArch64 opto stubs need to write to the pc slot in the thread anchor
4490 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
4491 %{
4492   constraint(ALLOC_IN_RC(ptr_reg));
4493   match(AddP reg off);
4494   op_cost(0);
4495   format %{ "[$reg, $off]" %}
4496   interface(MEMORY_INTER) %{
4497     base($reg);
4498     index(0xffffffff);
4499     scale(0x0);
4500     disp($off);
4501   %}
4502 %}
4503 
4504 //----------Special Memory Operands--------------------------------------------
4505 // Stack Slot Operand - This operand is used for loading and storing temporary
4506 //                      values on the stack where a match requires a value to
4507 //                      flow through memory.
4508 operand stackSlotP(sRegP reg)
4509 %{
4510   constraint(ALLOC_IN_RC(stack_slots));
4511   op_cost(100);
4512   // No match rule because this operand is only generated in matching
4513   // match(RegP);
4514   format %{ "[$reg]" %}
4515   interface(MEMORY_INTER) %{
4516     base(0x1e);  // RSP
4517     index(0x0);  // No Index
4518     scale(0x0);  // No Scale
4519     disp($reg);  // Stack Offset
4520   %}
4521 %}
4522 
4523 operand stackSlotI(sRegI reg)
4524 %{
4525   constraint(ALLOC_IN_RC(stack_slots));
4526   // No match rule because this operand is only generated in matching
4527   // match(RegI);
4528   format %{ "[$reg]" %}
4529   interface(MEMORY_INTER) %{
4530     base(0x1e);  // RSP
4531     index(0x0);  // No Index
4532     scale(0x0);  // No Scale
4533     disp($reg);  // Stack Offset
4534   %}
4535 %}
4536 
4537 operand stackSlotF(sRegF reg)
4538 %{
4539   constraint(ALLOC_IN_RC(stack_slots));
4540   // No match rule because this operand is only generated in matching
4541   // match(RegF);
4542   format %{ "[$reg]" %}
4543   interface(MEMORY_INTER) %{
4544     base(0x1e);  // RSP
4545     index(0x0);  // No Index
4546     scale(0x0);  // No Scale
4547     disp($reg);  // Stack Offset
4548   %}
4549 %}
4550 
4551 operand stackSlotD(sRegD reg)
4552 %{
4553   constraint(ALLOC_IN_RC(stack_slots));
4554   // No match rule because this operand is only generated in matching
4555   // match(RegD);
4556   format %{ "[$reg]" %}
4557   interface(MEMORY_INTER) %{
4558     base(0x1e);  // RSP
4559     index(0x0);  // No Index
4560     scale(0x0);  // No Scale
4561     disp($reg);  // Stack Offset
4562   %}
4563 %}
4564 
4565 operand stackSlotL(sRegL reg)
4566 %{
4567   constraint(ALLOC_IN_RC(stack_slots));
4568   // No match rule because this operand is only generated in matching
4569   // match(RegL);
4570   format %{ "[$reg]" %}
4571   interface(MEMORY_INTER) %{
4572     base(0x1e);  // RSP
4573     index(0x0);  // No Index
4574     scale(0x0);  // No Scale
4575     disp($reg);  // Stack Offset
4576   %}
4577 %}
4578 
4579 // Operands for expressing Control Flow
4580 // NOTE: Label is a predefined operand which should not be redefined in
4581 //       the AD file. It is generically handled within the ADLC.
4582 
4583 //----------Conditional Branch Operands----------------------------------------
4584 // Comparison Op  - This is the operation of the comparison, and is limited to
4585 //                  the following set of codes:
4586 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4587 //
4588 // Other attributes of the comparison, such as unsignedness, are specified
4589 // by the comparison instruction that sets a condition code flags register.
4590 // That result is represented by a flags operand whose subtype is appropriate
4591 // to the unsignedness (etc.) of the comparison.
4592 //
4593 // Later, the instruction which matches both the Comparison Op (a Bool) and
4594 // the flags (produced by the Cmp) specifies the coding of the comparison op
4595 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4596 
4597 // used for signed integral comparisons and fp comparisons
4598 
4599 operand cmpOp()
4600 %{
4601   match(Bool);
4602 
4603   format %{ "" %}
4604   interface(COND_INTER) %{
4605     equal(0x0, "eq");
4606     not_equal(0x1, "ne");
4607     less(0xb, "lt");
4608     greater_equal(0xa, "ge");
4609     less_equal(0xd, "le");
4610     greater(0xc, "gt");
4611     overflow(0x6, "vs");
4612     no_overflow(0x7, "vc");
4613   %}
4614 %}
4615 
4616 // used for unsigned integral comparisons
4617 
4618 operand cmpOpU()
4619 %{
4620   match(Bool);
4621 
4622   format %{ "" %}
4623   interface(COND_INTER) %{
4624     equal(0x0, "eq");
4625     not_equal(0x1, "ne");
4626     less(0x3, "lo");
4627     greater_equal(0x2, "hs");
4628     less_equal(0x9, "ls");
4629     greater(0x8, "hi");
4630     overflow(0x6, "vs");
4631     no_overflow(0x7, "vc");
4632   %}
4633 %}
4634 
4635 // Special operand allowing long args to int ops to be truncated for free
4636 
4637 operand iRegL2I(iRegL reg) %{
4638 
4639   op_cost(0);
4640 
4641   match(ConvL2I reg);
4642 
4643   format %{ "l2i($reg)" %}
4644 
4645   interface(REG_INTER)
4646 %}
4647 
4648 
4649 //----------OPERAND CLASSES----------------------------------------------------
4650 // Operand Classes are groups of operands that are used as to simplify
4651 // instruction definitions by not requiring the AD writer to specify
4652 // separate instructions for every form of operand when the
4653 // instruction accepts multiple operand types with the same basic
4654 // encoding and format. The classic case of this is memory operands.
4655 
4656 // memory is used to define read/write location for load/store
4657 // instruction defs. we can turn a memory op into an Address
4658 
4659 opclass memory(indirect, indIndexScaledOffsetI,  indIndexScaledOffsetL, indIndexScaledOffsetI2L, indIndexScaled, indIndexScaledI2L, indIndex, indOffI, indOffL,
4660                indirectN, indIndexScaledOffsetIN,  indIndexScaledOffsetLN, indIndexScaledOffsetI2LN, indIndexScaledN, indIndexScaledI2LN, indIndexN, indOffIN, indOffLN);
4661 
4662 
4663 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
4664 // operations. it allows the src to be either an iRegI or a (ConvL2I
4665 // iRegL). in the latter case the l2i normally planted for a ConvL2I
4666 // can be elided because the 32-bit instruction will just employ the
4667 // lower 32 bits anyway.
4668 //
4669 // n.b. this does not elide all L2I conversions. if the truncated
4670 // value is consumed by more than one operation then the ConvL2I
4671 // cannot be bundled into the consuming nodes so an l2i gets planted
4672 // (actually a movw $dst $src) and the downstream instructions consume
4673 // the result of the l2i as an iRegI input. That's a shame since the
4674 // movw is actually redundant but its not too costly.
4675 
4676 opclass iRegIorL2I(iRegI, iRegL2I);
4677 
4678 //----------PIPELINE-----------------------------------------------------------
4679 // Rules which define the behavior of the target architectures pipeline.
4680 // Integer ALU reg operation
4681 pipeline %{
4682 
4683 attributes %{
4684   // ARM instructions are of fixed length
4685   fixed_size_instructions;        // Fixed size instructions TODO does
4686   max_instructions_per_bundle = 2;   // A53 = 2, A57 = 4
4687   // ARM instructions come in 32-bit word units
4688   instruction_unit_size = 4;         // An instruction is 4 bytes long
4689   instruction_fetch_unit_size = 64;  // The processor fetches one line
4690   instruction_fetch_units = 1;       // of 64 bytes
4691 
4692   // List of nop instructions
4693   nops( MachNop );
4694 %}
4695 
4696 // We don't use an actual pipeline model so don't care about resources
4697 // or description. we do use pipeline classes to introduce fixed
4698 // latencies
4699 
4700 //----------RESOURCES----------------------------------------------------------
4701 // Resources are the functional units available to the machine
4702 
4703 resources( INS0, INS1, INS01 = INS0 | INS1,
4704            ALU0, ALU1, ALU = ALU0 | ALU1,
4705            MAC,
4706            DIV,
4707            BRANCH,
4708            LDST,
4709            NEON_FP);
4710 
4711 //----------PIPELINE DESCRIPTION-----------------------------------------------
4712 // Pipeline Description specifies the stages in the machine's pipeline
4713 
4714 pipe_desc(ISS, EX1, EX2, WR);
4715 
4716 //----------PIPELINE CLASSES---------------------------------------------------
4717 // Pipeline Classes describe the stages in which input and output are
4718 // referenced by the hardware pipeline.
4719 
4720 //------- Integer ALU operations --------------------------
4721 
4722 // Integer ALU reg-reg operation
4723 // Operands needed in EX1, result generated in EX2
4724 // Eg.  ADD     x0, x1, x2
4725 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
4726 %{
4727   single_instruction;
4728   dst    : EX2(write);
4729   src1   : EX1(read);
4730   src2   : EX1(read);
4731   INS01  : ISS; // Dual issue as instruction 0 or 1
4732   ALU    : EX2;
4733 %}
4734 
4735 // Integer ALU reg-reg operation with constant shift
4736 // Shifted register must be available in LATE_ISS instead of EX1
4737 // Eg.  ADD     x0, x1, x2, LSL #2
4738 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
4739 %{
4740   single_instruction;
4741   dst    : EX2(write);
4742   src1   : EX1(read);
4743   src2   : ISS(read);
4744   INS01  : ISS;
4745   ALU    : EX2;
4746 %}
4747 
4748 // Integer ALU reg operation with constant shift
4749 // Eg.  LSL     x0, x1, #shift
4750 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
4751 %{
4752   single_instruction;
4753   dst    : EX2(write);
4754   src1   : ISS(read);
4755   INS01  : ISS;
4756   ALU    : EX2;
4757 %}
4758 
4759 // Integer ALU reg-reg operation with variable shift
4760 // Both operands must be available in LATE_ISS instead of EX1
4761 // Result is available in EX1 instead of EX2
4762 // Eg.  LSLV    x0, x1, x2
4763 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
4764 %{
4765   single_instruction;
4766   dst    : EX1(write);
4767   src1   : ISS(read);
4768   src2   : ISS(read);
4769   INS01  : ISS;
4770   ALU    : EX1;
4771 %}
4772 
4773 // Integer ALU reg-reg operation with extract
4774 // As for _vshift above, but result generated in EX2
4775 // Eg.  EXTR    x0, x1, x2, #N
4776 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
4777 %{
4778   single_instruction;
4779   dst    : EX2(write);
4780   src1   : ISS(read);
4781   src2   : ISS(read);
4782   INS1   : ISS; // Can only dual issue as Instruction 1
4783   ALU    : EX1;
4784 %}
4785 
4786 // Integer ALU reg operation
4787 // Eg.  NEG     x0, x1
4788 pipe_class ialu_reg(iRegI dst, iRegI src)
4789 %{
4790   single_instruction;
4791   dst    : EX2(write);
4792   src    : EX1(read);
4793   INS01  : ISS;
4794   ALU    : EX2;
4795 %}
4796 
4797 // Integer ALU reg mmediate operation
4798 // Eg.  ADD     x0, x1, #N
4799 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
4800 %{
4801   single_instruction;
4802   dst    : EX2(write);
4803   src1   : EX1(read);
4804   INS01  : ISS;
4805   ALU    : EX2;
4806 %}
4807 
4808 // Integer ALU immediate operation (no source operands)
4809 // Eg.  MOV     x0, #N
4810 pipe_class ialu_imm(iRegI dst)
4811 %{
4812   single_instruction;
4813   dst    : EX1(write);
4814   INS01  : ISS;
4815   ALU    : EX1;
4816 %}
4817 
4818 //------- Compare operation -------------------------------
4819 
4820 // Compare reg-reg
4821 // Eg.  CMP     x0, x1
4822 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
4823 %{
4824   single_instruction;
4825 //  fixed_latency(16);
4826   cr     : EX2(write);
4827   op1    : EX1(read);
4828   op2    : EX1(read);
4829   INS01  : ISS;
4830   ALU    : EX2;
4831 %}
4832 
4833 // Compare reg-reg
4834 // Eg.  CMP     x0, #N
4835 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
4836 %{
4837   single_instruction;
4838 //  fixed_latency(16);
4839   cr     : EX2(write);
4840   op1    : EX1(read);
4841   INS01  : ISS;
4842   ALU    : EX2;
4843 %}
4844 
4845 //------- Conditional instructions ------------------------
4846 
4847 // Conditional no operands
4848 // Eg.  CSINC   x0, zr, zr, <cond>
4849 pipe_class icond_none(iRegI dst, rFlagsReg cr)
4850 %{
4851   single_instruction;
4852   cr     : EX1(read);
4853   dst    : EX2(write);
4854   INS01  : ISS;
4855   ALU    : EX2;
4856 %}
4857 
4858 // Conditional 2 operand
4859 // EG.  CSEL    X0, X1, X2, <cond>
4860 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
4861 %{
4862   single_instruction;
4863   cr     : EX1(read);
4864   src1   : EX1(read);
4865   src2   : EX1(read);
4866   dst    : EX2(write);
4867   INS01  : ISS;
4868   ALU    : EX2;
4869 %}
4870 
4871 // Conditional 2 operand
4872 // EG.  CSEL    X0, X1, X2, <cond>
4873 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
4874 %{
4875   single_instruction;
4876   cr     : EX1(read);
4877   src    : EX1(read);
4878   dst    : EX2(write);
4879   INS01  : ISS;
4880   ALU    : EX2;
4881 %}
4882 
4883 //------- Multiply pipeline operations --------------------
4884 
4885 // Multiply reg-reg
4886 // Eg.  MUL     w0, w1, w2
4887 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
4888 %{
4889   single_instruction;
4890   dst    : WR(write);
4891   src1   : ISS(read);
4892   src2   : ISS(read);
4893   INS01  : ISS;
4894   MAC    : WR;
4895 %}
4896 
4897 // Multiply accumulate
4898 // Eg.  MADD    w0, w1, w2, w3
4899 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
4900 %{
4901   single_instruction;
4902   dst    : WR(write);
4903   src1   : ISS(read);
4904   src2   : ISS(read);
4905   src3   : ISS(read);
4906   INS01  : ISS;
4907   MAC    : WR;
4908 %}
4909 
4910 // Eg.  MUL     w0, w1, w2
4911 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
4912 %{
4913   single_instruction;
4914   fixed_latency(3); // Maximum latency for 64 bit mul
4915   dst    : WR(write);
4916   src1   : ISS(read);
4917   src2   : ISS(read);
4918   INS01  : ISS;
4919   MAC    : WR;
4920 %}
4921 
4922 // Multiply accumulate
4923 // Eg.  MADD    w0, w1, w2, w3
4924 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
4925 %{
4926   single_instruction;
4927   fixed_latency(3); // Maximum latency for 64 bit mul
4928   dst    : WR(write);
4929   src1   : ISS(read);
4930   src2   : ISS(read);
4931   src3   : ISS(read);
4932   INS01  : ISS;
4933   MAC    : WR;
4934 %}
4935 
4936 //------- Divide pipeline operations --------------------
4937 
4938 // Eg.  SDIV    w0, w1, w2
4939 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
4940 %{
4941   single_instruction;
4942   fixed_latency(8); // Maximum latency for 32 bit divide
4943   dst    : WR(write);
4944   src1   : ISS(read);
4945   src2   : ISS(read);
4946   INS0   : ISS; // Can only dual issue as instruction 0
4947   DIV    : WR;
4948 %}
4949 
4950 // Eg.  SDIV    x0, x1, x2
4951 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
4952 %{
4953   single_instruction;
4954   fixed_latency(16); // Maximum latency for 64 bit divide
4955   dst    : WR(write);
4956   src1   : ISS(read);
4957   src2   : ISS(read);
4958   INS0   : ISS; // Can only dual issue as instruction 0
4959   DIV    : WR;
4960 %}
4961 
4962 //------- Load pipeline operations ------------------------
4963 
4964 // Load - prefetch
4965 // Eg.  PFRM    <mem>
4966 pipe_class iload_prefetch(memory mem)
4967 %{
4968   single_instruction;
4969   mem    : ISS(read);
4970   INS01  : ISS;
4971   LDST   : WR;
4972 %}
4973 
4974 // Load - reg, mem
4975 // Eg.  LDR     x0, <mem>
4976 pipe_class iload_reg_mem(iRegI dst, memory mem)
4977 %{
4978   single_instruction;
4979   dst    : WR(write);
4980   mem    : ISS(read);
4981   INS01  : ISS;
4982   LDST   : WR;
4983 %}
4984 
4985 // Load - reg, reg
4986 // Eg.  LDR     x0, [sp, x1]
4987 pipe_class iload_reg_reg(iRegI dst, iRegI src)
4988 %{
4989   single_instruction;
4990   dst    : WR(write);
4991   src    : ISS(read);
4992   INS01  : ISS;
4993   LDST   : WR;
4994 %}
4995 
4996 //------- Store pipeline operations -----------------------
4997 
4998 // Store - zr, mem
4999 // Eg.  STR     zr, <mem>
5000 pipe_class istore_mem(memory mem)
5001 %{
5002   single_instruction;
5003   mem    : ISS(read);
5004   INS01  : ISS;
5005   LDST   : WR;
5006 %}
5007 
5008 // Store - reg, mem
5009 // Eg.  STR     x0, <mem>
5010 pipe_class istore_reg_mem(iRegI src, memory mem)
5011 %{
5012   single_instruction;
5013   mem    : ISS(read);
5014   src    : EX2(read);
5015   INS01  : ISS;
5016   LDST   : WR;
5017 %}
5018 
5019 // Store - reg, reg
5020 // Eg. STR      x0, [sp, x1]
5021 pipe_class istore_reg_reg(iRegI dst, iRegI src)
5022 %{
5023   single_instruction;
5024   dst    : ISS(read);
5025   src    : EX2(read);
5026   INS01  : ISS;
5027   LDST   : WR;
5028 %}
5029 
5030 //------- Store pipeline operations -----------------------
5031 
5032 // Branch
5033 pipe_class pipe_branch()
5034 %{
5035   single_instruction;
5036   INS01  : ISS;
5037   BRANCH : EX1;
5038 %}
5039 
5040 // Conditional branch
5041 pipe_class pipe_branch_cond(rFlagsReg cr)
5042 %{
5043   single_instruction;
5044   cr     : EX1(read);
5045   INS01  : ISS;
5046   BRANCH : EX1;
5047 %}
5048 
5049 // Compare & Branch
5050 // EG.  CBZ/CBNZ
5051 pipe_class pipe_cmp_branch(iRegI op1)
5052 %{
5053   single_instruction;
5054   op1    : EX1(read);
5055   INS01  : ISS;
5056   BRANCH : EX1;
5057 %}
5058 
5059 //------- Synchronisation operations ----------------------
5060 
5061 // Any operation requiring serialization.
5062 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
5063 pipe_class pipe_serial()
5064 %{
5065   single_instruction;
5066   force_serialization;
5067   fixed_latency(16);
5068   INS01  : ISS(2); // Cannot dual issue with any other instruction
5069   LDST   : WR;
5070 %}
5071 
5072 // Generic big/slow expanded idiom - also serialized
5073 pipe_class pipe_slow()
5074 %{
5075   instruction_count(10);
5076   multiple_bundles;
5077   force_serialization;
5078   fixed_latency(16);
5079   INS01  : ISS(2); // Cannot dual issue with any other instruction
5080   LDST   : WR;
5081 %}
5082 
5083 // Empty pipeline class
5084 pipe_class pipe_class_empty()
5085 %{
5086   single_instruction;
5087   fixed_latency(0);
5088 %}
5089 
5090 // Default pipeline class.
5091 pipe_class pipe_class_default()
5092 %{
5093   single_instruction;
5094   fixed_latency(2);
5095 %}
5096 
5097 // Pipeline class for compares.
5098 pipe_class pipe_class_compare()
5099 %{
5100   single_instruction;
5101   fixed_latency(16);
5102 %}
5103 
5104 // Pipeline class for memory operations.
5105 pipe_class pipe_class_memory()
5106 %{
5107   single_instruction;
5108   fixed_latency(16);
5109 %}
5110 
5111 // Pipeline class for call.
5112 pipe_class pipe_class_call()
5113 %{
5114   single_instruction;
5115   fixed_latency(100);
5116 %}
5117 
5118 // Define the class for the Nop node.
5119 define %{
5120    MachNop = pipe_class_empty;
5121 %}
5122 
5123 %}
5124 //----------INSTRUCTIONS-------------------------------------------------------
5125 //
5126 // match      -- States which machine-independent subtree may be replaced
5127 //               by this instruction.
5128 // ins_cost   -- The estimated cost of this instruction is used by instruction
5129 //               selection to identify a minimum cost tree of machine
5130 //               instructions that matches a tree of machine-independent
5131 //               instructions.
5132 // format     -- A string providing the disassembly for this instruction.
5133 //               The value of an instruction's operand may be inserted
5134 //               by referring to it with a '$' prefix.
5135 // opcode     -- Three instruction opcodes may be provided.  These are referred
5136 //               to within an encode class as $primary, $secondary, and $tertiary
5137 //               rrspectively.  The primary opcode is commonly used to
5138 //               indicate the type of machine instruction, while secondary
5139 //               and tertiary are often used for prefix options or addressing
5140 //               modes.
5141 // ins_encode -- A list of encode classes with parameters. The encode class
5142 //               name must have been defined in an 'enc_class' specification
5143 //               in the encode section of the architecture description.
5144 
5145 // ============================================================================
5146 // Memory (Load/Store) Instructions
5147 
5148 // Load Instructions
5149 
5150 // Load Byte (8 bit signed)
5151 instruct loadB(iRegINoSp dst, memory mem)
5152 %{
5153   match(Set dst (LoadB mem));
5154   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5155 
5156   ins_cost(4 * INSN_COST);
5157   format %{ "ldrsbw  $dst, $mem\t# byte" %}
5158 
5159   ins_encode(aarch64_enc_ldrsbw(dst, mem));
5160 
5161   ins_pipe(iload_reg_mem);
5162 %}
5163 
5164 // Load Byte (8 bit signed) into long
5165 instruct loadB2L(iRegLNoSp dst, memory mem)
5166 %{
5167   match(Set dst (ConvI2L (LoadB mem)));
5168   predicate(UseBarriersForVolatile || n->in(1)->as_Load()->is_unordered());
5169 
5170   ins_cost(4 * INSN_COST);
5171   format %{ "ldrsb  $dst, $mem\t# byte" %}
5172 
5173   ins_encode(aarch64_enc_ldrsb(dst, mem));
5174 
5175   ins_pipe(iload_reg_mem);
5176 %}
5177 
5178 // Load Byte (8 bit unsigned)
5179 instruct loadUB(iRegINoSp dst, memory mem)
5180 %{
5181   match(Set dst (LoadUB mem));
5182   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5183 
5184   ins_cost(4 * INSN_COST);
5185   format %{ "ldrbw  $dst, $mem\t# byte" %}
5186 
5187   ins_encode(aarch64_enc_ldrb(dst, mem));
5188 
5189   ins_pipe(iload_reg_mem);
5190 %}
5191 
5192 // Load Byte (8 bit unsigned) into long
5193 instruct loadUB2L(iRegLNoSp dst, memory mem)
5194 %{
5195   match(Set dst (ConvI2L (LoadUB mem)));
5196   predicate(UseBarriersForVolatile || n->in(1)->as_Load()->is_unordered());
5197 
5198   ins_cost(4 * INSN_COST);
5199   format %{ "ldrb  $dst, $mem\t# byte" %}
5200 
5201   ins_encode(aarch64_enc_ldrb(dst, mem));
5202 
5203   ins_pipe(iload_reg_mem);
5204 %}
5205 
5206 // Load Short (16 bit signed)
5207 instruct loadS(iRegINoSp dst, memory mem)
5208 %{
5209   match(Set dst (LoadS mem));
5210   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5211 
5212   ins_cost(4 * INSN_COST);
5213   format %{ "ldrshw  $dst, $mem\t# short" %}
5214 
5215   ins_encode(aarch64_enc_ldrshw(dst, mem));
5216 
5217   ins_pipe(iload_reg_mem);
5218 %}
5219 
5220 // Load Short (16 bit signed) into long
5221 instruct loadS2L(iRegLNoSp dst, memory mem)
5222 %{
5223   match(Set dst (ConvI2L (LoadS mem)));
5224   predicate(UseBarriersForVolatile || n->in(1)->as_Load()->is_unordered());
5225 
5226   ins_cost(4 * INSN_COST);
5227   format %{ "ldrsh  $dst, $mem\t# short" %}
5228 
5229   ins_encode(aarch64_enc_ldrsh(dst, mem));
5230 
5231   ins_pipe(iload_reg_mem);
5232 %}
5233 
5234 // Load Char (16 bit unsigned)
5235 instruct loadUS(iRegINoSp dst, memory mem)
5236 %{
5237   match(Set dst (LoadUS mem));
5238   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5239 
5240   ins_cost(4 * INSN_COST);
5241   format %{ "ldrh  $dst, $mem\t# short" %}
5242 
5243   ins_encode(aarch64_enc_ldrh(dst, mem));
5244 
5245   ins_pipe(iload_reg_mem);
5246 %}
5247 
5248 // Load Short/Char (16 bit unsigned) into long
5249 instruct loadUS2L(iRegLNoSp dst, memory mem)
5250 %{
5251   match(Set dst (ConvI2L (LoadUS mem)));
5252   predicate(UseBarriersForVolatile || n->in(1)->as_Load()->is_unordered());
5253 
5254   ins_cost(4 * INSN_COST);
5255   format %{ "ldrh  $dst, $mem\t# short" %}
5256 
5257   ins_encode(aarch64_enc_ldrh(dst, mem));
5258 
5259   ins_pipe(iload_reg_mem);
5260 %}
5261 
5262 // Load Integer (32 bit signed)
5263 instruct loadI(iRegINoSp dst, memory mem)
5264 %{
5265   match(Set dst (LoadI mem));
5266   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5267 
5268   ins_cost(4 * INSN_COST);
5269   format %{ "ldrw  $dst, $mem\t# int" %}
5270 
5271   ins_encode(aarch64_enc_ldrw(dst, mem));
5272 
5273   ins_pipe(iload_reg_mem);
5274 %}
5275 
5276 // Load Integer (32 bit signed) into long
5277 instruct loadI2L(iRegLNoSp dst, memory mem)
5278 %{
5279   match(Set dst (ConvI2L (LoadI mem)));
5280   predicate(UseBarriersForVolatile || n->in(1)->as_Load()->is_unordered());
5281 
5282   ins_cost(4 * INSN_COST);
5283   format %{ "ldrsw  $dst, $mem\t# int" %}
5284 
5285   ins_encode(aarch64_enc_ldrsw(dst, mem));
5286 
5287   ins_pipe(iload_reg_mem);
5288 %}
5289 
5290 // Load Integer (32 bit unsigned) into long
5291 instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
5292 %{
5293   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5294   predicate(UseBarriersForVolatile || n->in(1)->in(1)->as_Load()->is_unordered());
5295 
5296   ins_cost(4 * INSN_COST);
5297   format %{ "ldrw  $dst, $mem\t# int" %}
5298 
5299   ins_encode(aarch64_enc_ldrw(dst, mem));
5300 
5301   ins_pipe(iload_reg_mem);
5302 %}
5303 
5304 // Load Long (64 bit signed)
5305 instruct loadL(iRegLNoSp dst, memory mem)
5306 %{
5307   match(Set dst (LoadL mem));
5308   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5309 
5310   ins_cost(4 * INSN_COST);
5311   format %{ "ldr  $dst, $mem\t# int" %}
5312 
5313   ins_encode(aarch64_enc_ldr(dst, mem));
5314 
5315   ins_pipe(iload_reg_mem);
5316 %}
5317 
5318 // Load Range
5319 instruct loadRange(iRegINoSp dst, memory mem)
5320 %{
5321   match(Set dst (LoadRange mem));
5322 
5323   ins_cost(4 * INSN_COST);
5324   format %{ "ldrw  $dst, $mem\t# range" %}
5325 
5326   ins_encode(aarch64_enc_ldrw(dst, mem));
5327 
5328   ins_pipe(iload_reg_mem);
5329 %}
5330 
5331 // Load Pointer
5332 instruct loadP(iRegPNoSp dst, memory mem)
5333 %{
5334   match(Set dst (LoadP mem));
5335   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5336 
5337   ins_cost(4 * INSN_COST);
5338   format %{ "ldr  $dst, $mem\t# ptr" %}
5339 
5340   ins_encode(aarch64_enc_ldr(dst, mem));
5341 
5342   ins_pipe(iload_reg_mem);
5343 %}
5344 
5345 // Load Compressed Pointer
5346 instruct loadN(iRegNNoSp dst, memory mem)
5347 %{
5348   match(Set dst (LoadN mem));
5349   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5350 
5351   ins_cost(4 * INSN_COST);
5352   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
5353 
5354   ins_encode(aarch64_enc_ldrw(dst, mem));
5355 
5356   ins_pipe(iload_reg_mem);
5357 %}
5358 
5359 // Load Klass Pointer
5360 instruct loadKlass(iRegPNoSp dst, memory mem)
5361 %{
5362   match(Set dst (LoadKlass mem));
5363   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5364 
5365   ins_cost(4 * INSN_COST);
5366   format %{ "ldr  $dst, $mem\t# class" %}
5367 
5368   ins_encode(aarch64_enc_ldr(dst, mem));
5369 
5370   ins_pipe(iload_reg_mem);
5371 %}
5372 
5373 // Load Narrow Klass Pointer
5374 instruct loadNKlass(iRegNNoSp dst, memory mem)
5375 %{
5376   match(Set dst (LoadNKlass mem));
5377   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5378 
5379   ins_cost(4 * INSN_COST);
5380   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
5381 
5382   ins_encode(aarch64_enc_ldrw(dst, mem));
5383 
5384   ins_pipe(iload_reg_mem);
5385 %}
5386 
5387 // Load Float
5388 instruct loadF(vRegF dst, memory mem)
5389 %{
5390   match(Set dst (LoadF mem));
5391   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5392 
5393   ins_cost(4 * INSN_COST);
5394   format %{ "ldrs  $dst, $mem\t# float" %}
5395 
5396   ins_encode( aarch64_enc_ldrs(dst, mem) );
5397 
5398   ins_pipe(pipe_class_memory);
5399 %}
5400 
5401 // Load Double
5402 instruct loadD(vRegD dst, memory mem)
5403 %{
5404   match(Set dst (LoadD mem));
5405   predicate(UseBarriersForVolatile || n->as_Load()->is_unordered());
5406 
5407   ins_cost(4 * INSN_COST);
5408   format %{ "ldrd  $dst, $mem\t# double" %}
5409 
5410   ins_encode( aarch64_enc_ldrd(dst, mem) );
5411 
5412   ins_pipe(pipe_class_memory);
5413 %}
5414 
5415 
5416 // Load Int Constant
5417 instruct loadConI(iRegINoSp dst, immI src)
5418 %{
5419   match(Set dst src);
5420 
5421   ins_cost(INSN_COST);
5422   format %{ "mov $dst, $src\t# int" %}
5423 
5424   ins_encode( aarch64_enc_movw_imm(dst, src) );
5425 
5426   ins_pipe(ialu_imm);
5427 %}
5428 
5429 // Load Long Constant
5430 instruct loadConL(iRegLNoSp dst, immL src)
5431 %{
5432   match(Set dst src);
5433 
5434   ins_cost(INSN_COST);
5435   format %{ "mov $dst, $src\t# long" %}
5436 
5437   ins_encode( aarch64_enc_mov_imm(dst, src) );
5438 
5439   ins_pipe(ialu_imm);
5440 %}
5441 
5442 // Load Pointer Constant
5443 
5444 instruct loadConP(iRegPNoSp dst, immP con)
5445 %{
5446   match(Set dst con);
5447 
5448   ins_cost(INSN_COST * 4);
5449   format %{
5450     "mov  $dst, $con\t# ptr\n\t"
5451   %}
5452 
5453   ins_encode(aarch64_enc_mov_p(dst, con));
5454 
5455   ins_pipe(ialu_imm);
5456 %}
5457 
5458 // Load Null Pointer Constant
5459 
5460 instruct loadConP0(iRegPNoSp dst, immP0 con)
5461 %{
5462   match(Set dst con);
5463 
5464   ins_cost(INSN_COST);
5465   format %{ "mov  $dst, $con\t# NULL ptr" %}
5466 
5467   ins_encode(aarch64_enc_mov_p0(dst, con));
5468 
5469   ins_pipe(ialu_imm);
5470 %}
5471 
5472 // Load Pointer Constant One
5473 
5474 instruct loadConP1(iRegPNoSp dst, immP_1 con)
5475 %{
5476   match(Set dst con);
5477 
5478   ins_cost(INSN_COST);
5479   format %{ "mov  $dst, $con\t# NULL ptr" %}
5480 
5481   ins_encode(aarch64_enc_mov_p1(dst, con));
5482 
5483   ins_pipe(ialu_imm);
5484 %}
5485 
5486 // Load Poll Page Constant
5487 
5488 instruct loadConPollPage(iRegPNoSp dst, immPollPage con)
5489 %{
5490   match(Set dst con);
5491 
5492   ins_cost(INSN_COST);
5493   format %{ "adr  $dst, $con\t# Poll Page Ptr" %}
5494 
5495   ins_encode(aarch64_enc_mov_poll_page(dst, con));
5496 
5497   ins_pipe(ialu_imm);
5498 %}
5499 
5500 // Load Byte Map Base Constant
5501 
5502 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
5503 %{
5504   match(Set dst con);
5505 
5506   ins_cost(INSN_COST);
5507   format %{ "adr  $dst, $con\t# Byte Map Base" %}
5508 
5509   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
5510 
5511   ins_pipe(ialu_imm);
5512 %}
5513 
5514 // Load Narrow Pointer Constant
5515 
5516 instruct loadConN(iRegNNoSp dst, immN con)
5517 %{
5518   match(Set dst con);
5519 
5520   ins_cost(INSN_COST * 4);
5521   format %{ "mov  $dst, $con\t# compressed ptr" %}
5522 
5523   ins_encode(aarch64_enc_mov_n(dst, con));
5524 
5525   ins_pipe(ialu_imm);
5526 %}
5527 
5528 // Load Narrow Null Pointer Constant
5529 
5530 instruct loadConN0(iRegNNoSp dst, immN0 con)
5531 %{
5532   match(Set dst con);
5533 
5534   ins_cost(INSN_COST);
5535   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
5536 
5537   ins_encode(aarch64_enc_mov_n0(dst, con));
5538 
5539   ins_pipe(ialu_imm);
5540 %}
5541 
5542 // Load Narrow Klass Constant
5543 
5544 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
5545 %{
5546   match(Set dst con);
5547 
5548   ins_cost(INSN_COST);
5549   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
5550 
5551   ins_encode(aarch64_enc_mov_nk(dst, con));
5552 
5553   ins_pipe(ialu_imm);
5554 %}
5555 
5556 // Load Packed Float Constant
5557 
5558 instruct loadConF_packed(vRegF dst, immFPacked con) %{
5559   match(Set dst con);
5560   ins_cost(INSN_COST * 4);
5561   format %{ "fmovs  $dst, $con"%}
5562   ins_encode %{
5563     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
5564   %}
5565 
5566   ins_pipe(pipe_class_default);
5567 %}
5568 
5569 // Load Float Constant
5570 
5571 instruct loadConF(vRegF dst, immF con) %{
5572   match(Set dst con);
5573 
5574   ins_cost(INSN_COST * 4);
5575 
5576   format %{
5577     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
5578   %}
5579 
5580   ins_encode %{
5581     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
5582   %}
5583 
5584   ins_pipe(pipe_class_default);
5585 %}
5586 
5587 // Load Packed Double Constant
5588 
5589 instruct loadConD_packed(vRegD dst, immDPacked con) %{
5590   match(Set dst con);
5591   ins_cost(INSN_COST);
5592   format %{ "fmovd  $dst, $con"%}
5593   ins_encode %{
5594     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
5595   %}
5596 
5597   ins_pipe(pipe_class_default);
5598 %}
5599 
5600 // Load Double Constant
5601 
5602 instruct loadConD(vRegD dst, immD con) %{
5603   match(Set dst con);
5604 
5605   ins_cost(INSN_COST * 5);
5606   format %{
5607     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
5608   %}
5609 
5610   ins_encode %{
5611     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
5612   %}
5613 
5614   ins_pipe(pipe_class_default);
5615 %}
5616 
5617 // Store Instructions
5618 
5619 // Store CMS card-mark Immediate
5620 instruct storeimmCM0(immI0 zero, memory mem)
5621 %{
5622   match(Set mem (StoreCM mem zero));
5623 
5624   ins_cost(INSN_COST);
5625   format %{ "strb zr, $mem\t# byte" %}
5626 
5627   ins_encode(aarch64_enc_strb0(mem));
5628 
5629   ins_pipe(istore_mem);
5630 %}
5631 
5632 // Store Byte
5633 instruct storeB(iRegIorL2I src, memory mem)
5634 %{
5635   match(Set mem (StoreB mem src));
5636   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5637 
5638   ins_cost(INSN_COST);
5639   format %{ "strb  $src, $mem\t# byte" %}
5640 
5641   ins_encode(aarch64_enc_strb(src, mem));
5642 
5643   ins_pipe(istore_reg_mem);
5644 %}
5645 
5646 
5647 instruct storeimmB0(immI0 zero, memory mem)
5648 %{
5649   match(Set mem (StoreB mem zero));
5650   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5651 
5652   ins_cost(INSN_COST);
5653   format %{ "strb zr, $mem\t# byte" %}
5654 
5655   ins_encode(aarch64_enc_strb0(mem));
5656 
5657   ins_pipe(istore_mem);
5658 %}
5659 
5660 // Store Char/Short
5661 instruct storeC(iRegIorL2I src, memory mem)
5662 %{
5663   match(Set mem (StoreC mem src));
5664   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5665 
5666   ins_cost(INSN_COST);
5667   format %{ "strh  $src, $mem\t# short" %}
5668 
5669   ins_encode(aarch64_enc_strh(src, mem));
5670 
5671   ins_pipe(istore_reg_mem);
5672 %}
5673 
5674 instruct storeimmC0(immI0 zero, memory mem)
5675 %{
5676   match(Set mem (StoreC mem zero));
5677   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5678 
5679   ins_cost(INSN_COST);
5680   format %{ "strh  zr, $mem\t# short" %}
5681 
5682   ins_encode(aarch64_enc_strh0(mem));
5683 
5684   ins_pipe(istore_mem);
5685 %}
5686 
5687 // Store Integer
5688 
5689 instruct storeI(iRegIorL2I src, memory mem)
5690 %{
5691   match(Set mem(StoreI mem src));
5692   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5693 
5694   ins_cost(INSN_COST);
5695   format %{ "strw  $src, $mem\t# int" %}
5696 
5697   ins_encode(aarch64_enc_strw(src, mem));
5698 
5699   ins_pipe(istore_reg_mem);
5700 %}
5701 
5702 instruct storeimmI0(immI0 zero, memory mem)
5703 %{
5704   match(Set mem(StoreI mem zero));
5705   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5706 
5707   ins_cost(INSN_COST);
5708   format %{ "strw  zr, $mem\t# int" %}
5709 
5710   ins_encode(aarch64_enc_strw0(mem));
5711 
5712   ins_pipe(istore_mem);
5713 %}
5714 
5715 // Store Long (64 bit signed)
5716 instruct storeL(iRegL src, memory mem)
5717 %{
5718   match(Set mem (StoreL mem src));
5719   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5720 
5721   ins_cost(INSN_COST);
5722   format %{ "str  $src, $mem\t# int" %}
5723 
5724   ins_encode(aarch64_enc_str(src, mem));
5725 
5726   ins_pipe(istore_reg_mem);
5727 %}
5728 
5729 // Store Long (64 bit signed)
5730 instruct storeimmL0(immL0 zero, memory mem)
5731 %{
5732   match(Set mem (StoreL mem zero));
5733   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5734 
5735   ins_cost(INSN_COST);
5736   format %{ "str  zr, $mem\t# int" %}
5737 
5738   ins_encode(aarch64_enc_str0(mem));
5739 
5740   ins_pipe(istore_mem);
5741 %}
5742 
5743 // Store Pointer
5744 instruct storeP(iRegP src, memory mem)
5745 %{
5746   match(Set mem (StoreP mem src));
5747   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5748 
5749   ins_cost(INSN_COST);
5750   format %{ "str  $src, $mem\t# ptr" %}
5751 
5752   ins_encode(aarch64_enc_str(src, mem));
5753 
5754   ins_pipe(istore_reg_mem);
5755 %}
5756 
5757 // Store Pointer
5758 instruct storeimmP0(immP0 zero, memory mem)
5759 %{
5760   match(Set mem (StoreP mem zero));
5761   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5762 
5763   ins_cost(INSN_COST);
5764   format %{ "str zr, $mem\t# ptr" %}
5765 
5766   ins_encode(aarch64_enc_str0(mem));
5767 
5768   ins_pipe(istore_mem);
5769 %}
5770 
5771 // Store Compressed Pointer
5772 instruct storeN(iRegN src, memory mem)
5773 %{
5774   match(Set mem (StoreN mem src));
5775   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5776 
5777   ins_cost(INSN_COST);
5778   format %{ "strw  $src, $mem\t# compressed ptr" %}
5779 
5780   ins_encode(aarch64_enc_strw(src, mem));
5781 
5782   ins_pipe(istore_reg_mem);
5783 %}
5784 
5785 instruct storeImmN0(iRegIHeapbase heapbase, immN0 zero, memory mem)
5786 %{
5787   match(Set mem (StoreN mem zero));
5788   predicate(Universe::narrow_oop_base() == NULL &&
5789             Universe::narrow_klass_base() == NULL &&
5790             (UseBarriersForVolatile || n->as_Store()->is_unordered()));
5791 
5792   ins_cost(INSN_COST);
5793   format %{ "strw  rheapbase, $mem\t# compressed ptr (rheapbase==0)" %}
5794 
5795   ins_encode(aarch64_enc_strw(heapbase, mem));
5796 
5797   ins_pipe(istore_reg_mem);
5798 %}
5799 
5800 // Store Float
5801 instruct storeF(vRegF src, memory mem)
5802 %{
5803   match(Set mem (StoreF mem src));
5804   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5805 
5806   ins_cost(INSN_COST);
5807   format %{ "strs  $src, $mem\t# float" %}
5808 
5809   ins_encode( aarch64_enc_strs(src, mem) );
5810 
5811   ins_pipe(pipe_class_memory);
5812 %}
5813 
5814 // TODO
5815 // implement storeImmF0 and storeFImmPacked
5816 
5817 // Store Double
5818 instruct storeD(vRegD src, memory mem)
5819 %{
5820   match(Set mem (StoreD mem src));
5821   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5822 
5823   ins_cost(INSN_COST);
5824   format %{ "strd  $src, $mem\t# double" %}
5825 
5826   ins_encode( aarch64_enc_strd(src, mem) );
5827 
5828   ins_pipe(pipe_class_memory);
5829 %}
5830 
5831 // Store Compressed Klass Pointer
5832 instruct storeNKlass(iRegN src, memory mem)
5833 %{
5834   predicate(UseBarriersForVolatile || n->as_Store()->is_unordered());
5835   match(Set mem (StoreNKlass mem src));
5836 
5837   ins_cost(INSN_COST);
5838   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
5839 
5840   ins_encode(aarch64_enc_strw(src, mem));
5841 
5842   ins_pipe(istore_reg_mem);
5843 %}
5844 
5845 // TODO
5846 // implement storeImmD0 and storeDImmPacked
5847 
5848 // prefetch instructions
5849 // Must be safe to execute with invalid address (cannot fault).
5850 
5851 instruct prefetchalloc( memory mem ) %{
5852   match(PrefetchAllocation mem);
5853 
5854   ins_cost(INSN_COST);
5855   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
5856 
5857   ins_encode( aarch64_enc_prefetchw(mem) );
5858 
5859   ins_pipe(iload_prefetch);
5860 %}
5861 
5862 //  ---------------- volatile loads and stores ----------------
5863 
5864 // Load Byte (8 bit signed)
5865 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
5866 %{
5867   match(Set dst (LoadB mem));
5868 
5869   ins_cost(VOLATILE_REF_COST);
5870   format %{ "ldarsb  $dst, $mem\t# byte" %}
5871 
5872   ins_encode(aarch64_enc_ldarsb(dst, mem));
5873 
5874   ins_pipe(pipe_serial);
5875 %}
5876 
5877 // Load Byte (8 bit signed) into long
5878 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
5879 %{
5880   match(Set dst (ConvI2L (LoadB mem)));
5881 
5882   ins_cost(VOLATILE_REF_COST);
5883   format %{ "ldarsb  $dst, $mem\t# byte" %}
5884 
5885   ins_encode(aarch64_enc_ldarsb(dst, mem));
5886 
5887   ins_pipe(pipe_serial);
5888 %}
5889 
5890 // Load Byte (8 bit unsigned)
5891 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
5892 %{
5893   match(Set dst (LoadUB mem));
5894 
5895   ins_cost(VOLATILE_REF_COST);
5896   format %{ "ldarb  $dst, $mem\t# byte" %}
5897 
5898   ins_encode(aarch64_enc_ldarb(dst, mem));
5899 
5900   ins_pipe(pipe_serial);
5901 %}
5902 
5903 // Load Byte (8 bit unsigned) into long
5904 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
5905 %{
5906   match(Set dst (ConvI2L (LoadUB mem)));
5907 
5908   ins_cost(VOLATILE_REF_COST);
5909   format %{ "ldarb  $dst, $mem\t# byte" %}
5910 
5911   ins_encode(aarch64_enc_ldarb(dst, mem));
5912 
5913   ins_pipe(pipe_serial);
5914 %}
5915 
5916 // Load Short (16 bit signed)
5917 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
5918 %{
5919   match(Set dst (LoadS mem));
5920 
5921   ins_cost(VOLATILE_REF_COST);
5922   format %{ "ldarshw  $dst, $mem\t# short" %}
5923 
5924   ins_encode(aarch64_enc_ldarshw(dst, mem));
5925 
5926   ins_pipe(pipe_serial);
5927 %}
5928 
5929 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
5930 %{
5931   match(Set dst (LoadUS mem));
5932 
5933   ins_cost(VOLATILE_REF_COST);
5934   format %{ "ldarhw  $dst, $mem\t# short" %}
5935 
5936   ins_encode(aarch64_enc_ldarhw(dst, mem));
5937 
5938   ins_pipe(pipe_serial);
5939 %}
5940 
5941 // Load Short/Char (16 bit unsigned) into long
5942 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
5943 %{
5944   match(Set dst (ConvI2L (LoadUS mem)));
5945 
5946   ins_cost(VOLATILE_REF_COST);
5947   format %{ "ldarh  $dst, $mem\t# short" %}
5948 
5949   ins_encode(aarch64_enc_ldarh(dst, mem));
5950 
5951   ins_pipe(pipe_serial);
5952 %}
5953 
5954 // Load Short/Char (16 bit signed) into long
5955 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
5956 %{
5957   match(Set dst (ConvI2L (LoadS mem)));
5958 
5959   ins_cost(VOLATILE_REF_COST);
5960   format %{ "ldarh  $dst, $mem\t# short" %}
5961 
5962   ins_encode(aarch64_enc_ldarsh(dst, mem));
5963 
5964   ins_pipe(pipe_serial);
5965 %}
5966 
5967 // Load Integer (32 bit signed)
5968 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
5969 %{
5970   match(Set dst (LoadI mem));
5971 
5972   ins_cost(VOLATILE_REF_COST);
5973   format %{ "ldarw  $dst, $mem\t# int" %}
5974 
5975   ins_encode(aarch64_enc_ldarw(dst, mem));
5976 
5977   ins_pipe(pipe_serial);
5978 %}
5979 
5980 // Load Integer (32 bit unsigned) into long
5981 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
5982 %{
5983   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5984 
5985   ins_cost(VOLATILE_REF_COST);
5986   format %{ "ldarw  $dst, $mem\t# int" %}
5987 
5988   ins_encode(aarch64_enc_ldarw(dst, mem));
5989 
5990   ins_pipe(pipe_serial);
5991 %}
5992 
5993 // Load Long (64 bit signed)
5994 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
5995 %{
5996   match(Set dst (LoadL mem));
5997 
5998   ins_cost(VOLATILE_REF_COST);
5999   format %{ "ldar  $dst, $mem\t# int" %}
6000 
6001   ins_encode(aarch64_enc_ldar(dst, mem));
6002 
6003   ins_pipe(pipe_serial);
6004 %}
6005 
6006 // Load Pointer
6007 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
6008 %{
6009   match(Set dst (LoadP mem));
6010 
6011   ins_cost(VOLATILE_REF_COST);
6012   format %{ "ldar  $dst, $mem\t# ptr" %}
6013 
6014   ins_encode(aarch64_enc_ldar(dst, mem));
6015 
6016   ins_pipe(pipe_serial);
6017 %}
6018 
6019 // Load Compressed Pointer
6020 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
6021 %{
6022   match(Set dst (LoadN mem));
6023 
6024   ins_cost(VOLATILE_REF_COST);
6025   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
6026 
6027   ins_encode(aarch64_enc_ldarw(dst, mem));
6028 
6029   ins_pipe(pipe_serial);
6030 %}
6031 
6032 // Load Float
6033 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
6034 %{
6035   match(Set dst (LoadF mem));
6036 
6037   ins_cost(VOLATILE_REF_COST);
6038   format %{ "ldars  $dst, $mem\t# float" %}
6039 
6040   ins_encode( aarch64_enc_fldars(dst, mem) );
6041 
6042   ins_pipe(pipe_serial);
6043 %}
6044 
6045 // Load Double
6046 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
6047 %{
6048   match(Set dst (LoadD mem));
6049 
6050   ins_cost(VOLATILE_REF_COST);
6051   format %{ "ldard  $dst, $mem\t# double" %}
6052 
6053   ins_encode( aarch64_enc_fldard(dst, mem) );
6054 
6055   ins_pipe(pipe_serial);
6056 %}
6057 
6058 // Store Byte
6059 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
6060 %{
6061   match(Set mem (StoreB mem src));
6062 
6063   ins_cost(VOLATILE_REF_COST);
6064   format %{ "stlrb  $src, $mem\t# byte" %}
6065 
6066   ins_encode(aarch64_enc_stlrb(src, mem));
6067 
6068   ins_pipe(pipe_class_memory);
6069 %}
6070 
6071 // Store Char/Short
6072 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
6073 %{
6074   match(Set mem (StoreC mem src));
6075 
6076   ins_cost(VOLATILE_REF_COST);
6077   format %{ "stlrh  $src, $mem\t# short" %}
6078 
6079   ins_encode(aarch64_enc_stlrh(src, mem));
6080 
6081   ins_pipe(pipe_class_memory);
6082 %}
6083 
6084 // Store Integer
6085 
6086 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
6087 %{
6088   match(Set mem(StoreI mem src));
6089 
6090   ins_cost(VOLATILE_REF_COST);
6091   format %{ "stlrw  $src, $mem\t# int" %}
6092 
6093   ins_encode(aarch64_enc_stlrw(src, mem));
6094 
6095   ins_pipe(pipe_class_memory);
6096 %}
6097 
6098 // Store Long (64 bit signed)
6099 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
6100 %{
6101   match(Set mem (StoreL mem src));
6102 
6103   ins_cost(VOLATILE_REF_COST);
6104   format %{ "stlr  $src, $mem\t# int" %}
6105 
6106   ins_encode(aarch64_enc_stlr(src, mem));
6107 
6108   ins_pipe(pipe_class_memory);
6109 %}
6110 
6111 // Store Pointer
6112 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
6113 %{
6114   match(Set mem (StoreP mem src));
6115 
6116   ins_cost(VOLATILE_REF_COST);
6117   format %{ "stlr  $src, $mem\t# ptr" %}
6118 
6119   ins_encode(aarch64_enc_stlr(src, mem));
6120 
6121   ins_pipe(pipe_class_memory);
6122 %}
6123 
6124 // Store Compressed Pointer
6125 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
6126 %{
6127   match(Set mem (StoreN mem src));
6128 
6129   ins_cost(VOLATILE_REF_COST);
6130   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
6131 
6132   ins_encode(aarch64_enc_stlrw(src, mem));
6133 
6134   ins_pipe(pipe_class_memory);
6135 %}
6136 
6137 // Store Float
6138 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
6139 %{
6140   match(Set mem (StoreF mem src));
6141 
6142   ins_cost(VOLATILE_REF_COST);
6143   format %{ "stlrs  $src, $mem\t# float" %}
6144 
6145   ins_encode( aarch64_enc_fstlrs(src, mem) );
6146 
6147   ins_pipe(pipe_class_memory);
6148 %}
6149 
6150 // TODO
6151 // implement storeImmF0 and storeFImmPacked
6152 
6153 // Store Double
6154 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
6155 %{
6156   match(Set mem (StoreD mem src));
6157 
6158   ins_cost(VOLATILE_REF_COST);
6159   format %{ "stlrd  $src, $mem\t# double" %}
6160 
6161   ins_encode( aarch64_enc_fstlrd(src, mem) );
6162 
6163   ins_pipe(pipe_class_memory);
6164 %}
6165 
6166 //  ---------------- end of volatile loads and stores ----------------
6167 
6168 // ============================================================================
6169 // BSWAP Instructions
6170 
6171 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
6172   match(Set dst (ReverseBytesI src));
6173 
6174   ins_cost(INSN_COST);
6175   format %{ "revw  $dst, $src" %}
6176 
6177   ins_encode %{
6178     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
6179   %}
6180 
6181   ins_pipe(ialu_reg);
6182 %}
6183 
6184 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
6185   match(Set dst (ReverseBytesL src));
6186 
6187   ins_cost(INSN_COST);
6188   format %{ "rev  $dst, $src" %}
6189 
6190   ins_encode %{
6191     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
6192   %}
6193 
6194   ins_pipe(ialu_reg);
6195 %}
6196 
6197 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
6198   match(Set dst (ReverseBytesUS src));
6199 
6200   ins_cost(INSN_COST);
6201   format %{ "rev16w  $dst, $src" %}
6202 
6203   ins_encode %{
6204     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
6205   %}
6206 
6207   ins_pipe(ialu_reg);
6208 %}
6209 
6210 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
6211   match(Set dst (ReverseBytesS src));
6212 
6213   ins_cost(INSN_COST);
6214   format %{ "rev16w  $dst, $src\n\t"
6215             "sbfmw $dst, $dst, #0, #15" %}
6216 
6217   ins_encode %{
6218     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
6219     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
6220   %}
6221 
6222   ins_pipe(ialu_reg);
6223 %}
6224 
6225 // ============================================================================
6226 // Zero Count Instructions
6227 
6228 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
6229   match(Set dst (CountLeadingZerosI src));
6230 
6231   ins_cost(INSN_COST);
6232   format %{ "clzw  $dst, $src" %}
6233   ins_encode %{
6234     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
6235   %}
6236 
6237   ins_pipe(ialu_reg);
6238 %}
6239 
6240 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
6241   match(Set dst (CountLeadingZerosL src));
6242 
6243   ins_cost(INSN_COST);
6244   format %{ "clz   $dst, $src" %}
6245   ins_encode %{
6246     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
6247   %}
6248 
6249   ins_pipe(ialu_reg);
6250 %}
6251 
6252 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
6253   match(Set dst (CountTrailingZerosI src));
6254 
6255   ins_cost(INSN_COST * 2);
6256   format %{ "rbitw  $dst, $src\n\t"
6257             "clzw   $dst, $dst" %}
6258   ins_encode %{
6259     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
6260     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
6261   %}
6262 
6263   ins_pipe(ialu_reg);
6264 %}
6265 
6266 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
6267   match(Set dst (CountTrailingZerosL src));
6268 
6269   ins_cost(INSN_COST * 2);
6270   format %{ "rbit   $dst, $src\n\t"
6271             "clz    $dst, $dst" %}
6272   ins_encode %{
6273     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
6274     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
6275   %}
6276 
6277   ins_pipe(ialu_reg);
6278 %}
6279 
6280 // ============================================================================
6281 // MemBar Instruction
6282 
6283 instruct load_fence() %{
6284   match(LoadFence);
6285   ins_cost(VOLATILE_REF_COST);
6286 
6287   format %{ "load_fence" %}
6288 
6289   ins_encode %{
6290     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
6291   %}
6292   ins_pipe(pipe_serial);
6293 %}
6294 
6295 instruct unnecessary_membar_acquire() %{
6296   predicate(! UseBarriersForVolatile && preceded_by_ordered_load(n));
6297   match(MemBarAcquire);
6298   ins_cost(0);
6299 
6300   format %{ "membar_acquire (elided)" %}
6301 
6302   ins_encode %{
6303     __ block_comment("membar_acquire (elided)");
6304   %}
6305 
6306   ins_pipe(pipe_class_empty);
6307 %}
6308 
6309 instruct membar_acquire() %{
6310   match(MemBarAcquire);
6311   ins_cost(VOLATILE_REF_COST);
6312 
6313   format %{ "membar_acquire" %}
6314 
6315   ins_encode %{
6316     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
6317   %}
6318 
6319   ins_pipe(pipe_serial);
6320 %}
6321 
6322 
6323 instruct membar_acquire_lock() %{
6324   match(MemBarAcquireLock);
6325   ins_cost(VOLATILE_REF_COST);
6326 
6327   format %{ "membar_acquire_lock" %}
6328 
6329   ins_encode %{
6330     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
6331   %}
6332 
6333   ins_pipe(pipe_serial);
6334 %}
6335 
6336 instruct store_fence() %{
6337   match(StoreFence);
6338   ins_cost(VOLATILE_REF_COST);
6339 
6340   format %{ "store_fence" %}
6341 
6342   ins_encode %{
6343     __ membar(Assembler::LoadStore|Assembler::StoreStore);
6344   %}
6345   ins_pipe(pipe_serial);
6346 %}
6347 
6348 instruct membar_release() %{
6349   match(MemBarRelease);
6350   ins_cost(VOLATILE_REF_COST);
6351 
6352   format %{ "membar_release" %}
6353 
6354   ins_encode %{
6355     __ membar(Assembler::LoadStore|Assembler::StoreStore);
6356   %}
6357   ins_pipe(pipe_serial);
6358 %}
6359 
6360 instruct membar_storestore() %{
6361   match(MemBarStoreStore);
6362   ins_cost(VOLATILE_REF_COST);
6363 
6364   format %{ "MEMBAR-store-store" %}
6365 
6366   ins_encode %{
6367     __ membar(Assembler::StoreStore);
6368   %}
6369   ins_pipe(pipe_serial);
6370 %}
6371 
6372 instruct membar_release_lock() %{
6373   match(MemBarReleaseLock);
6374   ins_cost(VOLATILE_REF_COST);
6375 
6376   format %{ "membar_release_lock" %}
6377 
6378   ins_encode %{
6379     __ membar(Assembler::LoadStore|Assembler::StoreStore);
6380   %}
6381 
6382   ins_pipe(pipe_serial);
6383 %}
6384 
6385 instruct membar_volatile() %{
6386   match(MemBarVolatile);
6387   ins_cost(VOLATILE_REF_COST*100);
6388 
6389   format %{ "membar_volatile" %}
6390 
6391   ins_encode %{
6392     __ membar(Assembler::StoreLoad);
6393   %}
6394 
6395   ins_pipe(pipe_serial);
6396 %}
6397 
6398 // ============================================================================
6399 // Cast/Convert Instructions
6400 
6401 instruct castX2P(iRegPNoSp dst, iRegL src) %{
6402   match(Set dst (CastX2P src));
6403 
6404   ins_cost(INSN_COST);
6405   format %{ "mov $dst, $src\t# long -> ptr" %}
6406 
6407   ins_encode %{
6408     if ($dst$$reg != $src$$reg) {
6409       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
6410     }
6411   %}
6412 
6413   ins_pipe(ialu_reg);
6414 %}
6415 
6416 instruct castP2X(iRegLNoSp dst, iRegP src) %{
6417   match(Set dst (CastP2X src));
6418 
6419   ins_cost(INSN_COST);
6420   format %{ "mov $dst, $src\t# ptr -> long" %}
6421 
6422   ins_encode %{
6423     if ($dst$$reg != $src$$reg) {
6424       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
6425     }
6426   %}
6427 
6428   ins_pipe(ialu_reg);
6429 %}
6430 
6431 // Convert oop into int for vectors alignment masking
6432 instruct convP2I(iRegINoSp dst, iRegP src) %{
6433   match(Set dst (ConvL2I (CastP2X src)));
6434 
6435   ins_cost(INSN_COST);
6436   format %{ "movw $dst, $src\t# ptr -> int" %}
6437   ins_encode %{
6438     __ movw($dst$$Register, $src$$Register);
6439   %}
6440 
6441   ins_pipe(ialu_reg);
6442 %}
6443 
6444 // Convert compressed oop into int for vectors alignment masking
6445 // in case of 32bit oops (heap < 4Gb).
6446 instruct convN2I(iRegINoSp dst, iRegN src)
6447 %{
6448   predicate(Universe::narrow_oop_shift() == 0);
6449   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6450 
6451   ins_cost(INSN_COST);
6452   format %{ "mov dst, $src\t# compressed ptr -> int" %}
6453   ins_encode %{
6454     __ movw($dst$$Register, $src$$Register);
6455   %}
6456 
6457   ins_pipe(ialu_reg);
6458 %}
6459 
6460 
6461 // Convert oop pointer into compressed form
6462 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
6463   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6464   match(Set dst (EncodeP src));
6465   effect(KILL cr);
6466   ins_cost(INSN_COST * 3);
6467   format %{ "encode_heap_oop $dst, $src" %}
6468   ins_encode %{
6469     Register s = $src$$Register;
6470     Register d = $dst$$Register;
6471     __ encode_heap_oop(d, s);
6472   %}
6473   ins_pipe(ialu_reg);
6474 %}
6475 
6476 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
6477   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6478   match(Set dst (EncodeP src));
6479   ins_cost(INSN_COST * 3);
6480   format %{ "encode_heap_oop_not_null $dst, $src" %}
6481   ins_encode %{
6482     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6483   %}
6484   ins_pipe(ialu_reg);
6485 %}
6486 
6487 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
6488   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6489             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6490   match(Set dst (DecodeN src));
6491   ins_cost(INSN_COST * 3);
6492   format %{ "decode_heap_oop $dst, $src" %}
6493   ins_encode %{
6494     Register s = $src$$Register;
6495     Register d = $dst$$Register;
6496     __ decode_heap_oop(d, s);
6497   %}
6498   ins_pipe(ialu_reg);
6499 %}
6500 
6501 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
6502   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6503             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6504   match(Set dst (DecodeN src));
6505   ins_cost(INSN_COST * 3);
6506   format %{ "decode_heap_oop_not_null $dst, $src" %}
6507   ins_encode %{
6508     Register s = $src$$Register;
6509     Register d = $dst$$Register;
6510     __ decode_heap_oop_not_null(d, s);
6511   %}
6512   ins_pipe(ialu_reg);
6513 %}
6514 
6515 // n.b. AArch64 implementations of encode_klass_not_null and
6516 // decode_klass_not_null do not modify the flags register so, unlike
6517 // Intel, we don't kill CR as a side effect here
6518 
6519 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
6520   match(Set dst (EncodePKlass src));
6521 
6522   ins_cost(INSN_COST * 3);
6523   format %{ "encode_klass_not_null $dst,$src" %}
6524 
6525   ins_encode %{
6526     Register src_reg = as_Register($src$$reg);
6527     Register dst_reg = as_Register($dst$$reg);
6528     __ encode_klass_not_null(dst_reg, src_reg);
6529   %}
6530 
6531    ins_pipe(ialu_reg);
6532 %}
6533 
6534 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
6535   match(Set dst (DecodeNKlass src));
6536 
6537   ins_cost(INSN_COST * 3);
6538   format %{ "decode_klass_not_null $dst,$src" %}
6539 
6540   ins_encode %{
6541     Register src_reg = as_Register($src$$reg);
6542     Register dst_reg = as_Register($dst$$reg);
6543     if (dst_reg != src_reg) {
6544       __ decode_klass_not_null(dst_reg, src_reg);
6545     } else {
6546       __ decode_klass_not_null(dst_reg);
6547     }
6548   %}
6549 
6550    ins_pipe(ialu_reg);
6551 %}
6552 
6553 instruct checkCastPP(iRegPNoSp dst)
6554 %{
6555   match(Set dst (CheckCastPP dst));
6556 
6557   size(0);
6558   format %{ "# checkcastPP of $dst" %}
6559   ins_encode(/* empty encoding */);
6560   ins_pipe(pipe_class_empty);
6561 %}
6562 
6563 instruct castPP(iRegPNoSp dst)
6564 %{
6565   match(Set dst (CastPP dst));
6566 
6567   size(0);
6568   format %{ "# castPP of $dst" %}
6569   ins_encode(/* empty encoding */);
6570   ins_pipe(pipe_class_empty);
6571 %}
6572 
6573 instruct castII(iRegI dst)
6574 %{
6575   match(Set dst (CastII dst));
6576 
6577   size(0);
6578   format %{ "# castII of $dst" %}
6579   ins_encode(/* empty encoding */);
6580   ins_cost(0);
6581   ins_pipe(pipe_class_empty);
6582 %}
6583 
6584 // ============================================================================
6585 // Atomic operation instructions
6586 //
6587 // Intel and SPARC both implement Ideal Node LoadPLocked and
6588 // Store{PIL}Conditional instructions using a normal load for the
6589 // LoadPLocked and a CAS for the Store{PIL}Conditional.
6590 //
6591 // The ideal code appears only to use LoadPLocked/StorePLocked as a
6592 // pair to lock object allocations from Eden space when not using
6593 // TLABs.
6594 //
6595 // There does not appear to be a Load{IL}Locked Ideal Node and the
6596 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
6597 // and to use StoreIConditional only for 32-bit and StoreLConditional
6598 // only for 64-bit.
6599 //
6600 // We implement LoadPLocked and StorePLocked instructions using,
6601 // respectively the AArch64 hw load-exclusive and store-conditional
6602 // instructions. Whereas we must implement each of
6603 // Store{IL}Conditional using a CAS which employs a pair of
6604 // instructions comprising a load-exclusive followed by a
6605 // store-conditional.
6606 
6607 
6608 // Locked-load (linked load) of the current heap-top
6609 // used when updating the eden heap top
6610 // implemented using ldaxr on AArch64
6611 
6612 instruct loadPLocked(iRegPNoSp dst, indirect mem)
6613 %{
6614   match(Set dst (LoadPLocked mem));
6615 
6616   ins_cost(VOLATILE_REF_COST);
6617 
6618   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
6619 
6620   ins_encode(aarch64_enc_ldaxr(dst, mem));
6621 
6622   ins_pipe(pipe_serial);
6623 %}
6624 
6625 // Conditional-store of the updated heap-top.
6626 // Used during allocation of the shared heap.
6627 // Sets flag (EQ) on success.
6628 // implemented using stlxr on AArch64.
6629 
6630 instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
6631 %{
6632   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
6633 
6634   ins_cost(VOLATILE_REF_COST);
6635 
6636  // TODO
6637  // do we need to do a store-conditional release or can we just use a
6638  // plain store-conditional?
6639 
6640   format %{
6641     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
6642     "cmpw rscratch1, zr\t# EQ on successful write"
6643   %}
6644 
6645   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
6646 
6647   ins_pipe(pipe_serial);
6648 %}
6649 
6650 // this has to be implemented as a CAS
6651 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
6652 %{
6653   match(Set cr (StoreLConditional mem (Binary oldval newval)));
6654 
6655   ins_cost(VOLATILE_REF_COST);
6656 
6657   format %{
6658     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
6659     "cmpw rscratch1, zr\t# EQ on successful write"
6660   %}
6661 
6662   ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval));
6663 
6664   ins_pipe(pipe_slow);
6665 %}
6666 
6667 // this has to be implemented as a CAS
6668 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
6669 %{
6670   match(Set cr (StoreIConditional mem (Binary oldval newval)));
6671 
6672   ins_cost(VOLATILE_REF_COST);
6673 
6674   format %{
6675     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
6676     "cmpw rscratch1, zr\t# EQ on successful write"
6677   %}
6678 
6679   ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval));
6680 
6681   ins_pipe(pipe_slow);
6682 %}
6683 
6684 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
6685 // can't match them
6686 
6687 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
6688 
6689   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
6690 
6691   effect(KILL cr);
6692 
6693  format %{
6694     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
6695     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
6696  %}
6697 
6698  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
6699             aarch64_enc_cset_eq(res));
6700 
6701   ins_pipe(pipe_slow);
6702 %}
6703 
6704 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
6705 
6706   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
6707 
6708   effect(KILL cr);
6709 
6710  format %{
6711     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
6712     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
6713  %}
6714 
6715  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
6716             aarch64_enc_cset_eq(res));
6717 
6718   ins_pipe(pipe_slow);
6719 %}
6720 
6721 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
6722 
6723   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
6724 
6725   effect(KILL cr);
6726 
6727  format %{
6728     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
6729     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
6730  %}
6731 
6732  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
6733             aarch64_enc_cset_eq(res));
6734 
6735   ins_pipe(pipe_slow);
6736 %}
6737 
6738 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
6739 
6740   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
6741 
6742   effect(KILL cr);
6743 
6744  format %{
6745     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
6746     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
6747  %}
6748 
6749  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
6750             aarch64_enc_cset_eq(res));
6751 
6752   ins_pipe(pipe_slow);
6753 %}
6754 
6755 
6756 instruct get_and_setI(indirect mem, iRegINoSp newv, iRegI prev) %{
6757   match(Set prev (GetAndSetI mem newv));
6758   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
6759   ins_encode %{
6760     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
6761   %}
6762   ins_pipe(pipe_serial);
6763 %}
6764 
6765 instruct get_and_setL(indirect mem, iRegLNoSp newv, iRegL prev) %{
6766   match(Set prev (GetAndSetL mem newv));
6767   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
6768   ins_encode %{
6769     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
6770   %}
6771   ins_pipe(pipe_serial);
6772 %}
6773 
6774 instruct get_and_setN(indirect mem, iRegNNoSp newv, iRegI prev) %{
6775   match(Set prev (GetAndSetN mem newv));
6776   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
6777   ins_encode %{
6778     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
6779   %}
6780   ins_pipe(pipe_serial);
6781 %}
6782 
6783 instruct get_and_setP(indirect mem, iRegPNoSp newv, iRegP prev) %{
6784   match(Set prev (GetAndSetP mem newv));
6785   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
6786   ins_encode %{
6787     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
6788   %}
6789   ins_pipe(pipe_serial);
6790 %}
6791 
6792 
6793 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
6794   match(Set newval (GetAndAddL mem incr));
6795   ins_cost(INSN_COST * 10);
6796   format %{ "get_and_addL $newval, [$mem], $incr" %}
6797   ins_encode %{
6798     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
6799   %}
6800   ins_pipe(pipe_serial);
6801 %}
6802 
6803 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
6804   predicate(n->as_LoadStore()->result_not_used());
6805   match(Set dummy (GetAndAddL mem incr));
6806   ins_cost(INSN_COST * 9);
6807   format %{ "get_and_addL [$mem], $incr" %}
6808   ins_encode %{
6809     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
6810   %}
6811   ins_pipe(pipe_serial);
6812 %}
6813 
6814 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
6815   match(Set newval (GetAndAddL mem incr));
6816   ins_cost(INSN_COST * 10);
6817   format %{ "get_and_addL $newval, [$mem], $incr" %}
6818   ins_encode %{
6819     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
6820   %}
6821   ins_pipe(pipe_serial);
6822 %}
6823 
6824 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
6825   predicate(n->as_LoadStore()->result_not_used());
6826   match(Set dummy (GetAndAddL mem incr));
6827   ins_cost(INSN_COST * 9);
6828   format %{ "get_and_addL [$mem], $incr" %}
6829   ins_encode %{
6830     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
6831   %}
6832   ins_pipe(pipe_serial);
6833 %}
6834 
6835 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
6836   match(Set newval (GetAndAddI mem incr));
6837   ins_cost(INSN_COST * 10);
6838   format %{ "get_and_addI $newval, [$mem], $incr" %}
6839   ins_encode %{
6840     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
6841   %}
6842   ins_pipe(pipe_serial);
6843 %}
6844 
6845 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
6846   predicate(n->as_LoadStore()->result_not_used());
6847   match(Set dummy (GetAndAddI mem incr));
6848   ins_cost(INSN_COST * 9);
6849   format %{ "get_and_addI [$mem], $incr" %}
6850   ins_encode %{
6851     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
6852   %}
6853   ins_pipe(pipe_serial);
6854 %}
6855 
6856 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
6857   match(Set newval (GetAndAddI mem incr));
6858   ins_cost(INSN_COST * 10);
6859   format %{ "get_and_addI $newval, [$mem], $incr" %}
6860   ins_encode %{
6861     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
6862   %}
6863   ins_pipe(pipe_serial);
6864 %}
6865 
6866 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
6867   predicate(n->as_LoadStore()->result_not_used());
6868   match(Set dummy (GetAndAddI mem incr));
6869   ins_cost(INSN_COST * 9);
6870   format %{ "get_and_addI [$mem], $incr" %}
6871   ins_encode %{
6872     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
6873   %}
6874   ins_pipe(pipe_serial);
6875 %}
6876 
6877 // Manifest a CmpL result in an integer register.
6878 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
6879 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
6880 %{
6881   match(Set dst (CmpL3 src1 src2));
6882   effect(KILL flags);
6883 
6884   ins_cost(INSN_COST * 6);
6885   format %{
6886       "cmp $src1, $src2"
6887       "csetw $dst, ne"
6888       "cnegw $dst, lt"
6889   %}
6890   // format %{ "CmpL3 $dst, $src1, $src2" %}
6891   ins_encode %{
6892     __ cmp($src1$$Register, $src2$$Register);
6893     __ csetw($dst$$Register, Assembler::NE);
6894     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
6895   %}
6896 
6897   ins_pipe(pipe_class_default);
6898 %}
6899 
6900 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
6901 %{
6902   match(Set dst (CmpL3 src1 src2));
6903   effect(KILL flags);
6904 
6905   ins_cost(INSN_COST * 6);
6906   format %{
6907       "cmp $src1, $src2"
6908       "csetw $dst, ne"
6909       "cnegw $dst, lt"
6910   %}
6911   ins_encode %{
6912     int32_t con = (int32_t)$src2$$constant;
6913      if (con < 0) {
6914       __ adds(zr, $src1$$Register, -con);
6915     } else {
6916       __ subs(zr, $src1$$Register, con);
6917     }
6918     __ csetw($dst$$Register, Assembler::NE);
6919     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
6920   %}
6921 
6922   ins_pipe(pipe_class_default);
6923 %}
6924 
6925 // ============================================================================
6926 // Conditional Move Instructions
6927 
6928 // n.b. we have identical rules for both a signed compare op (cmpOp)
6929 // and an unsigned compare op (cmpOpU). it would be nice if we could
6930 // define an op class which merged both inputs and use it to type the
6931 // argument to a single rule. unfortunatelyt his fails because the
6932 // opclass does not live up to the COND_INTER interface of its
6933 // component operands. When the generic code tries to negate the
6934 // operand it ends up running the generci Machoper::negate method
6935 // which throws a ShouldNotHappen. So, we have to provide two flavours
6936 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
6937 
6938 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
6939   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
6940 
6941   ins_cost(INSN_COST * 2);
6942   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
6943 
6944   ins_encode %{
6945     __ cselw(as_Register($dst$$reg),
6946              as_Register($src2$$reg),
6947              as_Register($src1$$reg),
6948              (Assembler::Condition)$cmp$$cmpcode);
6949   %}
6950 
6951   ins_pipe(icond_reg_reg);
6952 %}
6953 
6954 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
6955   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
6956 
6957   ins_cost(INSN_COST * 2);
6958   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
6959 
6960   ins_encode %{
6961     __ cselw(as_Register($dst$$reg),
6962              as_Register($src2$$reg),
6963              as_Register($src1$$reg),
6964              (Assembler::Condition)$cmp$$cmpcode);
6965   %}
6966 
6967   ins_pipe(icond_reg_reg);
6968 %}
6969 
6970 // special cases where one arg is zero
6971 
6972 // n.b. this is selected in preference to the rule above because it
6973 // avoids loading constant 0 into a source register
6974 
6975 // TODO
6976 // we ought only to be able to cull one of these variants as the ideal
6977 // transforms ought always to order the zero consistently (to left/right?)
6978 
6979 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
6980   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
6981 
6982   ins_cost(INSN_COST * 2);
6983   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
6984 
6985   ins_encode %{
6986     __ cselw(as_Register($dst$$reg),
6987              as_Register($src$$reg),
6988              zr,
6989              (Assembler::Condition)$cmp$$cmpcode);
6990   %}
6991 
6992   ins_pipe(icond_reg);
6993 %}
6994 
6995 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
6996   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
6997 
6998   ins_cost(INSN_COST * 2);
6999   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
7000 
7001   ins_encode %{
7002     __ cselw(as_Register($dst$$reg),
7003              as_Register($src$$reg),
7004              zr,
7005              (Assembler::Condition)$cmp$$cmpcode);
7006   %}
7007 
7008   ins_pipe(icond_reg);
7009 %}
7010 
7011 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
7012   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
7013 
7014   ins_cost(INSN_COST * 2);
7015   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
7016 
7017   ins_encode %{
7018     __ cselw(as_Register($dst$$reg),
7019              zr,
7020              as_Register($src$$reg),
7021              (Assembler::Condition)$cmp$$cmpcode);
7022   %}
7023 
7024   ins_pipe(icond_reg);
7025 %}
7026 
7027 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
7028   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
7029 
7030   ins_cost(INSN_COST * 2);
7031   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
7032 
7033   ins_encode %{
7034     __ cselw(as_Register($dst$$reg),
7035              zr,
7036              as_Register($src$$reg),
7037              (Assembler::Condition)$cmp$$cmpcode);
7038   %}
7039 
7040   ins_pipe(icond_reg);
7041 %}
7042 
7043 // special case for creating a boolean 0 or 1
7044 
7045 // n.b. this is selected in preference to the rule above because it
7046 // avoids loading constants 0 and 1 into a source register
7047 
7048 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
7049   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
7050 
7051   ins_cost(INSN_COST * 2);
7052   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
7053 
7054   ins_encode %{
7055     // equivalently
7056     // cset(as_Register($dst$$reg),
7057     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
7058     __ csincw(as_Register($dst$$reg),
7059              zr,
7060              zr,
7061              (Assembler::Condition)$cmp$$cmpcode);
7062   %}
7063 
7064   ins_pipe(icond_none);
7065 %}
7066 
7067 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
7068   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
7069 
7070   ins_cost(INSN_COST * 2);
7071   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
7072 
7073   ins_encode %{
7074     // equivalently
7075     // cset(as_Register($dst$$reg),
7076     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
7077     __ csincw(as_Register($dst$$reg),
7078              zr,
7079              zr,
7080              (Assembler::Condition)$cmp$$cmpcode);
7081   %}
7082 
7083   ins_pipe(icond_none);
7084 %}
7085 
7086 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
7087   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
7088 
7089   ins_cost(INSN_COST * 2);
7090   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
7091 
7092   ins_encode %{
7093     __ csel(as_Register($dst$$reg),
7094             as_Register($src2$$reg),
7095             as_Register($src1$$reg),
7096             (Assembler::Condition)$cmp$$cmpcode);
7097   %}
7098 
7099   ins_pipe(icond_reg_reg);
7100 %}
7101 
7102 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
7103   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
7104 
7105   ins_cost(INSN_COST * 2);
7106   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
7107 
7108   ins_encode %{
7109     __ csel(as_Register($dst$$reg),
7110             as_Register($src2$$reg),
7111             as_Register($src1$$reg),
7112             (Assembler::Condition)$cmp$$cmpcode);
7113   %}
7114 
7115   ins_pipe(icond_reg_reg);
7116 %}
7117 
7118 // special cases where one arg is zero
7119 
7120 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
7121   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
7122 
7123   ins_cost(INSN_COST * 2);
7124   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
7125 
7126   ins_encode %{
7127     __ csel(as_Register($dst$$reg),
7128             zr,
7129             as_Register($src$$reg),
7130             (Assembler::Condition)$cmp$$cmpcode);
7131   %}
7132 
7133   ins_pipe(icond_reg);
7134 %}
7135 
7136 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
7137   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
7138 
7139   ins_cost(INSN_COST * 2);
7140   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
7141 
7142   ins_encode %{
7143     __ csel(as_Register($dst$$reg),
7144             zr,
7145             as_Register($src$$reg),
7146             (Assembler::Condition)$cmp$$cmpcode);
7147   %}
7148 
7149   ins_pipe(icond_reg);
7150 %}
7151 
7152 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
7153   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
7154 
7155   ins_cost(INSN_COST * 2);
7156   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
7157 
7158   ins_encode %{
7159     __ csel(as_Register($dst$$reg),
7160             as_Register($src$$reg),
7161             zr,
7162             (Assembler::Condition)$cmp$$cmpcode);
7163   %}
7164 
7165   ins_pipe(icond_reg);
7166 %}
7167 
7168 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
7169   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
7170 
7171   ins_cost(INSN_COST * 2);
7172   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
7173 
7174   ins_encode %{
7175     __ csel(as_Register($dst$$reg),
7176             as_Register($src$$reg),
7177             zr,
7178             (Assembler::Condition)$cmp$$cmpcode);
7179   %}
7180 
7181   ins_pipe(icond_reg);
7182 %}
7183 
7184 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
7185   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
7186 
7187   ins_cost(INSN_COST * 2);
7188   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
7189 
7190   ins_encode %{
7191     __ csel(as_Register($dst$$reg),
7192             as_Register($src2$$reg),
7193             as_Register($src1$$reg),
7194             (Assembler::Condition)$cmp$$cmpcode);
7195   %}
7196 
7197   ins_pipe(icond_reg_reg);
7198 %}
7199 
7200 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
7201   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
7202 
7203   ins_cost(INSN_COST * 2);
7204   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
7205 
7206   ins_encode %{
7207     __ csel(as_Register($dst$$reg),
7208             as_Register($src2$$reg),
7209             as_Register($src1$$reg),
7210             (Assembler::Condition)$cmp$$cmpcode);
7211   %}
7212 
7213   ins_pipe(icond_reg_reg);
7214 %}
7215 
7216 // special cases where one arg is zero
7217 
7218 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
7219   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
7220 
7221   ins_cost(INSN_COST * 2);
7222   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
7223 
7224   ins_encode %{
7225     __ csel(as_Register($dst$$reg),
7226             zr,
7227             as_Register($src$$reg),
7228             (Assembler::Condition)$cmp$$cmpcode);
7229   %}
7230 
7231   ins_pipe(icond_reg);
7232 %}
7233 
7234 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
7235   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
7236 
7237   ins_cost(INSN_COST * 2);
7238   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
7239 
7240   ins_encode %{
7241     __ csel(as_Register($dst$$reg),
7242             zr,
7243             as_Register($src$$reg),
7244             (Assembler::Condition)$cmp$$cmpcode);
7245   %}
7246 
7247   ins_pipe(icond_reg);
7248 %}
7249 
7250 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
7251   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
7252 
7253   ins_cost(INSN_COST * 2);
7254   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
7255 
7256   ins_encode %{
7257     __ csel(as_Register($dst$$reg),
7258             as_Register($src$$reg),
7259             zr,
7260             (Assembler::Condition)$cmp$$cmpcode);
7261   %}
7262 
7263   ins_pipe(icond_reg);
7264 %}
7265 
7266 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
7267   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
7268 
7269   ins_cost(INSN_COST * 2);
7270   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
7271 
7272   ins_encode %{
7273     __ csel(as_Register($dst$$reg),
7274             as_Register($src$$reg),
7275             zr,
7276             (Assembler::Condition)$cmp$$cmpcode);
7277   %}
7278 
7279   ins_pipe(icond_reg);
7280 %}
7281 
7282 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
7283   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
7284 
7285   ins_cost(INSN_COST * 2);
7286   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
7287 
7288   ins_encode %{
7289     __ cselw(as_Register($dst$$reg),
7290              as_Register($src2$$reg),
7291              as_Register($src1$$reg),
7292              (Assembler::Condition)$cmp$$cmpcode);
7293   %}
7294 
7295   ins_pipe(icond_reg_reg);
7296 %}
7297 
7298 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
7299   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
7300 
7301   ins_cost(INSN_COST * 2);
7302   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
7303 
7304   ins_encode %{
7305     __ cselw(as_Register($dst$$reg),
7306              as_Register($src2$$reg),
7307              as_Register($src1$$reg),
7308              (Assembler::Condition)$cmp$$cmpcode);
7309   %}
7310 
7311   ins_pipe(icond_reg_reg);
7312 %}
7313 
7314 // special cases where one arg is zero
7315 
7316 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
7317   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
7318 
7319   ins_cost(INSN_COST * 2);
7320   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
7321 
7322   ins_encode %{
7323     __ cselw(as_Register($dst$$reg),
7324              zr,
7325              as_Register($src$$reg),
7326              (Assembler::Condition)$cmp$$cmpcode);
7327   %}
7328 
7329   ins_pipe(icond_reg);
7330 %}
7331 
7332 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
7333   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
7334 
7335   ins_cost(INSN_COST * 2);
7336   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
7337 
7338   ins_encode %{
7339     __ cselw(as_Register($dst$$reg),
7340              zr,
7341              as_Register($src$$reg),
7342              (Assembler::Condition)$cmp$$cmpcode);
7343   %}
7344 
7345   ins_pipe(icond_reg);
7346 %}
7347 
7348 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
7349   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
7350 
7351   ins_cost(INSN_COST * 2);
7352   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
7353 
7354   ins_encode %{
7355     __ cselw(as_Register($dst$$reg),
7356              as_Register($src$$reg),
7357              zr,
7358              (Assembler::Condition)$cmp$$cmpcode);
7359   %}
7360 
7361   ins_pipe(icond_reg);
7362 %}
7363 
7364 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
7365   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
7366 
7367   ins_cost(INSN_COST * 2);
7368   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
7369 
7370   ins_encode %{
7371     __ cselw(as_Register($dst$$reg),
7372              as_Register($src$$reg),
7373              zr,
7374              (Assembler::Condition)$cmp$$cmpcode);
7375   %}
7376 
7377   ins_pipe(icond_reg);
7378 %}
7379 
7380 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
7381 %{
7382   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
7383 
7384   ins_cost(INSN_COST * 3);
7385 
7386   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
7387   ins_encode %{
7388     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
7389     __ fcsels(as_FloatRegister($dst$$reg),
7390               as_FloatRegister($src2$$reg),
7391               as_FloatRegister($src1$$reg),
7392               cond);
7393   %}
7394 
7395   ins_pipe(pipe_class_default);
7396 %}
7397 
7398 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
7399 %{
7400   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
7401 
7402   ins_cost(INSN_COST * 3);
7403 
7404   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
7405   ins_encode %{
7406     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
7407     __ fcsels(as_FloatRegister($dst$$reg),
7408               as_FloatRegister($src2$$reg),
7409               as_FloatRegister($src1$$reg),
7410               cond);
7411   %}
7412 
7413   ins_pipe(pipe_class_default);
7414 %}
7415 
7416 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
7417 %{
7418   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
7419 
7420   ins_cost(INSN_COST * 3);
7421 
7422   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
7423   ins_encode %{
7424     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
7425     __ fcseld(as_FloatRegister($dst$$reg),
7426               as_FloatRegister($src2$$reg),
7427               as_FloatRegister($src1$$reg),
7428               cond);
7429   %}
7430 
7431   ins_pipe(pipe_class_default);
7432 %}
7433 
7434 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
7435 %{
7436   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
7437 
7438   ins_cost(INSN_COST * 3);
7439 
7440   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
7441   ins_encode %{
7442     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
7443     __ fcseld(as_FloatRegister($dst$$reg),
7444               as_FloatRegister($src2$$reg),
7445               as_FloatRegister($src1$$reg),
7446               cond);
7447   %}
7448 
7449   ins_pipe(pipe_class_default);
7450 %}
7451 
7452 // ============================================================================
7453 // Arithmetic Instructions
7454 //
7455 
7456 // Integer Addition
7457 
7458 // TODO
7459 // these currently employ operations which do not set CR and hence are
7460 // not flagged as killing CR but we would like to isolate the cases
7461 // where we want to set flags from those where we don't. need to work
7462 // out how to do that.
7463 
7464 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7465   match(Set dst (AddI src1 src2));
7466 
7467   ins_cost(INSN_COST);
7468   format %{ "addw  $dst, $src1, $src2" %}
7469 
7470   ins_encode %{
7471     __ addw(as_Register($dst$$reg),
7472             as_Register($src1$$reg),
7473             as_Register($src2$$reg));
7474   %}
7475 
7476   ins_pipe(ialu_reg_reg);
7477 %}
7478 
7479 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
7480   match(Set dst (AddI src1 src2));
7481 
7482   ins_cost(INSN_COST);
7483   format %{ "addw $dst, $src1, $src2" %}
7484 
7485   // use opcode to indicate that this is an add not a sub
7486   opcode(0x0);
7487 
7488   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
7489 
7490   ins_pipe(ialu_reg_imm);
7491 %}
7492 
7493 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
7494   match(Set dst (AddI (ConvL2I src1) src2));
7495 
7496   ins_cost(INSN_COST);
7497   format %{ "addw $dst, $src1, $src2" %}
7498 
7499   // use opcode to indicate that this is an add not a sub
7500   opcode(0x0);
7501 
7502   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
7503 
7504   ins_pipe(ialu_reg_imm);
7505 %}
7506 
7507 // Pointer Addition
7508 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
7509   match(Set dst (AddP src1 src2));
7510 
7511   ins_cost(INSN_COST);
7512   format %{ "add $dst, $src1, $src2\t# ptr" %}
7513 
7514   ins_encode %{
7515     __ add(as_Register($dst$$reg),
7516            as_Register($src1$$reg),
7517            as_Register($src2$$reg));
7518   %}
7519 
7520   ins_pipe(ialu_reg_reg);
7521 %}
7522 
7523 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
7524   match(Set dst (AddP src1 (ConvI2L src2)));
7525 
7526   ins_cost(INSN_COST);
7527   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
7528 
7529   ins_encode %{
7530     __ add(as_Register($dst$$reg),
7531            as_Register($src1$$reg),
7532            as_Register($src2$$reg), ext::sxtw);
7533   %}
7534 
7535   ins_pipe(ialu_reg_reg);
7536 %}
7537 
7538 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
7539   match(Set dst (AddP src1 (LShiftL src2 scale)));
7540 
7541   ins_cost(1.9 * INSN_COST);
7542   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
7543 
7544   ins_encode %{
7545     __ lea(as_Register($dst$$reg),
7546            Address(as_Register($src1$$reg), as_Register($src2$$reg),
7547                    Address::lsl($scale$$constant)));
7548   %}
7549 
7550   ins_pipe(ialu_reg_reg_shift);
7551 %}
7552 
7553 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
7554   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
7555 
7556   ins_cost(1.9 * INSN_COST);
7557   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
7558 
7559   ins_encode %{
7560     __ lea(as_Register($dst$$reg),
7561            Address(as_Register($src1$$reg), as_Register($src2$$reg),
7562                    Address::sxtw($scale$$constant)));
7563   %}
7564 
7565   ins_pipe(ialu_reg_reg_shift);
7566 %}
7567 
7568 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
7569   match(Set dst (LShiftL (ConvI2L src) scale));
7570 
7571   ins_cost(INSN_COST);
7572   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
7573 
7574   ins_encode %{
7575     __ sbfiz(as_Register($dst$$reg),
7576           as_Register($src$$reg),
7577           $scale$$constant & 63, MIN(32, (-$scale$$constant) & 63));
7578   %}
7579 
7580   ins_pipe(ialu_reg_shift);
7581 %}
7582 
7583 // Pointer Immediate Addition
7584 // n.b. this needs to be more expensive than using an indirect memory
7585 // operand
7586 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
7587   match(Set dst (AddP src1 src2));
7588 
7589   ins_cost(INSN_COST);
7590   format %{ "add $dst, $src1, $src2\t# ptr" %}
7591 
7592   // use opcode to indicate that this is an add not a sub
7593   opcode(0x0);
7594 
7595   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
7596 
7597   ins_pipe(ialu_reg_imm);
7598 %}
7599 
7600 // Long Addition
7601 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
7602 
7603   match(Set dst (AddL src1 src2));
7604 
7605   ins_cost(INSN_COST);
7606   format %{ "add  $dst, $src1, $src2" %}
7607 
7608   ins_encode %{
7609     __ add(as_Register($dst$$reg),
7610            as_Register($src1$$reg),
7611            as_Register($src2$$reg));
7612   %}
7613 
7614   ins_pipe(ialu_reg_reg);
7615 %}
7616 
7617 // No constant pool entries requiredLong Immediate Addition.
7618 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
7619   match(Set dst (AddL src1 src2));
7620 
7621   ins_cost(INSN_COST);
7622   format %{ "add $dst, $src1, $src2" %}
7623 
7624   // use opcode to indicate that this is an add not a sub
7625   opcode(0x0);
7626 
7627   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
7628 
7629   ins_pipe(ialu_reg_imm);
7630 %}
7631 
7632 // Integer Subtraction
7633 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7634   match(Set dst (SubI src1 src2));
7635 
7636   ins_cost(INSN_COST);
7637   format %{ "subw  $dst, $src1, $src2" %}
7638 
7639   ins_encode %{
7640     __ subw(as_Register($dst$$reg),
7641             as_Register($src1$$reg),
7642             as_Register($src2$$reg));
7643   %}
7644 
7645   ins_pipe(ialu_reg_reg);
7646 %}
7647 
7648 // Immediate Subtraction
7649 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
7650   match(Set dst (SubI src1 src2));
7651 
7652   ins_cost(INSN_COST);
7653   format %{ "subw $dst, $src1, $src2" %}
7654 
7655   // use opcode to indicate that this is a sub not an add
7656   opcode(0x1);
7657 
7658   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
7659 
7660   ins_pipe(ialu_reg_imm);
7661 %}
7662 
7663 // Long Subtraction
7664 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
7665 
7666   match(Set dst (SubL src1 src2));
7667 
7668   ins_cost(INSN_COST);
7669   format %{ "sub  $dst, $src1, $src2" %}
7670 
7671   ins_encode %{
7672     __ sub(as_Register($dst$$reg),
7673            as_Register($src1$$reg),
7674            as_Register($src2$$reg));
7675   %}
7676 
7677   ins_pipe(ialu_reg_reg);
7678 %}
7679 
7680 // No constant pool entries requiredLong Immediate Subtraction.
7681 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
7682   match(Set dst (SubL src1 src2));
7683 
7684   ins_cost(INSN_COST);
7685   format %{ "sub$dst, $src1, $src2" %}
7686 
7687   // use opcode to indicate that this is a sub not an add
7688   opcode(0x1);
7689 
7690   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
7691 
7692   ins_pipe(ialu_reg_imm);
7693 %}
7694 
7695 // Integer Negation (special case for sub)
7696 
7697 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
7698   match(Set dst (SubI zero src));
7699 
7700   ins_cost(INSN_COST);
7701   format %{ "negw $dst, $src\t# int" %}
7702 
7703   ins_encode %{
7704     __ negw(as_Register($dst$$reg),
7705             as_Register($src$$reg));
7706   %}
7707 
7708   ins_pipe(ialu_reg);
7709 %}
7710 
7711 // Long Negation
7712 
7713 instruct negL_reg(iRegLNoSp dst, iRegIorL2I src, immL0 zero, rFlagsReg cr) %{
7714   match(Set dst (SubL zero src));
7715 
7716   ins_cost(INSN_COST);
7717   format %{ "neg $dst, $src\t# long" %}
7718 
7719   ins_encode %{
7720     __ neg(as_Register($dst$$reg),
7721            as_Register($src$$reg));
7722   %}
7723 
7724   ins_pipe(ialu_reg);
7725 %}
7726 
7727 // Integer Multiply
7728 
7729 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7730   match(Set dst (MulI src1 src2));
7731 
7732   ins_cost(INSN_COST * 3);
7733   format %{ "mulw  $dst, $src1, $src2" %}
7734 
7735   ins_encode %{
7736     __ mulw(as_Register($dst$$reg),
7737             as_Register($src1$$reg),
7738             as_Register($src2$$reg));
7739   %}
7740 
7741   ins_pipe(imul_reg_reg);
7742 %}
7743 
7744 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7745   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
7746 
7747   ins_cost(INSN_COST * 3);
7748   format %{ "smull  $dst, $src1, $src2" %}
7749 
7750   ins_encode %{
7751     __ smull(as_Register($dst$$reg),
7752              as_Register($src1$$reg),
7753              as_Register($src2$$reg));
7754   %}
7755 
7756   ins_pipe(imul_reg_reg);
7757 %}
7758 
7759 // Long Multiply
7760 
7761 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
7762   match(Set dst (MulL src1 src2));
7763 
7764   ins_cost(INSN_COST * 5);
7765   format %{ "mul  $dst, $src1, $src2" %}
7766 
7767   ins_encode %{
7768     __ mul(as_Register($dst$$reg),
7769            as_Register($src1$$reg),
7770            as_Register($src2$$reg));
7771   %}
7772 
7773   ins_pipe(lmul_reg_reg);
7774 %}
7775 
7776 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
7777 %{
7778   match(Set dst (MulHiL src1 src2));
7779 
7780   ins_cost(INSN_COST * 7);
7781   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
7782 
7783   ins_encode %{
7784     __ smulh(as_Register($dst$$reg),
7785              as_Register($src1$$reg),
7786              as_Register($src2$$reg));
7787   %}
7788 
7789   ins_pipe(lmul_reg_reg);
7790 %}
7791 
7792 // Combined Integer Multiply & Add/Sub
7793 
7794 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
7795   match(Set dst (AddI src3 (MulI src1 src2)));
7796 
7797   ins_cost(INSN_COST * 3);
7798   format %{ "madd  $dst, $src1, $src2, $src3" %}
7799 
7800   ins_encode %{
7801     __ maddw(as_Register($dst$$reg),
7802              as_Register($src1$$reg),
7803              as_Register($src2$$reg),
7804              as_Register($src3$$reg));
7805   %}
7806 
7807   ins_pipe(imac_reg_reg);
7808 %}
7809 
7810 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
7811   match(Set dst (SubI src3 (MulI src1 src2)));
7812 
7813   ins_cost(INSN_COST * 3);
7814   format %{ "msub  $dst, $src1, $src2, $src3" %}
7815 
7816   ins_encode %{
7817     __ msubw(as_Register($dst$$reg),
7818              as_Register($src1$$reg),
7819              as_Register($src2$$reg),
7820              as_Register($src3$$reg));
7821   %}
7822 
7823   ins_pipe(imac_reg_reg);
7824 %}
7825 
7826 // Combined Long Multiply & Add/Sub
7827 
7828 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
7829   match(Set dst (AddL src3 (MulL src1 src2)));
7830 
7831   ins_cost(INSN_COST * 5);
7832   format %{ "madd  $dst, $src1, $src2, $src3" %}
7833 
7834   ins_encode %{
7835     __ madd(as_Register($dst$$reg),
7836             as_Register($src1$$reg),
7837             as_Register($src2$$reg),
7838             as_Register($src3$$reg));
7839   %}
7840 
7841   ins_pipe(lmac_reg_reg);
7842 %}
7843 
7844 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
7845   match(Set dst (SubL src3 (MulL src1 src2)));
7846 
7847   ins_cost(INSN_COST * 5);
7848   format %{ "msub  $dst, $src1, $src2, $src3" %}
7849 
7850   ins_encode %{
7851     __ msub(as_Register($dst$$reg),
7852             as_Register($src1$$reg),
7853             as_Register($src2$$reg),
7854             as_Register($src3$$reg));
7855   %}
7856 
7857   ins_pipe(lmac_reg_reg);
7858 %}
7859 
7860 // Integer Divide
7861 
7862 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7863   match(Set dst (DivI src1 src2));
7864 
7865   ins_cost(INSN_COST * 19);
7866   format %{ "sdivw  $dst, $src1, $src2" %}
7867 
7868   ins_encode(aarch64_enc_divw(dst, src1, src2));
7869   ins_pipe(idiv_reg_reg);
7870 %}
7871 
7872 instruct signExtract(iRegINoSp dst, iRegIorL2I src1, immI_31 div1, immI_31 div2) %{
7873   match(Set dst (URShiftI (RShiftI src1 div1) div2));
7874   ins_cost(INSN_COST);
7875   format %{ "lsrw $dst, $src1, $div1" %}
7876   ins_encode %{
7877     __ lsrw(as_Register($dst$$reg), as_Register($src1$$reg), 31);
7878   %}
7879   ins_pipe(ialu_reg_shift);
7880 %}
7881 
7882 instruct div2Round(iRegINoSp dst, iRegIorL2I src, immI_31 div1, immI_31 div2) %{
7883   match(Set dst (AddI src (URShiftI (RShiftI src div1) div2)));
7884   ins_cost(INSN_COST);
7885   format %{ "addw $dst, $src, LSR $div1" %}
7886 
7887   ins_encode %{
7888     __ addw(as_Register($dst$$reg),
7889               as_Register($src$$reg),
7890               as_Register($src$$reg),
7891               Assembler::LSR, 31);
7892   %}
7893   ins_pipe(ialu_reg);
7894 %}
7895 
7896 // Long Divide
7897 
7898 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
7899   match(Set dst (DivL src1 src2));
7900 
7901   ins_cost(INSN_COST * 35);
7902   format %{ "sdiv   $dst, $src1, $src2" %}
7903 
7904   ins_encode(aarch64_enc_div(dst, src1, src2));
7905   ins_pipe(ldiv_reg_reg);
7906 %}
7907 
7908 instruct signExtractL(iRegLNoSp dst, iRegL src1, immL_63 div1, immL_63 div2) %{
7909   match(Set dst (URShiftL (RShiftL src1 div1) div2));
7910   ins_cost(INSN_COST);
7911   format %{ "lsr $dst, $src1, $div1" %}
7912   ins_encode %{
7913     __ lsr(as_Register($dst$$reg), as_Register($src1$$reg), 63);
7914   %}
7915   ins_pipe(ialu_reg_shift);
7916 %}
7917 
7918 instruct div2RoundL(iRegLNoSp dst, iRegL src, immL_63 div1, immL_63 div2) %{
7919   match(Set dst (AddL src (URShiftL (RShiftL src div1) div2)));
7920   ins_cost(INSN_COST);
7921   format %{ "add $dst, $src, $div1" %}
7922 
7923   ins_encode %{
7924     __ add(as_Register($dst$$reg),
7925               as_Register($src$$reg),
7926               as_Register($src$$reg),
7927               Assembler::LSR, 63);
7928   %}
7929   ins_pipe(ialu_reg);
7930 %}
7931 
7932 // Integer Remainder
7933 
7934 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7935   match(Set dst (ModI src1 src2));
7936 
7937   ins_cost(INSN_COST * 22);
7938   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
7939             "msubw($dst, rscratch1, $src2, $src1" %}
7940 
7941   ins_encode(aarch64_enc_modw(dst, src1, src2));
7942   ins_pipe(idiv_reg_reg);
7943 %}
7944 
7945 // Long Remainder
7946 
7947 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
7948   match(Set dst (ModL src1 src2));
7949 
7950   ins_cost(INSN_COST * 38);
7951   format %{ "sdiv   rscratch1, $src1, $src2\n"
7952             "msub($dst, rscratch1, $src2, $src1" %}
7953 
7954   ins_encode(aarch64_enc_mod(dst, src1, src2));
7955   ins_pipe(ldiv_reg_reg);
7956 %}
7957 
7958 // Integer Shifts
7959 
7960 // Shift Left Register
7961 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7962   match(Set dst (LShiftI src1 src2));
7963 
7964   ins_cost(INSN_COST * 2);
7965   format %{ "lslvw  $dst, $src1, $src2" %}
7966 
7967   ins_encode %{
7968     __ lslvw(as_Register($dst$$reg),
7969              as_Register($src1$$reg),
7970              as_Register($src2$$reg));
7971   %}
7972 
7973   ins_pipe(ialu_reg_reg_vshift);
7974 %}
7975 
7976 // Shift Left Immediate
7977 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
7978   match(Set dst (LShiftI src1 src2));
7979 
7980   ins_cost(INSN_COST);
7981   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
7982 
7983   ins_encode %{
7984     __ lslw(as_Register($dst$$reg),
7985             as_Register($src1$$reg),
7986             $src2$$constant & 0x1f);
7987   %}
7988 
7989   ins_pipe(ialu_reg_shift);
7990 %}
7991 
7992 // Shift Right Logical Register
7993 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7994   match(Set dst (URShiftI src1 src2));
7995 
7996   ins_cost(INSN_COST * 2);
7997   format %{ "lsrvw  $dst, $src1, $src2" %}
7998 
7999   ins_encode %{
8000     __ lsrvw(as_Register($dst$$reg),
8001              as_Register($src1$$reg),
8002              as_Register($src2$$reg));
8003   %}
8004 
8005   ins_pipe(ialu_reg_reg_vshift);
8006 %}
8007 
8008 // Shift Right Logical Immediate
8009 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
8010   match(Set dst (URShiftI src1 src2));
8011 
8012   ins_cost(INSN_COST);
8013   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
8014 
8015   ins_encode %{
8016     __ lsrw(as_Register($dst$$reg),
8017             as_Register($src1$$reg),
8018             $src2$$constant & 0x1f);
8019   %}
8020 
8021   ins_pipe(ialu_reg_shift);
8022 %}
8023 
8024 // Shift Right Arithmetic Register
8025 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8026   match(Set dst (RShiftI src1 src2));
8027 
8028   ins_cost(INSN_COST * 2);
8029   format %{ "asrvw  $dst, $src1, $src2" %}
8030 
8031   ins_encode %{
8032     __ asrvw(as_Register($dst$$reg),
8033              as_Register($src1$$reg),
8034              as_Register($src2$$reg));
8035   %}
8036 
8037   ins_pipe(ialu_reg_reg_vshift);
8038 %}
8039 
8040 // Shift Right Arithmetic Immediate
8041 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
8042   match(Set dst (RShiftI src1 src2));
8043 
8044   ins_cost(INSN_COST);
8045   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
8046 
8047   ins_encode %{
8048     __ asrw(as_Register($dst$$reg),
8049             as_Register($src1$$reg),
8050             $src2$$constant & 0x1f);
8051   %}
8052 
8053   ins_pipe(ialu_reg_shift);
8054 %}
8055 
8056 // Combined Int Mask and Right Shift (using UBFM)
8057 // TODO
8058 
8059 // Long Shifts
8060 
8061 // Shift Left Register
8062 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
8063   match(Set dst (LShiftL src1 src2));
8064 
8065   ins_cost(INSN_COST * 2);
8066   format %{ "lslv  $dst, $src1, $src2" %}
8067 
8068   ins_encode %{
8069     __ lslv(as_Register($dst$$reg),
8070             as_Register($src1$$reg),
8071             as_Register($src2$$reg));
8072   %}
8073 
8074   ins_pipe(ialu_reg_reg_vshift);
8075 %}
8076 
8077 // Shift Left Immediate
8078 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
8079   match(Set dst (LShiftL src1 src2));
8080 
8081   ins_cost(INSN_COST);
8082   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
8083 
8084   ins_encode %{
8085     __ lsl(as_Register($dst$$reg),
8086             as_Register($src1$$reg),
8087             $src2$$constant & 0x3f);
8088   %}
8089 
8090   ins_pipe(ialu_reg_shift);
8091 %}
8092 
8093 // Shift Right Logical Register
8094 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
8095   match(Set dst (URShiftL src1 src2));
8096 
8097   ins_cost(INSN_COST * 2);
8098   format %{ "lsrv  $dst, $src1, $src2" %}
8099 
8100   ins_encode %{
8101     __ lsrv(as_Register($dst$$reg),
8102             as_Register($src1$$reg),
8103             as_Register($src2$$reg));
8104   %}
8105 
8106   ins_pipe(ialu_reg_reg_vshift);
8107 %}
8108 
8109 // Shift Right Logical Immediate
8110 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
8111   match(Set dst (URShiftL src1 src2));
8112 
8113   ins_cost(INSN_COST);
8114   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
8115 
8116   ins_encode %{
8117     __ lsr(as_Register($dst$$reg),
8118            as_Register($src1$$reg),
8119            $src2$$constant & 0x3f);
8120   %}
8121 
8122   ins_pipe(ialu_reg_shift);
8123 %}
8124 
8125 // A special-case pattern for card table stores.
8126 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
8127   match(Set dst (URShiftL (CastP2X src1) src2));
8128 
8129   ins_cost(INSN_COST);
8130   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
8131 
8132   ins_encode %{
8133     __ lsr(as_Register($dst$$reg),
8134            as_Register($src1$$reg),
8135            $src2$$constant & 0x3f);
8136   %}
8137 
8138   ins_pipe(ialu_reg_shift);
8139 %}
8140 
8141 // Shift Right Arithmetic Register
8142 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
8143   match(Set dst (RShiftL src1 src2));
8144 
8145   ins_cost(INSN_COST * 2);
8146   format %{ "asrv  $dst, $src1, $src2" %}
8147 
8148   ins_encode %{
8149     __ asrv(as_Register($dst$$reg),
8150             as_Register($src1$$reg),
8151             as_Register($src2$$reg));
8152   %}
8153 
8154   ins_pipe(ialu_reg_reg_vshift);
8155 %}
8156 
8157 // Shift Right Arithmetic Immediate
8158 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
8159   match(Set dst (RShiftL src1 src2));
8160 
8161   ins_cost(INSN_COST);
8162   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
8163 
8164   ins_encode %{
8165     __ asr(as_Register($dst$$reg),
8166            as_Register($src1$$reg),
8167            $src2$$constant & 0x3f);
8168   %}
8169 
8170   ins_pipe(ialu_reg_shift);
8171 %}
8172 
8173 // BEGIN This section of the file is automatically generated. Do not edit --------------
8174 
8175 instruct regL_not_reg(iRegLNoSp dst,
8176                          iRegL src1, immL_M1 m1,
8177                          rFlagsReg cr) %{
8178   match(Set dst (XorL src1 m1));
8179   ins_cost(INSN_COST);
8180   format %{ "eon  $dst, $src1, zr" %}
8181 
8182   ins_encode %{
8183     __ eon(as_Register($dst$$reg),
8184               as_Register($src1$$reg),
8185               zr,
8186               Assembler::LSL, 0);
8187   %}
8188 
8189   ins_pipe(ialu_reg);
8190 %}
8191 instruct regI_not_reg(iRegINoSp dst,
8192                          iRegIorL2I src1, immI_M1 m1,
8193                          rFlagsReg cr) %{
8194   match(Set dst (XorI src1 m1));
8195   ins_cost(INSN_COST);
8196   format %{ "eonw  $dst, $src1, zr" %}
8197 
8198   ins_encode %{
8199     __ eonw(as_Register($dst$$reg),
8200               as_Register($src1$$reg),
8201               zr,
8202               Assembler::LSL, 0);
8203   %}
8204 
8205   ins_pipe(ialu_reg);
8206 %}
8207 
8208 instruct AndI_reg_not_reg(iRegINoSp dst,
8209                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
8210                          rFlagsReg cr) %{
8211   match(Set dst (AndI src1 (XorI src2 m1)));
8212   ins_cost(INSN_COST);
8213   format %{ "bicw  $dst, $src1, $src2" %}
8214 
8215   ins_encode %{
8216     __ bicw(as_Register($dst$$reg),
8217               as_Register($src1$$reg),
8218               as_Register($src2$$reg),
8219               Assembler::LSL, 0);
8220   %}
8221 
8222   ins_pipe(ialu_reg_reg);
8223 %}
8224 
8225 instruct AndL_reg_not_reg(iRegLNoSp dst,
8226                          iRegL src1, iRegL src2, immL_M1 m1,
8227                          rFlagsReg cr) %{
8228   match(Set dst (AndL src1 (XorL src2 m1)));
8229   ins_cost(INSN_COST);
8230   format %{ "bic  $dst, $src1, $src2" %}
8231 
8232   ins_encode %{
8233     __ bic(as_Register($dst$$reg),
8234               as_Register($src1$$reg),
8235               as_Register($src2$$reg),
8236               Assembler::LSL, 0);
8237   %}
8238 
8239   ins_pipe(ialu_reg_reg);
8240 %}
8241 
8242 instruct OrI_reg_not_reg(iRegINoSp dst,
8243                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
8244                          rFlagsReg cr) %{
8245   match(Set dst (OrI src1 (XorI src2 m1)));
8246   ins_cost(INSN_COST);
8247   format %{ "ornw  $dst, $src1, $src2" %}
8248 
8249   ins_encode %{
8250     __ ornw(as_Register($dst$$reg),
8251               as_Register($src1$$reg),
8252               as_Register($src2$$reg),
8253               Assembler::LSL, 0);
8254   %}
8255 
8256   ins_pipe(ialu_reg_reg);
8257 %}
8258 
8259 instruct OrL_reg_not_reg(iRegLNoSp dst,
8260                          iRegL src1, iRegL src2, immL_M1 m1,
8261                          rFlagsReg cr) %{
8262   match(Set dst (OrL src1 (XorL src2 m1)));
8263   ins_cost(INSN_COST);
8264   format %{ "orn  $dst, $src1, $src2" %}
8265 
8266   ins_encode %{
8267     __ orn(as_Register($dst$$reg),
8268               as_Register($src1$$reg),
8269               as_Register($src2$$reg),
8270               Assembler::LSL, 0);
8271   %}
8272 
8273   ins_pipe(ialu_reg_reg);
8274 %}
8275 
8276 instruct XorI_reg_not_reg(iRegINoSp dst,
8277                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
8278                          rFlagsReg cr) %{
8279   match(Set dst (XorI m1 (XorI src2 src1)));
8280   ins_cost(INSN_COST);
8281   format %{ "eonw  $dst, $src1, $src2" %}
8282 
8283   ins_encode %{
8284     __ eonw(as_Register($dst$$reg),
8285               as_Register($src1$$reg),
8286               as_Register($src2$$reg),
8287               Assembler::LSL, 0);
8288   %}
8289 
8290   ins_pipe(ialu_reg_reg);
8291 %}
8292 
8293 instruct XorL_reg_not_reg(iRegLNoSp dst,
8294                          iRegL src1, iRegL src2, immL_M1 m1,
8295                          rFlagsReg cr) %{
8296   match(Set dst (XorL m1 (XorL src2 src1)));
8297   ins_cost(INSN_COST);
8298   format %{ "eon  $dst, $src1, $src2" %}
8299 
8300   ins_encode %{
8301     __ eon(as_Register($dst$$reg),
8302               as_Register($src1$$reg),
8303               as_Register($src2$$reg),
8304               Assembler::LSL, 0);
8305   %}
8306 
8307   ins_pipe(ialu_reg_reg);
8308 %}
8309 
8310 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
8311                          iRegIorL2I src1, iRegIorL2I src2,
8312                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8313   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
8314   ins_cost(1.9 * INSN_COST);
8315   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
8316 
8317   ins_encode %{
8318     __ bicw(as_Register($dst$$reg),
8319               as_Register($src1$$reg),
8320               as_Register($src2$$reg),
8321               Assembler::LSR,
8322               $src3$$constant & 0x3f);
8323   %}
8324 
8325   ins_pipe(ialu_reg_reg_shift);
8326 %}
8327 
8328 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
8329                          iRegL src1, iRegL src2,
8330                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8331   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
8332   ins_cost(1.9 * INSN_COST);
8333   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
8334 
8335   ins_encode %{
8336     __ bic(as_Register($dst$$reg),
8337               as_Register($src1$$reg),
8338               as_Register($src2$$reg),
8339               Assembler::LSR,
8340               $src3$$constant & 0x3f);
8341   %}
8342 
8343   ins_pipe(ialu_reg_reg_shift);
8344 %}
8345 
8346 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
8347                          iRegIorL2I src1, iRegIorL2I src2,
8348                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8349   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
8350   ins_cost(1.9 * INSN_COST);
8351   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
8352 
8353   ins_encode %{
8354     __ bicw(as_Register($dst$$reg),
8355               as_Register($src1$$reg),
8356               as_Register($src2$$reg),
8357               Assembler::ASR,
8358               $src3$$constant & 0x3f);
8359   %}
8360 
8361   ins_pipe(ialu_reg_reg_shift);
8362 %}
8363 
8364 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
8365                          iRegL src1, iRegL src2,
8366                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8367   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
8368   ins_cost(1.9 * INSN_COST);
8369   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
8370 
8371   ins_encode %{
8372     __ bic(as_Register($dst$$reg),
8373               as_Register($src1$$reg),
8374               as_Register($src2$$reg),
8375               Assembler::ASR,
8376               $src3$$constant & 0x3f);
8377   %}
8378 
8379   ins_pipe(ialu_reg_reg_shift);
8380 %}
8381 
8382 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
8383                          iRegIorL2I src1, iRegIorL2I src2,
8384                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8385   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
8386   ins_cost(1.9 * INSN_COST);
8387   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
8388 
8389   ins_encode %{
8390     __ bicw(as_Register($dst$$reg),
8391               as_Register($src1$$reg),
8392               as_Register($src2$$reg),
8393               Assembler::LSL,
8394               $src3$$constant & 0x3f);
8395   %}
8396 
8397   ins_pipe(ialu_reg_reg_shift);
8398 %}
8399 
8400 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
8401                          iRegL src1, iRegL src2,
8402                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8403   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
8404   ins_cost(1.9 * INSN_COST);
8405   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
8406 
8407   ins_encode %{
8408     __ bic(as_Register($dst$$reg),
8409               as_Register($src1$$reg),
8410               as_Register($src2$$reg),
8411               Assembler::LSL,
8412               $src3$$constant & 0x3f);
8413   %}
8414 
8415   ins_pipe(ialu_reg_reg_shift);
8416 %}
8417 
8418 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
8419                          iRegIorL2I src1, iRegIorL2I src2,
8420                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8421   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
8422   ins_cost(1.9 * INSN_COST);
8423   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
8424 
8425   ins_encode %{
8426     __ eonw(as_Register($dst$$reg),
8427               as_Register($src1$$reg),
8428               as_Register($src2$$reg),
8429               Assembler::LSR,
8430               $src3$$constant & 0x3f);
8431   %}
8432 
8433   ins_pipe(ialu_reg_reg_shift);
8434 %}
8435 
8436 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
8437                          iRegL src1, iRegL src2,
8438                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8439   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
8440   ins_cost(1.9 * INSN_COST);
8441   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
8442 
8443   ins_encode %{
8444     __ eon(as_Register($dst$$reg),
8445               as_Register($src1$$reg),
8446               as_Register($src2$$reg),
8447               Assembler::LSR,
8448               $src3$$constant & 0x3f);
8449   %}
8450 
8451   ins_pipe(ialu_reg_reg_shift);
8452 %}
8453 
8454 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
8455                          iRegIorL2I src1, iRegIorL2I src2,
8456                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8457   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
8458   ins_cost(1.9 * INSN_COST);
8459   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
8460 
8461   ins_encode %{
8462     __ eonw(as_Register($dst$$reg),
8463               as_Register($src1$$reg),
8464               as_Register($src2$$reg),
8465               Assembler::ASR,
8466               $src3$$constant & 0x3f);
8467   %}
8468 
8469   ins_pipe(ialu_reg_reg_shift);
8470 %}
8471 
8472 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
8473                          iRegL src1, iRegL src2,
8474                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8475   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
8476   ins_cost(1.9 * INSN_COST);
8477   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
8478 
8479   ins_encode %{
8480     __ eon(as_Register($dst$$reg),
8481               as_Register($src1$$reg),
8482               as_Register($src2$$reg),
8483               Assembler::ASR,
8484               $src3$$constant & 0x3f);
8485   %}
8486 
8487   ins_pipe(ialu_reg_reg_shift);
8488 %}
8489 
8490 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
8491                          iRegIorL2I src1, iRegIorL2I src2,
8492                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8493   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
8494   ins_cost(1.9 * INSN_COST);
8495   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
8496 
8497   ins_encode %{
8498     __ eonw(as_Register($dst$$reg),
8499               as_Register($src1$$reg),
8500               as_Register($src2$$reg),
8501               Assembler::LSL,
8502               $src3$$constant & 0x3f);
8503   %}
8504 
8505   ins_pipe(ialu_reg_reg_shift);
8506 %}
8507 
8508 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
8509                          iRegL src1, iRegL src2,
8510                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8511   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
8512   ins_cost(1.9 * INSN_COST);
8513   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
8514 
8515   ins_encode %{
8516     __ eon(as_Register($dst$$reg),
8517               as_Register($src1$$reg),
8518               as_Register($src2$$reg),
8519               Assembler::LSL,
8520               $src3$$constant & 0x3f);
8521   %}
8522 
8523   ins_pipe(ialu_reg_reg_shift);
8524 %}
8525 
8526 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
8527                          iRegIorL2I src1, iRegIorL2I src2,
8528                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8529   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
8530   ins_cost(1.9 * INSN_COST);
8531   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
8532 
8533   ins_encode %{
8534     __ ornw(as_Register($dst$$reg),
8535               as_Register($src1$$reg),
8536               as_Register($src2$$reg),
8537               Assembler::LSR,
8538               $src3$$constant & 0x3f);
8539   %}
8540 
8541   ins_pipe(ialu_reg_reg_shift);
8542 %}
8543 
8544 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
8545                          iRegL src1, iRegL src2,
8546                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8547   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
8548   ins_cost(1.9 * INSN_COST);
8549   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
8550 
8551   ins_encode %{
8552     __ orn(as_Register($dst$$reg),
8553               as_Register($src1$$reg),
8554               as_Register($src2$$reg),
8555               Assembler::LSR,
8556               $src3$$constant & 0x3f);
8557   %}
8558 
8559   ins_pipe(ialu_reg_reg_shift);
8560 %}
8561 
8562 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
8563                          iRegIorL2I src1, iRegIorL2I src2,
8564                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8565   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
8566   ins_cost(1.9 * INSN_COST);
8567   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
8568 
8569   ins_encode %{
8570     __ ornw(as_Register($dst$$reg),
8571               as_Register($src1$$reg),
8572               as_Register($src2$$reg),
8573               Assembler::ASR,
8574               $src3$$constant & 0x3f);
8575   %}
8576 
8577   ins_pipe(ialu_reg_reg_shift);
8578 %}
8579 
8580 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
8581                          iRegL src1, iRegL src2,
8582                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8583   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
8584   ins_cost(1.9 * INSN_COST);
8585   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
8586 
8587   ins_encode %{
8588     __ orn(as_Register($dst$$reg),
8589               as_Register($src1$$reg),
8590               as_Register($src2$$reg),
8591               Assembler::ASR,
8592               $src3$$constant & 0x3f);
8593   %}
8594 
8595   ins_pipe(ialu_reg_reg_shift);
8596 %}
8597 
8598 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
8599                          iRegIorL2I src1, iRegIorL2I src2,
8600                          immI src3, immI_M1 src4, rFlagsReg cr) %{
8601   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
8602   ins_cost(1.9 * INSN_COST);
8603   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
8604 
8605   ins_encode %{
8606     __ ornw(as_Register($dst$$reg),
8607               as_Register($src1$$reg),
8608               as_Register($src2$$reg),
8609               Assembler::LSL,
8610               $src3$$constant & 0x3f);
8611   %}
8612 
8613   ins_pipe(ialu_reg_reg_shift);
8614 %}
8615 
8616 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
8617                          iRegL src1, iRegL src2,
8618                          immI src3, immL_M1 src4, rFlagsReg cr) %{
8619   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
8620   ins_cost(1.9 * INSN_COST);
8621   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
8622 
8623   ins_encode %{
8624     __ orn(as_Register($dst$$reg),
8625               as_Register($src1$$reg),
8626               as_Register($src2$$reg),
8627               Assembler::LSL,
8628               $src3$$constant & 0x3f);
8629   %}
8630 
8631   ins_pipe(ialu_reg_reg_shift);
8632 %}
8633 
8634 instruct AndI_reg_URShift_reg(iRegINoSp dst,
8635                          iRegIorL2I src1, iRegIorL2I src2,
8636                          immI src3, rFlagsReg cr) %{
8637   match(Set dst (AndI src1 (URShiftI src2 src3)));
8638 
8639   ins_cost(1.9 * INSN_COST);
8640   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
8641 
8642   ins_encode %{
8643     __ andw(as_Register($dst$$reg),
8644               as_Register($src1$$reg),
8645               as_Register($src2$$reg),
8646               Assembler::LSR,
8647               $src3$$constant & 0x3f);
8648   %}
8649 
8650   ins_pipe(ialu_reg_reg_shift);
8651 %}
8652 
8653 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
8654                          iRegL src1, iRegL src2,
8655                          immI src3, rFlagsReg cr) %{
8656   match(Set dst (AndL src1 (URShiftL src2 src3)));
8657 
8658   ins_cost(1.9 * INSN_COST);
8659   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
8660 
8661   ins_encode %{
8662     __ andr(as_Register($dst$$reg),
8663               as_Register($src1$$reg),
8664               as_Register($src2$$reg),
8665               Assembler::LSR,
8666               $src3$$constant & 0x3f);
8667   %}
8668 
8669   ins_pipe(ialu_reg_reg_shift);
8670 %}
8671 
8672 instruct AndI_reg_RShift_reg(iRegINoSp dst,
8673                          iRegIorL2I src1, iRegIorL2I src2,
8674                          immI src3, rFlagsReg cr) %{
8675   match(Set dst (AndI src1 (RShiftI src2 src3)));
8676 
8677   ins_cost(1.9 * INSN_COST);
8678   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
8679 
8680   ins_encode %{
8681     __ andw(as_Register($dst$$reg),
8682               as_Register($src1$$reg),
8683               as_Register($src2$$reg),
8684               Assembler::ASR,
8685               $src3$$constant & 0x3f);
8686   %}
8687 
8688   ins_pipe(ialu_reg_reg_shift);
8689 %}
8690 
8691 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
8692                          iRegL src1, iRegL src2,
8693                          immI src3, rFlagsReg cr) %{
8694   match(Set dst (AndL src1 (RShiftL src2 src3)));
8695 
8696   ins_cost(1.9 * INSN_COST);
8697   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
8698 
8699   ins_encode %{
8700     __ andr(as_Register($dst$$reg),
8701               as_Register($src1$$reg),
8702               as_Register($src2$$reg),
8703               Assembler::ASR,
8704               $src3$$constant & 0x3f);
8705   %}
8706 
8707   ins_pipe(ialu_reg_reg_shift);
8708 %}
8709 
8710 instruct AndI_reg_LShift_reg(iRegINoSp dst,
8711                          iRegIorL2I src1, iRegIorL2I src2,
8712                          immI src3, rFlagsReg cr) %{
8713   match(Set dst (AndI src1 (LShiftI src2 src3)));
8714 
8715   ins_cost(1.9 * INSN_COST);
8716   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
8717 
8718   ins_encode %{
8719     __ andw(as_Register($dst$$reg),
8720               as_Register($src1$$reg),
8721               as_Register($src2$$reg),
8722               Assembler::LSL,
8723               $src3$$constant & 0x3f);
8724   %}
8725 
8726   ins_pipe(ialu_reg_reg_shift);
8727 %}
8728 
8729 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
8730                          iRegL src1, iRegL src2,
8731                          immI src3, rFlagsReg cr) %{
8732   match(Set dst (AndL src1 (LShiftL src2 src3)));
8733 
8734   ins_cost(1.9 * INSN_COST);
8735   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
8736 
8737   ins_encode %{
8738     __ andr(as_Register($dst$$reg),
8739               as_Register($src1$$reg),
8740               as_Register($src2$$reg),
8741               Assembler::LSL,
8742               $src3$$constant & 0x3f);
8743   %}
8744 
8745   ins_pipe(ialu_reg_reg_shift);
8746 %}
8747 
8748 instruct XorI_reg_URShift_reg(iRegINoSp dst,
8749                          iRegIorL2I src1, iRegIorL2I src2,
8750                          immI src3, rFlagsReg cr) %{
8751   match(Set dst (XorI src1 (URShiftI src2 src3)));
8752 
8753   ins_cost(1.9 * INSN_COST);
8754   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
8755 
8756   ins_encode %{
8757     __ eorw(as_Register($dst$$reg),
8758               as_Register($src1$$reg),
8759               as_Register($src2$$reg),
8760               Assembler::LSR,
8761               $src3$$constant & 0x3f);
8762   %}
8763 
8764   ins_pipe(ialu_reg_reg_shift);
8765 %}
8766 
8767 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
8768                          iRegL src1, iRegL src2,
8769                          immI src3, rFlagsReg cr) %{
8770   match(Set dst (XorL src1 (URShiftL src2 src3)));
8771 
8772   ins_cost(1.9 * INSN_COST);
8773   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
8774 
8775   ins_encode %{
8776     __ eor(as_Register($dst$$reg),
8777               as_Register($src1$$reg),
8778               as_Register($src2$$reg),
8779               Assembler::LSR,
8780               $src3$$constant & 0x3f);
8781   %}
8782 
8783   ins_pipe(ialu_reg_reg_shift);
8784 %}
8785 
8786 instruct XorI_reg_RShift_reg(iRegINoSp dst,
8787                          iRegIorL2I src1, iRegIorL2I src2,
8788                          immI src3, rFlagsReg cr) %{
8789   match(Set dst (XorI src1 (RShiftI src2 src3)));
8790 
8791   ins_cost(1.9 * INSN_COST);
8792   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
8793 
8794   ins_encode %{
8795     __ eorw(as_Register($dst$$reg),
8796               as_Register($src1$$reg),
8797               as_Register($src2$$reg),
8798               Assembler::ASR,
8799               $src3$$constant & 0x3f);
8800   %}
8801 
8802   ins_pipe(ialu_reg_reg_shift);
8803 %}
8804 
8805 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
8806                          iRegL src1, iRegL src2,
8807                          immI src3, rFlagsReg cr) %{
8808   match(Set dst (XorL src1 (RShiftL src2 src3)));
8809 
8810   ins_cost(1.9 * INSN_COST);
8811   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
8812 
8813   ins_encode %{
8814     __ eor(as_Register($dst$$reg),
8815               as_Register($src1$$reg),
8816               as_Register($src2$$reg),
8817               Assembler::ASR,
8818               $src3$$constant & 0x3f);
8819   %}
8820 
8821   ins_pipe(ialu_reg_reg_shift);
8822 %}
8823 
8824 instruct XorI_reg_LShift_reg(iRegINoSp dst,
8825                          iRegIorL2I src1, iRegIorL2I src2,
8826                          immI src3, rFlagsReg cr) %{
8827   match(Set dst (XorI src1 (LShiftI src2 src3)));
8828 
8829   ins_cost(1.9 * INSN_COST);
8830   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
8831 
8832   ins_encode %{
8833     __ eorw(as_Register($dst$$reg),
8834               as_Register($src1$$reg),
8835               as_Register($src2$$reg),
8836               Assembler::LSL,
8837               $src3$$constant & 0x3f);
8838   %}
8839 
8840   ins_pipe(ialu_reg_reg_shift);
8841 %}
8842 
8843 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
8844                          iRegL src1, iRegL src2,
8845                          immI src3, rFlagsReg cr) %{
8846   match(Set dst (XorL src1 (LShiftL src2 src3)));
8847 
8848   ins_cost(1.9 * INSN_COST);
8849   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
8850 
8851   ins_encode %{
8852     __ eor(as_Register($dst$$reg),
8853               as_Register($src1$$reg),
8854               as_Register($src2$$reg),
8855               Assembler::LSL,
8856               $src3$$constant & 0x3f);
8857   %}
8858 
8859   ins_pipe(ialu_reg_reg_shift);
8860 %}
8861 
8862 instruct OrI_reg_URShift_reg(iRegINoSp dst,
8863                          iRegIorL2I src1, iRegIorL2I src2,
8864                          immI src3, rFlagsReg cr) %{
8865   match(Set dst (OrI src1 (URShiftI src2 src3)));
8866 
8867   ins_cost(1.9 * INSN_COST);
8868   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
8869 
8870   ins_encode %{
8871     __ orrw(as_Register($dst$$reg),
8872               as_Register($src1$$reg),
8873               as_Register($src2$$reg),
8874               Assembler::LSR,
8875               $src3$$constant & 0x3f);
8876   %}
8877 
8878   ins_pipe(ialu_reg_reg_shift);
8879 %}
8880 
8881 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
8882                          iRegL src1, iRegL src2,
8883                          immI src3, rFlagsReg cr) %{
8884   match(Set dst (OrL src1 (URShiftL src2 src3)));
8885 
8886   ins_cost(1.9 * INSN_COST);
8887   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
8888 
8889   ins_encode %{
8890     __ orr(as_Register($dst$$reg),
8891               as_Register($src1$$reg),
8892               as_Register($src2$$reg),
8893               Assembler::LSR,
8894               $src3$$constant & 0x3f);
8895   %}
8896 
8897   ins_pipe(ialu_reg_reg_shift);
8898 %}
8899 
8900 instruct OrI_reg_RShift_reg(iRegINoSp dst,
8901                          iRegIorL2I src1, iRegIorL2I src2,
8902                          immI src3, rFlagsReg cr) %{
8903   match(Set dst (OrI src1 (RShiftI src2 src3)));
8904 
8905   ins_cost(1.9 * INSN_COST);
8906   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
8907 
8908   ins_encode %{
8909     __ orrw(as_Register($dst$$reg),
8910               as_Register($src1$$reg),
8911               as_Register($src2$$reg),
8912               Assembler::ASR,
8913               $src3$$constant & 0x3f);
8914   %}
8915 
8916   ins_pipe(ialu_reg_reg_shift);
8917 %}
8918 
8919 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
8920                          iRegL src1, iRegL src2,
8921                          immI src3, rFlagsReg cr) %{
8922   match(Set dst (OrL src1 (RShiftL src2 src3)));
8923 
8924   ins_cost(1.9 * INSN_COST);
8925   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
8926 
8927   ins_encode %{
8928     __ orr(as_Register($dst$$reg),
8929               as_Register($src1$$reg),
8930               as_Register($src2$$reg),
8931               Assembler::ASR,
8932               $src3$$constant & 0x3f);
8933   %}
8934 
8935   ins_pipe(ialu_reg_reg_shift);
8936 %}
8937 
8938 instruct OrI_reg_LShift_reg(iRegINoSp dst,
8939                          iRegIorL2I src1, iRegIorL2I src2,
8940                          immI src3, rFlagsReg cr) %{
8941   match(Set dst (OrI src1 (LShiftI src2 src3)));
8942 
8943   ins_cost(1.9 * INSN_COST);
8944   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
8945 
8946   ins_encode %{
8947     __ orrw(as_Register($dst$$reg),
8948               as_Register($src1$$reg),
8949               as_Register($src2$$reg),
8950               Assembler::LSL,
8951               $src3$$constant & 0x3f);
8952   %}
8953 
8954   ins_pipe(ialu_reg_reg_shift);
8955 %}
8956 
8957 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
8958                          iRegL src1, iRegL src2,
8959                          immI src3, rFlagsReg cr) %{
8960   match(Set dst (OrL src1 (LShiftL src2 src3)));
8961 
8962   ins_cost(1.9 * INSN_COST);
8963   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
8964 
8965   ins_encode %{
8966     __ orr(as_Register($dst$$reg),
8967               as_Register($src1$$reg),
8968               as_Register($src2$$reg),
8969               Assembler::LSL,
8970               $src3$$constant & 0x3f);
8971   %}
8972 
8973   ins_pipe(ialu_reg_reg_shift);
8974 %}
8975 
8976 instruct AddI_reg_URShift_reg(iRegINoSp dst,
8977                          iRegIorL2I src1, iRegIorL2I src2,
8978                          immI src3, rFlagsReg cr) %{
8979   match(Set dst (AddI src1 (URShiftI src2 src3)));
8980 
8981   ins_cost(1.9 * INSN_COST);
8982   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
8983 
8984   ins_encode %{
8985     __ addw(as_Register($dst$$reg),
8986               as_Register($src1$$reg),
8987               as_Register($src2$$reg),
8988               Assembler::LSR,
8989               $src3$$constant & 0x3f);
8990   %}
8991 
8992   ins_pipe(ialu_reg_reg_shift);
8993 %}
8994 
8995 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
8996                          iRegL src1, iRegL src2,
8997                          immI src3, rFlagsReg cr) %{
8998   match(Set dst (AddL src1 (URShiftL src2 src3)));
8999 
9000   ins_cost(1.9 * INSN_COST);
9001   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
9002 
9003   ins_encode %{
9004     __ add(as_Register($dst$$reg),
9005               as_Register($src1$$reg),
9006               as_Register($src2$$reg),
9007               Assembler::LSR,
9008               $src3$$constant & 0x3f);
9009   %}
9010 
9011   ins_pipe(ialu_reg_reg_shift);
9012 %}
9013 
9014 instruct AddI_reg_RShift_reg(iRegINoSp dst,
9015                          iRegIorL2I src1, iRegIorL2I src2,
9016                          immI src3, rFlagsReg cr) %{
9017   match(Set dst (AddI src1 (RShiftI src2 src3)));
9018 
9019   ins_cost(1.9 * INSN_COST);
9020   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
9021 
9022   ins_encode %{
9023     __ addw(as_Register($dst$$reg),
9024               as_Register($src1$$reg),
9025               as_Register($src2$$reg),
9026               Assembler::ASR,
9027               $src3$$constant & 0x3f);
9028   %}
9029 
9030   ins_pipe(ialu_reg_reg_shift);
9031 %}
9032 
9033 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
9034                          iRegL src1, iRegL src2,
9035                          immI src3, rFlagsReg cr) %{
9036   match(Set dst (AddL src1 (RShiftL src2 src3)));
9037 
9038   ins_cost(1.9 * INSN_COST);
9039   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
9040 
9041   ins_encode %{
9042     __ add(as_Register($dst$$reg),
9043               as_Register($src1$$reg),
9044               as_Register($src2$$reg),
9045               Assembler::ASR,
9046               $src3$$constant & 0x3f);
9047   %}
9048 
9049   ins_pipe(ialu_reg_reg_shift);
9050 %}
9051 
9052 instruct AddI_reg_LShift_reg(iRegINoSp dst,
9053                          iRegIorL2I src1, iRegIorL2I src2,
9054                          immI src3, rFlagsReg cr) %{
9055   match(Set dst (AddI src1 (LShiftI src2 src3)));
9056 
9057   ins_cost(1.9 * INSN_COST);
9058   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
9059 
9060   ins_encode %{
9061     __ addw(as_Register($dst$$reg),
9062               as_Register($src1$$reg),
9063               as_Register($src2$$reg),
9064               Assembler::LSL,
9065               $src3$$constant & 0x3f);
9066   %}
9067 
9068   ins_pipe(ialu_reg_reg_shift);
9069 %}
9070 
9071 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
9072                          iRegL src1, iRegL src2,
9073                          immI src3, rFlagsReg cr) %{
9074   match(Set dst (AddL src1 (LShiftL src2 src3)));
9075 
9076   ins_cost(1.9 * INSN_COST);
9077   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
9078 
9079   ins_encode %{
9080     __ add(as_Register($dst$$reg),
9081               as_Register($src1$$reg),
9082               as_Register($src2$$reg),
9083               Assembler::LSL,
9084               $src3$$constant & 0x3f);
9085   %}
9086 
9087   ins_pipe(ialu_reg_reg_shift);
9088 %}
9089 
9090 instruct SubI_reg_URShift_reg(iRegINoSp dst,
9091                          iRegIorL2I src1, iRegIorL2I src2,
9092                          immI src3, rFlagsReg cr) %{
9093   match(Set dst (SubI src1 (URShiftI src2 src3)));
9094 
9095   ins_cost(1.9 * INSN_COST);
9096   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
9097 
9098   ins_encode %{
9099     __ subw(as_Register($dst$$reg),
9100               as_Register($src1$$reg),
9101               as_Register($src2$$reg),
9102               Assembler::LSR,
9103               $src3$$constant & 0x3f);
9104   %}
9105 
9106   ins_pipe(ialu_reg_reg_shift);
9107 %}
9108 
9109 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
9110                          iRegL src1, iRegL src2,
9111                          immI src3, rFlagsReg cr) %{
9112   match(Set dst (SubL src1 (URShiftL src2 src3)));
9113 
9114   ins_cost(1.9 * INSN_COST);
9115   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
9116 
9117   ins_encode %{
9118     __ sub(as_Register($dst$$reg),
9119               as_Register($src1$$reg),
9120               as_Register($src2$$reg),
9121               Assembler::LSR,
9122               $src3$$constant & 0x3f);
9123   %}
9124 
9125   ins_pipe(ialu_reg_reg_shift);
9126 %}
9127 
9128 instruct SubI_reg_RShift_reg(iRegINoSp dst,
9129                          iRegIorL2I src1, iRegIorL2I src2,
9130                          immI src3, rFlagsReg cr) %{
9131   match(Set dst (SubI src1 (RShiftI src2 src3)));
9132 
9133   ins_cost(1.9 * INSN_COST);
9134   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
9135 
9136   ins_encode %{
9137     __ subw(as_Register($dst$$reg),
9138               as_Register($src1$$reg),
9139               as_Register($src2$$reg),
9140               Assembler::ASR,
9141               $src3$$constant & 0x3f);
9142   %}
9143 
9144   ins_pipe(ialu_reg_reg_shift);
9145 %}
9146 
9147 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
9148                          iRegL src1, iRegL src2,
9149                          immI src3, rFlagsReg cr) %{
9150   match(Set dst (SubL src1 (RShiftL src2 src3)));
9151 
9152   ins_cost(1.9 * INSN_COST);
9153   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
9154 
9155   ins_encode %{
9156     __ sub(as_Register($dst$$reg),
9157               as_Register($src1$$reg),
9158               as_Register($src2$$reg),
9159               Assembler::ASR,
9160               $src3$$constant & 0x3f);
9161   %}
9162 
9163   ins_pipe(ialu_reg_reg_shift);
9164 %}
9165 
9166 instruct SubI_reg_LShift_reg(iRegINoSp dst,
9167                          iRegIorL2I src1, iRegIorL2I src2,
9168                          immI src3, rFlagsReg cr) %{
9169   match(Set dst (SubI src1 (LShiftI src2 src3)));
9170 
9171   ins_cost(1.9 * INSN_COST);
9172   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
9173 
9174   ins_encode %{
9175     __ subw(as_Register($dst$$reg),
9176               as_Register($src1$$reg),
9177               as_Register($src2$$reg),
9178               Assembler::LSL,
9179               $src3$$constant & 0x3f);
9180   %}
9181 
9182   ins_pipe(ialu_reg_reg_shift);
9183 %}
9184 
9185 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
9186                          iRegL src1, iRegL src2,
9187                          immI src3, rFlagsReg cr) %{
9188   match(Set dst (SubL src1 (LShiftL src2 src3)));
9189 
9190   ins_cost(1.9 * INSN_COST);
9191   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
9192 
9193   ins_encode %{
9194     __ sub(as_Register($dst$$reg),
9195               as_Register($src1$$reg),
9196               as_Register($src2$$reg),
9197               Assembler::LSL,
9198               $src3$$constant & 0x3f);
9199   %}
9200 
9201   ins_pipe(ialu_reg_reg_shift);
9202 %}
9203 
9204 
9205 
9206 // Shift Left followed by Shift Right.
9207 // This idiom is used by the compiler for the i2b bytecode etc.
9208 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
9209 %{
9210   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
9211   // Make sure we are not going to exceed what sbfm can do.
9212   predicate((unsigned int)n->in(2)->get_int() <= 63
9213             && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
9214 
9215   ins_cost(INSN_COST * 2);
9216   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
9217   ins_encode %{
9218     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
9219     int s = 63 - lshift;
9220     int r = (rshift - lshift) & 63;
9221     __ sbfm(as_Register($dst$$reg),
9222             as_Register($src$$reg),
9223             r, s);
9224   %}
9225 
9226   ins_pipe(ialu_reg_shift);
9227 %}
9228 
9229 // Shift Left followed by Shift Right.
9230 // This idiom is used by the compiler for the i2b bytecode etc.
9231 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
9232 %{
9233   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
9234   // Make sure we are not going to exceed what sbfmw can do.
9235   predicate((unsigned int)n->in(2)->get_int() <= 31
9236             && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
9237 
9238   ins_cost(INSN_COST * 2);
9239   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
9240   ins_encode %{
9241     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
9242     int s = 31 - lshift;
9243     int r = (rshift - lshift) & 31;
9244     __ sbfmw(as_Register($dst$$reg),
9245             as_Register($src$$reg),
9246             r, s);
9247   %}
9248 
9249   ins_pipe(ialu_reg_shift);
9250 %}
9251 
9252 // Shift Left followed by Shift Right.
9253 // This idiom is used by the compiler for the i2b bytecode etc.
9254 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
9255 %{
9256   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
9257   // Make sure we are not going to exceed what ubfm can do.
9258   predicate((unsigned int)n->in(2)->get_int() <= 63
9259             && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
9260 
9261   ins_cost(INSN_COST * 2);
9262   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
9263   ins_encode %{
9264     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
9265     int s = 63 - lshift;
9266     int r = (rshift - lshift) & 63;
9267     __ ubfm(as_Register($dst$$reg),
9268             as_Register($src$$reg),
9269             r, s);
9270   %}
9271 
9272   ins_pipe(ialu_reg_shift);
9273 %}
9274 
9275 // Shift Left followed by Shift Right.
9276 // This idiom is used by the compiler for the i2b bytecode etc.
9277 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
9278 %{
9279   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
9280   // Make sure we are not going to exceed what ubfmw can do.
9281   predicate((unsigned int)n->in(2)->get_int() <= 31
9282             && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
9283 
9284   ins_cost(INSN_COST * 2);
9285   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
9286   ins_encode %{
9287     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
9288     int s = 31 - lshift;
9289     int r = (rshift - lshift) & 31;
9290     __ ubfmw(as_Register($dst$$reg),
9291             as_Register($src$$reg),
9292             r, s);
9293   %}
9294 
9295   ins_pipe(ialu_reg_shift);
9296 %}
9297 // Bitfield extract with shift & mask
9298 
9299 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
9300 %{
9301   match(Set dst (AndI (URShiftI src rshift) mask));
9302 
9303   ins_cost(INSN_COST);
9304   format %{ "ubfxw $dst, $src, $mask" %}
9305   ins_encode %{
9306     int rshift = $rshift$$constant;
9307     long mask = $mask$$constant;
9308     int width = exact_log2(mask+1);
9309     __ ubfxw(as_Register($dst$$reg),
9310             as_Register($src$$reg), rshift, width);
9311   %}
9312   ins_pipe(ialu_reg_shift);
9313 %}
9314 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
9315 %{
9316   match(Set dst (AndL (URShiftL src rshift) mask));
9317 
9318   ins_cost(INSN_COST);
9319   format %{ "ubfx $dst, $src, $mask" %}
9320   ins_encode %{
9321     int rshift = $rshift$$constant;
9322     long mask = $mask$$constant;
9323     int width = exact_log2(mask+1);
9324     __ ubfx(as_Register($dst$$reg),
9325             as_Register($src$$reg), rshift, width);
9326   %}
9327   ins_pipe(ialu_reg_shift);
9328 %}
9329 
9330 // We can use ubfx when extending an And with a mask when we know mask
9331 // is positive.  We know that because immI_bitmask guarantees it.
9332 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
9333 %{
9334   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
9335 
9336   ins_cost(INSN_COST * 2);
9337   format %{ "ubfx $dst, $src, $mask" %}
9338   ins_encode %{
9339     int rshift = $rshift$$constant;
9340     long mask = $mask$$constant;
9341     int width = exact_log2(mask+1);
9342     __ ubfx(as_Register($dst$$reg),
9343             as_Register($src$$reg), rshift, width);
9344   %}
9345   ins_pipe(ialu_reg_shift);
9346 %}
9347 
9348 // Rotations
9349 
9350 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
9351 %{
9352   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
9353   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
9354 
9355   ins_cost(INSN_COST);
9356   format %{ "extr $dst, $src1, $src2, #$rshift" %}
9357 
9358   ins_encode %{
9359     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
9360             $rshift$$constant & 63);
9361   %}
9362   ins_pipe(ialu_reg_reg_extr);
9363 %}
9364 
9365 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
9366 %{
9367   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
9368   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
9369 
9370   ins_cost(INSN_COST);
9371   format %{ "extr $dst, $src1, $src2, #$rshift" %}
9372 
9373   ins_encode %{
9374     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
9375             $rshift$$constant & 31);
9376   %}
9377   ins_pipe(ialu_reg_reg_extr);
9378 %}
9379 
9380 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
9381 %{
9382   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
9383   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
9384 
9385   ins_cost(INSN_COST);
9386   format %{ "extr $dst, $src1, $src2, #$rshift" %}
9387 
9388   ins_encode %{
9389     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
9390             $rshift$$constant & 63);
9391   %}
9392   ins_pipe(ialu_reg_reg_extr);
9393 %}
9394 
9395 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
9396 %{
9397   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
9398   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
9399 
9400   ins_cost(INSN_COST);
9401   format %{ "extr $dst, $src1, $src2, #$rshift" %}
9402 
9403   ins_encode %{
9404     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
9405             $rshift$$constant & 31);
9406   %}
9407   ins_pipe(ialu_reg_reg_extr);
9408 %}
9409 
9410 
9411 // rol expander
9412 
9413 instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
9414 %{
9415   effect(DEF dst, USE src, USE shift);
9416 
9417   format %{ "rol    $dst, $src, $shift" %}
9418   ins_cost(INSN_COST * 3);
9419   ins_encode %{
9420     __ subw(rscratch1, zr, as_Register($shift$$reg));
9421     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
9422             rscratch1);
9423     %}
9424   ins_pipe(ialu_reg_reg_vshift);
9425 %}
9426 
9427 // rol expander
9428 
9429 instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
9430 %{
9431   effect(DEF dst, USE src, USE shift);
9432 
9433   format %{ "rol    $dst, $src, $shift" %}
9434   ins_cost(INSN_COST * 3);
9435   ins_encode %{
9436     __ subw(rscratch1, zr, as_Register($shift$$reg));
9437     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
9438             rscratch1);
9439     %}
9440   ins_pipe(ialu_reg_reg_vshift);
9441 %}
9442 
9443 instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
9444 %{
9445   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
9446 
9447   expand %{
9448     rolL_rReg(dst, src, shift, cr);
9449   %}
9450 %}
9451 
9452 instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
9453 %{
9454   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
9455 
9456   expand %{
9457     rolL_rReg(dst, src, shift, cr);
9458   %}
9459 %}
9460 
9461 instruct rolI_rReg_Var_C_32(iRegLNoSp dst, iRegL src, iRegI shift, immI_32 c_32, rFlagsReg cr)
9462 %{
9463   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
9464 
9465   expand %{
9466     rolL_rReg(dst, src, shift, cr);
9467   %}
9468 %}
9469 
9470 instruct rolI_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
9471 %{
9472   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
9473 
9474   expand %{
9475     rolL_rReg(dst, src, shift, cr);
9476   %}
9477 %}
9478 
9479 // ror expander
9480 
9481 instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
9482 %{
9483   effect(DEF dst, USE src, USE shift);
9484 
9485   format %{ "ror    $dst, $src, $shift" %}
9486   ins_cost(INSN_COST);
9487   ins_encode %{
9488     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
9489             as_Register($shift$$reg));
9490     %}
9491   ins_pipe(ialu_reg_reg_vshift);
9492 %}
9493 
9494 // ror expander
9495 
9496 instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
9497 %{
9498   effect(DEF dst, USE src, USE shift);
9499 
9500   format %{ "ror    $dst, $src, $shift" %}
9501   ins_cost(INSN_COST);
9502   ins_encode %{
9503     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
9504             as_Register($shift$$reg));
9505     %}
9506   ins_pipe(ialu_reg_reg_vshift);
9507 %}
9508 
9509 instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
9510 %{
9511   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
9512 
9513   expand %{
9514     rorL_rReg(dst, src, shift, cr);
9515   %}
9516 %}
9517 
9518 instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
9519 %{
9520   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
9521 
9522   expand %{
9523     rorL_rReg(dst, src, shift, cr);
9524   %}
9525 %}
9526 
9527 instruct rorI_rReg_Var_C_32(iRegLNoSp dst, iRegL src, iRegI shift, immI_32 c_32, rFlagsReg cr)
9528 %{
9529   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
9530 
9531   expand %{
9532     rorL_rReg(dst, src, shift, cr);
9533   %}
9534 %}
9535 
9536 instruct rorI_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
9537 %{
9538   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
9539 
9540   expand %{
9541     rorL_rReg(dst, src, shift, cr);
9542   %}
9543 %}
9544 
9545 // Add/subtract (extended)
9546 
9547 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
9548 %{
9549   match(Set dst (AddL src1 (ConvI2L src2)));
9550   ins_cost(INSN_COST);
9551   format %{ "add  $dst, $src1, sxtw $src2" %}
9552 
9553    ins_encode %{
9554      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9555             as_Register($src2$$reg), ext::sxtw);
9556    %}
9557   ins_pipe(ialu_reg_reg);
9558 %};
9559 
9560 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
9561 %{
9562   match(Set dst (SubL src1 (ConvI2L src2)));
9563   ins_cost(INSN_COST);
9564   format %{ "sub  $dst, $src1, sxtw $src2" %}
9565 
9566    ins_encode %{
9567      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
9568             as_Register($src2$$reg), ext::sxtw);
9569    %}
9570   ins_pipe(ialu_reg_reg);
9571 %};
9572 
9573 
9574 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
9575 %{
9576   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
9577   ins_cost(INSN_COST);
9578   format %{ "add  $dst, $src1, sxth $src2" %}
9579 
9580    ins_encode %{
9581      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9582             as_Register($src2$$reg), ext::sxth);
9583    %}
9584   ins_pipe(ialu_reg_reg);
9585 %}
9586 
9587 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
9588 %{
9589   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
9590   ins_cost(INSN_COST);
9591   format %{ "add  $dst, $src1, sxtb $src2" %}
9592 
9593    ins_encode %{
9594      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9595             as_Register($src2$$reg), ext::sxtb);
9596    %}
9597   ins_pipe(ialu_reg_reg);
9598 %}
9599 
9600 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
9601 %{
9602   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
9603   ins_cost(INSN_COST);
9604   format %{ "add  $dst, $src1, uxtb $src2" %}
9605 
9606    ins_encode %{
9607      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9608             as_Register($src2$$reg), ext::uxtb);
9609    %}
9610   ins_pipe(ialu_reg_reg);
9611 %}
9612 
9613 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
9614 %{
9615   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
9616   ins_cost(INSN_COST);
9617   format %{ "add  $dst, $src1, sxth $src2" %}
9618 
9619    ins_encode %{
9620      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9621             as_Register($src2$$reg), ext::sxth);
9622    %}
9623   ins_pipe(ialu_reg_reg);
9624 %}
9625 
9626 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
9627 %{
9628   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
9629   ins_cost(INSN_COST);
9630   format %{ "add  $dst, $src1, sxtw $src2" %}
9631 
9632    ins_encode %{
9633      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9634             as_Register($src2$$reg), ext::sxtw);
9635    %}
9636   ins_pipe(ialu_reg_reg);
9637 %}
9638 
9639 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
9640 %{
9641   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
9642   ins_cost(INSN_COST);
9643   format %{ "add  $dst, $src1, sxtb $src2" %}
9644 
9645    ins_encode %{
9646      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9647             as_Register($src2$$reg), ext::sxtb);
9648    %}
9649   ins_pipe(ialu_reg_reg);
9650 %}
9651 
9652 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
9653 %{
9654   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
9655   ins_cost(INSN_COST);
9656   format %{ "add  $dst, $src1, uxtb $src2" %}
9657 
9658    ins_encode %{
9659      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9660             as_Register($src2$$reg), ext::uxtb);
9661    %}
9662   ins_pipe(ialu_reg_reg);
9663 %}
9664 
9665 
9666 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
9667 %{
9668   match(Set dst (AddI src1 (AndI src2 mask)));
9669   ins_cost(INSN_COST);
9670   format %{ "addw  $dst, $src1, $src2, uxtb" %}
9671 
9672    ins_encode %{
9673      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
9674             as_Register($src2$$reg), ext::uxtb);
9675    %}
9676   ins_pipe(ialu_reg_reg);
9677 %}
9678 
9679 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
9680 %{
9681   match(Set dst (AddI src1 (AndI src2 mask)));
9682   ins_cost(INSN_COST);
9683   format %{ "addw  $dst, $src1, $src2, uxth" %}
9684 
9685    ins_encode %{
9686      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
9687             as_Register($src2$$reg), ext::uxth);
9688    %}
9689   ins_pipe(ialu_reg_reg);
9690 %}
9691 
9692 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
9693 %{
9694   match(Set dst (AddL src1 (AndL src2 mask)));
9695   ins_cost(INSN_COST);
9696   format %{ "add  $dst, $src1, $src2, uxtb" %}
9697 
9698    ins_encode %{
9699      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9700             as_Register($src2$$reg), ext::uxtb);
9701    %}
9702   ins_pipe(ialu_reg_reg);
9703 %}
9704 
9705 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
9706 %{
9707   match(Set dst (AddL src1 (AndL src2 mask)));
9708   ins_cost(INSN_COST);
9709   format %{ "add  $dst, $src1, $src2, uxth" %}
9710 
9711    ins_encode %{
9712      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9713             as_Register($src2$$reg), ext::uxth);
9714    %}
9715   ins_pipe(ialu_reg_reg);
9716 %}
9717 
9718 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
9719 %{
9720   match(Set dst (AddL src1 (AndL src2 mask)));
9721   ins_cost(INSN_COST);
9722   format %{ "add  $dst, $src1, $src2, uxtw" %}
9723 
9724    ins_encode %{
9725      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
9726             as_Register($src2$$reg), ext::uxtw);
9727    %}
9728   ins_pipe(ialu_reg_reg);
9729 %}
9730 
9731 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
9732 %{
9733   match(Set dst (SubI src1 (AndI src2 mask)));
9734   ins_cost(INSN_COST);
9735   format %{ "subw  $dst, $src1, $src2, uxtb" %}
9736 
9737    ins_encode %{
9738      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
9739             as_Register($src2$$reg), ext::uxtb);
9740    %}
9741   ins_pipe(ialu_reg_reg);
9742 %}
9743 
9744 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
9745 %{
9746   match(Set dst (SubI src1 (AndI src2 mask)));
9747   ins_cost(INSN_COST);
9748   format %{ "subw  $dst, $src1, $src2, uxth" %}
9749 
9750    ins_encode %{
9751      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
9752             as_Register($src2$$reg), ext::uxth);
9753    %}
9754   ins_pipe(ialu_reg_reg);
9755 %}
9756 
9757 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
9758 %{
9759   match(Set dst (SubL src1 (AndL src2 mask)));
9760   ins_cost(INSN_COST);
9761   format %{ "sub  $dst, $src1, $src2, uxtb" %}
9762 
9763    ins_encode %{
9764      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
9765             as_Register($src2$$reg), ext::uxtb);
9766    %}
9767   ins_pipe(ialu_reg_reg);
9768 %}
9769 
9770 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
9771 %{
9772   match(Set dst (SubL src1 (AndL src2 mask)));
9773   ins_cost(INSN_COST);
9774   format %{ "sub  $dst, $src1, $src2, uxth" %}
9775 
9776    ins_encode %{
9777      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
9778             as_Register($src2$$reg), ext::uxth);
9779    %}
9780   ins_pipe(ialu_reg_reg);
9781 %}
9782 
9783 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
9784 %{
9785   match(Set dst (SubL src1 (AndL src2 mask)));
9786   ins_cost(INSN_COST);
9787   format %{ "sub  $dst, $src1, $src2, uxtw" %}
9788 
9789    ins_encode %{
9790      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
9791             as_Register($src2$$reg), ext::uxtw);
9792    %}
9793   ins_pipe(ialu_reg_reg);
9794 %}
9795 
9796 // END This section of the file is automatically generated. Do not edit --------------
9797 
9798 // ============================================================================
9799 // Floating Point Arithmetic Instructions
9800 
9801 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
9802   match(Set dst (AddF src1 src2));
9803 
9804   ins_cost(INSN_COST * 5);
9805   format %{ "fadds   $dst, $src1, $src2" %}
9806 
9807   ins_encode %{
9808     __ fadds(as_FloatRegister($dst$$reg),
9809              as_FloatRegister($src1$$reg),
9810              as_FloatRegister($src2$$reg));
9811   %}
9812 
9813   ins_pipe(pipe_class_default);
9814 %}
9815 
9816 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
9817   match(Set dst (AddD src1 src2));
9818 
9819   ins_cost(INSN_COST * 5);
9820   format %{ "faddd   $dst, $src1, $src2" %}
9821 
9822   ins_encode %{
9823     __ faddd(as_FloatRegister($dst$$reg),
9824              as_FloatRegister($src1$$reg),
9825              as_FloatRegister($src2$$reg));
9826   %}
9827 
9828   ins_pipe(pipe_class_default);
9829 %}
9830 
9831 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
9832   match(Set dst (SubF src1 src2));
9833 
9834   ins_cost(INSN_COST * 5);
9835   format %{ "fsubs   $dst, $src1, $src2" %}
9836 
9837   ins_encode %{
9838     __ fsubs(as_FloatRegister($dst$$reg),
9839              as_FloatRegister($src1$$reg),
9840              as_FloatRegister($src2$$reg));
9841   %}
9842 
9843   ins_pipe(pipe_class_default);
9844 %}
9845 
9846 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
9847   match(Set dst (SubD src1 src2));
9848 
9849   ins_cost(INSN_COST * 5);
9850   format %{ "fsubd   $dst, $src1, $src2" %}
9851 
9852   ins_encode %{
9853     __ fsubd(as_FloatRegister($dst$$reg),
9854              as_FloatRegister($src1$$reg),
9855              as_FloatRegister($src2$$reg));
9856   %}
9857 
9858   ins_pipe(pipe_class_default);
9859 %}
9860 
9861 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
9862   match(Set dst (MulF src1 src2));
9863 
9864   ins_cost(INSN_COST * 6);
9865   format %{ "fmuls   $dst, $src1, $src2" %}
9866 
9867   ins_encode %{
9868     __ fmuls(as_FloatRegister($dst$$reg),
9869              as_FloatRegister($src1$$reg),
9870              as_FloatRegister($src2$$reg));
9871   %}
9872 
9873   ins_pipe(pipe_class_default);
9874 %}
9875 
9876 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
9877   match(Set dst (MulD src1 src2));
9878 
9879   ins_cost(INSN_COST * 6);
9880   format %{ "fmuld   $dst, $src1, $src2" %}
9881 
9882   ins_encode %{
9883     __ fmuld(as_FloatRegister($dst$$reg),
9884              as_FloatRegister($src1$$reg),
9885              as_FloatRegister($src2$$reg));
9886   %}
9887 
9888   ins_pipe(pipe_class_default);
9889 %}
9890 
9891 // We cannot use these fused mul w add/sub ops because they don't
9892 // produce the same result as the equivalent separated ops
9893 // (essentially they don't round the intermediate result). that's a
9894 // shame. leaving them here in case we can idenitfy cases where it is
9895 // legitimate to use them
9896 
9897 
9898 // instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
9899 //   match(Set dst (AddF (MulF src1 src2) src3));
9900 
9901 //   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
9902 
9903 //   ins_encode %{
9904 //     __ fmadds(as_FloatRegister($dst$$reg),
9905 //              as_FloatRegister($src1$$reg),
9906 //              as_FloatRegister($src2$$reg),
9907 //              as_FloatRegister($src3$$reg));
9908 //   %}
9909 
9910 //   ins_pipe(pipe_class_default);
9911 // %}
9912 
9913 // instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
9914 //   match(Set dst (AddD (MulD src1 src2) src3));
9915 
9916 //   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
9917 
9918 //   ins_encode %{
9919 //     __ fmaddd(as_FloatRegister($dst$$reg),
9920 //              as_FloatRegister($src1$$reg),
9921 //              as_FloatRegister($src2$$reg),
9922 //              as_FloatRegister($src3$$reg));
9923 //   %}
9924 
9925 //   ins_pipe(pipe_class_default);
9926 // %}
9927 
9928 // instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
9929 //   match(Set dst (AddF (MulF (NegF src1) src2) src3));
9930 //   match(Set dst (AddF (NegF (MulF src1 src2)) src3));
9931 
9932 //   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
9933 
9934 //   ins_encode %{
9935 //     __ fmsubs(as_FloatRegister($dst$$reg),
9936 //               as_FloatRegister($src1$$reg),
9937 //               as_FloatRegister($src2$$reg),
9938 //              as_FloatRegister($src3$$reg));
9939 //   %}
9940 
9941 //   ins_pipe(pipe_class_default);
9942 // %}
9943 
9944 // instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
9945 //   match(Set dst (AddD (MulD (NegD src1) src2) src3));
9946 //   match(Set dst (AddD (NegD (MulD src1 src2)) src3));
9947 
9948 //   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
9949 
9950 //   ins_encode %{
9951 //     __ fmsubd(as_FloatRegister($dst$$reg),
9952 //               as_FloatRegister($src1$$reg),
9953 //               as_FloatRegister($src2$$reg),
9954 //               as_FloatRegister($src3$$reg));
9955 //   %}
9956 
9957 //   ins_pipe(pipe_class_default);
9958 // %}
9959 
9960 // instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
9961 //   match(Set dst (SubF (MulF (NegF src1) src2) src3));
9962 //   match(Set dst (SubF (NegF (MulF src1 src2)) src3));
9963 
9964 //   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
9965 
9966 //   ins_encode %{
9967 //     __ fnmadds(as_FloatRegister($dst$$reg),
9968 //                as_FloatRegister($src1$$reg),
9969 //                as_FloatRegister($src2$$reg),
9970 //                as_FloatRegister($src3$$reg));
9971 //   %}
9972 
9973 //   ins_pipe(pipe_class_default);
9974 // %}
9975 
9976 // instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
9977 //   match(Set dst (SubD (MulD (NegD src1) src2) src3));
9978 //   match(Set dst (SubD (NegD (MulD src1 src2)) src3));
9979 
9980 //   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
9981 
9982 //   ins_encode %{
9983 //     __ fnmaddd(as_FloatRegister($dst$$reg),
9984 //                as_FloatRegister($src1$$reg),
9985 //                as_FloatRegister($src2$$reg),
9986 //                as_FloatRegister($src3$$reg));
9987 //   %}
9988 
9989 //   ins_pipe(pipe_class_default);
9990 // %}
9991 
9992 // instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
9993 //   match(Set dst (SubF (MulF src1 src2) src3));
9994 
9995 //   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
9996 
9997 //   ins_encode %{
9998 //     __ fnmsubs(as_FloatRegister($dst$$reg),
9999 //                as_FloatRegister($src1$$reg),
10000 //                as_FloatRegister($src2$$reg),
10001 //                as_FloatRegister($src3$$reg));
10002 //   %}
10003 
10004 //   ins_pipe(pipe_class_default);
10005 // %}
10006 
10007 // instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
10008 //   match(Set dst (SubD (MulD src1 src2) src3));
10009 
10010 //   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
10011 
10012 //   ins_encode %{
10013 //   // n.b. insn name should be fnmsubd
10014 //     __ fnmsub(as_FloatRegister($dst$$reg),
10015 //                as_FloatRegister($src1$$reg),
10016 //                as_FloatRegister($src2$$reg),
10017 //                as_FloatRegister($src3$$reg));
10018 //   %}
10019 
10020 //   ins_pipe(pipe_class_default);
10021 // %}
10022 
10023 
10024 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
10025   match(Set dst (DivF src1  src2));
10026 
10027   ins_cost(INSN_COST * 18);
10028   format %{ "fdivs   $dst, $src1, $src2" %}
10029 
10030   ins_encode %{
10031     __ fdivs(as_FloatRegister($dst$$reg),
10032              as_FloatRegister($src1$$reg),
10033              as_FloatRegister($src2$$reg));
10034   %}
10035 
10036   ins_pipe(pipe_class_default);
10037 %}
10038 
10039 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
10040   match(Set dst (DivD src1  src2));
10041 
10042   ins_cost(INSN_COST * 32);
10043   format %{ "fdivd   $dst, $src1, $src2" %}
10044 
10045   ins_encode %{
10046     __ fdivd(as_FloatRegister($dst$$reg),
10047              as_FloatRegister($src1$$reg),
10048              as_FloatRegister($src2$$reg));
10049   %}
10050 
10051   ins_pipe(pipe_class_default);
10052 %}
10053 
10054 instruct negF_reg_reg(vRegF dst, vRegF src) %{
10055   match(Set dst (NegF src));
10056 
10057   ins_cost(INSN_COST * 3);
10058   format %{ "fneg   $dst, $src" %}
10059 
10060   ins_encode %{
10061     __ fnegs(as_FloatRegister($dst$$reg),
10062              as_FloatRegister($src$$reg));
10063   %}
10064 
10065   ins_pipe(pipe_class_default);
10066 %}
10067 
10068 instruct negD_reg_reg(vRegD dst, vRegD src) %{
10069   match(Set dst (NegD src));
10070 
10071   ins_cost(INSN_COST * 3);
10072   format %{ "fnegd   $dst, $src" %}
10073 
10074   ins_encode %{
10075     __ fnegd(as_FloatRegister($dst$$reg),
10076              as_FloatRegister($src$$reg));
10077   %}
10078 
10079   ins_pipe(pipe_class_default);
10080 %}
10081 
10082 instruct absF_reg(vRegF dst, vRegF src) %{
10083   match(Set dst (AbsF src));
10084 
10085   ins_cost(INSN_COST * 3);
10086   format %{ "fabss   $dst, $src" %}
10087   ins_encode %{
10088     __ fabss(as_FloatRegister($dst$$reg),
10089              as_FloatRegister($src$$reg));
10090   %}
10091 
10092   ins_pipe(pipe_class_default);
10093 %}
10094 
10095 instruct absD_reg(vRegD dst, vRegD src) %{
10096   match(Set dst (AbsD src));
10097 
10098   ins_cost(INSN_COST * 3);
10099   format %{ "fabsd   $dst, $src" %}
10100   ins_encode %{
10101     __ fabsd(as_FloatRegister($dst$$reg),
10102              as_FloatRegister($src$$reg));
10103   %}
10104 
10105   ins_pipe(pipe_class_default);
10106 %}
10107 
10108 instruct sqrtD_reg(vRegD dst, vRegD src) %{
10109   match(Set dst (SqrtD src));
10110 
10111   ins_cost(INSN_COST * 50);
10112   format %{ "fsqrtd  $dst, $src" %}
10113   ins_encode %{
10114     __ fsqrtd(as_FloatRegister($dst$$reg),
10115              as_FloatRegister($src$$reg));
10116   %}
10117 
10118   ins_pipe(pipe_class_default);
10119 %}
10120 
10121 instruct sqrtF_reg(vRegF dst, vRegF src) %{
10122   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
10123 
10124   ins_cost(INSN_COST * 50);
10125   format %{ "fsqrts  $dst, $src" %}
10126   ins_encode %{
10127     __ fsqrts(as_FloatRegister($dst$$reg),
10128              as_FloatRegister($src$$reg));
10129   %}
10130 
10131   ins_pipe(pipe_class_default);
10132 %}
10133 
10134 // ============================================================================
10135 // Logical Instructions
10136 
10137 // Integer Logical Instructions
10138 
10139 // And Instructions
10140 
10141 
10142 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
10143   match(Set dst (AndI src1 src2));
10144 
10145   format %{ "andw  $dst, $src1, $src2\t# int" %}
10146 
10147   ins_cost(INSN_COST);
10148   ins_encode %{
10149     __ andw(as_Register($dst$$reg),
10150             as_Register($src1$$reg),
10151             as_Register($src2$$reg));
10152   %}
10153 
10154   ins_pipe(ialu_reg_reg);
10155 %}
10156 
10157 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
10158   match(Set dst (AndI src1 src2));
10159 
10160   format %{ "andsw  $dst, $src1, $src2\t# int" %}
10161 
10162   ins_cost(INSN_COST);
10163   ins_encode %{
10164     __ andw(as_Register($dst$$reg),
10165             as_Register($src1$$reg),
10166             (unsigned long)($src2$$constant));
10167   %}
10168 
10169   ins_pipe(ialu_reg_imm);
10170 %}
10171 
10172 // Or Instructions
10173 
10174 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10175   match(Set dst (OrI src1 src2));
10176 
10177   format %{ "orrw  $dst, $src1, $src2\t# int" %}
10178 
10179   ins_cost(INSN_COST);
10180   ins_encode %{
10181     __ orrw(as_Register($dst$$reg),
10182             as_Register($src1$$reg),
10183             as_Register($src2$$reg));
10184   %}
10185 
10186   ins_pipe(ialu_reg_reg);
10187 %}
10188 
10189 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
10190   match(Set dst (OrI src1 src2));
10191 
10192   format %{ "orrw  $dst, $src1, $src2\t# int" %}
10193 
10194   ins_cost(INSN_COST);
10195   ins_encode %{
10196     __ orrw(as_Register($dst$$reg),
10197             as_Register($src1$$reg),
10198             (unsigned long)($src2$$constant));
10199   %}
10200 
10201   ins_pipe(ialu_reg_imm);
10202 %}
10203 
10204 // Xor Instructions
10205 
10206 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10207   match(Set dst (XorI src1 src2));
10208 
10209   format %{ "eorw  $dst, $src1, $src2\t# int" %}
10210 
10211   ins_cost(INSN_COST);
10212   ins_encode %{
10213     __ eorw(as_Register($dst$$reg),
10214             as_Register($src1$$reg),
10215             as_Register($src2$$reg));
10216   %}
10217 
10218   ins_pipe(ialu_reg_reg);
10219 %}
10220 
10221 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
10222   match(Set dst (XorI src1 src2));
10223 
10224   format %{ "eorw  $dst, $src1, $src2\t# int" %}
10225 
10226   ins_cost(INSN_COST);
10227   ins_encode %{
10228     __ eorw(as_Register($dst$$reg),
10229             as_Register($src1$$reg),
10230             (unsigned long)($src2$$constant));
10231   %}
10232 
10233   ins_pipe(ialu_reg_imm);
10234 %}
10235 
10236 // Long Logical Instructions
10237 // TODO
10238 
10239 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
10240   match(Set dst (AndL src1 src2));
10241 
10242   format %{ "and  $dst, $src1, $src2\t# int" %}
10243 
10244   ins_cost(INSN_COST);
10245   ins_encode %{
10246     __ andr(as_Register($dst$$reg),
10247             as_Register($src1$$reg),
10248             as_Register($src2$$reg));
10249   %}
10250 
10251   ins_pipe(ialu_reg_reg);
10252 %}
10253 
10254 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
10255   match(Set dst (AndL src1 src2));
10256 
10257   format %{ "and  $dst, $src1, $src2\t# int" %}
10258 
10259   ins_cost(INSN_COST);
10260   ins_encode %{
10261     __ andr(as_Register($dst$$reg),
10262             as_Register($src1$$reg),
10263             (unsigned long)($src2$$constant));
10264   %}
10265 
10266   ins_pipe(ialu_reg_imm);
10267 %}
10268 
10269 // Or Instructions
10270 
10271 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10272   match(Set dst (OrL src1 src2));
10273 
10274   format %{ "orr  $dst, $src1, $src2\t# int" %}
10275 
10276   ins_cost(INSN_COST);
10277   ins_encode %{
10278     __ orr(as_Register($dst$$reg),
10279            as_Register($src1$$reg),
10280            as_Register($src2$$reg));
10281   %}
10282 
10283   ins_pipe(ialu_reg_reg);
10284 %}
10285 
10286 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
10287   match(Set dst (OrL src1 src2));
10288 
10289   format %{ "orr  $dst, $src1, $src2\t# int" %}
10290 
10291   ins_cost(INSN_COST);
10292   ins_encode %{
10293     __ orr(as_Register($dst$$reg),
10294            as_Register($src1$$reg),
10295            (unsigned long)($src2$$constant));
10296   %}
10297 
10298   ins_pipe(ialu_reg_imm);
10299 %}
10300 
10301 // Xor Instructions
10302 
10303 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10304   match(Set dst (XorL src1 src2));
10305 
10306   format %{ "eor  $dst, $src1, $src2\t# int" %}
10307 
10308   ins_cost(INSN_COST);
10309   ins_encode %{
10310     __ eor(as_Register($dst$$reg),
10311            as_Register($src1$$reg),
10312            as_Register($src2$$reg));
10313   %}
10314 
10315   ins_pipe(ialu_reg_reg);
10316 %}
10317 
10318 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
10319   match(Set dst (XorL src1 src2));
10320 
10321   ins_cost(INSN_COST);
10322   format %{ "eor  $dst, $src1, $src2\t# int" %}
10323 
10324   ins_encode %{
10325     __ eor(as_Register($dst$$reg),
10326            as_Register($src1$$reg),
10327            (unsigned long)($src2$$constant));
10328   %}
10329 
10330   ins_pipe(ialu_reg_imm);
10331 %}
10332 
10333 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
10334 %{
10335   match(Set dst (ConvI2L src));
10336 
10337   ins_cost(INSN_COST);
10338   format %{ "sxtw  $dst, $src\t# i2l" %}
10339   ins_encode %{
10340     __ sbfm($dst$$Register, $src$$Register, 0, 31);
10341   %}
10342   ins_pipe(ialu_reg_shift);
10343 %}
10344 
10345 // this pattern occurs in bigmath arithmetic
10346 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
10347 %{
10348   match(Set dst (AndL (ConvI2L src) mask));
10349 
10350   ins_cost(INSN_COST);
10351   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
10352   ins_encode %{
10353     __ ubfm($dst$$Register, $src$$Register, 0, 31);
10354   %}
10355 
10356   ins_pipe(ialu_reg_shift);
10357 %}
10358 
10359 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
10360   match(Set dst (ConvL2I src));
10361 
10362   ins_cost(INSN_COST);
10363   format %{ "movw  $dst, $src \t// l2i" %}
10364 
10365   ins_encode %{
10366     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
10367   %}
10368 
10369   ins_pipe(ialu_reg);
10370 %}
10371 
10372 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
10373 %{
10374   match(Set dst (Conv2B src));
10375   effect(KILL cr);
10376 
10377   format %{
10378     "cmpw $src, zr\n\t"
10379     "cset $dst, ne"
10380   %}
10381 
10382   ins_encode %{
10383     __ cmpw(as_Register($src$$reg), zr);
10384     __ cset(as_Register($dst$$reg), Assembler::NE);
10385   %}
10386 
10387   ins_pipe(ialu_reg);
10388 %}
10389 
10390 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
10391 %{
10392   match(Set dst (Conv2B src));
10393   effect(KILL cr);
10394 
10395   format %{
10396     "cmp  $src, zr\n\t"
10397     "cset $dst, ne"
10398   %}
10399 
10400   ins_encode %{
10401     __ cmp(as_Register($src$$reg), zr);
10402     __ cset(as_Register($dst$$reg), Assembler::NE);
10403   %}
10404 
10405   ins_pipe(ialu_reg);
10406 %}
10407 
10408 instruct convD2F_reg(vRegF dst, vRegD src) %{
10409   match(Set dst (ConvD2F src));
10410 
10411   ins_cost(INSN_COST * 5);
10412   format %{ "fcvtd  $dst, $src \t// d2f" %}
10413 
10414   ins_encode %{
10415     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
10416   %}
10417 
10418   ins_pipe(pipe_class_default);
10419 %}
10420 
10421 instruct convF2D_reg(vRegD dst, vRegF src) %{
10422   match(Set dst (ConvF2D src));
10423 
10424   ins_cost(INSN_COST * 5);
10425   format %{ "fcvts  $dst, $src \t// f2d" %}
10426 
10427   ins_encode %{
10428     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
10429   %}
10430 
10431   ins_pipe(pipe_class_default);
10432 %}
10433 
10434 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
10435   match(Set dst (ConvF2I src));
10436 
10437   ins_cost(INSN_COST * 5);
10438   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
10439 
10440   ins_encode %{
10441     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
10442   %}
10443 
10444   ins_pipe(pipe_class_default);
10445 %}
10446 
10447 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
10448   match(Set dst (ConvF2L src));
10449 
10450   ins_cost(INSN_COST * 5);
10451   format %{ "fcvtzs  $dst, $src \t// f2l" %}
10452 
10453   ins_encode %{
10454     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
10455   %}
10456 
10457   ins_pipe(pipe_class_default);
10458 %}
10459 
10460 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
10461   match(Set dst (ConvI2F src));
10462 
10463   ins_cost(INSN_COST * 5);
10464   format %{ "scvtfws  $dst, $src \t// i2f" %}
10465 
10466   ins_encode %{
10467     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
10468   %}
10469 
10470   ins_pipe(pipe_class_default);
10471 %}
10472 
10473 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
10474   match(Set dst (ConvL2F src));
10475 
10476   ins_cost(INSN_COST * 5);
10477   format %{ "scvtfs  $dst, $src \t// l2f" %}
10478 
10479   ins_encode %{
10480     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
10481   %}
10482 
10483   ins_pipe(pipe_class_default);
10484 %}
10485 
10486 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
10487   match(Set dst (ConvD2I src));
10488 
10489   ins_cost(INSN_COST * 5);
10490   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
10491 
10492   ins_encode %{
10493     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
10494   %}
10495 
10496   ins_pipe(pipe_class_default);
10497 %}
10498 
10499 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
10500   match(Set dst (ConvD2L src));
10501 
10502   ins_cost(INSN_COST * 5);
10503   format %{ "fcvtzd  $dst, $src \t// d2l" %}
10504 
10505   ins_encode %{
10506     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
10507   %}
10508 
10509   ins_pipe(pipe_class_default);
10510 %}
10511 
10512 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
10513   match(Set dst (ConvI2D src));
10514 
10515   ins_cost(INSN_COST * 5);
10516   format %{ "scvtfwd  $dst, $src \t// i2d" %}
10517 
10518   ins_encode %{
10519     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
10520   %}
10521 
10522   ins_pipe(pipe_class_default);
10523 %}
10524 
10525 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
10526   match(Set dst (ConvL2D src));
10527 
10528   ins_cost(INSN_COST * 5);
10529   format %{ "scvtfd  $dst, $src \t// l2d" %}
10530 
10531   ins_encode %{
10532     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
10533   %}
10534 
10535   ins_pipe(pipe_class_default);
10536 %}
10537 
10538 // stack <-> reg and reg <-> reg shuffles with no conversion
10539 
10540 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
10541 
10542   match(Set dst (MoveF2I src));
10543 
10544   effect(DEF dst, USE src);
10545 
10546   ins_cost(4 * INSN_COST);
10547 
10548   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
10549 
10550   ins_encode %{
10551     __ ldrw($dst$$Register, Address(sp, $src$$disp));
10552   %}
10553 
10554   ins_pipe(iload_reg_reg);
10555 
10556 %}
10557 
10558 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
10559 
10560   match(Set dst (MoveI2F src));
10561 
10562   effect(DEF dst, USE src);
10563 
10564   ins_cost(4 * INSN_COST);
10565 
10566   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
10567 
10568   ins_encode %{
10569     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
10570   %}
10571 
10572   ins_pipe(pipe_class_memory);
10573 
10574 %}
10575 
10576 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
10577 
10578   match(Set dst (MoveD2L src));
10579 
10580   effect(DEF dst, USE src);
10581 
10582   ins_cost(4 * INSN_COST);
10583 
10584   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
10585 
10586   ins_encode %{
10587     __ ldr($dst$$Register, Address(sp, $src$$disp));
10588   %}
10589 
10590   ins_pipe(iload_reg_reg);
10591 
10592 %}
10593 
10594 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
10595 
10596   match(Set dst (MoveL2D src));
10597 
10598   effect(DEF dst, USE src);
10599 
10600   ins_cost(4 * INSN_COST);
10601 
10602   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
10603 
10604   ins_encode %{
10605     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
10606   %}
10607 
10608   ins_pipe(pipe_class_memory);
10609 
10610 %}
10611 
10612 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
10613 
10614   match(Set dst (MoveF2I src));
10615 
10616   effect(DEF dst, USE src);
10617 
10618   ins_cost(INSN_COST);
10619 
10620   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
10621 
10622   ins_encode %{
10623     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
10624   %}
10625 
10626   ins_pipe(pipe_class_memory);
10627 
10628 %}
10629 
10630 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
10631 
10632   match(Set dst (MoveI2F src));
10633 
10634   effect(DEF dst, USE src);
10635 
10636   ins_cost(INSN_COST);
10637 
10638   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
10639 
10640   ins_encode %{
10641     __ strw($src$$Register, Address(sp, $dst$$disp));
10642   %}
10643 
10644   ins_pipe(istore_reg_reg);
10645 
10646 %}
10647 
10648 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
10649 
10650   match(Set dst (MoveD2L src));
10651 
10652   effect(DEF dst, USE src);
10653 
10654   ins_cost(INSN_COST);
10655 
10656   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
10657 
10658   ins_encode %{
10659     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
10660   %}
10661 
10662   ins_pipe(pipe_class_memory);
10663 
10664 %}
10665 
10666 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
10667 
10668   match(Set dst (MoveL2D src));
10669 
10670   effect(DEF dst, USE src);
10671 
10672   ins_cost(INSN_COST);
10673 
10674   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
10675 
10676   ins_encode %{
10677     __ str($src$$Register, Address(sp, $dst$$disp));
10678   %}
10679 
10680   ins_pipe(istore_reg_reg);
10681 
10682 %}
10683 
10684 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
10685 
10686   match(Set dst (MoveF2I src));
10687 
10688   effect(DEF dst, USE src);
10689 
10690   ins_cost(INSN_COST);
10691 
10692   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
10693 
10694   ins_encode %{
10695     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
10696   %}
10697 
10698   ins_pipe(pipe_class_memory);
10699 
10700 %}
10701 
10702 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
10703 
10704   match(Set dst (MoveI2F src));
10705 
10706   effect(DEF dst, USE src);
10707 
10708   ins_cost(INSN_COST);
10709 
10710   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
10711 
10712   ins_encode %{
10713     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
10714   %}
10715 
10716   ins_pipe(pipe_class_memory);
10717 
10718 %}
10719 
10720 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
10721 
10722   match(Set dst (MoveD2L src));
10723 
10724   effect(DEF dst, USE src);
10725 
10726   ins_cost(INSN_COST);
10727 
10728   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
10729 
10730   ins_encode %{
10731     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
10732   %}
10733 
10734   ins_pipe(pipe_class_memory);
10735 
10736 %}
10737 
10738 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
10739 
10740   match(Set dst (MoveL2D src));
10741 
10742   effect(DEF dst, USE src);
10743 
10744   ins_cost(INSN_COST);
10745 
10746   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
10747 
10748   ins_encode %{
10749     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
10750   %}
10751 
10752   ins_pipe(pipe_class_memory);
10753 
10754 %}
10755 
10756 // ============================================================================
10757 // clearing of an array
10758 
10759 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
10760 %{
10761   match(Set dummy (ClearArray cnt base));
10762   effect(USE_KILL cnt, USE_KILL base);
10763 
10764   ins_cost(4 * INSN_COST);
10765   format %{ "ClearArray $cnt, $base" %}
10766 
10767   ins_encode(aarch64_enc_clear_array_reg_reg(cnt, base));
10768 
10769   ins_pipe(pipe_class_memory);
10770 %}
10771 
10772 // ============================================================================
10773 // Overflow Math Instructions
10774 
10775 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
10776 %{
10777   match(Set cr (OverflowAddI op1 op2));
10778 
10779   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
10780   ins_cost(INSN_COST);
10781   ins_encode %{
10782     __ cmnw($op1$$Register, $op2$$Register);
10783   %}
10784 
10785   ins_pipe(icmp_reg_reg);
10786 %}
10787 
10788 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
10789 %{
10790   match(Set cr (OverflowAddI op1 op2));
10791 
10792   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
10793   ins_cost(INSN_COST);
10794   ins_encode %{
10795     __ cmnw($op1$$Register, $op2$$constant);
10796   %}
10797 
10798   ins_pipe(icmp_reg_imm);
10799 %}
10800 
10801 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
10802 %{
10803   match(Set cr (OverflowAddL op1 op2));
10804 
10805   format %{ "cmn   $op1, $op2\t# overflow check long" %}
10806   ins_cost(INSN_COST);
10807   ins_encode %{
10808     __ cmn($op1$$Register, $op2$$Register);
10809   %}
10810 
10811   ins_pipe(icmp_reg_reg);
10812 %}
10813 
10814 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
10815 %{
10816   match(Set cr (OverflowAddL op1 op2));
10817 
10818   format %{ "cmn   $op1, $op2\t# overflow check long" %}
10819   ins_cost(INSN_COST);
10820   ins_encode %{
10821     __ cmn($op1$$Register, $op2$$constant);
10822   %}
10823 
10824   ins_pipe(icmp_reg_imm);
10825 %}
10826 
10827 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
10828 %{
10829   match(Set cr (OverflowSubI op1 op2));
10830 
10831   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
10832   ins_cost(INSN_COST);
10833   ins_encode %{
10834     __ cmpw($op1$$Register, $op2$$Register);
10835   %}
10836 
10837   ins_pipe(icmp_reg_reg);
10838 %}
10839 
10840 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
10841 %{
10842   match(Set cr (OverflowSubI op1 op2));
10843 
10844   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
10845   ins_cost(INSN_COST);
10846   ins_encode %{
10847     __ cmpw($op1$$Register, $op2$$constant);
10848   %}
10849 
10850   ins_pipe(icmp_reg_imm);
10851 %}
10852 
10853 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
10854 %{
10855   match(Set cr (OverflowSubL op1 op2));
10856 
10857   format %{ "cmp   $op1, $op2\t# overflow check long" %}
10858   ins_cost(INSN_COST);
10859   ins_encode %{
10860     __ cmp($op1$$Register, $op2$$Register);
10861   %}
10862 
10863   ins_pipe(icmp_reg_reg);
10864 %}
10865 
10866 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
10867 %{
10868   match(Set cr (OverflowSubL op1 op2));
10869 
10870   format %{ "cmp   $op1, $op2\t# overflow check long" %}
10871   ins_cost(INSN_COST);
10872   ins_encode %{
10873     __ cmp($op1$$Register, $op2$$constant);
10874   %}
10875 
10876   ins_pipe(icmp_reg_imm);
10877 %}
10878 
10879 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
10880 %{
10881   match(Set cr (OverflowSubI zero op1));
10882 
10883   format %{ "cmpw  zr, $op1\t# overflow check int" %}
10884   ins_cost(INSN_COST);
10885   ins_encode %{
10886     __ cmpw(zr, $op1$$Register);
10887   %}
10888 
10889   ins_pipe(icmp_reg_imm);
10890 %}
10891 
10892 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
10893 %{
10894   match(Set cr (OverflowSubL zero op1));
10895 
10896   format %{ "cmp   zr, $op1\t# overflow check long" %}
10897   ins_cost(INSN_COST);
10898   ins_encode %{
10899     __ cmp(zr, $op1$$Register);
10900   %}
10901 
10902   ins_pipe(icmp_reg_imm);
10903 %}
10904 
10905 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
10906 %{
10907   match(Set cr (OverflowMulI op1 op2));
10908 
10909   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
10910             "cmp   rscratch1, rscratch1, sxtw\n\t"
10911             "movw  rscratch1, #0x80000000\n\t"
10912             "cselw rscratch1, rscratch1, zr, NE\n\t"
10913             "cmpw  rscratch1, #1" %}
10914   ins_cost(5 * INSN_COST);
10915   ins_encode %{
10916     __ smull(rscratch1, $op1$$Register, $op2$$Register);
10917     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
10918     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
10919     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
10920     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
10921   %}
10922 
10923   ins_pipe(pipe_slow);
10924 %}
10925 
10926 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
10927 %{
10928   match(If cmp (OverflowMulI op1 op2));
10929   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
10930             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
10931   effect(USE labl, KILL cr);
10932 
10933   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
10934             "cmp   rscratch1, rscratch1, sxtw\n\t"
10935             "b$cmp   $labl" %}
10936   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
10937   ins_encode %{
10938     Label* L = $labl$$label;
10939     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10940     __ smull(rscratch1, $op1$$Register, $op2$$Register);
10941     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
10942     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
10943   %}
10944 
10945   ins_pipe(pipe_serial);
10946 %}
10947 
10948 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
10949 %{
10950   match(Set cr (OverflowMulL op1 op2));
10951 
10952   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
10953             "smulh rscratch2, $op1, $op2\n\t"
10954             "cmp   rscratch2, rscratch1, ASR #31\n\t"
10955             "movw  rscratch1, #0x80000000\n\t"
10956             "cselw rscratch1, rscratch1, zr, NE\n\t"
10957             "cmpw  rscratch1, #1" %}
10958   ins_cost(6 * INSN_COST);
10959   ins_encode %{
10960     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
10961     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
10962     __ cmp(rscratch2, rscratch1, Assembler::ASR, 31);    // Top is pure sign ext
10963     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
10964     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
10965     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
10966   %}
10967 
10968   ins_pipe(pipe_slow);
10969 %}
10970 
10971 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
10972 %{
10973   match(If cmp (OverflowMulL op1 op2));
10974   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
10975             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
10976   effect(USE labl, KILL cr);
10977 
10978   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
10979             "smulh rscratch2, $op1, $op2\n\t"
10980             "cmp   rscratch2, rscratch1, ASR #31\n\t"
10981             "b$cmp $labl" %}
10982   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
10983   ins_encode %{
10984     Label* L = $labl$$label;
10985     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10986     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
10987     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
10988     __ cmp(rscratch2, rscratch1, Assembler::ASR, 31);    // Top is pure sign ext
10989     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
10990   %}
10991 
10992   ins_pipe(pipe_serial);
10993 %}
10994 
10995 // ============================================================================
10996 // Compare Instructions
10997 
10998 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
10999 %{
11000   match(Set cr (CmpI op1 op2));
11001 
11002   effect(DEF cr, USE op1, USE op2);
11003 
11004   ins_cost(INSN_COST);
11005   format %{ "cmpw  $op1, $op2" %}
11006 
11007   ins_encode(aarch64_enc_cmpw(op1, op2));
11008 
11009   ins_pipe(icmp_reg_reg);
11010 %}
11011 
11012 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
11013 %{
11014   match(Set cr (CmpI op1 zero));
11015 
11016   effect(DEF cr, USE op1);
11017 
11018   ins_cost(INSN_COST);
11019   format %{ "cmpw $op1, 0" %}
11020 
11021   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
11022 
11023   ins_pipe(icmp_reg_imm);
11024 %}
11025 
11026 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
11027 %{
11028   match(Set cr (CmpI op1 op2));
11029 
11030   effect(DEF cr, USE op1);
11031 
11032   ins_cost(INSN_COST);
11033   format %{ "cmpw  $op1, $op2" %}
11034 
11035   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
11036 
11037   ins_pipe(icmp_reg_imm);
11038 %}
11039 
11040 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
11041 %{
11042   match(Set cr (CmpI op1 op2));
11043 
11044   effect(DEF cr, USE op1);
11045 
11046   ins_cost(INSN_COST * 2);
11047   format %{ "cmpw  $op1, $op2" %}
11048 
11049   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
11050 
11051   ins_pipe(icmp_reg_imm);
11052 %}
11053 
11054 // Unsigned compare Instructions; really, same as signed compare
11055 // except it should only be used to feed an If or a CMovI which takes a
11056 // cmpOpU.
11057 
11058 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
11059 %{
11060   match(Set cr (CmpU op1 op2));
11061 
11062   effect(DEF cr, USE op1, USE op2);
11063 
11064   ins_cost(INSN_COST);
11065   format %{ "cmpw  $op1, $op2\t# unsigned" %}
11066 
11067   ins_encode(aarch64_enc_cmpw(op1, op2));
11068 
11069   ins_pipe(icmp_reg_reg);
11070 %}
11071 
11072 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
11073 %{
11074   match(Set cr (CmpU op1 zero));
11075 
11076   effect(DEF cr, USE op1);
11077 
11078   ins_cost(INSN_COST);
11079   format %{ "cmpw $op1, #0\t# unsigned" %}
11080 
11081   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
11082 
11083   ins_pipe(icmp_reg_imm);
11084 %}
11085 
11086 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
11087 %{
11088   match(Set cr (CmpU op1 op2));
11089 
11090   effect(DEF cr, USE op1);
11091 
11092   ins_cost(INSN_COST);
11093   format %{ "cmpw  $op1, $op2\t# unsigned" %}
11094 
11095   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
11096 
11097   ins_pipe(icmp_reg_imm);
11098 %}
11099 
11100 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
11101 %{
11102   match(Set cr (CmpU op1 op2));
11103 
11104   effect(DEF cr, USE op1);
11105 
11106   ins_cost(INSN_COST * 2);
11107   format %{ "cmpw  $op1, $op2\t# unsigned" %}
11108 
11109   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
11110 
11111   ins_pipe(icmp_reg_imm);
11112 %}
11113 
11114 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
11115 %{
11116   match(Set cr (CmpL op1 op2));
11117 
11118   effect(DEF cr, USE op1, USE op2);
11119 
11120   ins_cost(INSN_COST);
11121   format %{ "cmp  $op1, $op2" %}
11122 
11123   ins_encode(aarch64_enc_cmp(op1, op2));
11124 
11125   ins_pipe(icmp_reg_reg);
11126 %}
11127 
11128 instruct compL_reg_immI0(rFlagsReg cr, iRegL op1, immI0 zero)
11129 %{
11130   match(Set cr (CmpL op1 zero));
11131 
11132   effect(DEF cr, USE op1);
11133 
11134   ins_cost(INSN_COST);
11135   format %{ "tst  $op1" %}
11136 
11137   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
11138 
11139   ins_pipe(icmp_reg_imm);
11140 %}
11141 
11142 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
11143 %{
11144   match(Set cr (CmpL op1 op2));
11145 
11146   effect(DEF cr, USE op1);
11147 
11148   ins_cost(INSN_COST);
11149   format %{ "cmp  $op1, $op2" %}
11150 
11151   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
11152 
11153   ins_pipe(icmp_reg_imm);
11154 %}
11155 
11156 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
11157 %{
11158   match(Set cr (CmpL op1 op2));
11159 
11160   effect(DEF cr, USE op1);
11161 
11162   ins_cost(INSN_COST * 2);
11163   format %{ "cmp  $op1, $op2" %}
11164 
11165   ins_encode(aarch64_enc_cmp_imm(op1, op2));
11166 
11167   ins_pipe(icmp_reg_imm);
11168 %}
11169 
11170 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
11171 %{
11172   match(Set cr (CmpP op1 op2));
11173 
11174   effect(DEF cr, USE op1, USE op2);
11175 
11176   ins_cost(INSN_COST);
11177   format %{ "cmp  $op1, $op2\t // ptr" %}
11178 
11179   ins_encode(aarch64_enc_cmpp(op1, op2));
11180 
11181   ins_pipe(icmp_reg_reg);
11182 %}
11183 
11184 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
11185 %{
11186   match(Set cr (CmpN op1 op2));
11187 
11188   effect(DEF cr, USE op1, USE op2);
11189 
11190   ins_cost(INSN_COST);
11191   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
11192 
11193   ins_encode(aarch64_enc_cmpn(op1, op2));
11194 
11195   ins_pipe(icmp_reg_reg);
11196 %}
11197 
11198 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
11199 %{
11200   match(Set cr (CmpP op1 zero));
11201 
11202   effect(DEF cr, USE op1, USE zero);
11203 
11204   ins_cost(INSN_COST);
11205   format %{ "cmp  $op1, 0\t // ptr" %}
11206 
11207   ins_encode(aarch64_enc_testp(op1));
11208 
11209   ins_pipe(icmp_reg_imm);
11210 %}
11211 
11212 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
11213 %{
11214   match(Set cr (CmpN op1 zero));
11215 
11216   effect(DEF cr, USE op1, USE zero);
11217 
11218   ins_cost(INSN_COST);
11219   format %{ "cmp  $op1, 0\t // compressed ptr" %}
11220 
11221   ins_encode(aarch64_enc_testn(op1));
11222 
11223   ins_pipe(icmp_reg_imm);
11224 %}
11225 
11226 // FP comparisons
11227 //
11228 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
11229 // using normal cmpOp. See declaration of rFlagsReg for details.
11230 
11231 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
11232 %{
11233   match(Set cr (CmpF src1 src2));
11234 
11235   ins_cost(3 * INSN_COST);
11236   format %{ "fcmps $src1, $src2" %}
11237 
11238   ins_encode %{
11239     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
11240   %}
11241 
11242   ins_pipe(pipe_class_compare);
11243 %}
11244 
11245 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
11246 %{
11247   match(Set cr (CmpF src1 src2));
11248 
11249   ins_cost(3 * INSN_COST);
11250   format %{ "fcmps $src1, 0.0" %}
11251 
11252   ins_encode %{
11253     __ fcmps(as_FloatRegister($src1$$reg), 0.0D);
11254   %}
11255 
11256   ins_pipe(pipe_class_compare);
11257 %}
11258 // FROM HERE
11259 
11260 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
11261 %{
11262   match(Set cr (CmpD src1 src2));
11263 
11264   ins_cost(3 * INSN_COST);
11265   format %{ "fcmpd $src1, $src2" %}
11266 
11267   ins_encode %{
11268     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
11269   %}
11270 
11271   ins_pipe(pipe_class_compare);
11272 %}
11273 
11274 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
11275 %{
11276   match(Set cr (CmpD src1 src2));
11277 
11278   ins_cost(3 * INSN_COST);
11279   format %{ "fcmpd $src1, 0.0" %}
11280 
11281   ins_encode %{
11282     __ fcmpd(as_FloatRegister($src1$$reg), 0.0D);
11283   %}
11284 
11285   ins_pipe(pipe_class_compare);
11286 %}
11287 
11288 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
11289 %{
11290   match(Set dst (CmpF3 src1 src2));
11291   effect(KILL cr);
11292 
11293   ins_cost(5 * INSN_COST);
11294   format %{ "fcmps $src1, $src2\n\t"
11295             "csinvw($dst, zr, zr, eq\n\t"
11296             "csnegw($dst, $dst, $dst, lt)"
11297   %}
11298 
11299   ins_encode %{
11300     Label done;
11301     FloatRegister s1 = as_FloatRegister($src1$$reg);
11302     FloatRegister s2 = as_FloatRegister($src2$$reg);
11303     Register d = as_Register($dst$$reg);
11304     __ fcmps(s1, s2);
11305     // installs 0 if EQ else -1
11306     __ csinvw(d, zr, zr, Assembler::EQ);
11307     // keeps -1 if less or unordered else installs 1
11308     __ csnegw(d, d, d, Assembler::LT);
11309     __ bind(done);
11310   %}
11311 
11312   ins_pipe(pipe_class_default);
11313 
11314 %}
11315 
11316 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
11317 %{
11318   match(Set dst (CmpD3 src1 src2));
11319   effect(KILL cr);
11320 
11321   ins_cost(5 * INSN_COST);
11322   format %{ "fcmpd $src1, $src2\n\t"
11323             "csinvw($dst, zr, zr, eq\n\t"
11324             "csnegw($dst, $dst, $dst, lt)"
11325   %}
11326 
11327   ins_encode %{
11328     Label done;
11329     FloatRegister s1 = as_FloatRegister($src1$$reg);
11330     FloatRegister s2 = as_FloatRegister($src2$$reg);
11331     Register d = as_Register($dst$$reg);
11332     __ fcmpd(s1, s2);
11333     // installs 0 if EQ else -1
11334     __ csinvw(d, zr, zr, Assembler::EQ);
11335     // keeps -1 if less or unordered else installs 1
11336     __ csnegw(d, d, d, Assembler::LT);
11337     __ bind(done);
11338   %}
11339   ins_pipe(pipe_class_default);
11340 
11341 %}
11342 
11343 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
11344 %{
11345   match(Set dst (CmpF3 src1 zero));
11346   effect(KILL cr);
11347 
11348   ins_cost(5 * INSN_COST);
11349   format %{ "fcmps $src1, 0.0\n\t"
11350             "csinvw($dst, zr, zr, eq\n\t"
11351             "csnegw($dst, $dst, $dst, lt)"
11352   %}
11353 
11354   ins_encode %{
11355     Label done;
11356     FloatRegister s1 = as_FloatRegister($src1$$reg);
11357     Register d = as_Register($dst$$reg);
11358     __ fcmps(s1, 0.0D);
11359     // installs 0 if EQ else -1
11360     __ csinvw(d, zr, zr, Assembler::EQ);
11361     // keeps -1 if less or unordered else installs 1
11362     __ csnegw(d, d, d, Assembler::LT);
11363     __ bind(done);
11364   %}
11365 
11366   ins_pipe(pipe_class_default);
11367 
11368 %}
11369 
11370 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
11371 %{
11372   match(Set dst (CmpD3 src1 zero));
11373   effect(KILL cr);
11374 
11375   ins_cost(5 * INSN_COST);
11376   format %{ "fcmpd $src1, 0.0\n\t"
11377             "csinvw($dst, zr, zr, eq\n\t"
11378             "csnegw($dst, $dst, $dst, lt)"
11379   %}
11380 
11381   ins_encode %{
11382     Label done;
11383     FloatRegister s1 = as_FloatRegister($src1$$reg);
11384     Register d = as_Register($dst$$reg);
11385     __ fcmpd(s1, 0.0D);
11386     // installs 0 if EQ else -1
11387     __ csinvw(d, zr, zr, Assembler::EQ);
11388     // keeps -1 if less or unordered else installs 1
11389     __ csnegw(d, d, d, Assembler::LT);
11390     __ bind(done);
11391   %}
11392   ins_pipe(pipe_class_default);
11393 
11394 %}
11395 
11396 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
11397 %{
11398   match(Set dst (CmpLTMask p q));
11399   effect(KILL cr);
11400 
11401   ins_cost(3 * INSN_COST);
11402 
11403   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
11404             "csetw $dst, lt\n\t"
11405             "subw $dst, zr, $dst"
11406   %}
11407 
11408   ins_encode %{
11409     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
11410     __ csetw(as_Register($dst$$reg), Assembler::LT);
11411     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
11412   %}
11413 
11414   ins_pipe(ialu_reg_reg);
11415 %}
11416 
11417 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
11418 %{
11419   match(Set dst (CmpLTMask src zero));
11420   effect(KILL cr);
11421 
11422   ins_cost(INSN_COST);
11423 
11424   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
11425 
11426   ins_encode %{
11427     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
11428   %}
11429 
11430   ins_pipe(ialu_reg_shift);
11431 %}
11432 
11433 // ============================================================================
11434 // Max and Min
11435 
11436 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
11437 %{
11438   match(Set dst (MinI src1 src2));
11439 
11440   effect(DEF dst, USE src1, USE src2, KILL cr);
11441   size(8);
11442 
11443   ins_cost(INSN_COST * 3);
11444   format %{
11445     "cmpw $src1 $src2\t signed int\n\t"
11446     "cselw $dst, $src1, $src2 lt\t"
11447   %}
11448 
11449   ins_encode %{
11450     __ cmpw(as_Register($src1$$reg),
11451             as_Register($src2$$reg));
11452     __ cselw(as_Register($dst$$reg),
11453              as_Register($src1$$reg),
11454              as_Register($src2$$reg),
11455              Assembler::LT);
11456   %}
11457 
11458   ins_pipe(ialu_reg_reg);
11459 %}
11460 // FROM HERE
11461 
11462 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
11463 %{
11464   match(Set dst (MaxI src1 src2));
11465 
11466   effect(DEF dst, USE src1, USE src2, KILL cr);
11467   size(8);
11468 
11469   ins_cost(INSN_COST * 3);
11470   format %{
11471     "cmpw $src1 $src2\t signed int\n\t"
11472     "cselw $dst, $src1, $src2 gt\t"
11473   %}
11474 
11475   ins_encode %{
11476     __ cmpw(as_Register($src1$$reg),
11477             as_Register($src2$$reg));
11478     __ cselw(as_Register($dst$$reg),
11479              as_Register($src1$$reg),
11480              as_Register($src2$$reg),
11481              Assembler::GT);
11482   %}
11483 
11484   ins_pipe(ialu_reg_reg);
11485 %}
11486 
11487 // ============================================================================
11488 // Branch Instructions
11489 
11490 // Direct Branch.
11491 instruct branch(label lbl)
11492 %{
11493   match(Goto);
11494 
11495   effect(USE lbl);
11496 
11497   ins_cost(BRANCH_COST);
11498   format %{ "b  $lbl" %}
11499 
11500   ins_encode(aarch64_enc_b(lbl));
11501 
11502   ins_pipe(pipe_branch);
11503 %}
11504 
11505 // Conditional Near Branch
11506 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
11507 %{
11508   // Same match rule as `branchConFar'.
11509   match(If cmp cr);
11510 
11511   effect(USE lbl);
11512 
11513   ins_cost(BRANCH_COST);
11514   // If set to 1 this indicates that the current instruction is a
11515   // short variant of a long branch. This avoids using this
11516   // instruction in first-pass matching. It will then only be used in
11517   // the `Shorten_branches' pass.
11518   // ins_short_branch(1);
11519   format %{ "b$cmp  $lbl" %}
11520 
11521   ins_encode(aarch64_enc_br_con(cmp, lbl));
11522 
11523   ins_pipe(pipe_branch_cond);
11524 %}
11525 
11526 // Conditional Near Branch Unsigned
11527 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
11528 %{
11529   // Same match rule as `branchConFar'.
11530   match(If cmp cr);
11531 
11532   effect(USE lbl);
11533 
11534   ins_cost(BRANCH_COST);
11535   // If set to 1 this indicates that the current instruction is a
11536   // short variant of a long branch. This avoids using this
11537   // instruction in first-pass matching. It will then only be used in
11538   // the `Shorten_branches' pass.
11539   // ins_short_branch(1);
11540   format %{ "b$cmp  $lbl\t# unsigned" %}
11541 
11542   ins_encode(aarch64_enc_br_conU(cmp, lbl));
11543 
11544   ins_pipe(pipe_branch_cond);
11545 %}
11546 
11547 // Make use of CBZ and CBNZ.  These instructions, as well as being
11548 // shorter than (cmp; branch), have the additional benefit of not
11549 // killing the flags.
11550 
11551 instruct cmpI_imm0_branch(cmpOp cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
11552   match(If cmp (CmpI op1 op2));
11553   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
11554             || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
11555   effect(USE labl);
11556 
11557   ins_cost(BRANCH_COST);
11558   format %{ "cbw$cmp   $op1, $labl" %}
11559   ins_encode %{
11560     Label* L = $labl$$label;
11561     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
11562     if (cond == Assembler::EQ)
11563       __ cbzw($op1$$Register, *L);
11564     else
11565       __ cbnzw($op1$$Register, *L);
11566   %}
11567   ins_pipe(pipe_cmp_branch);
11568 %}
11569 
11570 instruct cmpL_imm0_branch(cmpOp cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
11571   match(If cmp (CmpL op1 op2));
11572   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
11573             || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
11574   effect(USE labl);
11575 
11576   ins_cost(BRANCH_COST);
11577   format %{ "cb$cmp   $op1, $labl" %}
11578   ins_encode %{
11579     Label* L = $labl$$label;
11580     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
11581     if (cond == Assembler::EQ)
11582       __ cbz($op1$$Register, *L);
11583     else
11584       __ cbnz($op1$$Register, *L);
11585   %}
11586   ins_pipe(pipe_cmp_branch);
11587 %}
11588 
11589 instruct cmpP_imm0_branch(cmpOp cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
11590   match(If cmp (CmpP op1 op2));
11591   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
11592             || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
11593   effect(USE labl);
11594 
11595   ins_cost(BRANCH_COST);
11596   format %{ "cb$cmp   $op1, $labl" %}
11597   ins_encode %{
11598     Label* L = $labl$$label;
11599     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
11600     if (cond == Assembler::EQ)
11601       __ cbz($op1$$Register, *L);
11602     else
11603       __ cbnz($op1$$Register, *L);
11604   %}
11605   ins_pipe(pipe_cmp_branch);
11606 %}
11607 
11608 // Conditional Far Branch
11609 // Conditional Far Branch Unsigned
11610 // TODO: fixme
11611 
11612 // counted loop end branch near
11613 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
11614 %{
11615   match(CountedLoopEnd cmp cr);
11616 
11617   effect(USE lbl);
11618 
11619   ins_cost(BRANCH_COST);
11620   // short variant.
11621   // ins_short_branch(1);
11622   format %{ "b$cmp $lbl \t// counted loop end" %}
11623 
11624   ins_encode(aarch64_enc_br_con(cmp, lbl));
11625 
11626   ins_pipe(pipe_branch);
11627 %}
11628 
11629 // counted loop end branch near Unsigned
11630 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
11631 %{
11632   match(CountedLoopEnd cmp cr);
11633 
11634   effect(USE lbl);
11635 
11636   ins_cost(BRANCH_COST);
11637   // short variant.
11638   // ins_short_branch(1);
11639   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
11640 
11641   ins_encode(aarch64_enc_br_conU(cmp, lbl));
11642 
11643   ins_pipe(pipe_branch);
11644 %}
11645 
11646 // counted loop end branch far
11647 // counted loop end branch far unsigned
11648 // TODO: fixme
11649 
11650 // ============================================================================
11651 // inlined locking and unlocking
11652 
11653 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
11654 %{
11655   match(Set cr (FastLock object box));
11656   effect(TEMP tmp, TEMP tmp2);
11657 
11658   // TODO
11659   // identify correct cost
11660   ins_cost(5 * INSN_COST);
11661   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
11662 
11663   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
11664 
11665   ins_pipe(pipe_serial);
11666 %}
11667 
11668 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
11669 %{
11670   match(Set cr (FastUnlock object box));
11671   effect(TEMP tmp, TEMP tmp2);
11672 
11673   ins_cost(5 * INSN_COST);
11674   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
11675 
11676   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
11677 
11678   ins_pipe(pipe_serial);
11679 %}
11680 
11681 
11682 // ============================================================================
11683 // Safepoint Instructions
11684 
11685 // TODO
11686 // provide a near and far version of this code
11687 
11688 instruct safePoint(iRegP poll)
11689 %{
11690   match(SafePoint poll);
11691 
11692   format %{
11693     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
11694   %}
11695   ins_encode %{
11696     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
11697   %}
11698   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
11699 %}
11700 
11701 
11702 // ============================================================================
11703 // Procedure Call/Return Instructions
11704 
11705 // Call Java Static Instruction
11706 
11707 instruct CallStaticJavaDirect(method meth)
11708 %{
11709   match(CallStaticJava);
11710 
11711   effect(USE meth);
11712 
11713   predicate(!((CallStaticJavaNode*)n)->is_method_handle_invoke());
11714 
11715   ins_cost(CALL_COST);
11716 
11717   format %{ "call,static $meth \t// ==> " %}
11718 
11719   ins_encode( aarch64_enc_java_static_call(meth),
11720               aarch64_enc_call_epilog );
11721 
11722   ins_pipe(pipe_class_call);
11723 %}
11724 
11725 // TO HERE
11726 
11727 // Call Java Static Instruction (method handle version)
11728 
11729 instruct CallStaticJavaDirectHandle(method meth, iRegP_FP reg_mh_save)
11730 %{
11731   match(CallStaticJava);
11732 
11733   effect(USE meth);
11734 
11735   predicate(((CallStaticJavaNode*)n)->is_method_handle_invoke());
11736 
11737   ins_cost(CALL_COST);
11738 
11739   format %{ "call,static $meth \t// (methodhandle) ==> " %}
11740 
11741   ins_encode( aarch64_enc_java_handle_call(meth),
11742               aarch64_enc_call_epilog );
11743 
11744   ins_pipe(pipe_class_call);
11745 %}
11746 
11747 // Call Java Dynamic Instruction
11748 instruct CallDynamicJavaDirect(method meth)
11749 %{
11750   match(CallDynamicJava);
11751 
11752   effect(USE meth);
11753 
11754   ins_cost(CALL_COST);
11755 
11756   format %{ "CALL,dynamic $meth \t// ==> " %}
11757 
11758   ins_encode( aarch64_enc_java_dynamic_call(meth),
11759                aarch64_enc_call_epilog );
11760 
11761   ins_pipe(pipe_class_call);
11762 %}
11763 
11764 // Call Runtime Instruction
11765 
11766 instruct CallRuntimeDirect(method meth)
11767 %{
11768   match(CallRuntime);
11769 
11770   effect(USE meth);
11771 
11772   ins_cost(CALL_COST);
11773 
11774   format %{ "CALL, runtime $meth" %}
11775 
11776   ins_encode( aarch64_enc_java_to_runtime(meth) );
11777 
11778   ins_pipe(pipe_class_call);
11779 %}
11780 
11781 // Call Runtime Instruction
11782 
11783 instruct CallLeafDirect(method meth)
11784 %{
11785   match(CallLeaf);
11786 
11787   effect(USE meth);
11788 
11789   ins_cost(CALL_COST);
11790 
11791   format %{ "CALL, runtime leaf $meth" %}
11792 
11793   ins_encode( aarch64_enc_java_to_runtime(meth) );
11794 
11795   ins_pipe(pipe_class_call);
11796 %}
11797 
11798 // Call Runtime Instruction
11799 
11800 instruct CallLeafNoFPDirect(method meth)
11801 %{
11802   match(CallLeafNoFP);
11803 
11804   effect(USE meth);
11805 
11806   ins_cost(CALL_COST);
11807 
11808   format %{ "CALL, runtime leaf nofp $meth" %}
11809 
11810   ins_encode( aarch64_enc_java_to_runtime(meth) );
11811 
11812   ins_pipe(pipe_class_call);
11813 %}
11814 
11815 // Tail Call; Jump from runtime stub to Java code.
11816 // Also known as an 'interprocedural jump'.
11817 // Target of jump will eventually return to caller.
11818 // TailJump below removes the return address.
11819 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop)
11820 %{
11821   match(TailCall jump_target method_oop);
11822 
11823   ins_cost(CALL_COST);
11824 
11825   format %{ "br $jump_target\t# $method_oop holds method oop" %}
11826 
11827   ins_encode(aarch64_enc_tail_call(jump_target));
11828 
11829   ins_pipe(pipe_class_call);
11830 %}
11831 
11832 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
11833 %{
11834   match(TailJump jump_target ex_oop);
11835 
11836   ins_cost(CALL_COST);
11837 
11838   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
11839 
11840   ins_encode(aarch64_enc_tail_jmp(jump_target));
11841 
11842   ins_pipe(pipe_class_call);
11843 %}
11844 
11845 // Create exception oop: created by stack-crawling runtime code.
11846 // Created exception is now available to this handler, and is setup
11847 // just prior to jumping to this handler. No code emitted.
11848 // TODO check
11849 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
11850 instruct CreateException(iRegP_R0 ex_oop)
11851 %{
11852   match(Set ex_oop (CreateEx));
11853 
11854   format %{ " -- \t// exception oop; no code emitted" %}
11855 
11856   size(0);
11857 
11858   ins_encode( /*empty*/ );
11859 
11860   ins_pipe(pipe_class_empty);
11861 %}
11862 
11863 // Rethrow exception: The exception oop will come in the first
11864 // argument position. Then JUMP (not call) to the rethrow stub code.
11865 instruct RethrowException() %{
11866   match(Rethrow);
11867   ins_cost(CALL_COST);
11868 
11869   format %{ "b rethrow_stub" %}
11870 
11871   ins_encode( aarch64_enc_rethrow() );
11872 
11873   ins_pipe(pipe_class_call);
11874 %}
11875 
11876 
11877 // Return Instruction
11878 // epilog node loads ret address into lr as part of frame pop
11879 instruct Ret()
11880 %{
11881   match(Return);
11882 
11883   format %{ "ret\t// return register" %}
11884 
11885   ins_encode( aarch64_enc_ret() );
11886 
11887   ins_pipe(pipe_branch);
11888 %}
11889 
11890 // Die now.
11891 instruct ShouldNotReachHere() %{
11892   match(Halt);
11893 
11894   ins_cost(CALL_COST);
11895   format %{ "ShouldNotReachHere" %}
11896 
11897   ins_encode %{
11898     // TODO
11899     // implement proper trap call here
11900     __ brk(999);
11901   %}
11902 
11903   ins_pipe(pipe_class_default);
11904 %}
11905 
11906 // ============================================================================
11907 // Partial Subtype Check
11908 //
11909 // superklass array for an instance of the superklass.  Set a hidden
11910 // internal cache on a hit (cache is checked with exposed code in
11911 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11912 // encoding ALSO sets flags.
11913 
11914 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
11915 %{
11916   match(Set result (PartialSubtypeCheck sub super));
11917   effect(KILL cr, KILL temp);
11918 
11919   ins_cost(1100);  // slightly larger than the next version
11920   format %{ "partialSubtypeCheck $result, $sub, $super" %}
11921 
11922   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
11923 
11924   opcode(0x1); // Force zero of result reg on hit
11925 
11926   ins_pipe(pipe_class_memory);
11927 %}
11928 
11929 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
11930 %{
11931   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11932   effect(KILL temp, KILL result);
11933 
11934   ins_cost(1100);  // slightly larger than the next version
11935   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
11936 
11937   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
11938 
11939   opcode(0x0); // Don't zero result reg on hit
11940 
11941   ins_pipe(pipe_class_memory);
11942 %}
11943 
11944 instruct string_compare(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
11945                         iRegI_R0 result, iRegP_R10 tmp1, rFlagsReg cr)
11946 %{
11947   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11948   effect(KILL tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11949 
11950   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
11951   ins_encode %{
11952     __ string_compare($str1$$Register, $str2$$Register,
11953                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11954                       $tmp1$$Register);
11955   %}
11956   ins_pipe(pipe_class_memory);
11957 %}
11958 
11959 instruct string_indexof(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
11960        iRegI_R0 result, iRegI tmp1, iRegI tmp2, iRegI tmp3, iRegI tmp4, rFlagsReg cr)
11961 %{
11962   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11963   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
11964          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
11965   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result" %}
11966 
11967   ins_encode %{
11968     __ string_indexof($str1$$Register, $str2$$Register,
11969                       $cnt1$$Register, $cnt2$$Register,
11970                       $tmp1$$Register, $tmp2$$Register,
11971                       $tmp3$$Register, $tmp4$$Register,
11972                       -1, $result$$Register);
11973   %}
11974   ins_pipe(pipe_class_memory);
11975 %}
11976 
11977 instruct string_indexof_con(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
11978                  immI_le_4 int_cnt2, iRegI_R0 result, iRegI tmp1, iRegI tmp2,
11979                  iRegI tmp3, iRegI tmp4, rFlagsReg cr)
11980 %{
11981   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11982   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
11983          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
11984   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result" %}
11985 
11986   ins_encode %{
11987     int icnt2 = (int)$int_cnt2$$constant;
11988     __ string_indexof($str1$$Register, $str2$$Register,
11989                       $cnt1$$Register, zr,
11990                       $tmp1$$Register, $tmp2$$Register,
11991                       $tmp3$$Register, $tmp4$$Register,
11992                       icnt2, $result$$Register);
11993   %}
11994   ins_pipe(pipe_class_memory);
11995 %}
11996 
11997 instruct string_equals(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
11998                         iRegI_R0 result, iRegP_R10 tmp, rFlagsReg cr)
11999 %{
12000   match(Set result (StrEquals (Binary str1 str2) cnt));
12001   effect(KILL tmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
12002 
12003   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp" %}
12004   ins_encode %{
12005     __ string_equals($str1$$Register, $str2$$Register,
12006                       $cnt$$Register, $result$$Register,
12007                       $tmp$$Register);
12008   %}
12009   ins_pipe(pipe_class_memory);
12010 %}
12011 
12012 instruct array_equals(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
12013                       iRegP_R10 tmp, rFlagsReg cr)
12014 %{
12015   match(Set result (AryEq ary1 ary2));
12016   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, KILL cr);
12017 
12018   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
12019   ins_encode %{
12020     __ char_arrays_equals($ary1$$Register, $ary2$$Register,
12021                           $result$$Register, $tmp$$Register);
12022   %}
12023   ins_pipe(pipe_class_memory);
12024 %}
12025 
12026 // encode char[] to byte[] in ISO_8859_1
12027 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
12028                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
12029                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
12030                           iRegI_R0 result, rFlagsReg cr)
12031 %{
12032   match(Set result (EncodeISOArray src (Binary dst len)));
12033   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
12034          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
12035 
12036   format %{ "Encode array $src,$dst,$len -> $result" %}
12037   ins_encode %{
12038     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12039          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
12040          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
12041   %}
12042   ins_pipe( pipe_class_memory );
12043 %}
12044 
12045 // ============================================================================
12046 // This name is KNOWN by the ADLC and cannot be changed.
12047 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12048 // for this guy.
12049 instruct tlsLoadP(thread_RegP dst)
12050 %{
12051   match(Set dst (ThreadLocal));
12052 
12053   ins_cost(0);
12054 
12055   format %{ " -- \t// $dst=Thread::current(), empty" %}
12056 
12057   size(0);
12058 
12059   ins_encode( /*empty*/ );
12060 
12061   ins_pipe(pipe_class_empty);
12062 %}
12063 
12064 
12065 
12066 //----------PEEPHOLE RULES-----------------------------------------------------
12067 // These must follow all instruction definitions as they use the names
12068 // defined in the instructions definitions.
12069 //
12070 // peepmatch ( root_instr_name [preceding_instruction]* );
12071 //
12072 // peepconstraint %{
12073 // (instruction_number.operand_name relational_op instruction_number.operand_name
12074 //  [, ...] );
12075 // // instruction numbers are zero-based using left to right order in peepmatch
12076 //
12077 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12078 // // provide an instruction_number.operand_name for each operand that appears
12079 // // in the replacement instruction's match rule
12080 //
12081 // ---------VM FLAGS---------------------------------------------------------
12082 //
12083 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12084 //
12085 // Each peephole rule is given an identifying number starting with zero and
12086 // increasing by one in the order seen by the parser.  An individual peephole
12087 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12088 // on the command-line.
12089 //
12090 // ---------CURRENT LIMITATIONS----------------------------------------------
12091 //
12092 // Only match adjacent instructions in same basic block
12093 // Only equality constraints
12094 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12095 // Only one replacement instruction
12096 //
12097 // ---------EXAMPLE----------------------------------------------------------
12098 //
12099 // // pertinent parts of existing instructions in architecture description
12100 // instruct movI(iRegINoSp dst, iRegI src)
12101 // %{
12102 //   match(Set dst (CopyI src));
12103 // %}
12104 //
12105 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
12106 // %{
12107 //   match(Set dst (AddI dst src));
12108 //   effect(KILL cr);
12109 // %}
12110 //
12111 // // Change (inc mov) to lea
12112 // peephole %{
12113 //   // increment preceeded by register-register move
12114 //   peepmatch ( incI_iReg movI );
12115 //   // require that the destination register of the increment
12116 //   // match the destination register of the move
12117 //   peepconstraint ( 0.dst == 1.dst );
12118 //   // construct a replacement instruction that sets
12119 //   // the destination to ( move's source register + one )
12120 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
12121 // %}
12122 //
12123 
12124 // Implementation no longer uses movX instructions since
12125 // machine-independent system no longer uses CopyX nodes.
12126 //
12127 // peephole
12128 // %{
12129 //   peepmatch (incI_iReg movI);
12130 //   peepconstraint (0.dst == 1.dst);
12131 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
12132 // %}
12133 
12134 // peephole
12135 // %{
12136 //   peepmatch (decI_iReg movI);
12137 //   peepconstraint (0.dst == 1.dst);
12138 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
12139 // %}
12140 
12141 // peephole
12142 // %{
12143 //   peepmatch (addI_iReg_imm movI);
12144 //   peepconstraint (0.dst == 1.dst);
12145 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
12146 // %}
12147 
12148 // peephole
12149 // %{
12150 //   peepmatch (incL_iReg movL);
12151 //   peepconstraint (0.dst == 1.dst);
12152 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
12153 // %}
12154 
12155 // peephole
12156 // %{
12157 //   peepmatch (decL_iReg movL);
12158 //   peepconstraint (0.dst == 1.dst);
12159 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
12160 // %}
12161 
12162 // peephole
12163 // %{
12164 //   peepmatch (addL_iReg_imm movL);
12165 //   peepconstraint (0.dst == 1.dst);
12166 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
12167 // %}
12168 
12169 // peephole
12170 // %{
12171 //   peepmatch (addP_iReg_imm movP);
12172 //   peepconstraint (0.dst == 1.dst);
12173 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
12174 // %}
12175 
12176 // // Change load of spilled value to only a spill
12177 // instruct storeI(memory mem, iRegI src)
12178 // %{
12179 //   match(Set mem (StoreI mem src));
12180 // %}
12181 //
12182 // instruct loadI(iRegINoSp dst, memory mem)
12183 // %{
12184 //   match(Set dst (LoadI mem));
12185 // %}
12186 //
12187 
12188 //----------SMARTSPILL RULES---------------------------------------------------
12189 // These must follow all instruction definitions as they use the names
12190 // defined in the instructions definitions.
12191 
12192 // Local Variables:
12193 // mode: c++
12194 // End: