1 //
   2 // Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 // AArch64 Architecture Description File
  27 
  28 //----------REGISTER DEFINITION BLOCK------------------------------------------
  29 // This information is used by the matcher and the register allocator to
  30 // describe individual registers and classes of registers within the target
  31 // archtecture.
  32 
  33 register %{
  34 //----------Architecture Description Register Definitions----------------------
  35 // General Registers
  36 // "reg_def"  name ( register save type, C convention save type,
  37 //                   ideal register type, encoding );
  38 // Register Save Types:
  39 //
  40 // NS  = No-Save:       The register allocator assumes that these registers
  41 //                      can be used without saving upon entry to the method, &
  42 //                      that they do not need to be saved at call sites.
  43 //
  44 // SOC = Save-On-Call:  The register allocator assumes that these registers
  45 //                      can be used without saving upon entry to the method,
  46 //                      but that they must be saved at call sites.
  47 //
  48 // SOE = Save-On-Entry: The register allocator assumes that these registers
  49 //                      must be saved before using them upon entry to the
  50 //                      method, but they do not need to be saved at call
  51 //                      sites.
  52 //
  53 // AS  = Always-Save:   The register allocator assumes that these registers
  54 //                      must be saved before using them upon entry to the
  55 //                      method, & that they must be saved at call sites.
  56 //
  57 // Ideal Register Type is used to determine how to save & restore a
  58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  60 //
  61 // The encoding number is the actual bit-pattern placed into the opcodes.
  62 
  63 // We must define the 64 bit int registers in two 32 bit halves, the
  64 // real lower register and a virtual upper half register. upper halves
  65 // are used by the register allocator but are not actually supplied as
  66 // operands to memory ops.
  67 //
  68 // follow the C1 compiler in making registers
  69 //
  70 //   r0-r7,r10-r26 volatile (caller save)
  71 //   r27-r32 system (no save, no allocate)
  72 //   r8-r9 invisible to the allocator (so we can use them as scratch regs)
  73 //
  74 // as regards Java usage. we don't use any callee save registers
  75 // because this makes it difficult to de-optimise a frame (see comment
  76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
  77 //
  78 
  79 // General Registers
  80 
  81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
  82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
  83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
  84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
  85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
  86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
  87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
  88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
  89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
  90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
  91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
  92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
  93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
  94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
  95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
  96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
  97 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  98 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  99 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
 100 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 101 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
 102 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
 103 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
 104 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
 105 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
 106 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
 107 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
 108 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
 109 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
 110 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
 111 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
 112 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
 113 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()        );
 114 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
 115 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
 116 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
 117 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
 118 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
 119 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
 120 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
 121 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
 122 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
 123 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
 124 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
 125 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
 126 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
 127 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
 128 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
 129 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
 130 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
 131 reg_def R27     (  NS, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
 132 reg_def R27_H   (  NS, SOE, Op_RegI, 27, r27->as_VMReg()->next());
 133 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
 134 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
 135 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
 136 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
 137 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
 138 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
 139 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
 140 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
 141 
 142 // ----------------------------
 143 // Float/Double Registers
 144 // ----------------------------
 145 
 146 // Double Registers
 147 
 148 // The rules of ADL require that double registers be defined in pairs.
 149 // Each pair must be two 32-bit values, but not necessarily a pair of
 150 // single float registers. In each pair, ADLC-assigned register numbers
 151 // must be adjacent, with the lower number even. Finally, when the
 152 // CPU stores such a register pair to memory, the word associated with
 153 // the lower ADLC-assigned number must be stored to the lower address.
 154 
 155 // AArch64 has 32 floating-point registers. Each can store a vector of
 156 // single or double precision floating-point values up to 8 * 32
 157 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
 158 // use the first float or double element of the vector.
 159 
 160 // for Java use float registers v0-v15 are always save on call whereas
 161 // the platform ABI treats v8-v15 as callee save). float registers
 162 // v16-v31 are SOC as per the platform spec
 163 
 164   reg_def V0   ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()         );
 165   reg_def V0_H ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next() );
 166   reg_def V1   ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()         );
 167   reg_def V1_H ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next() );
 168   reg_def V2   ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()         );
 169   reg_def V2_H ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next() );
 170   reg_def V3   ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()         );
 171   reg_def V3_H ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next() );
 172   reg_def V4   ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()         );
 173   reg_def V4_H ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next() );
 174   reg_def V5   ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()         );
 175   reg_def V5_H ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next() );
 176   reg_def V6   ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()         );
 177   reg_def V6_H ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next() );
 178   reg_def V7   ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()         );
 179   reg_def V7_H ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next() );
 180   reg_def V8   ( SOC, SOE, Op_RegF,  8, v8->as_VMReg()         );
 181   reg_def V8_H ( SOC, SOE, Op_RegF,  8, v8->as_VMReg()->next() );
 182   reg_def V9   ( SOC, SOE, Op_RegF,  9, v9->as_VMReg()         );
 183   reg_def V9_H ( SOC, SOE, Op_RegF,  9, v9->as_VMReg()->next() );
 184   reg_def V10  ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()        );
 185   reg_def V10_H( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next());
 186   reg_def V11  ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()        );
 187   reg_def V11_H( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next());
 188   reg_def V12  ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()        );
 189   reg_def V12_H( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next());
 190   reg_def V13  ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()        );
 191   reg_def V13_H( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next());
 192   reg_def V14  ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()        );
 193   reg_def V14_H( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next());
 194   reg_def V15  ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()        );
 195   reg_def V15_H( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next());
 196   reg_def V16  ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()        );
 197   reg_def V16_H( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next());
 198   reg_def V17  ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()        );
 199   reg_def V17_H( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next());
 200   reg_def V18  ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()        );
 201   reg_def V18_H( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next());
 202   reg_def V19  ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()        );
 203   reg_def V19_H( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next());
 204   reg_def V20  ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()        );
 205   reg_def V20_H( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next());
 206   reg_def V21  ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()        );
 207   reg_def V21_H( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next());
 208   reg_def V22  ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()        );
 209   reg_def V22_H( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next());
 210   reg_def V23  ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()        );
 211   reg_def V23_H( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next());
 212   reg_def V24  ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()        );
 213   reg_def V24_H( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next());
 214   reg_def V25  ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()        );
 215   reg_def V25_H( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next());
 216   reg_def V26  ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()        );
 217   reg_def V26_H( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next());
 218   reg_def V27  ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()        );
 219   reg_def V27_H( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next());
 220   reg_def V28  ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()        );
 221   reg_def V28_H( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next());
 222   reg_def V29  ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()        );
 223   reg_def V29_H( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next());
 224   reg_def V30  ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()        );
 225   reg_def V30_H( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next());
 226   reg_def V31  ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()        );
 227   reg_def V31_H( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next());
 228 
 229 // ----------------------------
 230 // Special Registers
 231 // ----------------------------
 232 
 233 // the AArch64 CSPR status flag register is not directly acessible as
 234 // instruction operand. the FPSR status flag register is a system
 235 // register which can be written/read using MSR/MRS but again does not
 236 // appear as an operand (a code identifying the FSPR occurs as an
 237 // immediate value in the instruction).
 238 
 239 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
 240 
 241 
 242 // Specify priority of register selection within phases of register
 243 // allocation.  Highest priority is first.  A useful heuristic is to
 244 // give registers a low priority when they are required by machine
 245 // instructions, like EAX and EDX on I486, and choose no-save registers
 246 // before save-on-call, & save-on-call before save-on-entry.  Registers
 247 // which participate in fixed calling sequences should come last.
 248 // Registers which are used as pairs must fall on an even boundary.
 249 
 250 alloc_class chunk0(
 251     // volatiles
 252     R10, R10_H,
 253     R11, R11_H,
 254     R12, R12_H,
 255     R13, R13_H,
 256     R14, R14_H,
 257     R15, R15_H,
 258     R16, R16_H,
 259     R17, R17_H,
 260     R18, R18_H,
 261 
 262     // arg registers
 263     R0, R0_H,
 264     R1, R1_H,
 265     R2, R2_H,
 266     R3, R3_H,
 267     R4, R4_H,
 268     R5, R5_H,
 269     R6, R6_H,
 270     R7, R7_H,
 271 
 272     // non-volatiles
 273     R19, R19_H,
 274     R20, R20_H,
 275     R21, R21_H,
 276     R22, R22_H,
 277     R23, R23_H,
 278     R24, R24_H,
 279     R25, R25_H,
 280     R26, R26_H,
 281 
 282     // non-allocatable registers
 283 
 284     R27, R27_H, // heapbase
 285     R28, R28_H, // thread
 286     R29, R29_H, // fp
 287     R30, R30_H, // lr
 288     R31, R31_H, // sp
 289 );
 290 
 291 alloc_class chunk1(
 292 
 293     // no save
 294     V16, V16_H,
 295     V17, V17_H,
 296     V18, V18_H,
 297     V19, V19_H,
 298     V20, V20_H,
 299     V21, V21_H,
 300     V22, V22_H,
 301     V23, V23_H,
 302     V24, V24_H,
 303     V25, V25_H,
 304     V26, V26_H,
 305     V27, V27_H,
 306     V28, V28_H,
 307     V29, V29_H,
 308     V30, V30_H,
 309     V31, V31_H,
 310 
 311     // arg registers
 312     V0, V0_H,
 313     V1, V1_H,
 314     V2, V2_H,
 315     V3, V3_H,
 316     V4, V4_H,
 317     V5, V5_H,
 318     V6, V6_H,
 319     V7, V7_H,
 320 
 321     // non-volatiles
 322     V8, V8_H,
 323     V9, V9_H,
 324     V10, V10_H,
 325     V11, V11_H,
 326     V12, V12_H,
 327     V13, V13_H,
 328     V14, V14_H,
 329     V15, V15_H,
 330 );
 331 
 332 alloc_class chunk2(RFLAGS);
 333 
 334 //----------Architecture Description Register Classes--------------------------
 335 // Several register classes are automatically defined based upon information in
 336 // this architecture description.
 337 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 338 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 339 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 340 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 341 //
 342 
 343 // Class for all 32 bit integer registers -- excludes SP which will
 344 // never be used as an integer register
 345 reg_class any_reg32(
 346     R0,
 347     R1,
 348     R2,
 349     R3,
 350     R4,
 351     R5,
 352     R6,
 353     R7,
 354     R10,
 355     R11,
 356     R12,
 357     R13,
 358     R14,
 359     R15,
 360     R16,
 361     R17,
 362     R18,
 363     R19,
 364     R20,
 365     R21,
 366     R22,
 367     R23,
 368     R24,
 369     R25,
 370     R26,
 371     R27,
 372     R28,
 373     R29,
 374     R30
 375 );
 376 
 377 // Singleton class for R0 int register
 378 reg_class int_r0_reg(R0);
 379 
 380 // Singleton class for R2 int register
 381 reg_class int_r2_reg(R2);
 382 
 383 // Singleton class for R3 int register
 384 reg_class int_r3_reg(R3);
 385 
 386 // Singleton class for R4 int register
 387 reg_class int_r4_reg(R4);
 388 
 389 // Class for all long integer registers (including RSP)
 390 reg_class any_reg(
 391     R0, R0_H,
 392     R1, R1_H,
 393     R2, R2_H,
 394     R3, R3_H,
 395     R4, R4_H,
 396     R5, R5_H,
 397     R6, R6_H,
 398     R7, R7_H,
 399     R10, R10_H,
 400     R11, R11_H,
 401     R12, R12_H,
 402     R13, R13_H,
 403     R14, R14_H,
 404     R15, R15_H,
 405     R16, R16_H,
 406     R17, R17_H,
 407     R18, R18_H,
 408     R19, R19_H,
 409     R20, R20_H,
 410     R21, R21_H,
 411     R22, R22_H,
 412     R23, R23_H,
 413     R24, R24_H,
 414     R25, R25_H,
 415     R26, R26_H,
 416     R27, R27_H,
 417     R28, R28_H,
 418     R29, R29_H,
 419     R30, R30_H,
 420     R31, R31_H
 421 );
 422 
 423 // Class for all non-special integer registers
 424 reg_class no_special_reg32(
 425     R0,
 426     R1,
 427     R2,
 428     R3,
 429     R4,
 430     R5,
 431     R6,
 432     R7,
 433     R10,
 434     R11,
 435     R12,                        // rmethod
 436     R13,
 437     R14,
 438     R15,
 439     R16,
 440     R17,
 441     R18,
 442     R19,
 443     R20,
 444     R21,
 445     R22,
 446     R23,
 447     R24,
 448     R25,
 449     R26
 450  /* R27, */                     // heapbase
 451  /* R28, */                     // thread
 452     R29,                        // fp
 453  /* R30, */                     // lr
 454  /* R31 */                      // sp
 455 );
 456 
 457 // Class for all non-special long integer registers
 458 reg_class no_special_reg(
 459     R0, R0_H,
 460     R1, R1_H,
 461     R2, R2_H,
 462     R3, R3_H,
 463     R4, R4_H,
 464     R5, R5_H,
 465     R6, R6_H,
 466     R7, R7_H,
 467     R10, R10_H,
 468     R11, R11_H,
 469     R12, R12_H,                 // rmethod
 470     R13, R13_H,
 471     R14, R14_H,
 472     R15, R15_H,
 473     R16, R16_H,
 474     R17, R17_H,
 475     R18, R18_H,
 476     R19, R19_H,
 477     R20, R20_H,
 478     R21, R21_H,
 479     R22, R22_H,
 480     R23, R23_H,
 481     R24, R24_H,
 482     R25, R25_H,
 483     R26, R26_H,
 484  /* R27, R27_H, */              // heapbase
 485  /* R28, R28_H, */              // thread
 486     R29, R29_H,                 // fp
 487  /* R30, R30_H, */              // lr
 488  /* R31, R31_H */               // sp
 489 );
 490 
 491 // Class for 64 bit register r0
 492 reg_class r0_reg(
 493     R0, R0_H
 494 );
 495 
 496 // Class for 64 bit register r1
 497 reg_class r1_reg(
 498     R1, R1_H
 499 );
 500 
 501 // Class for 64 bit register r2
 502 reg_class r2_reg(
 503     R2, R2_H
 504 );
 505 
 506 // Class for 64 bit register r3
 507 reg_class r3_reg(
 508     R3, R3_H
 509 );
 510 
 511 // Class for 64 bit register r4
 512 reg_class r4_reg(
 513     R4, R4_H
 514 );
 515 
 516 // Class for 64 bit register r5
 517 reg_class r5_reg(
 518     R5, R5_H
 519 );
 520 
 521 // Class for 64 bit register r10
 522 reg_class r10_reg(
 523     R10, R10_H
 524 );
 525 
 526 // Class for 64 bit register r11
 527 reg_class r11_reg(
 528     R11, R11_H
 529 );
 530 
 531 // Class for method register
 532 reg_class method_reg(
 533     R12, R12_H
 534 );
 535 
 536 // Class for heapbase register
 537 reg_class heapbase_reg(
 538     R27, R27_H
 539 );
 540 
 541 // Class for thread register
 542 reg_class thread_reg(
 543     R28, R28_H
 544 );
 545 
 546 // Class for frame pointer register
 547 reg_class fp_reg(
 548     R29, R29_H
 549 );
 550 
 551 // Class for link register
 552 reg_class lr_reg(
 553     R30, R30_H
 554 );
 555 
 556 // Class for long sp register
 557 reg_class sp_reg(
 558   R31, R31_H
 559 );
 560 
 561 // Class for all pointer registers
 562 reg_class ptr_reg(
 563     R0, R0_H,
 564     R1, R1_H,
 565     R2, R2_H,
 566     R3, R3_H,
 567     R4, R4_H,
 568     R5, R5_H,
 569     R6, R6_H,
 570     R7, R7_H,
 571     R10, R10_H,
 572     R11, R11_H,
 573     R12, R12_H,
 574     R13, R13_H,
 575     R14, R14_H,
 576     R15, R15_H,
 577     R16, R16_H,
 578     R17, R17_H,
 579     R18, R18_H,
 580     R19, R19_H,
 581     R20, R20_H,
 582     R21, R21_H,
 583     R22, R22_H,
 584     R23, R23_H,
 585     R24, R24_H,
 586     R25, R25_H,
 587     R26, R26_H,
 588     R27, R27_H,
 589     R28, R28_H,
 590     R29, R29_H,
 591     R30, R30_H,
 592     R31, R31_H
 593 );
 594 
 595 // Class for all non_special pointer registers
 596 reg_class no_special_ptr_reg(
 597     R0, R0_H,
 598     R1, R1_H,
 599     R2, R2_H,
 600     R3, R3_H,
 601     R4, R4_H,
 602     R5, R5_H,
 603     R6, R6_H,
 604     R7, R7_H,
 605     R10, R10_H,
 606     R11, R11_H,
 607     R12, R12_H,
 608     R13, R13_H,
 609     R14, R14_H,
 610     R15, R15_H,
 611     R16, R16_H,
 612     R17, R17_H,
 613     R18, R18_H,
 614     R19, R19_H,
 615     R20, R20_H,
 616     R21, R21_H,
 617     R22, R22_H,
 618     R23, R23_H,
 619     R24, R24_H,
 620     R25, R25_H,
 621     R26, R26_H,
 622  /* R27, R27_H, */              // heapbase
 623  /* R28, R28_H, */              // thread
 624  /* R29, R29_H, */              // fp
 625  /* R30, R30_H, */              // lr
 626  /* R31, R31_H */               // sp
 627 );
 628 
 629 // Class for all float registers
 630 reg_class float_reg(
 631     V0,
 632     V1,
 633     V2,
 634     V3,
 635     V4,
 636     V5,
 637     V6,
 638     V7,
 639     V8,
 640     V9,
 641     V10,
 642     V11,
 643     V12,
 644     V13,
 645     V14,
 646     V15,
 647     V16,
 648     V17,
 649     V18,
 650     V19,
 651     V20,
 652     V21,
 653     V22,
 654     V23,
 655     V24,
 656     V25,
 657     V26,
 658     V27,
 659     V28,
 660     V29,
 661     V30,
 662     V31
 663 );
 664 
 665 // Double precision float registers have virtual `high halves' that
 666 // are needed by the allocator.
 667 // Class for all double registers
 668 reg_class double_reg(
 669     V0, V0_H,
 670     V1, V1_H,
 671     V2, V2_H,
 672     V3, V3_H,
 673     V4, V4_H,
 674     V5, V5_H,
 675     V6, V6_H,
 676     V7, V7_H,
 677     V8, V8_H,
 678     V9, V9_H,
 679     V10, V10_H,
 680     V11, V11_H,
 681     V12, V12_H,
 682     V13, V13_H,
 683     V14, V14_H,
 684     V15, V15_H,
 685     V16, V16_H,
 686     V17, V17_H,
 687     V18, V18_H,
 688     V19, V19_H,
 689     V20, V20_H,
 690     V21, V21_H,
 691     V22, V22_H,
 692     V23, V23_H,
 693     V24, V24_H,
 694     V25, V25_H,
 695     V26, V26_H,
 696     V27, V27_H,
 697     V28, V28_H,
 698     V29, V29_H,
 699     V30, V30_H,
 700     V31, V31_H
 701 );
 702 
 703 // Class for 128 bit register v0
 704 reg_class v0_reg(
 705     V0, V0_H
 706 );
 707 
 708 // Class for 128 bit register v1
 709 reg_class v1_reg(
 710     V1, V1_H
 711 );
 712 
 713 // Class for 128 bit register v2
 714 reg_class v2_reg(
 715     V2, V2_H
 716 );
 717 
 718 // Class for 128 bit register v3
 719 reg_class v3_reg(
 720     V3, V3_H
 721 );
 722 
 723 // Singleton class for condition codes
 724 reg_class int_flags(RFLAGS);
 725 
 726 %}
 727 
 728 //----------DEFINITION BLOCK---------------------------------------------------
 729 // Define name --> value mappings to inform the ADLC of an integer valued name
 730 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 731 // Format:
 732 //        int_def  <name>         ( <int_value>, <expression>);
 733 // Generated Code in ad_<arch>.hpp
 734 //        #define  <name>   (<expression>)
 735 //        // value == <int_value>
 736 // Generated code in ad_<arch>.cpp adlc_verification()
 737 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 738 //
 739 
 740 // we follow the ppc-aix port in using a simple cost model which ranks
 741 // register operations as cheap, memory ops as more expensive and
 742 // branches as most expensive. the first two have a low as well as a
 743 // normal cost. huge cost appears to be a way of saying don't do
 744 // something
 745 
 746 definitions %{
 747   // The default cost (of a register move instruction).
 748   int_def INSN_COST            (    100,     100);
 749   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 750   int_def CALL_COST            (    200,     2 * INSN_COST);
 751   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 752 %}
 753 
 754 
 755 //----------SOURCE BLOCK-------------------------------------------------------
 756 // This is a block of C++ code which provides values, functions, and
 757 // definitions necessary in the rest of the architecture description
 758 
 759 source_hpp %{
 760 
 761 #include "memory/cardTableModRefBS.hpp"
 762 
 763 class CallStubImpl {
 764 
 765   //--------------------------------------------------------------
 766   //---<  Used for optimization in Compile::shorten_branches  >---
 767   //--------------------------------------------------------------
 768 
 769  public:
 770   // Size of call trampoline stub.
 771   static uint size_call_trampoline() {
 772     return 0; // no call trampolines on this platform
 773   }
 774 
 775   // number of relocations needed by a call trampoline stub
 776   static uint reloc_call_trampoline() {
 777     return 0; // no call trampolines on this platform
 778   }
 779 };
 780 
 781 class HandlerImpl {
 782 
 783  public:
 784 
 785   static int emit_exception_handler(CodeBuffer &cbuf);
 786   static int emit_deopt_handler(CodeBuffer& cbuf);
 787 
 788   static uint size_exception_handler() {
 789     return MacroAssembler::far_branch_size();
 790   }
 791 
 792   static uint size_deopt_handler() {
 793     // count one adr and one far branch instruction
 794     return 4 * NativeInstruction::instruction_size;
 795   }
 796 };
 797 
 798   // graph traversal helpers
 799   MemBarNode *has_parent_membar(const Node *n,
 800                                 ProjNode *&ctl, ProjNode *&mem);
 801   MemBarNode *has_child_membar(const MemBarNode *n,
 802                                ProjNode *&ctl, ProjNode *&mem);
 803 
 804   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 805   bool unnecessary_acquire(const Node *barrier);
 806   bool needs_acquiring_load(const Node *load);
 807 
 808   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 809   bool unnecessary_release(const Node *barrier);
 810   bool unnecessary_volatile(const Node *barrier);
 811   bool needs_releasing_store(const Node *store);
 812 
 813   // Use barrier instructions for unsafe volatile gets rather than
 814   // trying to identify an exact signature for them
 815   const bool UseBarriersForUnsafeVolatileGet = false;
 816 %}
 817 
 818 source %{
 819 
 820   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 821   // use to implement volatile reads and writes. For a volatile read
 822   // we simply need
 823   //
 824   //   ldar<x>
 825   //
 826   // and for a volatile write we need
 827   //
 828   //   stlr<x>
 829   // 
 830   // Alternatively, we can implement them by pairing a normal
 831   // load/store with a memory barrier. For a volatile read we need
 832   // 
 833   //   ldr<x>
 834   //   dmb ishld
 835   //
 836   // for a volatile write
 837   //
 838   //   dmb ish
 839   //   str<x>
 840   //   dmb ish
 841   //
 842   // In order to generate the desired instruction sequence we need to
 843   // be able to identify specific 'signature' ideal graph node
 844   // sequences which i) occur as a translation of a volatile reads or
 845   // writes and ii) do not occur through any other translation or
 846   // graph transformation. We can then provide alternative aldc
 847   // matching rules which translate these node sequences to the
 848   // desired machine code sequences. Selection of the alternative
 849   // rules can be implemented by predicates which identify the
 850   // relevant node sequences.
 851   //
 852   // The ideal graph generator translates a volatile read to the node
 853   // sequence
 854   //
 855   //   LoadX[mo_acquire]
 856   //   MemBarAcquire
 857   //
 858   // As a special case when using the compressed oops optimization we
 859   // may also see this variant
 860   //
 861   //   LoadN[mo_acquire]
 862   //   DecodeN
 863   //   MemBarAcquire
 864   //
 865   // A volatile write is translated to the node sequence
 866   //
 867   //   MemBarRelease
 868   //   StoreX[mo_release]
 869   //   MemBarVolatile
 870   //
 871   // n.b. the above node patterns are generated with a strict
 872   // 'signature' configuration of input and output dependencies (see
 873   // the predicates below for exact details). The two signatures are
 874   // unique to translated volatile reads/stores -- they will not
 875   // appear as a result of any other bytecode translation or inlining
 876   // nor as a consequence of optimizing transforms.
 877   //
 878   // We also want to catch inlined unsafe volatile gets and puts and
 879   // be able to implement them using either ldar<x>/stlr<x> or some
 880   // combination of ldr<x>/stlr<x> and dmb instructions.
 881   //
 882   // Inlined unsafe volatiles puts manifest as a minor variant of the
 883   // normal volatile put node sequence containing an extra cpuorder
 884   // membar
 885   //
 886   //   MemBarRelease
 887   //   MemBarCPUOrder
 888   //   StoreX[mo_release]
 889   //   MemBarVolatile
 890   //
 891   // n.b. as an aside, the cpuorder membar is not itself subject to
 892   // matching and translation by adlc rules.  However, the rule
 893   // predicates need to detect its presence in order to correctly
 894   // select the desired adlc rules.
 895   //
 896   // Inlined unsafe volatiles gets manifest as a somewhat different
 897   // node sequence to a normal volatile get
 898   //
 899   //   MemBarCPUOrder
 900   //        ||       \\
 901   //   MemBarAcquire LoadX[mo_acquire]
 902   //        ||
 903   //   MemBarCPUOrder
 904   //
 905   // In this case the acquire membar does not directly depend on the
 906   // load. However, we can be sure that the load is generated from an
 907   // inlined unsafe volatile get if we see it dependent on this unique
 908   // sequence of membar nodes. Similarly, given an acquire membar we
 909   // can know that it was added because of an inlined unsafe volatile
 910   // get if it is fed and feeds a cpuorder membar and if its feed
 911   // membar also feeds an acquiring load.
 912   //
 913   // So, where we can identify these volatile read and write
 914   // signatures we can choose to plant either of the above two code
 915   // sequences. For a volatile read we can simply plant a normal
 916   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 917   // also choose to inhibit translation of the MemBarAcquire and
 918   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 919   //
 920   // When we recognise a volatile store signature we can choose to
 921   // plant at a dmb ish as a translation for the MemBarRelease, a
 922   // normal str<x> and then a dmb ish for the MemBarVolatile.
 923   // Alternatively, we can inhibit translation of the MemBarRelease
 924   // and MemBarVolatile and instead plant a simple stlr<x>
 925   // instruction.
 926   //
 927   // Of course, the above only applies when we see these signature
 928   // configurations. We still want to plant dmb instructions in any
 929   // other cases where we may see a MemBarAcquire, MemBarRelease or
 930   // MemBarVolatile. For example, at the end of a constructor which
 931   // writes final/volatile fields we will see a MemBarRelease
 932   // instruction and this needs a 'dmb ish' lest we risk the
 933   // constructed object being visible without making the
 934   // final/volatile field writes visible.
 935   //
 936   // n.b. the translation rules below which rely on detection of the
 937   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 938   // If we see anything other than the signature configurations we
 939   // always just translate the loads and stors to ldr<x> and str<x>
 940   // and translate acquire, release and volatile membars to the
 941   // relevant dmb instructions.
 942   //
 943   // n.b.b as a case in point for the above comment, the current
 944   // predicates don't detect the precise signature for certain types
 945   // of volatile object stores (where the heap_base input type is not
 946   // known at compile-time to be non-NULL). In those cases the
 947   // MemBarRelease and MemBarVolatile bracket an if-then-else sequence
 948   // with a store in each branch (we need a different store depending
 949   // on whether heap_base is actually NULL). In such a case we will
 950   // just plant a dmb both before and after the branch/merge. The
 951   // predicate could (and probably should) be fixed later to also
 952   // detect this case.
 953 
 954   // graph traversal helpers
 955 
 956   // if node n is linked to a parent MemBarNode by an intervening
 957   // Control or Memory ProjNode return the MemBarNode otherwise return
 958   // NULL.
 959   //
 960   // n may only be a Load or a MemBar.
 961   //
 962   // The ProjNode* references c and m are used to return the relevant
 963   // nodes.
 964 
 965   MemBarNode *has_parent_membar(const Node *n, ProjNode *&c, ProjNode *&m)
 966   {
 967     Node *ctl = NULL;
 968     Node *mem = NULL;
 969     Node *membar = NULL;
 970 
 971     if (n->is_Load()) {
 972       ctl = n->lookup(LoadNode::Control);
 973       mem = n->lookup(LoadNode::Memory);
 974     } else if (n->is_MemBar()) {
 975       ctl = n->lookup(TypeFunc::Control);
 976       mem = n->lookup(TypeFunc::Memory);
 977     } else {
 978         return NULL;
 979     }
 980 
 981     if (!ctl || !mem || !ctl->is_Proj() || !mem->is_Proj())
 982       return NULL;
 983 
 984     c = ctl->as_Proj();
 985 
 986     membar = ctl->lookup(0);
 987 
 988     if (!membar || !membar->is_MemBar())
 989       return NULL;
 990 
 991     m = mem->as_Proj();
 992 
 993     if (mem->lookup(0) != membar)
 994       return NULL;
 995 
 996     return membar->as_MemBar();
 997   }
 998 
 999   // if n is linked to a child MemBarNode by intervening Control and
1000   // Memory ProjNodes return the MemBarNode otherwise return NULL.
1001   //
1002   // The ProjNode** arguments c and m are used to return pointers to
1003   // the relevant nodes. A null argument means don't don't return a
1004   // value.
1005 
1006   MemBarNode *has_child_membar(const MemBarNode *n, ProjNode *&c, ProjNode *&m)
1007   {
1008     ProjNode *ctl = n->proj_out(TypeFunc::Control);
1009     ProjNode *mem = n->proj_out(TypeFunc::Memory);
1010 
1011     // MemBar needs to have both a Ctl and Mem projection
1012     if (! ctl || ! mem)
1013       return NULL;
1014 
1015     c = ctl;
1016     m = mem;
1017 
1018     MemBarNode *child = NULL;
1019     Node *x;
1020 
1021     for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {
1022       x = ctl->fast_out(i);
1023       // if we see a membar we keep hold of it. we may also see a new
1024       // arena copy of the original but it will appear later
1025       if (x->is_MemBar()) {
1026           child = x->as_MemBar();
1027           break;
1028       }
1029     }
1030 
1031     if (child == NULL)
1032       return NULL;
1033 
1034     for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
1035       x = mem->fast_out(i);
1036       // if we see a membar we keep hold of it. we may also see a new
1037       // arena copy of the original but it will appear later
1038       if (x == child) {
1039         return child;
1040       }
1041     }
1042     return NULL;
1043   }
1044 
1045   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1046 
1047 bool unnecessary_acquire(const Node *barrier) {
1048   // assert barrier->is_MemBar();
1049   if (UseBarriersForVolatile)
1050     // we need to plant a dmb
1051     return false;
1052 
1053   // a volatile read derived from bytecode (or also from an inlined
1054   // SHA field read via LibraryCallKit::load_field_from_object)
1055   // manifests as a LoadX[mo_acquire] followed by an acquire membar
1056   // with a bogus read dependency on it's preceding load. so in those
1057   // cases we will find the load node at the PARMS offset of the
1058   // acquire membar.  n.b. there may be an intervening DecodeN node.
1059   //
1060   // a volatile load derived from an inlined unsafe field access
1061   // manifests as a cpuorder membar with Ctl and Mem projections
1062   // feeding both an acquire membar and a LoadX[mo_acquire]. The
1063   // acquire then feeds another cpuorder membar via Ctl and Mem
1064   // projections. The load has no output dependency on these trailing
1065   // membars because subsequent nodes inserted into the graph take
1066   // their control feed from the final membar cpuorder meaning they
1067   // are all ordered after the load.
1068 
1069   Node *x = barrier->lookup(TypeFunc::Parms);
1070   if (x) {
1071     // we are starting from an acquire and it has a fake dependency
1072     //
1073     // need to check for
1074     //
1075     //   LoadX[mo_acquire]
1076     //   {  |1   }
1077     //   {DecodeN}
1078     //      |Parms
1079     //   MemBarAcquire*
1080     //
1081     // where * tags node we were passed
1082     // and |k means input k
1083     if (x->is_DecodeNarrowPtr())
1084       x = x->in(1);
1085 
1086     return (x->is_Load() && x->as_Load()->is_acquire());
1087   }
1088   
1089   // only continue if we want to try to match unsafe volatile gets
1090   if (UseBarriersForUnsafeVolatileGet)
1091     return false;
1092 
1093   // need to check for
1094   //
1095   //     MemBarCPUOrder
1096   //        ||       \\
1097   //   MemBarAcquire* LoadX[mo_acquire]
1098   //        ||
1099   //   MemBarCPUOrder
1100   //
1101   // where * tags node we were passed
1102   // and || or \\ are Ctl+Mem feeds via intermediate Proj Nodes
1103 
1104   // check for a parent MemBarCPUOrder
1105   ProjNode *ctl;
1106   ProjNode *mem;
1107   MemBarNode *parent = has_parent_membar(barrier, ctl, mem);
1108   if (!parent || parent->Opcode() != Op_MemBarCPUOrder)
1109     return false;
1110   // ensure the proj nodes both feed a LoadX[mo_acquire]
1111   LoadNode *ld = NULL;
1112   for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {
1113     x = ctl->fast_out(i);
1114     // if we see a load we keep hold of it and stop searching
1115     if (x->is_Load()) {
1116       ld = x->as_Load();
1117       break;
1118     }
1119   }
1120   // it must be an acquiring load
1121   if (! ld || ! ld->is_acquire())
1122     return false;
1123   for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
1124     x = mem->fast_out(i);
1125     // if we see the same load we drop it and stop searching
1126     if (x == ld) {
1127       ld = NULL;
1128       break;
1129     }
1130   }
1131   // we must have dropped the load
1132   if (ld)
1133     return false;
1134   // check for a child cpuorder membar
1135   MemBarNode *child  = has_child_membar(barrier->as_MemBar(), ctl, mem);
1136   if (!child || child->Opcode() != Op_MemBarCPUOrder)
1137     return false;
1138 
1139   return true;
1140 }
1141 
1142 bool needs_acquiring_load(const Node *n)
1143 {
1144   // assert n->is_Load();
1145   if (UseBarriersForVolatile)
1146     // we use a normal load and a dmb
1147     return false;
1148 
1149   LoadNode *ld = n->as_Load();
1150 
1151   if (!ld->is_acquire())
1152     return false;
1153 
1154   // check if this load is feeding an acquire membar
1155   //
1156   //   LoadX[mo_acquire]
1157   //   {  |1   }
1158   //   {DecodeN}
1159   //      |Parms
1160   //   MemBarAcquire*
1161   //
1162   // where * tags node we were passed
1163   // and |k means input k
1164 
1165   Node *start = ld;
1166   Node *mbacq = NULL;
1167 
1168   // if we hit a DecodeNarrowPtr we reset the start node and restart
1169   // the search through the outputs
1170  restart:
1171 
1172   for (DUIterator_Fast imax, i = start->fast_outs(imax); i < imax; i++) {
1173     Node *x = start->fast_out(i);
1174     if (x->is_MemBar() && x->Opcode() == Op_MemBarAcquire) {
1175       mbacq = x;
1176     } else if (!mbacq &&
1177                (x->is_DecodeNarrowPtr() ||
1178                 (x->is_Mach() && x->Opcode() == Op_DecodeN))) {
1179       start = x;
1180       goto restart;
1181     }
1182   }
1183 
1184   if (mbacq) {
1185     return true;
1186   }
1187 
1188   // only continue if we want to try to match unsafe volatile gets
1189   if (UseBarriersForUnsafeVolatileGet)
1190     return false;
1191 
1192   // check if Ctl and Proj feed comes from a MemBarCPUOrder
1193   //
1194   //     MemBarCPUOrder
1195   //        ||       \\
1196   //   MemBarAcquire* LoadX[mo_acquire]
1197   //        ||
1198   //   MemBarCPUOrder
1199 
1200   MemBarNode *membar;
1201   ProjNode *ctl;
1202   ProjNode *mem;
1203 
1204   membar = has_parent_membar(ld, ctl, mem);
1205 
1206   if (!membar || !membar->Opcode() == Op_MemBarCPUOrder)
1207     return false;
1208 
1209   // ensure that there is a CPUOrder->Acquire->CPUOrder membar chain
1210 
1211   membar = has_child_membar(membar, ctl, mem);
1212 
1213   if (!membar || !membar->Opcode() == Op_MemBarAcquire)
1214     return false;
1215 
1216   membar = has_child_membar(membar, ctl, mem);
1217   
1218   if (!membar || !membar->Opcode() == Op_MemBarCPUOrder)
1219     return false;
1220 
1221   return true;
1222 }
1223 
1224 bool unnecessary_release(const Node *n) {
1225   // assert n->is_MemBar();
1226   if (UseBarriersForVolatile)
1227     // we need to plant a dmb
1228     return false;
1229 
1230   // ok, so we can omit this release barrier if it has been inserted
1231   // as part of a volatile store sequence
1232   //
1233   //   MemBarRelease
1234   //  {      ||      }
1235   //  {MemBarCPUOrder} -- optional
1236   //         ||     \\
1237   //         ||     StoreX[mo_release]
1238   //         | \     /
1239   //         | MergeMem
1240   //         | /
1241   //   MemBarVolatile
1242   //
1243   // where
1244   //  || and \\ represent Ctl and Mem feeds via Proj nodes
1245   //  | \ and / indicate further routing of the Ctl and Mem feeds
1246   // 
1247   // so we need to check that
1248   //
1249   // ia) the release membar (or its dependent cpuorder membar) feeds
1250   // control to a store node (via a Control project node)
1251   //
1252   // ii) the store is ordered release
1253   //
1254   // iii) the release membar (or its dependent cpuorder membar) feeds
1255   // control to a volatile membar (via the same Control project node)
1256   //
1257   // iv) the release membar feeds memory to a merge mem and to the
1258   // same store (both via a single Memory proj node)
1259   //
1260   // v) the store outputs to the merge mem
1261   //
1262   // vi) the merge mem outputs to the same volatile membar
1263   //
1264   // n.b. if this is an inlined unsafe node then the release membar
1265   // may feed its control and memory links via an intervening cpuorder
1266   // membar. this case can be dealt with when we check the release
1267   // membar projections. if they both feed a single cpuorder membar
1268   // node continue to make the same checks as above but with the
1269   // cpuorder membar substituted for the release membar. if they don't
1270   // both feed a cpuorder membar then the check fails.
1271   //
1272   // n.b.b. for an inlined unsafe store of an object in the case where
1273   // !TypePtr::NULL_PTR->higher_equal(type(heap_base_oop)) we may see
1274   // an embedded if then else where we expect the store. this is
1275   // needed to do the right type of store depending on whether
1276   // heap_base is NULL. We could check for that but for now we can
1277   // just take the hit of on inserting a redundant dmb for this
1278   // redundant volatile membar
1279 
1280   MemBarNode *barrier = n->as_MemBar();
1281   ProjNode *ctl;
1282   ProjNode *mem;
1283   // check for an intervening cpuorder membar
1284   MemBarNode *b = has_child_membar(barrier, ctl, mem);
1285   if (b && b->Opcode() == Op_MemBarCPUOrder) {
1286     // ok, so start form the dependent cpuorder barrier
1287     barrier = b;
1288   }
1289   // check the ctl and mem flow
1290   ctl = barrier->proj_out(TypeFunc::Control);
1291   mem = barrier->proj_out(TypeFunc::Memory);
1292 
1293   // the barrier needs to have both a Ctl and Mem projection
1294   if (! ctl || ! mem)
1295     return false;
1296 
1297   Node *x = NULL;
1298   Node *mbvol = NULL;
1299   StoreNode * st = NULL;
1300 
1301   // For a normal volatile write the Ctl ProjNode should have output
1302   // to a MemBarVolatile and a Store marked as releasing
1303   //
1304   // n.b. for an inlined unsafe store of an object in the case where
1305   // !TypePtr::NULL_PTR->higher_equal(type(heap_base_oop)) we may see
1306   // an embedded if then else where we expect the store. this is
1307   // needed to do the right type of store depending on whether
1308   // heap_base is NULL. We could check for that case too but for now
1309   // we can just take the hit of inserting a dmb and a non-volatile
1310   // store to implement the volatile store
1311 
1312   for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {
1313     x = ctl->fast_out(i);
1314     if (x->is_MemBar() && x->Opcode() == Op_MemBarVolatile) {
1315       if (mbvol) {
1316         return false;
1317       }
1318       mbvol = x;
1319     } else if (x->is_Store()) {
1320       st = x->as_Store();
1321       if (! st->is_release()) {
1322         return false;
1323       }
1324     } else if (!x->is_Mach()) {
1325       // we may see mach nodes added during matching but nothing else
1326       return false;
1327     }
1328   }
1329 
1330   if (!mbvol || !st)
1331     return false;
1332 
1333   // the Mem ProjNode should output to a MergeMem and the same Store
1334   Node *mm = NULL;
1335   for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
1336     x = mem->fast_out(i);
1337     if (!mm && x->is_MergeMem()) {
1338       mm = x;
1339     } else if (x != st && !x->is_Mach()) {
1340       // we may see mach nodes added during matching but nothing else
1341       return false;
1342     }
1343   }
1344 
1345   if (!mm)
1346     return false;
1347 
1348   // the MergeMem should output to the MemBarVolatile
1349   for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {
1350     x = mm->fast_out(i);
1351     if (x != mbvol && !x->is_Mach()) {
1352       // we may see mach nodes added during matching but nothing else
1353       return false;
1354     }
1355   }
1356 
1357   return true;
1358 }
1359 
1360 bool unnecessary_volatile(const Node *n) {
1361   // assert n->is_MemBar();
1362   if (UseBarriersForVolatile)
1363     // we need to plant a dmb
1364     return false;
1365 
1366   // ok, so we can omit this volatile barrier if it has been inserted
1367   // as part of a volatile store sequence
1368   //
1369   //   MemBarRelease
1370   //  {      ||      }
1371   //  {MemBarCPUOrder} -- optional
1372   //         ||     \\
1373   //         ||     StoreX[mo_release]
1374   //         | \     /
1375   //         | MergeMem
1376   //         | /
1377   //   MemBarVolatile
1378   //
1379   // where
1380   //  || and \\ represent Ctl and Mem feeds via Proj nodes
1381   //  | \ and / indicate further routing of the Ctl and Mem feeds
1382   // 
1383   // we need to check that
1384   //
1385   // i) the volatile membar gets its control feed from a release
1386   // membar (or its dependent cpuorder membar) via a Control project
1387   // node
1388   //
1389   // ii) the release membar (or its dependent cpuorder membar) also
1390   // feeds control to a store node via the same proj node
1391   //
1392   // iii) the store is ordered release
1393   //
1394   // iv) the release membar (or its dependent cpuorder membar) feeds
1395   // memory to a merge mem and to the same store (both via a single
1396   // Memory proj node)
1397   //
1398   // v) the store outputs to the merge mem
1399   //
1400   // vi) the merge mem outputs to the volatile membar
1401   //
1402   // n.b. for an inlined unsafe store of an object in the case where
1403   // !TypePtr::NULL_PTR->higher_equal(type(heap_base_oop)) we may see
1404   // an embedded if then else where we expect the store. this is
1405   // needed to do the right type of store depending on whether
1406   // heap_base is NULL. We could check for that but for now we can
1407   // just take the hit of on inserting a redundant dmb for this
1408   // redundant volatile membar
1409 
1410   MemBarNode *mbvol = n->as_MemBar();
1411   Node *x = n->lookup(TypeFunc::Control);
1412 
1413   if (! x || !x->is_Proj())
1414     return false;
1415 
1416   ProjNode *proj = x->as_Proj();
1417 
1418   x = proj->lookup(0);
1419 
1420   if (!x || !x->is_MemBar())
1421     return false;
1422 
1423   MemBarNode *barrier = x->as_MemBar();
1424 
1425   // if the barrier is a release membar we have what we want. if it is
1426   // a cpuorder membar then we need to ensure that it is fed by a
1427   // release membar in which case we proceed to check the graph below
1428   // this cpuorder membar as the feed
1429 
1430   if (x->Opcode() != Op_MemBarRelease) {
1431     if (x->Opcode() != Op_MemBarCPUOrder)
1432       return false;
1433     ProjNode *ctl;
1434     ProjNode *mem;
1435     MemBarNode *b = has_parent_membar(x, ctl, mem);
1436     if (!b || !b->Opcode() == Op_MemBarRelease)
1437       return false;
1438   }
1439 
1440   ProjNode *ctl = barrier->proj_out(TypeFunc::Control);
1441   ProjNode *mem = barrier->proj_out(TypeFunc::Memory);
1442 
1443   // barrier needs to have both a Ctl and Mem projection
1444   // and we need to have reached it via the Ctl projection
1445   if (! ctl || ! mem || ctl != proj)
1446     return false;
1447 
1448   StoreNode * st = NULL;
1449 
1450   // The Ctl ProjNode should have output to a MemBarVolatile and
1451   // a Store marked as releasing
1452   for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {
1453     x = ctl->fast_out(i);
1454     if (x->is_MemBar() && x->Opcode() == Op_MemBarVolatile) {
1455       if (x != mbvol) {
1456         return false;
1457       }
1458     } else if (x->is_Store()) {
1459       st = x->as_Store();
1460       if (! st->is_release()) {
1461         return false;
1462       }
1463     } else if (!x->is_Mach()){
1464       // we may see mach nodes added during matching but nothing else
1465       return false;
1466     }
1467   }
1468 
1469   if (!st)
1470     return false;
1471 
1472   // the Mem ProjNode should output to a MergeMem and the same Store
1473   Node *mm = NULL;
1474   for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
1475     x = mem->fast_out(i);
1476     if (!mm && x->is_MergeMem()) {
1477       mm = x;
1478     } else if (x != st && !x->is_Mach()) {
1479       // we may see mach nodes added during matching but nothing else
1480       return false;
1481     }
1482   }
1483 
1484   if (!mm)
1485     return false;
1486 
1487   // the MergeMem should output to the MemBarVolatile
1488   for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {
1489     x = mm->fast_out(i);
1490     if (x != mbvol && !x->is_Mach()) {
1491       // we may see mach nodes added during matching but nothing else
1492       return false;
1493     }
1494   }
1495 
1496   return true;
1497 }
1498 
1499 
1500 
1501 bool needs_releasing_store(const Node *n)
1502 {
1503   // assert n->is_Store();
1504   if (UseBarriersForVolatile)
1505     // we use a normal store and dmb combination
1506     return false;
1507 
1508   StoreNode *st = n->as_Store();
1509 
1510   if (!st->is_release())
1511     return false;
1512 
1513   // check if this store is bracketed by a release (or its dependent
1514   // cpuorder membar) and a volatile membar
1515   //
1516   //   MemBarRelease
1517   //  {      ||      }
1518   //  {MemBarCPUOrder} -- optional
1519   //         ||     \\
1520   //         ||     StoreX[mo_release]
1521   //         | \     /
1522   //         | MergeMem
1523   //         | /
1524   //   MemBarVolatile
1525   //
1526   // where
1527   //  || and \\ represent Ctl and Mem feeds via Proj nodes
1528   //  | \ and / indicate further routing of the Ctl and Mem feeds
1529   // 
1530 
1531 
1532   Node *x = st->lookup(TypeFunc::Control);
1533 
1534   if (! x || !x->is_Proj())
1535     return false;
1536 
1537   ProjNode *proj = x->as_Proj();
1538 
1539   x = proj->lookup(0);
1540 
1541   if (!x || !x->is_MemBar())
1542     return false;
1543 
1544   MemBarNode *barrier = x->as_MemBar();
1545 
1546   // if the barrier is a release membar we have what we want. if it is
1547   // a cpuorder membar then we need to ensure that it is fed by a
1548   // release membar in which case we proceed to check the graph below
1549   // this cpuorder membar as the feed
1550 
1551   if (x->Opcode() != Op_MemBarRelease) {
1552     if (x->Opcode() != Op_MemBarCPUOrder)
1553       return false;
1554     Node *ctl = x->lookup(TypeFunc::Control);
1555     Node *mem = x->lookup(TypeFunc::Memory);
1556     if (!ctl || !ctl->is_Proj() || !mem || !mem->is_Proj())
1557       return false;
1558     x = ctl->lookup(0);
1559     if (!x || !x->is_MemBar() || !x->Opcode() == Op_MemBarRelease)
1560       return false;
1561     Node *y = mem->lookup(0);
1562     if (!y || y != x)
1563       return false;
1564   }
1565 
1566   ProjNode *ctl = barrier->proj_out(TypeFunc::Control);
1567   ProjNode *mem = barrier->proj_out(TypeFunc::Memory);
1568 
1569   // MemBarRelease needs to have both a Ctl and Mem projection
1570   // and we need to have reached it via the Ctl projection
1571   if (! ctl || ! mem || ctl != proj)
1572     return false;
1573 
1574   MemBarNode *mbvol = NULL;
1575 
1576   // The Ctl ProjNode should have output to a MemBarVolatile and
1577   // a Store marked as releasing
1578   for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {
1579     x = ctl->fast_out(i);
1580     if (x->is_MemBar() && x->Opcode() == Op_MemBarVolatile) {
1581       mbvol = x->as_MemBar();
1582     } else if (x->is_Store()) {
1583       if (x != st) {
1584         return false;
1585       }
1586     } else if (!x->is_Mach()){
1587       return false;
1588     }
1589   }
1590 
1591   if (!mbvol)
1592     return false;
1593 
1594   // the Mem ProjNode should output to a MergeMem and the same Store
1595   Node *mm = NULL;
1596   for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
1597     x = mem->fast_out(i);
1598     if (!mm && x->is_MergeMem()) {
1599       mm = x;
1600     } else if (x != st && !x->is_Mach()) {
1601       return false;
1602     }
1603   }
1604 
1605   if (!mm)
1606     return false;
1607 
1608   // the MergeMem should output to the MemBarVolatile
1609   for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {
1610     x = mm->fast_out(i);
1611     if (x != mbvol && !x->is_Mach()) {
1612       return false;
1613     }
1614   }
1615 
1616   return true;
1617 }
1618 
1619 
1620 
1621 #define __ _masm.
1622 
1623 // advance declarations for helper functions to convert register
1624 // indices to register objects
1625 
1626 // the ad file has to provide implementations of certain methods
1627 // expected by the generic code
1628 //
1629 // REQUIRED FUNCTIONALITY
1630 
1631 //=============================================================================
1632 
1633 // !!!!! Special hack to get all types of calls to specify the byte offset
1634 //       from the start of the call to the point where the return address
1635 //       will point.
1636 
1637 int MachCallStaticJavaNode::ret_addr_offset()
1638 {
1639   // call should be a simple bl
1640   // unless this is a method handle invoke in which case it is
1641   // mov(rfp, sp), bl, mov(sp, rfp)
1642   int off = 4;
1643   if (_method_handle_invoke) {
1644     off += 4;
1645   }
1646   return off;
1647 }
1648 
1649 int MachCallDynamicJavaNode::ret_addr_offset()
1650 {
1651   return 16; // movz, movk, movk, bl
1652 }
1653 
1654 int MachCallRuntimeNode::ret_addr_offset() {
1655   // for generated stubs the call will be
1656   //   far_call(addr)
1657   // for real runtime callouts it will be six instructions
1658   // see aarch64_enc_java_to_runtime
1659   //   adr(rscratch2, retaddr)
1660   //   lea(rscratch1, RuntimeAddress(addr)
1661   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1662   //   blrt rscratch1
1663   CodeBlob *cb = CodeCache::find_blob(_entry_point);
1664   if (cb) {
1665     return MacroAssembler::far_branch_size();
1666   } else {
1667     return 6 * NativeInstruction::instruction_size;
1668   }
1669 }
1670 
1671 // Indicate if the safepoint node needs the polling page as an input
1672 
1673 // the shared code plants the oop data at the start of the generated
1674 // code for the safepoint node and that needs ot be at the load
1675 // instruction itself. so we cannot plant a mov of the safepoint poll
1676 // address followed by a load. setting this to true means the mov is
1677 // scheduled as a prior instruction. that's better for scheduling
1678 // anyway.
1679 
1680 bool SafePointNode::needs_polling_address_input()
1681 {
1682   return true;
1683 }
1684 
1685 //=============================================================================
1686 
1687 #ifndef PRODUCT
1688 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1689   st->print("BREAKPOINT");
1690 }
1691 #endif
1692 
1693 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1694   MacroAssembler _masm(&cbuf);
1695   __ brk(0);
1696 }
1697 
1698 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1699   return MachNode::size(ra_);
1700 }
1701 
1702 //=============================================================================
1703 
1704 #ifndef PRODUCT
1705   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1706     st->print("nop \t# %d bytes pad for loops and calls", _count);
1707   }
1708 #endif
1709 
1710   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1711     MacroAssembler _masm(&cbuf);
1712     for (int i = 0; i < _count; i++) {
1713       __ nop();
1714     }
1715   }
1716 
1717   uint MachNopNode::size(PhaseRegAlloc*) const {
1718     return _count * NativeInstruction::instruction_size;
1719   }
1720 
1721 //=============================================================================
1722 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1723 
1724 int Compile::ConstantTable::calculate_table_base_offset() const {
1725   return 0;  // absolute addressing, no offset
1726 }
1727 
1728 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1729 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1730   ShouldNotReachHere();
1731 }
1732 
1733 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1734   // Empty encoding
1735 }
1736 
1737 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1738   return 0;
1739 }
1740 
1741 #ifndef PRODUCT
1742 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1743   st->print("-- \t// MachConstantBaseNode (empty encoding)");
1744 }
1745 #endif
1746 
1747 #ifndef PRODUCT
1748 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1749   Compile* C = ra_->C;
1750 
1751   int framesize = C->frame_slots() << LogBytesPerInt;
1752 
1753   if (C->need_stack_bang(framesize))
1754     st->print("# stack bang size=%d\n\t", framesize);
1755 
1756   if (framesize == 0) {
1757     // Is this even possible?
1758     st->print("stp  lr, rfp, [sp, #%d]!", -(2 * wordSize));
1759   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1760     st->print("sub  sp, sp, #%d\n\t", framesize);
1761     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1762   } else {
1763     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1764     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1765     st->print("sub  sp, sp, rscratch1");
1766   }
1767 }
1768 #endif
1769 
1770 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1771   Compile* C = ra_->C;
1772   MacroAssembler _masm(&cbuf);
1773 
1774   // n.b. frame size includes space for return pc and rfp
1775   const long framesize = C->frame_size_in_bytes();
1776   assert(framesize%(2*wordSize) == 0, "must preserve 2*wordSize alignment");
1777 
1778   // insert a nop at the start of the prolog so we can patch in a
1779   // branch if we need to invalidate the method later
1780   __ nop();
1781 
1782   int bangsize = C->bang_size_in_bytes();
1783   if (C->need_stack_bang(bangsize) && UseStackBanging)
1784     __ generate_stack_overflow_check(bangsize);
1785 
1786   __ build_frame(framesize);
1787 
1788   if (NotifySimulator) {
1789     __ notify(Assembler::method_entry);
1790   }
1791 
1792   if (VerifyStackAtCalls) {
1793     Unimplemented();
1794   }
1795 
1796   C->set_frame_complete(cbuf.insts_size());
1797 
1798   if (C->has_mach_constant_base_node()) {
1799     // NOTE: We set the table base offset here because users might be
1800     // emitted before MachConstantBaseNode.
1801     Compile::ConstantTable& constant_table = C->constant_table();
1802     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1803   }
1804 }
1805 
1806 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1807 {
1808   return MachNode::size(ra_); // too many variables; just compute it
1809                               // the hard way
1810 }
1811 
1812 int MachPrologNode::reloc() const
1813 {
1814   return 0;
1815 }
1816 
1817 //=============================================================================
1818 
1819 #ifndef PRODUCT
1820 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1821   Compile* C = ra_->C;
1822   int framesize = C->frame_slots() << LogBytesPerInt;
1823 
1824   st->print("# pop frame %d\n\t",framesize);
1825 
1826   if (framesize == 0) {
1827     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1828   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1829     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1830     st->print("add  sp, sp, #%d\n\t", framesize);
1831   } else {
1832     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1833     st->print("add  sp, sp, rscratch1\n\t");
1834     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1835   }
1836 
1837   if (do_polling() && C->is_method_compilation()) {
1838     st->print("# touch polling page\n\t");
1839     st->print("mov  rscratch1, #0x%lx\n\t", p2i(os::get_polling_page()));
1840     st->print("ldr zr, [rscratch1]");
1841   }
1842 }
1843 #endif
1844 
1845 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1846   Compile* C = ra_->C;
1847   MacroAssembler _masm(&cbuf);
1848   int framesize = C->frame_slots() << LogBytesPerInt;
1849 
1850   __ remove_frame(framesize);
1851 
1852   if (NotifySimulator) {
1853     __ notify(Assembler::method_reentry);
1854   }
1855 
1856   if (do_polling() && C->is_method_compilation()) {
1857     __ read_polling_page(rscratch1, os::get_polling_page(), relocInfo::poll_return_type);
1858   }
1859 }
1860 
1861 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1862   // Variable size. Determine dynamically.
1863   return MachNode::size(ra_);
1864 }
1865 
1866 int MachEpilogNode::reloc() const {
1867   // Return number of relocatable values contained in this instruction.
1868   return 1; // 1 for polling page.
1869 }
1870 
1871 const Pipeline * MachEpilogNode::pipeline() const {
1872   return MachNode::pipeline_class();
1873 }
1874 
1875 // This method seems to be obsolete. It is declared in machnode.hpp
1876 // and defined in all *.ad files, but it is never called. Should we
1877 // get rid of it?
1878 int MachEpilogNode::safepoint_offset() const {
1879   assert(do_polling(), "no return for this epilog node");
1880   return 4;
1881 }
1882 
1883 //=============================================================================
1884 
1885 // Figure out which register class each belongs in: rc_int, rc_float or
1886 // rc_stack.
1887 enum RC { rc_bad, rc_int, rc_float, rc_stack };
1888 
1889 static enum RC rc_class(OptoReg::Name reg) {
1890 
1891   if (reg == OptoReg::Bad) {
1892     return rc_bad;
1893   }
1894 
1895   // we have 30 int registers * 2 halves
1896   // (rscratch1 and rscratch2 are omitted)
1897 
1898   if (reg < 60) {
1899     return rc_int;
1900   }
1901 
1902   // we have 32 float register * 2 halves
1903   if (reg < 60 + 64) {
1904     return rc_float;
1905   }
1906 
1907   // Between float regs & stack is the flags regs.
1908   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1909 
1910   return rc_stack;
1911 }
1912 
1913 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1914   Compile* C = ra_->C;
1915 
1916   // Get registers to move.
1917   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1918   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1919   OptoReg::Name dst_hi = ra_->get_reg_second(this);
1920   OptoReg::Name dst_lo = ra_->get_reg_first(this);
1921 
1922   enum RC src_hi_rc = rc_class(src_hi);
1923   enum RC src_lo_rc = rc_class(src_lo);
1924   enum RC dst_hi_rc = rc_class(dst_hi);
1925   enum RC dst_lo_rc = rc_class(dst_lo);
1926 
1927   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1928 
1929   if (src_hi != OptoReg::Bad) {
1930     assert((src_lo&1)==0 && src_lo+1==src_hi &&
1931            (dst_lo&1)==0 && dst_lo+1==dst_hi,
1932            "expected aligned-adjacent pairs");
1933   }
1934 
1935   if (src_lo == dst_lo && src_hi == dst_hi) {
1936     return 0;            // Self copy, no move.
1937   }
1938 
1939   switch (src_lo_rc) {
1940   case rc_int:
1941     if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
1942       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1943           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1944           // 64 bit
1945         if (cbuf) {
1946           MacroAssembler _masm(cbuf);
1947           __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1948                  as_Register(Matcher::_regEncode[src_lo]));
1949         } else if (st) {
1950           st->print("mov  %s, %s\t# shuffle",
1951                     Matcher::regName[dst_lo],
1952                     Matcher::regName[src_lo]);
1953         }
1954       } else {
1955         // 32 bit
1956         if (cbuf) {
1957           MacroAssembler _masm(cbuf);
1958           __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1959                   as_Register(Matcher::_regEncode[src_lo]));
1960         } else if (st) {
1961           st->print("movw  %s, %s\t# shuffle",
1962                     Matcher::regName[dst_lo],
1963                     Matcher::regName[src_lo]);
1964         }
1965       }
1966     } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1967       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1968           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1969           // 64 bit
1970         if (cbuf) {
1971           MacroAssembler _masm(cbuf);
1972           __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1973                    as_Register(Matcher::_regEncode[src_lo]));
1974         } else if (st) {
1975           st->print("fmovd  %s, %s\t# shuffle",
1976                     Matcher::regName[dst_lo],
1977                     Matcher::regName[src_lo]);
1978         }
1979       } else {
1980         // 32 bit
1981         if (cbuf) {
1982           MacroAssembler _masm(cbuf);
1983           __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1984                    as_Register(Matcher::_regEncode[src_lo]));
1985         } else if (st) {
1986           st->print("fmovs  %s, %s\t# shuffle",
1987                     Matcher::regName[dst_lo],
1988                     Matcher::regName[src_lo]);
1989         }
1990       }
1991     } else {                    // gpr --> stack spill
1992       assert(dst_lo_rc == rc_stack, "spill to bad register class");
1993       int dst_offset = ra_->reg2offset(dst_lo);
1994       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
1995           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
1996           // 64 bit
1997         if (cbuf) {
1998           MacroAssembler _masm(cbuf);
1999           __ str(as_Register(Matcher::_regEncode[src_lo]),
2000                  Address(sp, dst_offset));
2001         } else if (st) {
2002           st->print("str  %s, [sp, #%d]\t# spill",
2003                     Matcher::regName[src_lo],
2004                     dst_offset);
2005         }
2006       } else {
2007         // 32 bit
2008         if (cbuf) {
2009           MacroAssembler _masm(cbuf);
2010           __ strw(as_Register(Matcher::_regEncode[src_lo]),
2011                  Address(sp, dst_offset));
2012         } else if (st) {
2013           st->print("strw  %s, [sp, #%d]\t# spill",
2014                     Matcher::regName[src_lo],
2015                     dst_offset);
2016         }
2017       }
2018     }
2019     return 4;
2020   case rc_float:
2021     if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
2022       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2023           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2024           // 64 bit
2025         if (cbuf) {
2026           MacroAssembler _masm(cbuf);
2027           __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2028                    as_FloatRegister(Matcher::_regEncode[src_lo]));
2029         } else if (st) {
2030           st->print("fmovd  %s, %s\t# shuffle",
2031                     Matcher::regName[dst_lo],
2032                     Matcher::regName[src_lo]);
2033         }
2034       } else {
2035         // 32 bit
2036         if (cbuf) {
2037           MacroAssembler _masm(cbuf);
2038           __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2039                    as_FloatRegister(Matcher::_regEncode[src_lo]));
2040         } else if (st) {
2041           st->print("fmovs  %s, %s\t# shuffle",
2042                     Matcher::regName[dst_lo],
2043                     Matcher::regName[src_lo]);
2044         }
2045       }
2046     } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2047       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2048           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2049           // 64 bit
2050         if (cbuf) {
2051           MacroAssembler _masm(cbuf);
2052           __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2053                    as_FloatRegister(Matcher::_regEncode[src_lo]));
2054         } else if (st) {
2055           st->print("fmovd  %s, %s\t# shuffle",
2056                     Matcher::regName[dst_lo],
2057                     Matcher::regName[src_lo]);
2058         }
2059       } else {
2060         // 32 bit
2061         if (cbuf) {
2062           MacroAssembler _masm(cbuf);
2063           __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2064                    as_FloatRegister(Matcher::_regEncode[src_lo]));
2065         } else if (st) {
2066           st->print("fmovs  %s, %s\t# shuffle",
2067                     Matcher::regName[dst_lo],
2068                     Matcher::regName[src_lo]);
2069         }
2070       }
2071     } else {                    // fpr --> stack spill
2072       assert(dst_lo_rc == rc_stack, "spill to bad register class");
2073       int dst_offset = ra_->reg2offset(dst_lo);
2074       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2075           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2076           // 64 bit
2077         if (cbuf) {
2078           MacroAssembler _masm(cbuf);
2079           __ strd(as_FloatRegister(Matcher::_regEncode[src_lo]),
2080                  Address(sp, dst_offset));
2081         } else if (st) {
2082           st->print("strd  %s, [sp, #%d]\t# spill",
2083                     Matcher::regName[src_lo],
2084                     dst_offset);
2085         }
2086       } else {
2087         // 32 bit
2088         if (cbuf) {
2089           MacroAssembler _masm(cbuf);
2090           __ strs(as_FloatRegister(Matcher::_regEncode[src_lo]),
2091                  Address(sp, dst_offset));
2092         } else if (st) {
2093           st->print("strs  %s, [sp, #%d]\t# spill",
2094                     Matcher::regName[src_lo],
2095                     dst_offset);
2096         }
2097       }
2098     }
2099     return 4;
2100   case rc_stack:
2101     int src_offset = ra_->reg2offset(src_lo);
2102     if (dst_lo_rc == rc_int) {  // stack --> gpr load
2103       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2104           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2105           // 64 bit
2106         if (cbuf) {
2107           MacroAssembler _masm(cbuf);
2108           __ ldr(as_Register(Matcher::_regEncode[dst_lo]),
2109                  Address(sp, src_offset));
2110         } else if (st) {
2111           st->print("ldr  %s, [sp, %d]\t# restore",
2112                     Matcher::regName[dst_lo],
2113                     src_offset);
2114         }
2115       } else {
2116         // 32 bit
2117         if (cbuf) {
2118           MacroAssembler _masm(cbuf);
2119           __ ldrw(as_Register(Matcher::_regEncode[dst_lo]),
2120                   Address(sp, src_offset));
2121         } else if (st) {
2122           st->print("ldr  %s, [sp, %d]\t# restore",
2123                     Matcher::regName[dst_lo],
2124                    src_offset);
2125         }
2126       }
2127       return 4;
2128     } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2129       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2130           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2131           // 64 bit
2132         if (cbuf) {
2133           MacroAssembler _masm(cbuf);
2134           __ ldrd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2135                  Address(sp, src_offset));
2136         } else if (st) {
2137           st->print("ldrd  %s, [sp, %d]\t# restore",
2138                     Matcher::regName[dst_lo],
2139                     src_offset);
2140         }
2141       } else {
2142         // 32 bit
2143         if (cbuf) {
2144           MacroAssembler _masm(cbuf);
2145           __ ldrs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2146                   Address(sp, src_offset));
2147         } else if (st) {
2148           st->print("ldrs  %s, [sp, %d]\t# restore",
2149                     Matcher::regName[dst_lo],
2150                    src_offset);
2151         }
2152       }
2153       return 4;
2154     } else {                    // stack --> stack copy
2155       assert(dst_lo_rc == rc_stack, "spill to bad register class");
2156       int dst_offset = ra_->reg2offset(dst_lo);
2157       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2158           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2159           // 64 bit
2160         if (cbuf) {
2161           MacroAssembler _masm(cbuf);
2162           __ ldr(rscratch1, Address(sp, src_offset));
2163           __ str(rscratch1, Address(sp, dst_offset));
2164         } else if (st) {
2165           st->print("ldr  rscratch1, [sp, %d]\t# mem-mem spill",
2166                     src_offset);
2167           st->print("\n\t");
2168           st->print("str  rscratch1, [sp, %d]",
2169                     dst_offset);
2170         }
2171       } else {
2172         // 32 bit
2173         if (cbuf) {
2174           MacroAssembler _masm(cbuf);
2175           __ ldrw(rscratch1, Address(sp, src_offset));
2176           __ strw(rscratch1, Address(sp, dst_offset));
2177         } else if (st) {
2178           st->print("ldrw  rscratch1, [sp, %d]\t# mem-mem spill",
2179                     src_offset);
2180           st->print("\n\t");
2181           st->print("strw  rscratch1, [sp, %d]",
2182                     dst_offset);
2183         }
2184       }
2185       return 8;
2186     }
2187   }
2188 
2189   assert(false," bad rc_class for spill ");
2190   Unimplemented();
2191   return 0;
2192 
2193 }
2194 
2195 #ifndef PRODUCT
2196 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2197   if (!ra_)
2198     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2199   else
2200     implementation(NULL, ra_, false, st);
2201 }
2202 #endif
2203 
2204 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2205   implementation(&cbuf, ra_, false, NULL);
2206 }
2207 
2208 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2209   return implementation(NULL, ra_, true, NULL);
2210 }
2211 
2212 //=============================================================================
2213 
2214 #ifndef PRODUCT
2215 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2216   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2217   int reg = ra_->get_reg_first(this);
2218   st->print("add %s, rsp, #%d]\t# box lock",
2219             Matcher::regName[reg], offset);
2220 }
2221 #endif
2222 
2223 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2224   MacroAssembler _masm(&cbuf);
2225 
2226   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2227   int reg    = ra_->get_encode(this);
2228 
2229   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2230     __ add(as_Register(reg), sp, offset);
2231   } else {
2232     ShouldNotReachHere();
2233   }
2234 }
2235 
2236 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2237   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2238   return 4;
2239 }
2240 
2241 //=============================================================================
2242 
2243 #ifndef PRODUCT
2244 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2245 {
2246   st->print_cr("# MachUEPNode");
2247   if (UseCompressedClassPointers) {
2248     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2249     if (Universe::narrow_klass_shift() != 0) {
2250       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2251     }
2252   } else {
2253    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2254   }
2255   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2256   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2257 }
2258 #endif
2259 
2260 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2261 {
2262   // This is the unverified entry point.
2263   MacroAssembler _masm(&cbuf);
2264 
2265   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2266   Label skip;
2267   // TODO
2268   // can we avoid this skip and still use a reloc?
2269   __ br(Assembler::EQ, skip);
2270   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2271   __ bind(skip);
2272 }
2273 
2274 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2275 {
2276   return MachNode::size(ra_);
2277 }
2278 
2279 // REQUIRED EMIT CODE
2280 
2281 //=============================================================================
2282 
2283 // Emit exception handler code.
2284 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2285 {
2286   // mov rscratch1 #exception_blob_entry_point
2287   // br rscratch1
2288   // Note that the code buffer's insts_mark is always relative to insts.
2289   // That's why we must use the macroassembler to generate a handler.
2290   MacroAssembler _masm(&cbuf);
2291   address base =
2292   __ start_a_stub(size_exception_handler());
2293   if (base == NULL)  return 0;  // CodeBuffer::expand failed
2294   int offset = __ offset();
2295   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2296   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2297   __ end_a_stub();
2298   return offset;
2299 }
2300 
2301 // Emit deopt handler code.
2302 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2303 {
2304   // Note that the code buffer's insts_mark is always relative to insts.
2305   // That's why we must use the macroassembler to generate a handler.
2306   MacroAssembler _masm(&cbuf);
2307   address base =
2308   __ start_a_stub(size_deopt_handler());
2309   if (base == NULL)  return 0;  // CodeBuffer::expand failed
2310   int offset = __ offset();
2311 
2312   __ adr(lr, __ pc());
2313   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2314 
2315   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
2316   __ end_a_stub();
2317   return offset;
2318 }
2319 
2320 // REQUIRED MATCHER CODE
2321 
2322 //=============================================================================
2323 
2324 const bool Matcher::match_rule_supported(int opcode) {
2325 
2326   // TODO
2327   // identify extra cases that we might want to provide match rules for
2328   // e.g. Op_StrEquals and other intrinsics
2329   if (!has_match_rule(opcode)) {
2330     return false;
2331   }
2332 
2333   return true;  // Per default match rules are supported.
2334 }
2335 
2336 int Matcher::regnum_to_fpu_offset(int regnum)
2337 {
2338   Unimplemented();
2339   return 0;
2340 }
2341 
2342 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset)
2343 {
2344   Unimplemented();
2345   return false;
2346 }
2347 
2348 const bool Matcher::isSimpleConstant64(jlong value) {
2349   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
2350   // Probably always true, even if a temp register is required.
2351   return true;
2352 }
2353 
2354 // true just means we have fast l2f conversion
2355 const bool Matcher::convL2FSupported(void) {
2356   return true;
2357 }
2358 
2359 // Vector width in bytes.
2360 const int Matcher::vector_width_in_bytes(BasicType bt) {
2361   // TODO fixme
2362   return 0;
2363 }
2364 
2365 // Limits on vector size (number of elements) loaded into vector.
2366 const int Matcher::max_vector_size(const BasicType bt) {
2367   return vector_width_in_bytes(bt)/type2aelembytes(bt);
2368 }
2369 const int Matcher::min_vector_size(const BasicType bt) {
2370   int max_size = max_vector_size(bt);
2371   // Min size which can be loaded into vector is 4 bytes.
2372   int size = (type2aelembytes(bt) == 1) ? 4 : 2;
2373   return MIN2(size,max_size);
2374 }
2375 
2376 // Vector ideal reg.
2377 const int Matcher::vector_ideal_reg(int len) {
2378   // TODO fixme
2379   return Op_RegD;
2380 }
2381 
2382 // Only lowest bits of xmm reg are used for vector shift count.
2383 const int Matcher::vector_shift_count_ideal_reg(int size) {
2384   // TODO fixme
2385   return Op_RegL;
2386 }
2387 
2388 // AES support not yet implemented
2389 const bool Matcher::pass_original_key_for_aes() {
2390   return false;
2391 }
2392 
2393 // x86 supports misaligned vectors store/load.
2394 const bool Matcher::misaligned_vectors_ok() {
2395   // TODO fixme
2396   // return !AlignVector; // can be changed by flag
2397   return false;
2398 }
2399 
2400 // false => size gets scaled to BytesPerLong, ok.
2401 const bool Matcher::init_array_count_is_in_bytes = false;
2402 
2403 // Threshold size for cleararray.
2404 const int Matcher::init_array_short_size = 18 * BytesPerLong;
2405 
2406 // Use conditional move (CMOVL)
2407 const int Matcher::long_cmove_cost() {
2408   // long cmoves are no more expensive than int cmoves
2409   return 0;
2410 }
2411 
2412 const int Matcher::float_cmove_cost() {
2413   // float cmoves are no more expensive than int cmoves
2414   return 0;
2415 }
2416 
2417 // Does the CPU require late expand (see block.cpp for description of late expand)?
2418 const bool Matcher::require_postalloc_expand = false;
2419 
2420 // Should the Matcher clone shifts on addressing modes, expecting them
2421 // to be subsumed into complex addressing expressions or compute them
2422 // into registers?  True for Intel but false for most RISCs
2423 const bool Matcher::clone_shift_expressions = false;
2424 
2425 // Do we need to mask the count passed to shift instructions or does
2426 // the cpu only look at the lower 5/6 bits anyway?
2427 const bool Matcher::need_masked_shift_count = false;
2428 
2429 // This affects two different things:
2430 //  - how Decode nodes are matched
2431 //  - how ImplicitNullCheck opportunities are recognized
2432 // If true, the matcher will try to remove all Decodes and match them
2433 // (as operands) into nodes. NullChecks are not prepared to deal with
2434 // Decodes by final_graph_reshaping().
2435 // If false, final_graph_reshaping() forces the decode behind the Cmp
2436 // for a NullCheck. The matcher matches the Decode node into a register.
2437 // Implicit_null_check optimization moves the Decode along with the
2438 // memory operation back up before the NullCheck.
2439 bool Matcher::narrow_oop_use_complex_address() {
2440   return Universe::narrow_oop_shift() == 0;
2441 }
2442 
2443 bool Matcher::narrow_klass_use_complex_address() {
2444 // TODO
2445 // decide whether we need to set this to true
2446   return false;
2447 }
2448 
2449 // Is it better to copy float constants, or load them directly from
2450 // memory?  Intel can load a float constant from a direct address,
2451 // requiring no extra registers.  Most RISCs will have to materialize
2452 // an address into a register first, so they would do better to copy
2453 // the constant from stack.
2454 const bool Matcher::rematerialize_float_constants = false;
2455 
2456 // If CPU can load and store mis-aligned doubles directly then no
2457 // fixup is needed.  Else we split the double into 2 integer pieces
2458 // and move it piece-by-piece.  Only happens when passing doubles into
2459 // C code as the Java calling convention forces doubles to be aligned.
2460 const bool Matcher::misaligned_doubles_ok = true;
2461 
2462 // No-op on amd64
2463 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
2464   Unimplemented();
2465 }
2466 
2467 // Advertise here if the CPU requires explicit rounding operations to
2468 // implement the UseStrictFP mode.
2469 const bool Matcher::strict_fp_requires_explicit_rounding = false;
2470 
2471 // Are floats converted to double when stored to stack during
2472 // deoptimization?
2473 bool Matcher::float_in_double() { return true; }
2474 
2475 // Do ints take an entire long register or just half?
2476 // The relevant question is how the int is callee-saved:
2477 // the whole long is written but de-opt'ing will have to extract
2478 // the relevant 32 bits.
2479 const bool Matcher::int_in_long = true;
2480 
2481 // Return whether or not this register is ever used as an argument.
2482 // This function is used on startup to build the trampoline stubs in
2483 // generateOptoStub.  Registers not mentioned will be killed by the VM
2484 // call in the trampoline, and arguments in those registers not be
2485 // available to the callee.
2486 bool Matcher::can_be_java_arg(int reg)
2487 {
2488   return
2489     reg ==  R0_num || reg == R0_H_num ||
2490     reg ==  R1_num || reg == R1_H_num ||
2491     reg ==  R2_num || reg == R2_H_num ||
2492     reg ==  R3_num || reg == R3_H_num ||
2493     reg ==  R4_num || reg == R4_H_num ||
2494     reg ==  R5_num || reg == R5_H_num ||
2495     reg ==  R6_num || reg == R6_H_num ||
2496     reg ==  R7_num || reg == R7_H_num ||
2497     reg ==  V0_num || reg == V0_H_num ||
2498     reg ==  V1_num || reg == V1_H_num ||
2499     reg ==  V2_num || reg == V2_H_num ||
2500     reg ==  V3_num || reg == V3_H_num ||
2501     reg ==  V4_num || reg == V4_H_num ||
2502     reg ==  V5_num || reg == V5_H_num ||
2503     reg ==  V6_num || reg == V6_H_num ||
2504     reg ==  V7_num || reg == V7_H_num;
2505 }
2506 
2507 bool Matcher::is_spillable_arg(int reg)
2508 {
2509   return can_be_java_arg(reg);
2510 }
2511 
2512 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2513   return false;
2514 }
2515 
2516 RegMask Matcher::divI_proj_mask() {
2517   ShouldNotReachHere();
2518   return RegMask();
2519 }
2520 
2521 // Register for MODI projection of divmodI.
2522 RegMask Matcher::modI_proj_mask() {
2523   ShouldNotReachHere();
2524   return RegMask();
2525 }
2526 
2527 // Register for DIVL projection of divmodL.
2528 RegMask Matcher::divL_proj_mask() {
2529   ShouldNotReachHere();
2530   return RegMask();
2531 }
2532 
2533 // Register for MODL projection of divmodL.
2534 RegMask Matcher::modL_proj_mask() {
2535   ShouldNotReachHere();
2536   return RegMask();
2537 }
2538 
2539 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2540   return FP_REG_mask();
2541 }
2542 
2543 // helper for encoding java_to_runtime calls on sim
2544 //
2545 // this is needed to compute the extra arguments required when
2546 // planting a call to the simulator blrt instruction. the TypeFunc
2547 // can be queried to identify the counts for integral, and floating
2548 // arguments and the return type
2549 
2550 static void getCallInfo(const TypeFunc *tf, int &gpcnt, int &fpcnt, int &rtype)
2551 {
2552   int gps = 0;
2553   int fps = 0;
2554   const TypeTuple *domain = tf->domain();
2555   int max = domain->cnt();
2556   for (int i = TypeFunc::Parms; i < max; i++) {
2557     const Type *t = domain->field_at(i);
2558     switch(t->basic_type()) {
2559     case T_FLOAT:
2560     case T_DOUBLE:
2561       fps++;
2562     default:
2563       gps++;
2564     }
2565   }
2566   gpcnt = gps;
2567   fpcnt = fps;
2568   BasicType rt = tf->return_type();
2569   switch (rt) {
2570   case T_VOID:
2571     rtype = MacroAssembler::ret_type_void;
2572     break;
2573   default:
2574     rtype = MacroAssembler::ret_type_integral;
2575     break;
2576   case T_FLOAT:
2577     rtype = MacroAssembler::ret_type_float;
2578     break;
2579   case T_DOUBLE:
2580     rtype = MacroAssembler::ret_type_double;
2581     break;
2582   }
2583 }
2584 
2585 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
2586   MacroAssembler _masm(&cbuf);                                          \
2587   {                                                                     \
2588     guarantee(INDEX == -1, "mode not permitted for volatile");          \
2589     guarantee(DISP == 0, "mode not permitted for volatile");            \
2590     guarantee(SCALE == 0, "mode not permitted for volatile");           \
2591     __ INSN(REG, as_Register(BASE));                                    \
2592   }
2593 
2594 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2595 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2596 
2597   // Used for all non-volatile memory accesses.  The use of
2598   // $mem->opcode() to discover whether this pattern uses sign-extended
2599   // offsets is something of a kludge.
2600   static void loadStore(MacroAssembler masm, mem_insn insn,
2601                          Register reg, int opcode,
2602                          Register base, int index, int size, int disp)
2603   {
2604     Address::extend scale;
2605 
2606     // Hooboy, this is fugly.  We need a way to communicate to the
2607     // encoder that the index needs to be sign extended, so we have to
2608     // enumerate all the cases.
2609     switch (opcode) {
2610     case INDINDEXSCALEDOFFSETI2L:
2611     case INDINDEXSCALEDI2L:
2612     case INDINDEXSCALEDOFFSETI2LN:
2613     case INDINDEXSCALEDI2LN:
2614       scale = Address::sxtw(size);
2615       break;
2616     default:
2617       scale = Address::lsl(size);
2618     }
2619 
2620     if (index == -1) {
2621       (masm.*insn)(reg, Address(base, disp));
2622     } else {
2623       if (disp == 0) {
2624         (masm.*insn)(reg, Address(base, as_Register(index), scale));
2625       } else {
2626         masm.lea(rscratch1, Address(base, disp));
2627         (masm.*insn)(reg, Address(rscratch1, as_Register(index), scale));
2628       }
2629     }
2630   }
2631 
2632   static void loadStore(MacroAssembler masm, mem_float_insn insn,
2633                          FloatRegister reg, int opcode,
2634                          Register base, int index, int size, int disp)
2635   {
2636     Address::extend scale;
2637 
2638     switch (opcode) {
2639     case INDINDEXSCALEDOFFSETI2L:
2640     case INDINDEXSCALEDI2L:
2641     case INDINDEXSCALEDOFFSETI2LN:
2642     case INDINDEXSCALEDI2LN:
2643       scale = Address::sxtw(size);
2644       break;
2645     default:
2646       scale = Address::lsl(size);
2647     }
2648 
2649      if (index == -1) {
2650       (masm.*insn)(reg, Address(base, disp));
2651     } else {
2652       if (disp == 0) {
2653         (masm.*insn)(reg, Address(base, as_Register(index), scale));
2654       } else {
2655         masm.lea(rscratch1, Address(base, disp));
2656         (masm.*insn)(reg, Address(rscratch1, as_Register(index), scale));
2657       }
2658     }
2659   }
2660 
2661 %}
2662 
2663 
2664 
2665 //----------ENCODING BLOCK-----------------------------------------------------
2666 // This block specifies the encoding classes used by the compiler to
2667 // output byte streams.  Encoding classes are parameterized macros
2668 // used by Machine Instruction Nodes in order to generate the bit
2669 // encoding of the instruction.  Operands specify their base encoding
2670 // interface with the interface keyword.  There are currently
2671 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2672 // COND_INTER.  REG_INTER causes an operand to generate a function
2673 // which returns its register number when queried.  CONST_INTER causes
2674 // an operand to generate a function which returns the value of the
2675 // constant when queried.  MEMORY_INTER causes an operand to generate
2676 // four functions which return the Base Register, the Index Register,
2677 // the Scale Value, and the Offset Value of the operand when queried.
2678 // COND_INTER causes an operand to generate six functions which return
2679 // the encoding code (ie - encoding bits for the instruction)
2680 // associated with each basic boolean condition for a conditional
2681 // instruction.
2682 //
2683 // Instructions specify two basic values for encoding.  Again, a
2684 // function is available to check if the constant displacement is an
2685 // oop. They use the ins_encode keyword to specify their encoding
2686 // classes (which must be a sequence of enc_class names, and their
2687 // parameters, specified in the encoding block), and they use the
2688 // opcode keyword to specify, in order, their primary, secondary, and
2689 // tertiary opcode.  Only the opcode sections which a particular
2690 // instruction needs for encoding need to be specified.
2691 encode %{
2692   // Build emit functions for each basic byte or larger field in the
2693   // intel encoding scheme (opcode, rm, sib, immediate), and call them
2694   // from C++ code in the enc_class source block.  Emit functions will
2695   // live in the main source block for now.  In future, we can
2696   // generalize this by adding a syntax that specifies the sizes of
2697   // fields in an order, so that the adlc can build the emit functions
2698   // automagically
2699 
2700   // catch all for unimplemented encodings
2701   enc_class enc_unimplemented %{
2702     MacroAssembler _masm(&cbuf);
2703     __ unimplemented("C2 catch all");
2704   %}
2705 
2706   // BEGIN Non-volatile memory access
2707 
2708   enc_class aarch64_enc_ldrsbw(iRegI dst, memory mem) %{
2709     Register dst_reg = as_Register($dst$$reg);
2710     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2711                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2712   %}
2713 
2714   enc_class aarch64_enc_ldrsb(iRegI dst, memory mem) %{
2715     Register dst_reg = as_Register($dst$$reg);
2716     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2717                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2718   %}
2719 
2720   enc_class aarch64_enc_ldrb(iRegI dst, memory mem) %{
2721     Register dst_reg = as_Register($dst$$reg);
2722     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2723                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2724   %}
2725 
2726   enc_class aarch64_enc_ldrb(iRegL dst, memory mem) %{
2727     Register dst_reg = as_Register($dst$$reg);
2728     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2729                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2730   %}
2731 
2732   enc_class aarch64_enc_ldrshw(iRegI dst, memory mem) %{
2733     Register dst_reg = as_Register($dst$$reg);
2734     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2735                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2736   %}
2737 
2738   enc_class aarch64_enc_ldrsh(iRegI dst, memory mem) %{
2739     Register dst_reg = as_Register($dst$$reg);
2740     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2741                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2742   %}
2743 
2744   enc_class aarch64_enc_ldrh(iRegI dst, memory mem) %{
2745     Register dst_reg = as_Register($dst$$reg);
2746     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2747                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2748   %}
2749 
2750   enc_class aarch64_enc_ldrh(iRegL dst, memory mem) %{
2751     Register dst_reg = as_Register($dst$$reg);
2752     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2753                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2754   %}
2755 
2756   enc_class aarch64_enc_ldrw(iRegI dst, memory mem) %{
2757     Register dst_reg = as_Register($dst$$reg);
2758     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2759                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2760   %}
2761 
2762   enc_class aarch64_enc_ldrw(iRegL dst, memory mem) %{
2763     Register dst_reg = as_Register($dst$$reg);
2764     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2765                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2766   %}
2767 
2768   enc_class aarch64_enc_ldrsw(iRegL dst, memory mem) %{
2769     Register dst_reg = as_Register($dst$$reg);
2770     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2771                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2772   %}
2773 
2774   enc_class aarch64_enc_ldr(iRegL dst, memory mem) %{
2775     Register dst_reg = as_Register($dst$$reg);
2776     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2777                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2778   %}
2779 
2780   enc_class aarch64_enc_ldrs(vRegF dst, memory mem) %{
2781     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2782     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2783                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2784   %}
2785 
2786   enc_class aarch64_enc_ldrd(vRegD dst, memory mem) %{
2787     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2788     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2789                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2790   %}
2791 
2792   enc_class aarch64_enc_strb(iRegI src, memory mem) %{
2793     Register src_reg = as_Register($src$$reg);
2794     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2795                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2796   %}
2797 
2798   enc_class aarch64_enc_strb0(memory mem) %{
2799     MacroAssembler _masm(&cbuf);
2800     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2801                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2802   %}
2803 
2804   enc_class aarch64_enc_strh(iRegI src, memory mem) %{
2805     Register src_reg = as_Register($src$$reg);
2806     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2807                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2808   %}
2809 
2810   enc_class aarch64_enc_strh0(memory mem) %{
2811     MacroAssembler _masm(&cbuf);
2812     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2813                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2814   %}
2815 
2816   enc_class aarch64_enc_strw(iRegI src, memory mem) %{
2817     Register src_reg = as_Register($src$$reg);
2818     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2819                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2820   %}
2821 
2822   enc_class aarch64_enc_strw0(memory mem) %{
2823     MacroAssembler _masm(&cbuf);
2824     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2825                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2826   %}
2827 
2828   enc_class aarch64_enc_str(iRegL src, memory mem) %{
2829     Register src_reg = as_Register($src$$reg);
2830     // we sometimes get asked to store the stack pointer into the
2831     // current thread -- we cannot do that directly on AArch64
2832     if (src_reg == r31_sp) {
2833       MacroAssembler _masm(&cbuf);
2834       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2835       __ mov(rscratch2, sp);
2836       src_reg = rscratch2;
2837     }
2838     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2839                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2840   %}
2841 
2842   enc_class aarch64_enc_str0(memory mem) %{
2843     MacroAssembler _masm(&cbuf);
2844     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2845                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2846   %}
2847 
2848   enc_class aarch64_enc_strs(vRegF src, memory mem) %{
2849     FloatRegister src_reg = as_FloatRegister($src$$reg);
2850     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2851                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2852   %}
2853 
2854   enc_class aarch64_enc_strd(vRegD src, memory mem) %{
2855     FloatRegister src_reg = as_FloatRegister($src$$reg);
2856     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2857                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2858   %}
2859 
2860   // END Non-volatile memory access
2861 
2862   // volatile loads and stores
2863 
2864   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
2865     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2866                  rscratch1, stlrb);
2867   %}
2868 
2869   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
2870     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2871                  rscratch1, stlrh);
2872   %}
2873 
2874   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
2875     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2876                  rscratch1, stlrw);
2877   %}
2878 
2879 
2880   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
2881     Register dst_reg = as_Register($dst$$reg);
2882     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2883              rscratch1, ldarb);
2884     __ sxtbw(dst_reg, dst_reg);
2885   %}
2886 
2887   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
2888     Register dst_reg = as_Register($dst$$reg);
2889     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2890              rscratch1, ldarb);
2891     __ sxtb(dst_reg, dst_reg);
2892   %}
2893 
2894   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
2895     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2896              rscratch1, ldarb);
2897   %}
2898 
2899   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
2900     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2901              rscratch1, ldarb);
2902   %}
2903 
2904   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
2905     Register dst_reg = as_Register($dst$$reg);
2906     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2907              rscratch1, ldarh);
2908     __ sxthw(dst_reg, dst_reg);
2909   %}
2910 
2911   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
2912     Register dst_reg = as_Register($dst$$reg);
2913     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2914              rscratch1, ldarh);
2915     __ sxth(dst_reg, dst_reg);
2916   %}
2917 
2918   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
2919     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2920              rscratch1, ldarh);
2921   %}
2922 
2923   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
2924     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2925              rscratch1, ldarh);
2926   %}
2927 
2928   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
2929     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2930              rscratch1, ldarw);
2931   %}
2932 
2933   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
2934     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2935              rscratch1, ldarw);
2936   %}
2937 
2938   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
2939     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2940              rscratch1, ldar);
2941   %}
2942 
2943   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
2944     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2945              rscratch1, ldarw);
2946     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
2947   %}
2948 
2949   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
2950     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2951              rscratch1, ldar);
2952     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
2953   %}
2954 
2955   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
2956     Register src_reg = as_Register($src$$reg);
2957     // we sometimes get asked to store the stack pointer into the
2958     // current thread -- we cannot do that directly on AArch64
2959     if (src_reg == r31_sp) {
2960         MacroAssembler _masm(&cbuf);
2961       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2962       __ mov(rscratch2, sp);
2963       src_reg = rscratch2;
2964     }
2965     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2966                  rscratch1, stlr);
2967   %}
2968 
2969   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
2970     {
2971       MacroAssembler _masm(&cbuf);
2972       FloatRegister src_reg = as_FloatRegister($src$$reg);
2973       __ fmovs(rscratch2, src_reg);
2974     }
2975     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2976                  rscratch1, stlrw);
2977   %}
2978 
2979   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
2980     {
2981       MacroAssembler _masm(&cbuf);
2982       FloatRegister src_reg = as_FloatRegister($src$$reg);
2983       __ fmovd(rscratch2, src_reg);
2984     }
2985     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2986                  rscratch1, stlr);
2987   %}
2988 
2989   // synchronized read/update encodings
2990 
2991   enc_class aarch64_enc_ldaxr(iRegL dst, memory mem) %{
2992     MacroAssembler _masm(&cbuf);
2993     Register dst_reg = as_Register($dst$$reg);
2994     Register base = as_Register($mem$$base);
2995     int index = $mem$$index;
2996     int scale = $mem$$scale;
2997     int disp = $mem$$disp;
2998     if (index == -1) {
2999        if (disp != 0) {
3000         __ lea(rscratch1, Address(base, disp));
3001         __ ldaxr(dst_reg, rscratch1);
3002       } else {
3003         // TODO
3004         // should we ever get anything other than this case?
3005         __ ldaxr(dst_reg, base);
3006       }
3007     } else {
3008       Register index_reg = as_Register(index);
3009       if (disp == 0) {
3010         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3011         __ ldaxr(dst_reg, rscratch1);
3012       } else {
3013         __ lea(rscratch1, Address(base, disp));
3014         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3015         __ ldaxr(dst_reg, rscratch1);
3016       }
3017     }
3018   %}
3019 
3020   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory mem) %{
3021     MacroAssembler _masm(&cbuf);
3022     Register src_reg = as_Register($src$$reg);
3023     Register base = as_Register($mem$$base);
3024     int index = $mem$$index;
3025     int scale = $mem$$scale;
3026     int disp = $mem$$disp;
3027     if (index == -1) {
3028        if (disp != 0) {
3029         __ lea(rscratch2, Address(base, disp));
3030         __ stlxr(rscratch1, src_reg, rscratch2);
3031       } else {
3032         // TODO
3033         // should we ever get anything other than this case?
3034         __ stlxr(rscratch1, src_reg, base);
3035       }
3036     } else {
3037       Register index_reg = as_Register(index);
3038       if (disp == 0) {
3039         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3040         __ stlxr(rscratch1, src_reg, rscratch2);
3041       } else {
3042         __ lea(rscratch2, Address(base, disp));
3043         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3044         __ stlxr(rscratch1, src_reg, rscratch2);
3045       }
3046     }
3047     __ cmpw(rscratch1, zr);
3048   %}
3049 
3050   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3051     MacroAssembler _masm(&cbuf);
3052     Register old_reg = as_Register($oldval$$reg);
3053     Register new_reg = as_Register($newval$$reg);
3054     Register base = as_Register($mem$$base);
3055     Register addr_reg;
3056     int index = $mem$$index;
3057     int scale = $mem$$scale;
3058     int disp = $mem$$disp;
3059     if (index == -1) {
3060        if (disp != 0) {
3061         __ lea(rscratch2, Address(base, disp));
3062         addr_reg = rscratch2;
3063       } else {
3064         // TODO
3065         // should we ever get anything other than this case?
3066         addr_reg = base;
3067       }
3068     } else {
3069       Register index_reg = as_Register(index);
3070       if (disp == 0) {
3071         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3072         addr_reg = rscratch2;
3073       } else {
3074         __ lea(rscratch2, Address(base, disp));
3075         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3076         addr_reg = rscratch2;
3077       }
3078     }
3079     Label retry_load, done;
3080     __ bind(retry_load);
3081     __ ldxr(rscratch1, addr_reg);
3082     __ cmp(rscratch1, old_reg);
3083     __ br(Assembler::NE, done);
3084     __ stlxr(rscratch1, new_reg, addr_reg);
3085     __ cbnzw(rscratch1, retry_load);
3086     __ bind(done);
3087   %}
3088 
3089   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3090     MacroAssembler _masm(&cbuf);
3091     Register old_reg = as_Register($oldval$$reg);
3092     Register new_reg = as_Register($newval$$reg);
3093     Register base = as_Register($mem$$base);
3094     Register addr_reg;
3095     int index = $mem$$index;
3096     int scale = $mem$$scale;
3097     int disp = $mem$$disp;
3098     if (index == -1) {
3099        if (disp != 0) {
3100         __ lea(rscratch2, Address(base, disp));
3101         addr_reg = rscratch2;
3102       } else {
3103         // TODO
3104         // should we ever get anything other than this case?
3105         addr_reg = base;
3106       }
3107     } else {
3108       Register index_reg = as_Register(index);
3109       if (disp == 0) {
3110         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3111         addr_reg = rscratch2;
3112       } else {
3113         __ lea(rscratch2, Address(base, disp));
3114         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3115         addr_reg = rscratch2;
3116       }
3117     }
3118     Label retry_load, done;
3119     __ bind(retry_load);
3120     __ ldxrw(rscratch1, addr_reg);
3121     __ cmpw(rscratch1, old_reg);
3122     __ br(Assembler::NE, done);
3123     __ stlxrw(rscratch1, new_reg, addr_reg);
3124     __ cbnzw(rscratch1, retry_load);
3125     __ bind(done);
3126   %}
3127 
3128   // auxiliary used for CompareAndSwapX to set result register
3129   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3130     MacroAssembler _masm(&cbuf);
3131     Register res_reg = as_Register($res$$reg);
3132     __ cset(res_reg, Assembler::EQ);
3133   %}
3134 
3135   // prefetch encodings
3136 
3137   enc_class aarch64_enc_prefetchw(memory mem) %{
3138     MacroAssembler _masm(&cbuf);
3139     Register base = as_Register($mem$$base);
3140     int index = $mem$$index;
3141     int scale = $mem$$scale;
3142     int disp = $mem$$disp;
3143     if (index == -1) {
3144       __ prfm(Address(base, disp), PSTL1KEEP);
3145       __ nop();
3146     } else {
3147       Register index_reg = as_Register(index);
3148       if (disp == 0) {
3149         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3150       } else {
3151         __ lea(rscratch1, Address(base, disp));
3152         __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3153       }
3154     }
3155   %}
3156 
3157   enc_class aarch64_enc_clear_array_reg_reg(iRegL_R11 cnt, iRegP_R10 base) %{
3158     MacroAssembler _masm(&cbuf);
3159     Register cnt_reg = as_Register($cnt$$reg);
3160     Register base_reg = as_Register($base$$reg);
3161     // base is word aligned
3162     // cnt is count of words
3163 
3164     Label loop;
3165     Label entry;
3166 
3167 //  Algorithm:
3168 //
3169 //    scratch1 = cnt & 7;
3170 //    cnt -= scratch1;
3171 //    p += scratch1;
3172 //    switch (scratch1) {
3173 //      do {
3174 //        cnt -= 8;
3175 //          p[-8] = 0;
3176 //        case 7:
3177 //          p[-7] = 0;
3178 //        case 6:
3179 //          p[-6] = 0;
3180 //          // ...
3181 //        case 1:
3182 //          p[-1] = 0;
3183 //        case 0:
3184 //          p += 8;
3185 //      } while (cnt);
3186 //    }
3187 
3188     const int unroll = 8; // Number of str(zr) instructions we'll unroll
3189 
3190     __ andr(rscratch1, cnt_reg, unroll - 1);  // tmp1 = cnt % unroll
3191     __ sub(cnt_reg, cnt_reg, rscratch1);      // cnt -= unroll
3192     // base_reg always points to the end of the region we're about to zero
3193     __ add(base_reg, base_reg, rscratch1, Assembler::LSL, exact_log2(wordSize));
3194     __ adr(rscratch2, entry);
3195     __ sub(rscratch2, rscratch2, rscratch1, Assembler::LSL, 2);
3196     __ br(rscratch2);
3197     __ bind(loop);
3198     __ sub(cnt_reg, cnt_reg, unroll);
3199     for (int i = -unroll; i < 0; i++)
3200       __ str(zr, Address(base_reg, i * wordSize));
3201     __ bind(entry);
3202     __ add(base_reg, base_reg, unroll * wordSize);
3203     __ cbnz(cnt_reg, loop);
3204   %}
3205 
3206   /// mov envcodings
3207 
3208   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3209     MacroAssembler _masm(&cbuf);
3210     u_int32_t con = (u_int32_t)$src$$constant;
3211     Register dst_reg = as_Register($dst$$reg);
3212     if (con == 0) {
3213       __ movw(dst_reg, zr);
3214     } else {
3215       __ movw(dst_reg, con);
3216     }
3217   %}
3218 
3219   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3220     MacroAssembler _masm(&cbuf);
3221     Register dst_reg = as_Register($dst$$reg);
3222     u_int64_t con = (u_int64_t)$src$$constant;
3223     if (con == 0) {
3224       __ mov(dst_reg, zr);
3225     } else {
3226       __ mov(dst_reg, con);
3227     }
3228   %}
3229 
3230   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3231     MacroAssembler _masm(&cbuf);
3232     Register dst_reg = as_Register($dst$$reg);
3233     address con = (address)$src$$constant;
3234     if (con == NULL || con == (address)1) {
3235       ShouldNotReachHere();
3236     } else {
3237       relocInfo::relocType rtype = $src->constant_reloc();
3238       if (rtype == relocInfo::oop_type) {
3239         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
3240       } else if (rtype == relocInfo::metadata_type) {
3241         __ mov_metadata(dst_reg, (Metadata*)con);
3242       } else {
3243         assert(rtype == relocInfo::none, "unexpected reloc type");
3244         if (con < (address)(uintptr_t)os::vm_page_size()) {
3245           __ mov(dst_reg, con);
3246         } else {
3247           unsigned long offset;
3248           __ adrp(dst_reg, con, offset);
3249           __ add(dst_reg, dst_reg, offset);
3250         }
3251       }
3252     }
3253   %}
3254 
3255   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3256     MacroAssembler _masm(&cbuf);
3257     Register dst_reg = as_Register($dst$$reg);
3258     __ mov(dst_reg, zr);
3259   %}
3260 
3261   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3262     MacroAssembler _masm(&cbuf);
3263     Register dst_reg = as_Register($dst$$reg);
3264     __ mov(dst_reg, (u_int64_t)1);
3265   %}
3266 
3267   enc_class aarch64_enc_mov_poll_page(iRegP dst, immPollPage src) %{
3268     MacroAssembler _masm(&cbuf);
3269     address page = (address)$src$$constant;
3270     Register dst_reg = as_Register($dst$$reg);
3271     unsigned long off;
3272     __ adrp(dst_reg, Address(page, relocInfo::poll_type), off);
3273     assert(off == 0, "assumed offset == 0");
3274   %}
3275 
3276   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3277     MacroAssembler _masm(&cbuf);
3278     address page = (address)$src$$constant;
3279     Register dst_reg = as_Register($dst$$reg);
3280     unsigned long off;
3281     __ adrp(dst_reg, ExternalAddress(page), off);
3282     assert(off == 0, "assumed offset == 0");
3283   %}
3284 
3285   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3286     MacroAssembler _masm(&cbuf);
3287     Register dst_reg = as_Register($dst$$reg);
3288     address con = (address)$src$$constant;
3289     if (con == NULL) {
3290       ShouldNotReachHere();
3291     } else {
3292       relocInfo::relocType rtype = $src->constant_reloc();
3293       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3294       __ set_narrow_oop(dst_reg, (jobject)con);
3295     }
3296   %}
3297 
3298   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3299     MacroAssembler _masm(&cbuf);
3300     Register dst_reg = as_Register($dst$$reg);
3301     __ mov(dst_reg, zr);
3302   %}
3303 
3304   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3305     MacroAssembler _masm(&cbuf);
3306     Register dst_reg = as_Register($dst$$reg);
3307     address con = (address)$src$$constant;
3308     if (con == NULL) {
3309       ShouldNotReachHere();
3310     } else {
3311       relocInfo::relocType rtype = $src->constant_reloc();
3312       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3313       __ set_narrow_klass(dst_reg, (Klass *)con);
3314     }
3315   %}
3316 
3317   // arithmetic encodings
3318 
3319   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3320     MacroAssembler _masm(&cbuf);
3321     Register dst_reg = as_Register($dst$$reg);
3322     Register src_reg = as_Register($src1$$reg);
3323     int32_t con = (int32_t)$src2$$constant;
3324     // add has primary == 0, subtract has primary == 1
3325     if ($primary) { con = -con; }
3326     if (con < 0) {
3327       __ subw(dst_reg, src_reg, -con);
3328     } else {
3329       __ addw(dst_reg, src_reg, con);
3330     }
3331   %}
3332 
3333   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3334     MacroAssembler _masm(&cbuf);
3335     Register dst_reg = as_Register($dst$$reg);
3336     Register src_reg = as_Register($src1$$reg);
3337     int32_t con = (int32_t)$src2$$constant;
3338     // add has primary == 0, subtract has primary == 1
3339     if ($primary) { con = -con; }
3340     if (con < 0) {
3341       __ sub(dst_reg, src_reg, -con);
3342     } else {
3343       __ add(dst_reg, src_reg, con);
3344     }
3345   %}
3346 
3347   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3348     MacroAssembler _masm(&cbuf);
3349    Register dst_reg = as_Register($dst$$reg);
3350    Register src1_reg = as_Register($src1$$reg);
3351    Register src2_reg = as_Register($src2$$reg);
3352     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3353   %}
3354 
3355   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3356     MacroAssembler _masm(&cbuf);
3357    Register dst_reg = as_Register($dst$$reg);
3358    Register src1_reg = as_Register($src1$$reg);
3359    Register src2_reg = as_Register($src2$$reg);
3360     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3361   %}
3362 
3363   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3364     MacroAssembler _masm(&cbuf);
3365    Register dst_reg = as_Register($dst$$reg);
3366    Register src1_reg = as_Register($src1$$reg);
3367    Register src2_reg = as_Register($src2$$reg);
3368     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3369   %}
3370 
3371   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3372     MacroAssembler _masm(&cbuf);
3373    Register dst_reg = as_Register($dst$$reg);
3374    Register src1_reg = as_Register($src1$$reg);
3375    Register src2_reg = as_Register($src2$$reg);
3376     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3377   %}
3378 
3379   // compare instruction encodings
3380 
3381   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3382     MacroAssembler _masm(&cbuf);
3383     Register reg1 = as_Register($src1$$reg);
3384     Register reg2 = as_Register($src2$$reg);
3385     __ cmpw(reg1, reg2);
3386   %}
3387 
3388   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3389     MacroAssembler _masm(&cbuf);
3390     Register reg = as_Register($src1$$reg);
3391     int32_t val = $src2$$constant;
3392     if (val >= 0) {
3393       __ subsw(zr, reg, val);
3394     } else {
3395       __ addsw(zr, reg, -val);
3396     }
3397   %}
3398 
3399   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3400     MacroAssembler _masm(&cbuf);
3401     Register reg1 = as_Register($src1$$reg);
3402     u_int32_t val = (u_int32_t)$src2$$constant;
3403     __ movw(rscratch1, val);
3404     __ cmpw(reg1, rscratch1);
3405   %}
3406 
3407   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3408     MacroAssembler _masm(&cbuf);
3409     Register reg1 = as_Register($src1$$reg);
3410     Register reg2 = as_Register($src2$$reg);
3411     __ cmp(reg1, reg2);
3412   %}
3413 
3414   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3415     MacroAssembler _masm(&cbuf);
3416     Register reg = as_Register($src1$$reg);
3417     int64_t val = $src2$$constant;
3418     if (val >= 0) {
3419       __ subs(zr, reg, val);
3420     } else if (val != -val) {
3421       __ adds(zr, reg, -val);
3422     } else {
3423     // aargh, Long.MIN_VALUE is a special case
3424       __ orr(rscratch1, zr, (u_int64_t)val);
3425       __ subs(zr, reg, rscratch1);
3426     }
3427   %}
3428 
3429   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3430     MacroAssembler _masm(&cbuf);
3431     Register reg1 = as_Register($src1$$reg);
3432     u_int64_t val = (u_int64_t)$src2$$constant;
3433     __ mov(rscratch1, val);
3434     __ cmp(reg1, rscratch1);
3435   %}
3436 
3437   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3438     MacroAssembler _masm(&cbuf);
3439     Register reg1 = as_Register($src1$$reg);
3440     Register reg2 = as_Register($src2$$reg);
3441     __ cmp(reg1, reg2);
3442   %}
3443 
3444   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3445     MacroAssembler _masm(&cbuf);
3446     Register reg1 = as_Register($src1$$reg);
3447     Register reg2 = as_Register($src2$$reg);
3448     __ cmpw(reg1, reg2);
3449   %}
3450 
3451   enc_class aarch64_enc_testp(iRegP src) %{
3452     MacroAssembler _masm(&cbuf);
3453     Register reg = as_Register($src$$reg);
3454     __ cmp(reg, zr);
3455   %}
3456 
3457   enc_class aarch64_enc_testn(iRegN src) %{
3458     MacroAssembler _masm(&cbuf);
3459     Register reg = as_Register($src$$reg);
3460     __ cmpw(reg, zr);
3461   %}
3462 
3463   enc_class aarch64_enc_b(label lbl) %{
3464     MacroAssembler _masm(&cbuf);
3465     Label *L = $lbl$$label;
3466     __ b(*L);
3467   %}
3468 
3469   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3470     MacroAssembler _masm(&cbuf);
3471     Label *L = $lbl$$label;
3472     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3473   %}
3474 
3475   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3476     MacroAssembler _masm(&cbuf);
3477     Label *L = $lbl$$label;
3478     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3479   %}
3480 
3481   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3482   %{
3483      Register sub_reg = as_Register($sub$$reg);
3484      Register super_reg = as_Register($super$$reg);
3485      Register temp_reg = as_Register($temp$$reg);
3486      Register result_reg = as_Register($result$$reg);
3487 
3488      Label miss;
3489      MacroAssembler _masm(&cbuf);
3490      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3491                                      NULL, &miss,
3492                                      /*set_cond_codes:*/ true);
3493      if ($primary) {
3494        __ mov(result_reg, zr);
3495      }
3496      __ bind(miss);
3497   %}
3498 
3499   enc_class aarch64_enc_java_static_call(method meth) %{
3500     MacroAssembler _masm(&cbuf);
3501 
3502     address addr = (address)$meth$$method;
3503     if (!_method) {
3504       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3505       __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3506     } else if (_optimized_virtual) {
3507       __ trampoline_call(Address(addr, relocInfo::opt_virtual_call_type), &cbuf);
3508     } else {
3509       __ trampoline_call(Address(addr, relocInfo::static_call_type), &cbuf);
3510     }
3511 
3512     if (_method) {
3513       // Emit stub for static call
3514       CompiledStaticCall::emit_to_interp_stub(cbuf);
3515     }
3516   %}
3517 
3518   enc_class aarch64_enc_java_handle_call(method meth) %{
3519     MacroAssembler _masm(&cbuf);
3520     relocInfo::relocType reloc;
3521 
3522     // RFP is preserved across all calls, even compiled calls.
3523     // Use it to preserve SP.
3524     __ mov(rfp, sp);
3525 
3526     const int start_offset = __ offset();
3527     address addr = (address)$meth$$method;
3528     if (!_method) {
3529       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3530       __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3531     } else if (_optimized_virtual) {
3532       __ trampoline_call(Address(addr, relocInfo::opt_virtual_call_type), &cbuf);
3533     } else {
3534       __ trampoline_call(Address(addr, relocInfo::static_call_type), &cbuf);
3535     }
3536 
3537     if (_method) {
3538       // Emit stub for static call
3539       CompiledStaticCall::emit_to_interp_stub(cbuf);
3540     }
3541 
3542     // now restore sp
3543     __ mov(sp, rfp);
3544   %}
3545 
3546   enc_class aarch64_enc_java_dynamic_call(method meth) %{
3547     MacroAssembler _masm(&cbuf);
3548     __ ic_call((address)$meth$$method);
3549   %}
3550 
3551   enc_class aarch64_enc_call_epilog() %{
3552     MacroAssembler _masm(&cbuf);
3553     if (VerifyStackAtCalls) {
3554       // Check that stack depth is unchanged: find majik cookie on stack
3555       __ call_Unimplemented();
3556     }
3557   %}
3558 
3559   enc_class aarch64_enc_java_to_runtime(method meth) %{
3560     MacroAssembler _masm(&cbuf);
3561 
3562     // some calls to generated routines (arraycopy code) are scheduled
3563     // by C2 as runtime calls. if so we can call them using a br (they
3564     // will be in a reachable segment) otherwise we have to use a blrt
3565     // which loads the absolute address into a register.
3566     address entry = (address)$meth$$method;
3567     CodeBlob *cb = CodeCache::find_blob(entry);
3568     if (cb) {
3569       __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3570     } else {
3571       int gpcnt;
3572       int fpcnt;
3573       int rtype;
3574       getCallInfo(tf(), gpcnt, fpcnt, rtype);
3575       Label retaddr;
3576       __ adr(rscratch2, retaddr);
3577       __ lea(rscratch1, RuntimeAddress(entry));
3578       // Leave a breadcrumb for JavaThread::pd_last_frame().
3579       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3580       __ blrt(rscratch1, gpcnt, fpcnt, rtype);
3581       __ bind(retaddr);
3582       __ add(sp, sp, 2 * wordSize);
3583     }
3584   %}
3585 
3586   enc_class aarch64_enc_rethrow() %{
3587     MacroAssembler _masm(&cbuf);
3588     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3589   %}
3590 
3591   enc_class aarch64_enc_ret() %{
3592     MacroAssembler _masm(&cbuf);
3593     __ ret(lr);
3594   %}
3595 
3596   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3597     MacroAssembler _masm(&cbuf);
3598     Register target_reg = as_Register($jump_target$$reg);
3599     __ br(target_reg);
3600   %}
3601 
3602   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3603     MacroAssembler _masm(&cbuf);
3604     Register target_reg = as_Register($jump_target$$reg);
3605     // exception oop should be in r0
3606     // ret addr has been popped into lr
3607     // callee expects it in r3
3608     __ mov(r3, lr);
3609     __ br(target_reg);
3610   %}
3611 
3612   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3613     MacroAssembler _masm(&cbuf);
3614     Register oop = as_Register($object$$reg);
3615     Register box = as_Register($box$$reg);
3616     Register disp_hdr = as_Register($tmp$$reg);
3617     Register tmp = as_Register($tmp2$$reg);
3618     Label cont;
3619     Label object_has_monitor;
3620     Label cas_failed;
3621 
3622     assert_different_registers(oop, box, tmp, disp_hdr);
3623 
3624     // Load markOop from object into displaced_header.
3625     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3626 
3627     // Always do locking in runtime.
3628     if (EmitSync & 0x01) {
3629       __ cmp(oop, zr);
3630       return;
3631     }
3632 
3633     if (UseBiasedLocking) {
3634       __ biased_locking_enter(disp_hdr, oop, box, tmp, true, cont);
3635     }
3636 
3637     // Handle existing monitor
3638     if (EmitSync & 0x02) {
3639       // we can use AArch64's bit test and branch here but
3640       // markoopDesc does not define a bit index just the bit value
3641       // so assert in case the bit pos changes
3642 #     define __monitor_value_log2 1
3643       assert(markOopDesc::monitor_value == (1 << __monitor_value_log2), "incorrect bit position");
3644       __ tbnz(disp_hdr, __monitor_value_log2, object_has_monitor);
3645 #     undef __monitor_value_log2
3646     }
3647 
3648     // Set displaced_header to be (markOop of object | UNLOCK_VALUE).
3649     __ orr(disp_hdr, disp_hdr, markOopDesc::unlocked_value);
3650 
3651     // Load Compare Value application register.
3652 
3653     // Initialize the box. (Must happen before we update the object mark!)
3654     __ str(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3655 
3656     // Compare object markOop with mark and if equal exchange scratch1
3657     // with object markOop.
3658     // Note that this is simply a CAS: it does not generate any
3659     // barriers.  These are separately generated by
3660     // membar_acquire_lock().
3661     {
3662       Label retry_load;
3663       __ bind(retry_load);
3664       __ ldxr(tmp, oop);
3665       __ cmp(tmp, disp_hdr);
3666       __ br(Assembler::NE, cas_failed);
3667       // use stlxr to ensure update is immediately visible
3668       __ stlxr(tmp, box, oop);
3669       __ cbzw(tmp, cont);
3670       __ b(retry_load);
3671     }
3672 
3673     // Formerly:
3674     // __ cmpxchgptr(/*oldv=*/disp_hdr,
3675     //               /*newv=*/box,
3676     //               /*addr=*/oop,
3677     //               /*tmp=*/tmp,
3678     //               cont,
3679     //               /*fail*/NULL);
3680 
3681     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3682 
3683     // If the compare-and-exchange succeeded, then we found an unlocked
3684     // object, will have now locked it will continue at label cont
3685 
3686     __ bind(cas_failed);
3687     // We did not see an unlocked object so try the fast recursive case.
3688 
3689     // Check if the owner is self by comparing the value in the
3690     // markOop of object (disp_hdr) with the stack pointer.
3691     __ mov(rscratch1, sp);
3692     __ sub(disp_hdr, disp_hdr, rscratch1);
3693     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place));
3694     // If condition is true we are cont and hence we can store 0 as the
3695     // displaced header in the box, which indicates that it is a recursive lock.
3696     __ ands(tmp/*==0?*/, disp_hdr, tmp);
3697     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3698 
3699     // Handle existing monitor.
3700     if ((EmitSync & 0x02) == 0) {
3701       __ b(cont);
3702 
3703       __ bind(object_has_monitor);
3704       // The object's monitor m is unlocked iff m->owner == NULL,
3705       // otherwise m->owner may contain a thread or a stack address.
3706       //
3707       // Try to CAS m->owner from NULL to current thread.
3708       __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
3709       __ mov(disp_hdr, zr);
3710 
3711       {
3712         Label retry_load, fail;
3713         __ bind(retry_load);
3714         __ ldxr(rscratch1, tmp);
3715         __ cmp(disp_hdr, rscratch1);
3716         __ br(Assembler::NE, fail);
3717         // use stlxr to ensure update is immediately visible
3718         __ stlxr(rscratch1, rthread, tmp);
3719         __ cbnzw(rscratch1, retry_load);
3720         __ bind(fail);
3721       }
3722 
3723       // Label next;
3724       // __ cmpxchgptr(/*oldv=*/disp_hdr,
3725       //               /*newv=*/rthread,
3726       //               /*addr=*/tmp,
3727       //               /*tmp=*/rscratch1,
3728       //               /*succeed*/next,
3729       //               /*fail*/NULL);
3730       // __ bind(next);
3731 
3732       // store a non-null value into the box.
3733       __ str(box, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3734 
3735       // PPC port checks the following invariants
3736       // #ifdef ASSERT
3737       // bne(flag, cont);
3738       // We have acquired the monitor, check some invariants.
3739       // addw(/*monitor=*/tmp, tmp, -ObjectMonitor::owner_offset_in_bytes());
3740       // Invariant 1: _recursions should be 0.
3741       // assert(ObjectMonitor::recursions_size_in_bytes() == 8, "unexpected size");
3742       // assert_mem8_is_zero(ObjectMonitor::recursions_offset_in_bytes(), tmp,
3743       //                        "monitor->_recursions should be 0", -1);
3744       // Invariant 2: OwnerIsThread shouldn't be 0.
3745       // assert(ObjectMonitor::OwnerIsThread_size_in_bytes() == 4, "unexpected size");
3746       //assert_mem4_isnot_zero(ObjectMonitor::OwnerIsThread_offset_in_bytes(), tmp,
3747       //                           "monitor->OwnerIsThread shouldn't be 0", -1);
3748       // #endif
3749     }
3750 
3751     __ bind(cont);
3752     // flag == EQ indicates success
3753     // flag == NE indicates failure
3754 
3755   %}
3756 
3757   // TODO
3758   // reimplement this with custom cmpxchgptr code
3759   // which avoids some of the unnecessary branching
3760   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3761     MacroAssembler _masm(&cbuf);
3762     Register oop = as_Register($object$$reg);
3763     Register box = as_Register($box$$reg);
3764     Register disp_hdr = as_Register($tmp$$reg);
3765     Register tmp = as_Register($tmp2$$reg);
3766     Label cont;
3767     Label object_has_monitor;
3768     Label cas_failed;
3769 
3770     assert_different_registers(oop, box, tmp, disp_hdr);
3771 
3772     // Always do locking in runtime.
3773     if (EmitSync & 0x01) {
3774       __ cmp(oop, zr); // Oop can't be 0 here => always false.
3775       return;
3776     }
3777 
3778     if (UseBiasedLocking) {
3779       __ biased_locking_exit(oop, tmp, cont);
3780     }
3781 
3782     // Find the lock address and load the displaced header from the stack.
3783     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3784 
3785     // If the displaced header is 0, we have a recursive unlock.
3786     __ cmp(disp_hdr, zr);
3787     __ br(Assembler::EQ, cont);
3788 
3789 
3790     // Handle existing monitor.
3791     if ((EmitSync & 0x02) == 0) {
3792       __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3793       __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3794     }
3795 
3796     // Check if it is still a light weight lock, this is is true if we
3797     // see the stack address of the basicLock in the markOop of the
3798     // object.
3799 
3800       {
3801         Label retry_load;
3802         __ bind(retry_load);
3803         __ ldxr(tmp, oop);
3804         __ cmp(box, tmp);
3805         __ br(Assembler::NE, cas_failed);
3806         // use stlxr to ensure update is immediately visible
3807         __ stlxr(tmp, disp_hdr, oop);
3808         __ cbzw(tmp, cont);
3809         __ b(retry_load);
3810       }
3811 
3812     // __ cmpxchgptr(/*compare_value=*/box,
3813     //               /*exchange_value=*/disp_hdr,
3814     //               /*where=*/oop,
3815     //               /*result=*/tmp,
3816     //               cont,
3817     //               /*cas_failed*/NULL);
3818     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3819 
3820     __ bind(cas_failed);
3821 
3822     // Handle existing monitor.
3823     if ((EmitSync & 0x02) == 0) {
3824       __ b(cont);
3825 
3826       __ bind(object_has_monitor);
3827       __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor
3828       __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3829       __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3830       __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3831       __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3832       __ cmp(rscratch1, zr);
3833       __ br(Assembler::NE, cont);
3834 
3835       __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3836       __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3837       __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3838       __ cmp(rscratch1, zr);
3839       __ cbnz(rscratch1, cont);
3840       // need a release store here
3841       __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3842       __ stlr(rscratch1, tmp); // rscratch1 is zero
3843     }
3844 
3845     __ bind(cont);
3846     // flag == EQ indicates success
3847     // flag == NE indicates failure
3848   %}
3849 
3850 %}
3851 
3852 //----------FRAME--------------------------------------------------------------
3853 // Definition of frame structure and management information.
3854 //
3855 //  S T A C K   L A Y O U T    Allocators stack-slot number
3856 //                             |   (to get allocators register number
3857 //  G  Owned by    |        |  v    add OptoReg::stack0())
3858 //  r   CALLER     |        |
3859 //  o     |        +--------+      pad to even-align allocators stack-slot
3860 //  w     V        |  pad0  |        numbers; owned by CALLER
3861 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
3862 //  h     ^        |   in   |  5
3863 //        |        |  args  |  4   Holes in incoming args owned by SELF
3864 //  |     |        |        |  3
3865 //  |     |        +--------+
3866 //  V     |        | old out|      Empty on Intel, window on Sparc
3867 //        |    old |preserve|      Must be even aligned.
3868 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
3869 //        |        |   in   |  3   area for Intel ret address
3870 //     Owned by    |preserve|      Empty on Sparc.
3871 //       SELF      +--------+
3872 //        |        |  pad2  |  2   pad to align old SP
3873 //        |        +--------+  1
3874 //        |        | locks  |  0
3875 //        |        +--------+----> OptoReg::stack0(), even aligned
3876 //        |        |  pad1  | 11   pad to align new SP
3877 //        |        +--------+
3878 //        |        |        | 10
3879 //        |        | spills |  9   spills
3880 //        V        |        |  8   (pad0 slot for callee)
3881 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
3882 //        ^        |  out   |  7
3883 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
3884 //     Owned by    +--------+
3885 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
3886 //        |    new |preserve|      Must be even-aligned.
3887 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
3888 //        |        |        |
3889 //
3890 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
3891 //         known from SELF's arguments and the Java calling convention.
3892 //         Region 6-7 is determined per call site.
3893 // Note 2: If the calling convention leaves holes in the incoming argument
3894 //         area, those holes are owned by SELF.  Holes in the outgoing area
3895 //         are owned by the CALLEE.  Holes should not be nessecary in the
3896 //         incoming area, as the Java calling convention is completely under
3897 //         the control of the AD file.  Doubles can be sorted and packed to
3898 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
3899 //         varargs C calling conventions.
3900 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
3901 //         even aligned with pad0 as needed.
3902 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
3903 //           (the latter is true on Intel but is it false on AArch64?)
3904 //         region 6-11 is even aligned; it may be padded out more so that
3905 //         the region from SP to FP meets the minimum stack alignment.
3906 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3907 //         alignment.  Region 11, pad1, may be dynamically extended so that
3908 //         SP meets the minimum alignment.
3909 
3910 frame %{
3911   // What direction does stack grow in (assumed to be same for C & Java)
3912   stack_direction(TOWARDS_LOW);
3913 
3914   // These three registers define part of the calling convention
3915   // between compiled code and the interpreter.
3916 
3917   // Inline Cache Register or methodOop for I2C.
3918   inline_cache_reg(R12);
3919 
3920   // Method Oop Register when calling interpreter.
3921   interpreter_method_oop_reg(R12);
3922 
3923   // Number of stack slots consumed by locking an object
3924   sync_stack_slots(2);
3925 
3926   // Compiled code's Frame Pointer
3927   frame_pointer(R31);
3928 
3929   // Interpreter stores its frame pointer in a register which is
3930   // stored to the stack by I2CAdaptors.
3931   // I2CAdaptors convert from interpreted java to compiled java.
3932   interpreter_frame_pointer(R29);
3933 
3934   // Stack alignment requirement
3935   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3936 
3937   // Number of stack slots between incoming argument block and the start of
3938   // a new frame.  The PROLOG must add this many slots to the stack.  The
3939   // EPILOG must remove this many slots. aarch64 needs two slots for
3940   // return address and fp.
3941   // TODO think this is correct but check
3942   in_preserve_stack_slots(4);
3943 
3944   // Number of outgoing stack slots killed above the out_preserve_stack_slots
3945   // for calls to C.  Supports the var-args backing area for register parms.
3946   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3947 
3948   // The after-PROLOG location of the return address.  Location of
3949   // return address specifies a type (REG or STACK) and a number
3950   // representing the register number (i.e. - use a register name) or
3951   // stack slot.
3952   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3953   // Otherwise, it is above the locks and verification slot and alignment word
3954   // TODO this may well be correct but need to check why that - 2 is there
3955   // ppc port uses 0 but we definitely need to allow for fixed_slots
3956   // which folds in the space used for monitors
3957   return_addr(STACK - 2 +
3958               round_to((Compile::current()->in_preserve_stack_slots() +
3959                         Compile::current()->fixed_slots()),
3960                        stack_alignment_in_slots()));
3961 
3962   // Body of function which returns an integer array locating
3963   // arguments either in registers or in stack slots.  Passed an array
3964   // of ideal registers called "sig" and a "length" count.  Stack-slot
3965   // offsets are based on outgoing arguments, i.e. a CALLER setting up
3966   // arguments for a CALLEE.  Incoming stack arguments are
3967   // automatically biased by the preserve_stack_slots field above.
3968 
3969   calling_convention
3970   %{
3971     // No difference between ingoing/outgoing just pass false
3972     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3973   %}
3974 
3975   c_calling_convention
3976   %{
3977     // This is obviously always outgoing
3978     (void) SharedRuntime::c_calling_convention(sig_bt, regs, NULL, length);
3979   %}
3980 
3981   // Location of compiled Java return values.  Same as C for now.
3982   return_value
3983   %{
3984     // TODO do we allow ideal_reg == Op_RegN???
3985     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3986            "only return normal values");
3987 
3988     static const int lo[Op_RegL + 1] = { // enum name
3989       0,                                 // Op_Node
3990       0,                                 // Op_Set
3991       R0_num,                            // Op_RegN
3992       R0_num,                            // Op_RegI
3993       R0_num,                            // Op_RegP
3994       V0_num,                            // Op_RegF
3995       V0_num,                            // Op_RegD
3996       R0_num                             // Op_RegL
3997     };
3998 
3999     static const int hi[Op_RegL + 1] = { // enum name
4000       0,                                 // Op_Node
4001       0,                                 // Op_Set
4002       OptoReg::Bad,                       // Op_RegN
4003       OptoReg::Bad,                      // Op_RegI
4004       R0_H_num,                          // Op_RegP
4005       OptoReg::Bad,                      // Op_RegF
4006       V0_H_num,                          // Op_RegD
4007       R0_H_num                           // Op_RegL
4008     };
4009 
4010     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4011   %}
4012 %}
4013 
4014 //----------ATTRIBUTES---------------------------------------------------------
4015 //----------Operand Attributes-------------------------------------------------
4016 op_attrib op_cost(1);        // Required cost attribute
4017 
4018 //----------Instruction Attributes---------------------------------------------
4019 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4020 ins_attrib ins_size(32);        // Required size attribute (in bits)
4021 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4022                                 // a non-matching short branch variant
4023                                 // of some long branch?
4024 ins_attrib ins_alignment(4);    // Required alignment attribute (must
4025                                 // be a power of 2) specifies the
4026                                 // alignment that some part of the
4027                                 // instruction (not necessarily the
4028                                 // start) requires.  If > 1, a
4029                                 // compute_padding() function must be
4030                                 // provided for the instruction
4031 
4032 //----------OPERANDS-----------------------------------------------------------
4033 // Operand definitions must precede instruction definitions for correct parsing
4034 // in the ADLC because operands constitute user defined types which are used in
4035 // instruction definitions.
4036 
4037 //----------Simple Operands----------------------------------------------------
4038 
4039 // Integer operands 32 bit
4040 // 32 bit immediate
4041 operand immI()
4042 %{
4043   match(ConI);
4044 
4045   op_cost(0);
4046   format %{ %}
4047   interface(CONST_INTER);
4048 %}
4049 
4050 // 32 bit zero
4051 operand immI0()
4052 %{
4053   predicate(n->get_int() == 0);
4054   match(ConI);
4055 
4056   op_cost(0);
4057   format %{ %}
4058   interface(CONST_INTER);
4059 %}
4060 
4061 // 32 bit unit increment
4062 operand immI_1()
4063 %{
4064   predicate(n->get_int() == 1);
4065   match(ConI);
4066 
4067   op_cost(0);
4068   format %{ %}
4069   interface(CONST_INTER);
4070 %}
4071 
4072 // 32 bit unit decrement
4073 operand immI_M1()
4074 %{
4075   predicate(n->get_int() == -1);
4076   match(ConI);
4077 
4078   op_cost(0);
4079   format %{ %}
4080   interface(CONST_INTER);
4081 %}
4082 
4083 operand immI_le_4()
4084 %{
4085   predicate(n->get_int() <= 4);
4086   match(ConI);
4087 
4088   op_cost(0);
4089   format %{ %}
4090   interface(CONST_INTER);
4091 %}
4092 
4093 operand immI_31()
4094 %{
4095   predicate(n->get_int() == 31);
4096   match(ConI);
4097 
4098   op_cost(0);
4099   format %{ %}
4100   interface(CONST_INTER);
4101 %}
4102 
4103 operand immI_8()
4104 %{
4105   predicate(n->get_int() == 8);
4106   match(ConI);
4107 
4108   op_cost(0);
4109   format %{ %}
4110   interface(CONST_INTER);
4111 %}
4112 
4113 operand immI_16()
4114 %{
4115   predicate(n->get_int() == 16);
4116   match(ConI);
4117 
4118   op_cost(0);
4119   format %{ %}
4120   interface(CONST_INTER);
4121 %}
4122 
4123 operand immI_24()
4124 %{
4125   predicate(n->get_int() == 24);
4126   match(ConI);
4127 
4128   op_cost(0);
4129   format %{ %}
4130   interface(CONST_INTER);
4131 %}
4132 
4133 operand immI_32()
4134 %{
4135   predicate(n->get_int() == 32);
4136   match(ConI);
4137 
4138   op_cost(0);
4139   format %{ %}
4140   interface(CONST_INTER);
4141 %}
4142 
4143 operand immI_48()
4144 %{
4145   predicate(n->get_int() == 48);
4146   match(ConI);
4147 
4148   op_cost(0);
4149   format %{ %}
4150   interface(CONST_INTER);
4151 %}
4152 
4153 operand immI_56()
4154 %{
4155   predicate(n->get_int() == 56);
4156   match(ConI);
4157 
4158   op_cost(0);
4159   format %{ %}
4160   interface(CONST_INTER);
4161 %}
4162 
4163 operand immI_64()
4164 %{
4165   predicate(n->get_int() == 64);
4166   match(ConI);
4167 
4168   op_cost(0);
4169   format %{ %}
4170   interface(CONST_INTER);
4171 %}
4172 
4173 operand immI_255()
4174 %{
4175   predicate(n->get_int() == 255);
4176   match(ConI);
4177 
4178   op_cost(0);
4179   format %{ %}
4180   interface(CONST_INTER);
4181 %}
4182 
4183 operand immI_65535()
4184 %{
4185   predicate(n->get_int() == 65535);
4186   match(ConI);
4187 
4188   op_cost(0);
4189   format %{ %}
4190   interface(CONST_INTER);
4191 %}
4192 
4193 operand immL_63()
4194 %{
4195   predicate(n->get_int() == 63);
4196   match(ConI);
4197 
4198   op_cost(0);
4199   format %{ %}
4200   interface(CONST_INTER);
4201 %}
4202 
4203 operand immL_255()
4204 %{
4205   predicate(n->get_int() == 255);
4206   match(ConI);
4207 
4208   op_cost(0);
4209   format %{ %}
4210   interface(CONST_INTER);
4211 %}
4212 
4213 operand immL_65535()
4214 %{
4215   predicate(n->get_long() == 65535L);
4216   match(ConL);
4217 
4218   op_cost(0);
4219   format %{ %}
4220   interface(CONST_INTER);
4221 %}
4222 
4223 operand immL_4294967295()
4224 %{
4225   predicate(n->get_long() == 4294967295L);
4226   match(ConL);
4227 
4228   op_cost(0);
4229   format %{ %}
4230   interface(CONST_INTER);
4231 %}
4232 
4233 operand immL_bitmask()
4234 %{
4235   predicate(((n->get_long() & 0xc000000000000000l) == 0)
4236             && is_power_of_2(n->get_long() + 1));
4237   match(ConL);
4238 
4239   op_cost(0);
4240   format %{ %}
4241   interface(CONST_INTER);
4242 %}
4243 
4244 operand immI_bitmask()
4245 %{
4246   predicate(((n->get_int() & 0xc0000000) == 0)
4247             && is_power_of_2(n->get_int() + 1));
4248   match(ConI);
4249 
4250   op_cost(0);
4251   format %{ %}
4252   interface(CONST_INTER);
4253 %}
4254 
4255 // Scale values for scaled offset addressing modes (up to long but not quad)
4256 operand immIScale()
4257 %{
4258   predicate(0 <= n->get_int() && (n->get_int() <= 3));
4259   match(ConI);
4260 
4261   op_cost(0);
4262   format %{ %}
4263   interface(CONST_INTER);
4264 %}
4265 
4266 // 26 bit signed offset -- for pc-relative branches
4267 operand immI26()
4268 %{
4269   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4270   match(ConI);
4271 
4272   op_cost(0);
4273   format %{ %}
4274   interface(CONST_INTER);
4275 %}
4276 
4277 // 19 bit signed offset -- for pc-relative loads
4278 operand immI19()
4279 %{
4280   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4281   match(ConI);
4282 
4283   op_cost(0);
4284   format %{ %}
4285   interface(CONST_INTER);
4286 %}
4287 
4288 // 12 bit unsigned offset -- for base plus immediate loads
4289 operand immIU12()
4290 %{
4291   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4292   match(ConI);
4293 
4294   op_cost(0);
4295   format %{ %}
4296   interface(CONST_INTER);
4297 %}
4298 
4299 operand immLU12()
4300 %{
4301   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4302   match(ConL);
4303 
4304   op_cost(0);
4305   format %{ %}
4306   interface(CONST_INTER);
4307 %}
4308 
4309 // Offset for scaled or unscaled immediate loads and stores
4310 operand immIOffset()
4311 %{
4312   predicate(Address::offset_ok_for_immed(n->get_int()));
4313   match(ConI);
4314 
4315   op_cost(0);
4316   format %{ %}
4317   interface(CONST_INTER);
4318 %}
4319 
4320 operand immLoffset()
4321 %{
4322   predicate(Address::offset_ok_for_immed(n->get_long()));
4323   match(ConL);
4324 
4325   op_cost(0);
4326   format %{ %}
4327   interface(CONST_INTER);
4328 %}
4329 
4330 // 32 bit integer valid for add sub immediate
4331 operand immIAddSub()
4332 %{
4333   predicate(Assembler::operand_valid_for_add_sub_immediate((long)n->get_int()));
4334   match(ConI);
4335   op_cost(0);
4336   format %{ %}
4337   interface(CONST_INTER);
4338 %}
4339 
4340 // 32 bit unsigned integer valid for logical immediate
4341 // TODO -- check this is right when e.g the mask is 0x80000000
4342 operand immILog()
4343 %{
4344   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (unsigned long)n->get_int()));
4345   match(ConI);
4346 
4347   op_cost(0);
4348   format %{ %}
4349   interface(CONST_INTER);
4350 %}
4351 
4352 // Integer operands 64 bit
4353 // 64 bit immediate
4354 operand immL()
4355 %{
4356   match(ConL);
4357 
4358   op_cost(0);
4359   format %{ %}
4360   interface(CONST_INTER);
4361 %}
4362 
4363 // 64 bit zero
4364 operand immL0()
4365 %{
4366   predicate(n->get_long() == 0);
4367   match(ConL);
4368 
4369   op_cost(0);
4370   format %{ %}
4371   interface(CONST_INTER);
4372 %}
4373 
4374 // 64 bit unit increment
4375 operand immL_1()
4376 %{
4377   predicate(n->get_long() == 1);
4378   match(ConL);
4379 
4380   op_cost(0);
4381   format %{ %}
4382   interface(CONST_INTER);
4383 %}
4384 
4385 // 64 bit unit decrement
4386 operand immL_M1()
4387 %{
4388   predicate(n->get_long() == -1);
4389   match(ConL);
4390 
4391   op_cost(0);
4392   format %{ %}
4393   interface(CONST_INTER);
4394 %}
4395 
4396 // 32 bit offset of pc in thread anchor
4397 
4398 operand immL_pc_off()
4399 %{
4400   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4401                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4402   match(ConL);
4403 
4404   op_cost(0);
4405   format %{ %}
4406   interface(CONST_INTER);
4407 %}
4408 
4409 // 64 bit integer valid for add sub immediate
4410 operand immLAddSub()
4411 %{
4412   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4413   match(ConL);
4414   op_cost(0);
4415   format %{ %}
4416   interface(CONST_INTER);
4417 %}
4418 
4419 // 64 bit integer valid for logical immediate
4420 operand immLLog()
4421 %{
4422   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (unsigned long)n->get_long()));
4423   match(ConL);
4424   op_cost(0);
4425   format %{ %}
4426   interface(CONST_INTER);
4427 %}
4428 
4429 // Long Immediate: low 32-bit mask
4430 operand immL_32bits()
4431 %{
4432   predicate(n->get_long() == 0xFFFFFFFFL);
4433   match(ConL);
4434   op_cost(0);
4435   format %{ %}
4436   interface(CONST_INTER);
4437 %}
4438 
4439 // Pointer operands
4440 // Pointer Immediate
4441 operand immP()
4442 %{
4443   match(ConP);
4444 
4445   op_cost(0);
4446   format %{ %}
4447   interface(CONST_INTER);
4448 %}
4449 
4450 // NULL Pointer Immediate
4451 operand immP0()
4452 %{
4453   predicate(n->get_ptr() == 0);
4454   match(ConP);
4455 
4456   op_cost(0);
4457   format %{ %}
4458   interface(CONST_INTER);
4459 %}
4460 
4461 // Pointer Immediate One
4462 // this is used in object initialization (initial object header)
4463 operand immP_1()
4464 %{
4465   predicate(n->get_ptr() == 1);
4466   match(ConP);
4467 
4468   op_cost(0);
4469   format %{ %}
4470   interface(CONST_INTER);
4471 %}
4472 
4473 // Polling Page Pointer Immediate
4474 operand immPollPage()
4475 %{
4476   predicate((address)n->get_ptr() == os::get_polling_page());
4477   match(ConP);
4478 
4479   op_cost(0);
4480   format %{ %}
4481   interface(CONST_INTER);
4482 %}
4483 
4484 // Card Table Byte Map Base
4485 operand immByteMapBase()
4486 %{
4487   // Get base of card map
4488   predicate((jbyte*)n->get_ptr() ==
4489         ((CardTableModRefBS*)(Universe::heap()->barrier_set()))->byte_map_base);
4490   match(ConP);
4491 
4492   op_cost(0);
4493   format %{ %}
4494   interface(CONST_INTER);
4495 %}
4496 
4497 // Pointer Immediate Minus One
4498 // this is used when we want to write the current PC to the thread anchor
4499 operand immP_M1()
4500 %{
4501   predicate(n->get_ptr() == -1);
4502   match(ConP);
4503 
4504   op_cost(0);
4505   format %{ %}
4506   interface(CONST_INTER);
4507 %}
4508 
4509 // Pointer Immediate Minus Two
4510 // this is used when we want to write the current PC to the thread anchor
4511 operand immP_M2()
4512 %{
4513   predicate(n->get_ptr() == -2);
4514   match(ConP);
4515 
4516   op_cost(0);
4517   format %{ %}
4518   interface(CONST_INTER);
4519 %}
4520 
4521 // Float and Double operands
4522 // Double Immediate
4523 operand immD()
4524 %{
4525   match(ConD);
4526   op_cost(0);
4527   format %{ %}
4528   interface(CONST_INTER);
4529 %}
4530 
4531 // Double Immediate: +0.0d
4532 operand immD0()
4533 %{
4534   predicate(jlong_cast(n->getd()) == 0);
4535   match(ConD);
4536 
4537   op_cost(0);
4538   format %{ %}
4539   interface(CONST_INTER);
4540 %}
4541 
4542 // constant 'double +0.0'.
4543 operand immDPacked()
4544 %{
4545   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4546   match(ConD);
4547   op_cost(0);
4548   format %{ %}
4549   interface(CONST_INTER);
4550 %}
4551 
4552 // Float Immediate
4553 operand immF()
4554 %{
4555   match(ConF);
4556   op_cost(0);
4557   format %{ %}
4558   interface(CONST_INTER);
4559 %}
4560 
4561 // Float Immediate: +0.0f.
4562 operand immF0()
4563 %{
4564   predicate(jint_cast(n->getf()) == 0);
4565   match(ConF);
4566 
4567   op_cost(0);
4568   format %{ %}
4569   interface(CONST_INTER);
4570 %}
4571 
4572 //
4573 operand immFPacked()
4574 %{
4575   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4576   match(ConF);
4577   op_cost(0);
4578   format %{ %}
4579   interface(CONST_INTER);
4580 %}
4581 
4582 // Narrow pointer operands
4583 // Narrow Pointer Immediate
4584 operand immN()
4585 %{
4586   match(ConN);
4587 
4588   op_cost(0);
4589   format %{ %}
4590   interface(CONST_INTER);
4591 %}
4592 
4593 // Narrow NULL Pointer Immediate
4594 operand immN0()
4595 %{
4596   predicate(n->get_narrowcon() == 0);
4597   match(ConN);
4598 
4599   op_cost(0);
4600   format %{ %}
4601   interface(CONST_INTER);
4602 %}
4603 
4604 operand immNKlass()
4605 %{
4606   match(ConNKlass);
4607 
4608   op_cost(0);
4609   format %{ %}
4610   interface(CONST_INTER);
4611 %}
4612 
4613 // Integer 32 bit Register Operands
4614 // Integer 32 bitRegister (excludes SP)
4615 operand iRegI()
4616 %{
4617   constraint(ALLOC_IN_RC(any_reg32));
4618   match(RegI);
4619   match(iRegINoSp);
4620   op_cost(0);
4621   format %{ %}
4622   interface(REG_INTER);
4623 %}
4624 
4625 // Integer 32 bit Register not Special
4626 operand iRegINoSp()
4627 %{
4628   constraint(ALLOC_IN_RC(no_special_reg32));
4629   match(RegI);
4630   op_cost(0);
4631   format %{ %}
4632   interface(REG_INTER);
4633 %}
4634 
4635 // Integer 64 bit Register Operands
4636 // Integer 64 bit Register (includes SP)
4637 operand iRegL()
4638 %{
4639   constraint(ALLOC_IN_RC(any_reg));
4640   match(RegL);
4641   match(iRegLNoSp);
4642   op_cost(0);
4643   format %{ %}
4644   interface(REG_INTER);
4645 %}
4646 
4647 // Integer 64 bit Register not Special
4648 operand iRegLNoSp()
4649 %{
4650   constraint(ALLOC_IN_RC(no_special_reg));
4651   match(RegL);
4652   format %{ %}
4653   interface(REG_INTER);
4654 %}
4655 
4656 // Pointer Register Operands
4657 // Pointer Register
4658 operand iRegP()
4659 %{
4660   constraint(ALLOC_IN_RC(ptr_reg));
4661   match(RegP);
4662   match(iRegPNoSp);
4663   match(iRegP_R0);
4664   //match(iRegP_R2);
4665   //match(iRegP_R4);
4666   //match(iRegP_R5);
4667   match(thread_RegP);
4668   op_cost(0);
4669   format %{ %}
4670   interface(REG_INTER);
4671 %}
4672 
4673 // Pointer 64 bit Register not Special
4674 operand iRegPNoSp()
4675 %{
4676   constraint(ALLOC_IN_RC(no_special_ptr_reg));
4677   match(RegP);
4678   // match(iRegP);
4679   // match(iRegP_R0);
4680   // match(iRegP_R2);
4681   // match(iRegP_R4);
4682   // match(iRegP_R5);
4683   // match(thread_RegP);
4684   op_cost(0);
4685   format %{ %}
4686   interface(REG_INTER);
4687 %}
4688 
4689 // Pointer 64 bit Register R0 only
4690 operand iRegP_R0()
4691 %{
4692   constraint(ALLOC_IN_RC(r0_reg));
4693   match(RegP);
4694   // match(iRegP);
4695   match(iRegPNoSp);
4696   op_cost(0);
4697   format %{ %}
4698   interface(REG_INTER);
4699 %}
4700 
4701 // Pointer 64 bit Register R1 only
4702 operand iRegP_R1()
4703 %{
4704   constraint(ALLOC_IN_RC(r1_reg));
4705   match(RegP);
4706   // match(iRegP);
4707   match(iRegPNoSp);
4708   op_cost(0);
4709   format %{ %}
4710   interface(REG_INTER);
4711 %}
4712 
4713 // Pointer 64 bit Register R2 only
4714 operand iRegP_R2()
4715 %{
4716   constraint(ALLOC_IN_RC(r2_reg));
4717   match(RegP);
4718   // match(iRegP);
4719   match(iRegPNoSp);
4720   op_cost(0);
4721   format %{ %}
4722   interface(REG_INTER);
4723 %}
4724 
4725 // Pointer 64 bit Register R3 only
4726 operand iRegP_R3()
4727 %{
4728   constraint(ALLOC_IN_RC(r3_reg));
4729   match(RegP);
4730   // match(iRegP);
4731   match(iRegPNoSp);
4732   op_cost(0);
4733   format %{ %}
4734   interface(REG_INTER);
4735 %}
4736 
4737 // Pointer 64 bit Register R4 only
4738 operand iRegP_R4()
4739 %{
4740   constraint(ALLOC_IN_RC(r4_reg));
4741   match(RegP);
4742   // match(iRegP);
4743   match(iRegPNoSp);
4744   op_cost(0);
4745   format %{ %}
4746   interface(REG_INTER);
4747 %}
4748 
4749 // Pointer 64 bit Register R5 only
4750 operand iRegP_R5()
4751 %{
4752   constraint(ALLOC_IN_RC(r5_reg));
4753   match(RegP);
4754   // match(iRegP);
4755   match(iRegPNoSp);
4756   op_cost(0);
4757   format %{ %}
4758   interface(REG_INTER);
4759 %}
4760 
4761 // Pointer 64 bit Register R10 only
4762 operand iRegP_R10()
4763 %{
4764   constraint(ALLOC_IN_RC(r10_reg));
4765   match(RegP);
4766   // match(iRegP);
4767   match(iRegPNoSp);
4768   op_cost(0);
4769   format %{ %}
4770   interface(REG_INTER);
4771 %}
4772 
4773 // Long 64 bit Register R11 only
4774 operand iRegL_R11()
4775 %{
4776   constraint(ALLOC_IN_RC(r11_reg));
4777   match(RegL);
4778   match(iRegLNoSp);
4779   op_cost(0);
4780   format %{ %}
4781   interface(REG_INTER);
4782 %}
4783 
4784 // Pointer 64 bit Register FP only
4785 operand iRegP_FP()
4786 %{
4787   constraint(ALLOC_IN_RC(fp_reg));
4788   match(RegP);
4789   // match(iRegP);
4790   op_cost(0);
4791   format %{ %}
4792   interface(REG_INTER);
4793 %}
4794 
4795 // Register R0 only
4796 operand iRegI_R0()
4797 %{
4798   constraint(ALLOC_IN_RC(int_r0_reg));
4799   match(RegI);
4800   match(iRegINoSp);
4801   op_cost(0);
4802   format %{ %}
4803   interface(REG_INTER);
4804 %}
4805 
4806 // Register R2 only
4807 operand iRegI_R2()
4808 %{
4809   constraint(ALLOC_IN_RC(int_r2_reg));
4810   match(RegI);
4811   match(iRegINoSp);
4812   op_cost(0);
4813   format %{ %}
4814   interface(REG_INTER);
4815 %}
4816 
4817 // Register R3 only
4818 operand iRegI_R3()
4819 %{
4820   constraint(ALLOC_IN_RC(int_r3_reg));
4821   match(RegI);
4822   match(iRegINoSp);
4823   op_cost(0);
4824   format %{ %}
4825   interface(REG_INTER);
4826 %}
4827 
4828 
4829 // Register R2 only
4830 operand iRegI_R4()
4831 %{
4832   constraint(ALLOC_IN_RC(int_r4_reg));
4833   match(RegI);
4834   match(iRegINoSp);
4835   op_cost(0);
4836   format %{ %}
4837   interface(REG_INTER);
4838 %}
4839 
4840 
4841 // Pointer Register Operands
4842 // Narrow Pointer Register
4843 operand iRegN()
4844 %{
4845   constraint(ALLOC_IN_RC(any_reg32));
4846   match(RegN);
4847   match(iRegNNoSp);
4848   op_cost(0);
4849   format %{ %}
4850   interface(REG_INTER);
4851 %}
4852 
4853 // Integer 64 bit Register not Special
4854 operand iRegNNoSp()
4855 %{
4856   constraint(ALLOC_IN_RC(no_special_reg32));
4857   match(RegN);
4858   op_cost(0);
4859   format %{ %}
4860   interface(REG_INTER);
4861 %}
4862 
4863 // heap base register -- used for encoding immN0
4864 
4865 operand iRegIHeapbase()
4866 %{
4867   constraint(ALLOC_IN_RC(heapbase_reg));
4868   match(RegI);
4869   op_cost(0);
4870   format %{ %}
4871   interface(REG_INTER);
4872 %}
4873 
4874 // Float Register
4875 // Float register operands
4876 operand vRegF()
4877 %{
4878   constraint(ALLOC_IN_RC(float_reg));
4879   match(RegF);
4880 
4881   op_cost(0);
4882   format %{ %}
4883   interface(REG_INTER);
4884 %}
4885 
4886 // Double Register
4887 // Double register operands
4888 operand vRegD()
4889 %{
4890   constraint(ALLOC_IN_RC(double_reg));
4891   match(RegD);
4892 
4893   op_cost(0);
4894   format %{ %}
4895   interface(REG_INTER);
4896 %}
4897 
4898 operand vRegD_V0()
4899 %{
4900   constraint(ALLOC_IN_RC(v0_reg));
4901   match(RegD);
4902   op_cost(0);
4903   format %{ %}
4904   interface(REG_INTER);
4905 %}
4906 
4907 operand vRegD_V1()
4908 %{
4909   constraint(ALLOC_IN_RC(v1_reg));
4910   match(RegD);
4911   op_cost(0);
4912   format %{ %}
4913   interface(REG_INTER);
4914 %}
4915 
4916 operand vRegD_V2()
4917 %{
4918   constraint(ALLOC_IN_RC(v2_reg));
4919   match(RegD);
4920   op_cost(0);
4921   format %{ %}
4922   interface(REG_INTER);
4923 %}
4924 
4925 operand vRegD_V3()
4926 %{
4927   constraint(ALLOC_IN_RC(v3_reg));
4928   match(RegD);
4929   op_cost(0);
4930   format %{ %}
4931   interface(REG_INTER);
4932 %}
4933 
4934 // Flags register, used as output of signed compare instructions
4935 
4936 // note that on AArch64 we also use this register as the output for
4937 // for floating point compare instructions (CmpF CmpD). this ensures
4938 // that ordered inequality tests use GT, GE, LT or LE none of which
4939 // pass through cases where the result is unordered i.e. one or both
4940 // inputs to the compare is a NaN. this means that the ideal code can
4941 // replace e.g. a GT with an LE and not end up capturing the NaN case
4942 // (where the comparison should always fail). EQ and NE tests are
4943 // always generated in ideal code so that unordered folds into the NE
4944 // case, matching the behaviour of AArch64 NE.
4945 //
4946 // This differs from x86 where the outputs of FP compares use a
4947 // special FP flags registers and where compares based on this
4948 // register are distinguished into ordered inequalities (cmpOpUCF) and
4949 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
4950 // to explicitly handle the unordered case in branches. x86 also has
4951 // to include extra CMoveX rules to accept a cmpOpUCF input.
4952 
4953 operand rFlagsReg()
4954 %{
4955   constraint(ALLOC_IN_RC(int_flags));
4956   match(RegFlags);
4957 
4958   op_cost(0);
4959   format %{ "RFLAGS" %}
4960   interface(REG_INTER);
4961 %}
4962 
4963 // Flags register, used as output of unsigned compare instructions
4964 operand rFlagsRegU()
4965 %{
4966   constraint(ALLOC_IN_RC(int_flags));
4967   match(RegFlags);
4968 
4969   op_cost(0);
4970   format %{ "RFLAGSU" %}
4971   interface(REG_INTER);
4972 %}
4973 
4974 // Special Registers
4975 
4976 // Method Register
4977 operand inline_cache_RegP(iRegP reg)
4978 %{
4979   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
4980   match(reg);
4981   match(iRegPNoSp);
4982   op_cost(0);
4983   format %{ %}
4984   interface(REG_INTER);
4985 %}
4986 
4987 operand interpreter_method_oop_RegP(iRegP reg)
4988 %{
4989   constraint(ALLOC_IN_RC(method_reg)); // interpreter_method_oop_reg
4990   match(reg);
4991   match(iRegPNoSp);
4992   op_cost(0);
4993   format %{ %}
4994   interface(REG_INTER);
4995 %}
4996 
4997 // Thread Register
4998 operand thread_RegP(iRegP reg)
4999 %{
5000   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5001   match(reg);
5002   op_cost(0);
5003   format %{ %}
5004   interface(REG_INTER);
5005 %}
5006 
5007 operand lr_RegP(iRegP reg)
5008 %{
5009   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5010   match(reg);
5011   op_cost(0);
5012   format %{ %}
5013   interface(REG_INTER);
5014 %}
5015 
5016 //----------Memory Operands----------------------------------------------------
5017 
5018 operand indirect(iRegP reg)
5019 %{
5020   constraint(ALLOC_IN_RC(ptr_reg));
5021   match(reg);
5022   op_cost(0);
5023   format %{ "[$reg]" %}
5024   interface(MEMORY_INTER) %{
5025     base($reg);
5026     index(0xffffffff);
5027     scale(0x0);
5028     disp(0x0);
5029   %}
5030 %}
5031 
5032 operand indIndexScaledOffsetI(iRegP reg, iRegL lreg, immIScale scale, immIU12 off)
5033 %{
5034   constraint(ALLOC_IN_RC(ptr_reg));
5035   match(AddP (AddP reg (LShiftL lreg scale)) off);
5036   op_cost(INSN_COST);
5037   format %{ "$reg, $lreg lsl($scale), $off" %}
5038   interface(MEMORY_INTER) %{
5039     base($reg);
5040     index($lreg);
5041     scale($scale);
5042     disp($off);
5043   %}
5044 %}
5045 
5046 operand indIndexScaledOffsetL(iRegP reg, iRegL lreg, immIScale scale, immLU12 off)
5047 %{
5048   constraint(ALLOC_IN_RC(ptr_reg));
5049   match(AddP (AddP reg (LShiftL lreg scale)) off);
5050   op_cost(INSN_COST);
5051   format %{ "$reg, $lreg lsl($scale), $off" %}
5052   interface(MEMORY_INTER) %{
5053     base($reg);
5054     index($lreg);
5055     scale($scale);
5056     disp($off);
5057   %}
5058 %}
5059 
5060 operand indIndexScaledOffsetI2L(iRegP reg, iRegI ireg, immIScale scale, immLU12 off)
5061 %{
5062   constraint(ALLOC_IN_RC(ptr_reg));
5063   match(AddP (AddP reg (LShiftL (ConvI2L ireg) scale)) off);
5064   op_cost(INSN_COST);
5065   format %{ "$reg, $ireg sxtw($scale), $off I2L" %}
5066   interface(MEMORY_INTER) %{
5067     base($reg);
5068     index($ireg);
5069     scale($scale);
5070     disp($off);
5071   %}
5072 %}
5073 
5074 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5075 %{
5076   constraint(ALLOC_IN_RC(ptr_reg));
5077   match(AddP reg (LShiftL (ConvI2L ireg) scale));
5078   op_cost(0);
5079   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5080   interface(MEMORY_INTER) %{
5081     base($reg);
5082     index($ireg);
5083     scale($scale);
5084     disp(0x0);
5085   %}
5086 %}
5087 
5088 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5089 %{
5090   constraint(ALLOC_IN_RC(ptr_reg));
5091   match(AddP reg (LShiftL lreg scale));
5092   op_cost(0);
5093   format %{ "$reg, $lreg lsl($scale)" %}
5094   interface(MEMORY_INTER) %{
5095     base($reg);
5096     index($lreg);
5097     scale($scale);
5098     disp(0x0);
5099   %}
5100 %}
5101 
5102 operand indIndex(iRegP reg, iRegL lreg)
5103 %{
5104   constraint(ALLOC_IN_RC(ptr_reg));
5105   match(AddP reg lreg);
5106   op_cost(0);
5107   format %{ "$reg, $lreg" %}
5108   interface(MEMORY_INTER) %{
5109     base($reg);
5110     index($lreg);
5111     scale(0x0);
5112     disp(0x0);
5113   %}
5114 %}
5115 
5116 operand indOffI(iRegP reg, immIOffset off)
5117 %{
5118   constraint(ALLOC_IN_RC(ptr_reg));
5119   match(AddP reg off);
5120   op_cost(INSN_COST);
5121   format %{ "[$reg, $off]" %}
5122   interface(MEMORY_INTER) %{
5123     base($reg);
5124     index(0xffffffff);
5125     scale(0x0);
5126     disp($off);
5127   %}
5128 %}
5129 
5130 operand indOffL(iRegP reg, immLoffset off)
5131 %{
5132   constraint(ALLOC_IN_RC(ptr_reg));
5133   match(AddP reg off);
5134   op_cost(0);
5135   format %{ "[$reg, $off]" %}
5136   interface(MEMORY_INTER) %{
5137     base($reg);
5138     index(0xffffffff);
5139     scale(0x0);
5140     disp($off);
5141   %}
5142 %}
5143 
5144 
5145 operand indirectN(iRegN reg)
5146 %{
5147   predicate(Universe::narrow_oop_shift() == 0);
5148   constraint(ALLOC_IN_RC(ptr_reg));
5149   match(DecodeN reg);
5150   op_cost(0);
5151   format %{ "[$reg]\t# narrow" %}
5152   interface(MEMORY_INTER) %{
5153     base($reg);
5154     index(0xffffffff);
5155     scale(0x0);
5156     disp(0x0);
5157   %}
5158 %}
5159 
5160 operand indIndexScaledOffsetIN(iRegN reg, iRegL lreg, immIScale scale, immIU12 off)
5161 %{
5162   predicate(Universe::narrow_oop_shift() == 0);
5163   constraint(ALLOC_IN_RC(ptr_reg));
5164   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5165   op_cost(0);
5166   format %{ "$reg, $lreg lsl($scale), $off\t# narrow" %}
5167   interface(MEMORY_INTER) %{
5168     base($reg);
5169     index($lreg);
5170     scale($scale);
5171     disp($off);
5172   %}
5173 %}
5174 
5175 operand indIndexScaledOffsetLN(iRegN reg, iRegL lreg, immIScale scale, immLU12 off)
5176 %{
5177   predicate(Universe::narrow_oop_shift() == 0);
5178   constraint(ALLOC_IN_RC(ptr_reg));
5179   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5180   op_cost(INSN_COST);
5181   format %{ "$reg, $lreg lsl($scale), $off\t# narrow" %}
5182   interface(MEMORY_INTER) %{
5183     base($reg);
5184     index($lreg);
5185     scale($scale);
5186     disp($off);
5187   %}
5188 %}
5189 
5190 operand indIndexScaledOffsetI2LN(iRegN reg, iRegI ireg, immIScale scale, immLU12 off)
5191 %{
5192   predicate(Universe::narrow_oop_shift() == 0);
5193   constraint(ALLOC_IN_RC(ptr_reg));
5194   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale)) off);
5195   op_cost(INSN_COST);
5196   format %{ "$reg, $ireg sxtw($scale), $off I2L\t# narrow" %}
5197   interface(MEMORY_INTER) %{
5198     base($reg);
5199     index($ireg);
5200     scale($scale);
5201     disp($off);
5202   %}
5203 %}
5204 
5205 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5206 %{
5207   predicate(Universe::narrow_oop_shift() == 0);
5208   constraint(ALLOC_IN_RC(ptr_reg));
5209   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5210   op_cost(0);
5211   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5212   interface(MEMORY_INTER) %{
5213     base($reg);
5214     index($ireg);
5215     scale($scale);
5216     disp(0x0);
5217   %}
5218 %}
5219 
5220 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5221 %{
5222   predicate(Universe::narrow_oop_shift() == 0);
5223   constraint(ALLOC_IN_RC(ptr_reg));
5224   match(AddP (DecodeN reg) (LShiftL lreg scale));
5225   op_cost(0);
5226   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5227   interface(MEMORY_INTER) %{
5228     base($reg);
5229     index($lreg);
5230     scale($scale);
5231     disp(0x0);
5232   %}
5233 %}
5234 
5235 operand indIndexN(iRegN reg, iRegL lreg)
5236 %{
5237   predicate(Universe::narrow_oop_shift() == 0);
5238   constraint(ALLOC_IN_RC(ptr_reg));
5239   match(AddP (DecodeN reg) lreg);
5240   op_cost(0);
5241   format %{ "$reg, $lreg\t# narrow" %}
5242   interface(MEMORY_INTER) %{
5243     base($reg);
5244     index($lreg);
5245     scale(0x0);
5246     disp(0x0);
5247   %}
5248 %}
5249 
5250 operand indOffIN(iRegN reg, immIOffset off)
5251 %{
5252   predicate(Universe::narrow_oop_shift() == 0);
5253   constraint(ALLOC_IN_RC(ptr_reg));
5254   match(AddP (DecodeN reg) off);
5255   op_cost(0);
5256   format %{ "[$reg, $off]\t# narrow" %}
5257   interface(MEMORY_INTER) %{
5258     base($reg);
5259     index(0xffffffff);
5260     scale(0x0);
5261     disp($off);
5262   %}
5263 %}
5264 
5265 operand indOffLN(iRegN reg, immLoffset off)
5266 %{
5267   predicate(Universe::narrow_oop_shift() == 0);
5268   constraint(ALLOC_IN_RC(ptr_reg));
5269   match(AddP (DecodeN reg) off);
5270   op_cost(0);
5271   format %{ "[$reg, $off]\t# narrow" %}
5272   interface(MEMORY_INTER) %{
5273     base($reg);
5274     index(0xffffffff);
5275     scale(0x0);
5276     disp($off);
5277   %}
5278 %}
5279 
5280 
5281 
5282 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5283 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5284 %{
5285   constraint(ALLOC_IN_RC(ptr_reg));
5286   match(AddP reg off);
5287   op_cost(0);
5288   format %{ "[$reg, $off]" %}
5289   interface(MEMORY_INTER) %{
5290     base($reg);
5291     index(0xffffffff);
5292     scale(0x0);
5293     disp($off);
5294   %}
5295 %}
5296 
5297 //----------Special Memory Operands--------------------------------------------
5298 // Stack Slot Operand - This operand is used for loading and storing temporary
5299 //                      values on the stack where a match requires a value to
5300 //                      flow through memory.
5301 operand stackSlotP(sRegP reg)
5302 %{
5303   constraint(ALLOC_IN_RC(stack_slots));
5304   op_cost(100);
5305   // No match rule because this operand is only generated in matching
5306   // match(RegP);
5307   format %{ "[$reg]" %}
5308   interface(MEMORY_INTER) %{
5309     base(0x1e);  // RSP
5310     index(0x0);  // No Index
5311     scale(0x0);  // No Scale
5312     disp($reg);  // Stack Offset
5313   %}
5314 %}
5315 
5316 operand stackSlotI(sRegI reg)
5317 %{
5318   constraint(ALLOC_IN_RC(stack_slots));
5319   // No match rule because this operand is only generated in matching
5320   // match(RegI);
5321   format %{ "[$reg]" %}
5322   interface(MEMORY_INTER) %{
5323     base(0x1e);  // RSP
5324     index(0x0);  // No Index
5325     scale(0x0);  // No Scale
5326     disp($reg);  // Stack Offset
5327   %}
5328 %}
5329 
5330 operand stackSlotF(sRegF reg)
5331 %{
5332   constraint(ALLOC_IN_RC(stack_slots));
5333   // No match rule because this operand is only generated in matching
5334   // match(RegF);
5335   format %{ "[$reg]" %}
5336   interface(MEMORY_INTER) %{
5337     base(0x1e);  // RSP
5338     index(0x0);  // No Index
5339     scale(0x0);  // No Scale
5340     disp($reg);  // Stack Offset
5341   %}
5342 %}
5343 
5344 operand stackSlotD(sRegD reg)
5345 %{
5346   constraint(ALLOC_IN_RC(stack_slots));
5347   // No match rule because this operand is only generated in matching
5348   // match(RegD);
5349   format %{ "[$reg]" %}
5350   interface(MEMORY_INTER) %{
5351     base(0x1e);  // RSP
5352     index(0x0);  // No Index
5353     scale(0x0);  // No Scale
5354     disp($reg);  // Stack Offset
5355   %}
5356 %}
5357 
5358 operand stackSlotL(sRegL reg)
5359 %{
5360   constraint(ALLOC_IN_RC(stack_slots));
5361   // No match rule because this operand is only generated in matching
5362   // match(RegL);
5363   format %{ "[$reg]" %}
5364   interface(MEMORY_INTER) %{
5365     base(0x1e);  // RSP
5366     index(0x0);  // No Index
5367     scale(0x0);  // No Scale
5368     disp($reg);  // Stack Offset
5369   %}
5370 %}
5371 
5372 // Operands for expressing Control Flow
5373 // NOTE: Label is a predefined operand which should not be redefined in
5374 //       the AD file. It is generically handled within the ADLC.
5375 
5376 //----------Conditional Branch Operands----------------------------------------
5377 // Comparison Op  - This is the operation of the comparison, and is limited to
5378 //                  the following set of codes:
5379 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5380 //
5381 // Other attributes of the comparison, such as unsignedness, are specified
5382 // by the comparison instruction that sets a condition code flags register.
5383 // That result is represented by a flags operand whose subtype is appropriate
5384 // to the unsignedness (etc.) of the comparison.
5385 //
5386 // Later, the instruction which matches both the Comparison Op (a Bool) and
5387 // the flags (produced by the Cmp) specifies the coding of the comparison op
5388 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5389 
5390 // used for signed integral comparisons and fp comparisons
5391 
5392 operand cmpOp()
5393 %{
5394   match(Bool);
5395 
5396   format %{ "" %}
5397   interface(COND_INTER) %{
5398     equal(0x0, "eq");
5399     not_equal(0x1, "ne");
5400     less(0xb, "lt");
5401     greater_equal(0xa, "ge");
5402     less_equal(0xd, "le");
5403     greater(0xc, "gt");
5404     overflow(0x6, "vs");
5405     no_overflow(0x7, "vc");
5406   %}
5407 %}
5408 
5409 // used for unsigned integral comparisons
5410 
5411 operand cmpOpU()
5412 %{
5413   match(Bool);
5414 
5415   format %{ "" %}
5416   interface(COND_INTER) %{
5417     equal(0x0, "eq");
5418     not_equal(0x1, "ne");
5419     less(0x3, "lo");
5420     greater_equal(0x2, "hs");
5421     less_equal(0x9, "ls");
5422     greater(0x8, "hi");
5423     overflow(0x6, "vs");
5424     no_overflow(0x7, "vc");
5425   %}
5426 %}
5427 
5428 // Special operand allowing long args to int ops to be truncated for free
5429 
5430 operand iRegL2I(iRegL reg) %{
5431 
5432   op_cost(0);
5433 
5434   match(ConvL2I reg);
5435 
5436   format %{ "l2i($reg)" %}
5437 
5438   interface(REG_INTER)
5439 %}
5440 
5441 
5442 //----------OPERAND CLASSES----------------------------------------------------
5443 // Operand Classes are groups of operands that are used as to simplify
5444 // instruction definitions by not requiring the AD writer to specify
5445 // separate instructions for every form of operand when the
5446 // instruction accepts multiple operand types with the same basic
5447 // encoding and format. The classic case of this is memory operands.
5448 
5449 // memory is used to define read/write location for load/store
5450 // instruction defs. we can turn a memory op into an Address
5451 
5452 opclass memory(indirect, indIndexScaledOffsetI,  indIndexScaledOffsetL, indIndexScaledOffsetI2L, indIndexScaled, indIndexScaledI2L, indIndex, indOffI, indOffL,
5453                indirectN, indIndexScaledOffsetIN,  indIndexScaledOffsetLN, indIndexScaledOffsetI2LN, indIndexScaledN, indIndexScaledI2LN, indIndexN, indOffIN, indOffLN);
5454 
5455 
5456 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5457 // operations. it allows the src to be either an iRegI or a (ConvL2I
5458 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5459 // can be elided because the 32-bit instruction will just employ the
5460 // lower 32 bits anyway.
5461 //
5462 // n.b. this does not elide all L2I conversions. if the truncated
5463 // value is consumed by more than one operation then the ConvL2I
5464 // cannot be bundled into the consuming nodes so an l2i gets planted
5465 // (actually a movw $dst $src) and the downstream instructions consume
5466 // the result of the l2i as an iRegI input. That's a shame since the
5467 // movw is actually redundant but its not too costly.
5468 
5469 opclass iRegIorL2I(iRegI, iRegL2I);
5470 
5471 //----------PIPELINE-----------------------------------------------------------
5472 // Rules which define the behavior of the target architectures pipeline.
5473 // Integer ALU reg operation
5474 pipeline %{
5475 
5476 attributes %{
5477   // ARM instructions are of fixed length
5478   fixed_size_instructions;        // Fixed size instructions TODO does
5479   max_instructions_per_bundle = 2;   // A53 = 2, A57 = 4
5480   // ARM instructions come in 32-bit word units
5481   instruction_unit_size = 4;         // An instruction is 4 bytes long
5482   instruction_fetch_unit_size = 64;  // The processor fetches one line
5483   instruction_fetch_units = 1;       // of 64 bytes
5484 
5485   // List of nop instructions
5486   nops( MachNop );
5487 %}
5488 
5489 // We don't use an actual pipeline model so don't care about resources
5490 // or description. we do use pipeline classes to introduce fixed
5491 // latencies
5492 
5493 //----------RESOURCES----------------------------------------------------------
5494 // Resources are the functional units available to the machine
5495 
5496 resources( INS0, INS1, INS01 = INS0 | INS1,
5497            ALU0, ALU1, ALU = ALU0 | ALU1,
5498            MAC,
5499            DIV,
5500            BRANCH,
5501            LDST,
5502            NEON_FP);
5503 
5504 //----------PIPELINE DESCRIPTION-----------------------------------------------
5505 // Pipeline Description specifies the stages in the machine's pipeline
5506 
5507 pipe_desc(ISS, EX1, EX2, WR);
5508 
5509 //----------PIPELINE CLASSES---------------------------------------------------
5510 // Pipeline Classes describe the stages in which input and output are
5511 // referenced by the hardware pipeline.
5512 
5513 //------- Integer ALU operations --------------------------
5514 
5515 // Integer ALU reg-reg operation
5516 // Operands needed in EX1, result generated in EX2
5517 // Eg.  ADD     x0, x1, x2
5518 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5519 %{
5520   single_instruction;
5521   dst    : EX2(write);
5522   src1   : EX1(read);
5523   src2   : EX1(read);
5524   INS01  : ISS; // Dual issue as instruction 0 or 1
5525   ALU    : EX2;
5526 %}
5527 
5528 // Integer ALU reg-reg operation with constant shift
5529 // Shifted register must be available in LATE_ISS instead of EX1
5530 // Eg.  ADD     x0, x1, x2, LSL #2
5531 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
5532 %{
5533   single_instruction;
5534   dst    : EX2(write);
5535   src1   : EX1(read);
5536   src2   : ISS(read);
5537   INS01  : ISS;
5538   ALU    : EX2;
5539 %}
5540 
5541 // Integer ALU reg operation with constant shift
5542 // Eg.  LSL     x0, x1, #shift
5543 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
5544 %{
5545   single_instruction;
5546   dst    : EX2(write);
5547   src1   : ISS(read);
5548   INS01  : ISS;
5549   ALU    : EX2;
5550 %}
5551 
5552 // Integer ALU reg-reg operation with variable shift
5553 // Both operands must be available in LATE_ISS instead of EX1
5554 // Result is available in EX1 instead of EX2
5555 // Eg.  LSLV    x0, x1, x2
5556 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
5557 %{
5558   single_instruction;
5559   dst    : EX1(write);
5560   src1   : ISS(read);
5561   src2   : ISS(read);
5562   INS01  : ISS;
5563   ALU    : EX1;
5564 %}
5565 
5566 // Integer ALU reg-reg operation with extract
5567 // As for _vshift above, but result generated in EX2
5568 // Eg.  EXTR    x0, x1, x2, #N
5569 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
5570 %{
5571   single_instruction;
5572   dst    : EX2(write);
5573   src1   : ISS(read);
5574   src2   : ISS(read);
5575   INS1   : ISS; // Can only dual issue as Instruction 1
5576   ALU    : EX1;
5577 %}
5578 
5579 // Integer ALU reg operation
5580 // Eg.  NEG     x0, x1
5581 pipe_class ialu_reg(iRegI dst, iRegI src)
5582 %{
5583   single_instruction;
5584   dst    : EX2(write);
5585   src    : EX1(read);
5586   INS01  : ISS;
5587   ALU    : EX2;
5588 %}
5589 
5590 // Integer ALU reg mmediate operation
5591 // Eg.  ADD     x0, x1, #N
5592 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
5593 %{
5594   single_instruction;
5595   dst    : EX2(write);
5596   src1   : EX1(read);
5597   INS01  : ISS;
5598   ALU    : EX2;
5599 %}
5600 
5601 // Integer ALU immediate operation (no source operands)
5602 // Eg.  MOV     x0, #N
5603 pipe_class ialu_imm(iRegI dst)
5604 %{
5605   single_instruction;
5606   dst    : EX1(write);
5607   INS01  : ISS;
5608   ALU    : EX1;
5609 %}
5610 
5611 //------- Compare operation -------------------------------
5612 
5613 // Compare reg-reg
5614 // Eg.  CMP     x0, x1
5615 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
5616 %{
5617   single_instruction;
5618 //  fixed_latency(16);
5619   cr     : EX2(write);
5620   op1    : EX1(read);
5621   op2    : EX1(read);
5622   INS01  : ISS;
5623   ALU    : EX2;
5624 %}
5625 
5626 // Compare reg-reg
5627 // Eg.  CMP     x0, #N
5628 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
5629 %{
5630   single_instruction;
5631 //  fixed_latency(16);
5632   cr     : EX2(write);
5633   op1    : EX1(read);
5634   INS01  : ISS;
5635   ALU    : EX2;
5636 %}
5637 
5638 //------- Conditional instructions ------------------------
5639 
5640 // Conditional no operands
5641 // Eg.  CSINC   x0, zr, zr, <cond>
5642 pipe_class icond_none(iRegI dst, rFlagsReg cr)
5643 %{
5644   single_instruction;
5645   cr     : EX1(read);
5646   dst    : EX2(write);
5647   INS01  : ISS;
5648   ALU    : EX2;
5649 %}
5650 
5651 // Conditional 2 operand
5652 // EG.  CSEL    X0, X1, X2, <cond>
5653 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
5654 %{
5655   single_instruction;
5656   cr     : EX1(read);
5657   src1   : EX1(read);
5658   src2   : EX1(read);
5659   dst    : EX2(write);
5660   INS01  : ISS;
5661   ALU    : EX2;
5662 %}
5663 
5664 // Conditional 2 operand
5665 // EG.  CSEL    X0, X1, X2, <cond>
5666 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
5667 %{
5668   single_instruction;
5669   cr     : EX1(read);
5670   src    : EX1(read);
5671   dst    : EX2(write);
5672   INS01  : ISS;
5673   ALU    : EX2;
5674 %}
5675 
5676 //------- Multiply pipeline operations --------------------
5677 
5678 // Multiply reg-reg
5679 // Eg.  MUL     w0, w1, w2
5680 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5681 %{
5682   single_instruction;
5683   dst    : WR(write);
5684   src1   : ISS(read);
5685   src2   : ISS(read);
5686   INS01  : ISS;
5687   MAC    : WR;
5688 %}
5689 
5690 // Multiply accumulate
5691 // Eg.  MADD    w0, w1, w2, w3
5692 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
5693 %{
5694   single_instruction;
5695   dst    : WR(write);
5696   src1   : ISS(read);
5697   src2   : ISS(read);
5698   src3   : ISS(read);
5699   INS01  : ISS;
5700   MAC    : WR;
5701 %}
5702 
5703 // Eg.  MUL     w0, w1, w2
5704 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5705 %{
5706   single_instruction;
5707   fixed_latency(3); // Maximum latency for 64 bit mul
5708   dst    : WR(write);
5709   src1   : ISS(read);
5710   src2   : ISS(read);
5711   INS01  : ISS;
5712   MAC    : WR;
5713 %}
5714 
5715 // Multiply accumulate
5716 // Eg.  MADD    w0, w1, w2, w3
5717 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
5718 %{
5719   single_instruction;
5720   fixed_latency(3); // Maximum latency for 64 bit mul
5721   dst    : WR(write);
5722   src1   : ISS(read);
5723   src2   : ISS(read);
5724   src3   : ISS(read);
5725   INS01  : ISS;
5726   MAC    : WR;
5727 %}
5728 
5729 //------- Divide pipeline operations --------------------
5730 
5731 // Eg.  SDIV    w0, w1, w2
5732 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5733 %{
5734   single_instruction;
5735   fixed_latency(8); // Maximum latency for 32 bit divide
5736   dst    : WR(write);
5737   src1   : ISS(read);
5738   src2   : ISS(read);
5739   INS0   : ISS; // Can only dual issue as instruction 0
5740   DIV    : WR;
5741 %}
5742 
5743 // Eg.  SDIV    x0, x1, x2
5744 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5745 %{
5746   single_instruction;
5747   fixed_latency(16); // Maximum latency for 64 bit divide
5748   dst    : WR(write);
5749   src1   : ISS(read);
5750   src2   : ISS(read);
5751   INS0   : ISS; // Can only dual issue as instruction 0
5752   DIV    : WR;
5753 %}
5754 
5755 //------- Load pipeline operations ------------------------
5756 
5757 // Load - prefetch
5758 // Eg.  PFRM    <mem>
5759 pipe_class iload_prefetch(memory mem)
5760 %{
5761   single_instruction;
5762   mem    : ISS(read);
5763   INS01  : ISS;
5764   LDST   : WR;
5765 %}
5766 
5767 // Load - reg, mem
5768 // Eg.  LDR     x0, <mem>
5769 pipe_class iload_reg_mem(iRegI dst, memory mem)
5770 %{
5771   single_instruction;
5772   dst    : WR(write);
5773   mem    : ISS(read);
5774   INS01  : ISS;
5775   LDST   : WR;
5776 %}
5777 
5778 // Load - reg, reg
5779 // Eg.  LDR     x0, [sp, x1]
5780 pipe_class iload_reg_reg(iRegI dst, iRegI src)
5781 %{
5782   single_instruction;
5783   dst    : WR(write);
5784   src    : ISS(read);
5785   INS01  : ISS;
5786   LDST   : WR;
5787 %}
5788 
5789 //------- Store pipeline operations -----------------------
5790 
5791 // Store - zr, mem
5792 // Eg.  STR     zr, <mem>
5793 pipe_class istore_mem(memory mem)
5794 %{
5795   single_instruction;
5796   mem    : ISS(read);
5797   INS01  : ISS;
5798   LDST   : WR;
5799 %}
5800 
5801 // Store - reg, mem
5802 // Eg.  STR     x0, <mem>
5803 pipe_class istore_reg_mem(iRegI src, memory mem)
5804 %{
5805   single_instruction;
5806   mem    : ISS(read);
5807   src    : EX2(read);
5808   INS01  : ISS;
5809   LDST   : WR;
5810 %}
5811 
5812 // Store - reg, reg
5813 // Eg. STR      x0, [sp, x1]
5814 pipe_class istore_reg_reg(iRegI dst, iRegI src)
5815 %{
5816   single_instruction;
5817   dst    : ISS(read);
5818   src    : EX2(read);
5819   INS01  : ISS;
5820   LDST   : WR;
5821 %}
5822 
5823 //------- Store pipeline operations -----------------------
5824 
5825 // Branch
5826 pipe_class pipe_branch()
5827 %{
5828   single_instruction;
5829   INS01  : ISS;
5830   BRANCH : EX1;
5831 %}
5832 
5833 // Conditional branch
5834 pipe_class pipe_branch_cond(rFlagsReg cr)
5835 %{
5836   single_instruction;
5837   cr     : EX1(read);
5838   INS01  : ISS;
5839   BRANCH : EX1;
5840 %}
5841 
5842 // Compare & Branch
5843 // EG.  CBZ/CBNZ
5844 pipe_class pipe_cmp_branch(iRegI op1)
5845 %{
5846   single_instruction;
5847   op1    : EX1(read);
5848   INS01  : ISS;
5849   BRANCH : EX1;
5850 %}
5851 
5852 //------- Synchronisation operations ----------------------
5853 
5854 // Any operation requiring serialization.
5855 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
5856 pipe_class pipe_serial()
5857 %{
5858   single_instruction;
5859   force_serialization;
5860   fixed_latency(16);
5861   INS01  : ISS(2); // Cannot dual issue with any other instruction
5862   LDST   : WR;
5863 %}
5864 
5865 // Generic big/slow expanded idiom - also serialized
5866 pipe_class pipe_slow()
5867 %{
5868   instruction_count(10);
5869   multiple_bundles;
5870   force_serialization;
5871   fixed_latency(16);
5872   INS01  : ISS(2); // Cannot dual issue with any other instruction
5873   LDST   : WR;
5874 %}
5875 
5876 // Empty pipeline class
5877 pipe_class pipe_class_empty()
5878 %{
5879   single_instruction;
5880   fixed_latency(0);
5881 %}
5882 
5883 // Default pipeline class.
5884 pipe_class pipe_class_default()
5885 %{
5886   single_instruction;
5887   fixed_latency(2);
5888 %}
5889 
5890 // Pipeline class for compares.
5891 pipe_class pipe_class_compare()
5892 %{
5893   single_instruction;
5894   fixed_latency(16);
5895 %}
5896 
5897 // Pipeline class for memory operations.
5898 pipe_class pipe_class_memory()
5899 %{
5900   single_instruction;
5901   fixed_latency(16);
5902 %}
5903 
5904 // Pipeline class for call.
5905 pipe_class pipe_class_call()
5906 %{
5907   single_instruction;
5908   fixed_latency(100);
5909 %}
5910 
5911 // Define the class for the Nop node.
5912 define %{
5913    MachNop = pipe_class_empty;
5914 %}
5915 
5916 %}
5917 //----------INSTRUCTIONS-------------------------------------------------------
5918 //
5919 // match      -- States which machine-independent subtree may be replaced
5920 //               by this instruction.
5921 // ins_cost   -- The estimated cost of this instruction is used by instruction
5922 //               selection to identify a minimum cost tree of machine
5923 //               instructions that matches a tree of machine-independent
5924 //               instructions.
5925 // format     -- A string providing the disassembly for this instruction.
5926 //               The value of an instruction's operand may be inserted
5927 //               by referring to it with a '$' prefix.
5928 // opcode     -- Three instruction opcodes may be provided.  These are referred
5929 //               to within an encode class as $primary, $secondary, and $tertiary
5930 //               rrspectively.  The primary opcode is commonly used to
5931 //               indicate the type of machine instruction, while secondary
5932 //               and tertiary are often used for prefix options or addressing
5933 //               modes.
5934 // ins_encode -- A list of encode classes with parameters. The encode class
5935 //               name must have been defined in an 'enc_class' specification
5936 //               in the encode section of the architecture description.
5937 
5938 // ============================================================================
5939 // Memory (Load/Store) Instructions
5940 
5941 // Load Instructions
5942 
5943 // Load Byte (8 bit signed)
5944 instruct loadB(iRegINoSp dst, memory mem)
5945 %{
5946   match(Set dst (LoadB mem));
5947   predicate(!needs_acquiring_load(n));
5948 
5949   ins_cost(4 * INSN_COST);
5950   format %{ "ldrsbw  $dst, $mem\t# byte" %}
5951 
5952   ins_encode(aarch64_enc_ldrsbw(dst, mem));
5953 
5954   ins_pipe(iload_reg_mem);
5955 %}
5956 
5957 // Load Byte (8 bit signed) into long
5958 instruct loadB2L(iRegLNoSp dst, memory mem)
5959 %{
5960   match(Set dst (ConvI2L (LoadB mem)));
5961   predicate(!needs_acquiring_load(n->in(1)));
5962 
5963   ins_cost(4 * INSN_COST);
5964   format %{ "ldrsb  $dst, $mem\t# byte" %}
5965 
5966   ins_encode(aarch64_enc_ldrsb(dst, mem));
5967 
5968   ins_pipe(iload_reg_mem);
5969 %}
5970 
5971 // Load Byte (8 bit unsigned)
5972 instruct loadUB(iRegINoSp dst, memory mem)
5973 %{
5974   match(Set dst (LoadUB mem));
5975   predicate(!needs_acquiring_load(n));
5976 
5977   ins_cost(4 * INSN_COST);
5978   format %{ "ldrbw  $dst, $mem\t# byte" %}
5979 
5980   ins_encode(aarch64_enc_ldrb(dst, mem));
5981 
5982   ins_pipe(iload_reg_mem);
5983 %}
5984 
5985 // Load Byte (8 bit unsigned) into long
5986 instruct loadUB2L(iRegLNoSp dst, memory mem)
5987 %{
5988   match(Set dst (ConvI2L (LoadUB mem)));
5989   predicate(!needs_acquiring_load(n->in(1)));
5990 
5991   ins_cost(4 * INSN_COST);
5992   format %{ "ldrb  $dst, $mem\t# byte" %}
5993 
5994   ins_encode(aarch64_enc_ldrb(dst, mem));
5995 
5996   ins_pipe(iload_reg_mem);
5997 %}
5998 
5999 // Load Short (16 bit signed)
6000 instruct loadS(iRegINoSp dst, memory mem)
6001 %{
6002   match(Set dst (LoadS mem));
6003   predicate(!needs_acquiring_load(n));
6004 
6005   ins_cost(4 * INSN_COST);
6006   format %{ "ldrshw  $dst, $mem\t# short" %}
6007 
6008   ins_encode(aarch64_enc_ldrshw(dst, mem));
6009 
6010   ins_pipe(iload_reg_mem);
6011 %}
6012 
6013 // Load Short (16 bit signed) into long
6014 instruct loadS2L(iRegLNoSp dst, memory mem)
6015 %{
6016   match(Set dst (ConvI2L (LoadS mem)));
6017   predicate(!needs_acquiring_load(n->in(1)));
6018 
6019   ins_cost(4 * INSN_COST);
6020   format %{ "ldrsh  $dst, $mem\t# short" %}
6021 
6022   ins_encode(aarch64_enc_ldrsh(dst, mem));
6023 
6024   ins_pipe(iload_reg_mem);
6025 %}
6026 
6027 // Load Char (16 bit unsigned)
6028 instruct loadUS(iRegINoSp dst, memory mem)
6029 %{
6030   match(Set dst (LoadUS mem));
6031   predicate(!needs_acquiring_load(n));
6032 
6033   ins_cost(4 * INSN_COST);
6034   format %{ "ldrh  $dst, $mem\t# short" %}
6035 
6036   ins_encode(aarch64_enc_ldrh(dst, mem));
6037 
6038   ins_pipe(iload_reg_mem);
6039 %}
6040 
6041 // Load Short/Char (16 bit unsigned) into long
6042 instruct loadUS2L(iRegLNoSp dst, memory mem)
6043 %{
6044   match(Set dst (ConvI2L (LoadUS mem)));
6045   predicate(!needs_acquiring_load(n->in(1)));
6046 
6047   ins_cost(4 * INSN_COST);
6048   format %{ "ldrh  $dst, $mem\t# short" %}
6049 
6050   ins_encode(aarch64_enc_ldrh(dst, mem));
6051 
6052   ins_pipe(iload_reg_mem);
6053 %}
6054 
6055 // Load Integer (32 bit signed)
6056 instruct loadI(iRegINoSp dst, memory mem)
6057 %{
6058   match(Set dst (LoadI mem));
6059   predicate(!needs_acquiring_load(n));
6060 
6061   ins_cost(4 * INSN_COST);
6062   format %{ "ldrw  $dst, $mem\t# int" %}
6063 
6064   ins_encode(aarch64_enc_ldrw(dst, mem));
6065 
6066   ins_pipe(iload_reg_mem);
6067 %}
6068 
6069 // Load Integer (32 bit signed) into long
6070 instruct loadI2L(iRegLNoSp dst, memory mem)
6071 %{
6072   match(Set dst (ConvI2L (LoadI mem)));
6073   predicate(!needs_acquiring_load(n->in(1)));
6074 
6075   ins_cost(4 * INSN_COST);
6076   format %{ "ldrsw  $dst, $mem\t# int" %}
6077 
6078   ins_encode(aarch64_enc_ldrsw(dst, mem));
6079 
6080   ins_pipe(iload_reg_mem);
6081 %}
6082 
6083 // Load Integer (32 bit unsigned) into long
6084 instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
6085 %{
6086   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6087   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6088 
6089   ins_cost(4 * INSN_COST);
6090   format %{ "ldrw  $dst, $mem\t# int" %}
6091 
6092   ins_encode(aarch64_enc_ldrw(dst, mem));
6093 
6094   ins_pipe(iload_reg_mem);
6095 %}
6096 
6097 // Load Long (64 bit signed)
6098 instruct loadL(iRegLNoSp dst, memory mem)
6099 %{
6100   match(Set dst (LoadL mem));
6101   predicate(!needs_acquiring_load(n));
6102 
6103   ins_cost(4 * INSN_COST);
6104   format %{ "ldr  $dst, $mem\t# int" %}
6105 
6106   ins_encode(aarch64_enc_ldr(dst, mem));
6107 
6108   ins_pipe(iload_reg_mem);
6109 %}
6110 
6111 // Load Range
6112 instruct loadRange(iRegINoSp dst, memory mem)
6113 %{
6114   match(Set dst (LoadRange mem));
6115 
6116   ins_cost(4 * INSN_COST);
6117   format %{ "ldrw  $dst, $mem\t# range" %}
6118 
6119   ins_encode(aarch64_enc_ldrw(dst, mem));
6120 
6121   ins_pipe(iload_reg_mem);
6122 %}
6123 
6124 // Load Pointer
6125 instruct loadP(iRegPNoSp dst, memory mem)
6126 %{
6127   match(Set dst (LoadP mem));
6128   predicate(!needs_acquiring_load(n));
6129 
6130   ins_cost(4 * INSN_COST);
6131   format %{ "ldr  $dst, $mem\t# ptr" %}
6132 
6133   ins_encode(aarch64_enc_ldr(dst, mem));
6134 
6135   ins_pipe(iload_reg_mem);
6136 %}
6137 
6138 // Load Compressed Pointer
6139 instruct loadN(iRegNNoSp dst, memory mem)
6140 %{
6141   match(Set dst (LoadN mem));
6142   predicate(!needs_acquiring_load(n));
6143 
6144   ins_cost(4 * INSN_COST);
6145   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
6146 
6147   ins_encode(aarch64_enc_ldrw(dst, mem));
6148 
6149   ins_pipe(iload_reg_mem);
6150 %}
6151 
6152 // Load Klass Pointer
6153 instruct loadKlass(iRegPNoSp dst, memory mem)
6154 %{
6155   match(Set dst (LoadKlass mem));
6156   predicate(!needs_acquiring_load(n));
6157 
6158   ins_cost(4 * INSN_COST);
6159   format %{ "ldr  $dst, $mem\t# class" %}
6160 
6161   ins_encode(aarch64_enc_ldr(dst, mem));
6162 
6163   ins_pipe(iload_reg_mem);
6164 %}
6165 
6166 // Load Narrow Klass Pointer
6167 instruct loadNKlass(iRegNNoSp dst, memory mem)
6168 %{
6169   match(Set dst (LoadNKlass mem));
6170   predicate(!needs_acquiring_load(n));
6171 
6172   ins_cost(4 * INSN_COST);
6173   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
6174 
6175   ins_encode(aarch64_enc_ldrw(dst, mem));
6176 
6177   ins_pipe(iload_reg_mem);
6178 %}
6179 
6180 // Load Float
6181 instruct loadF(vRegF dst, memory mem)
6182 %{
6183   match(Set dst (LoadF mem));
6184   predicate(!needs_acquiring_load(n));
6185 
6186   ins_cost(4 * INSN_COST);
6187   format %{ "ldrs  $dst, $mem\t# float" %}
6188 
6189   ins_encode( aarch64_enc_ldrs(dst, mem) );
6190 
6191   ins_pipe(pipe_class_memory);
6192 %}
6193 
6194 // Load Double
6195 instruct loadD(vRegD dst, memory mem)
6196 %{
6197   match(Set dst (LoadD mem));
6198   predicate(!needs_acquiring_load(n));
6199 
6200   ins_cost(4 * INSN_COST);
6201   format %{ "ldrd  $dst, $mem\t# double" %}
6202 
6203   ins_encode( aarch64_enc_ldrd(dst, mem) );
6204 
6205   ins_pipe(pipe_class_memory);
6206 %}
6207 
6208 
6209 // Load Int Constant
6210 instruct loadConI(iRegINoSp dst, immI src)
6211 %{
6212   match(Set dst src);
6213 
6214   ins_cost(INSN_COST);
6215   format %{ "mov $dst, $src\t# int" %}
6216 
6217   ins_encode( aarch64_enc_movw_imm(dst, src) );
6218 
6219   ins_pipe(ialu_imm);
6220 %}
6221 
6222 // Load Long Constant
6223 instruct loadConL(iRegLNoSp dst, immL src)
6224 %{
6225   match(Set dst src);
6226 
6227   ins_cost(INSN_COST);
6228   format %{ "mov $dst, $src\t# long" %}
6229 
6230   ins_encode( aarch64_enc_mov_imm(dst, src) );
6231 
6232   ins_pipe(ialu_imm);
6233 %}
6234 
6235 // Load Pointer Constant
6236 
6237 instruct loadConP(iRegPNoSp dst, immP con)
6238 %{
6239   match(Set dst con);
6240 
6241   ins_cost(INSN_COST * 4);
6242   format %{
6243     "mov  $dst, $con\t# ptr\n\t"
6244   %}
6245 
6246   ins_encode(aarch64_enc_mov_p(dst, con));
6247 
6248   ins_pipe(ialu_imm);
6249 %}
6250 
6251 // Load Null Pointer Constant
6252 
6253 instruct loadConP0(iRegPNoSp dst, immP0 con)
6254 %{
6255   match(Set dst con);
6256 
6257   ins_cost(INSN_COST);
6258   format %{ "mov  $dst, $con\t# NULL ptr" %}
6259 
6260   ins_encode(aarch64_enc_mov_p0(dst, con));
6261 
6262   ins_pipe(ialu_imm);
6263 %}
6264 
6265 // Load Pointer Constant One
6266 
6267 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6268 %{
6269   match(Set dst con);
6270 
6271   ins_cost(INSN_COST);
6272   format %{ "mov  $dst, $con\t# NULL ptr" %}
6273 
6274   ins_encode(aarch64_enc_mov_p1(dst, con));
6275 
6276   ins_pipe(ialu_imm);
6277 %}
6278 
6279 // Load Poll Page Constant
6280 
6281 instruct loadConPollPage(iRegPNoSp dst, immPollPage con)
6282 %{
6283   match(Set dst con);
6284 
6285   ins_cost(INSN_COST);
6286   format %{ "adr  $dst, $con\t# Poll Page Ptr" %}
6287 
6288   ins_encode(aarch64_enc_mov_poll_page(dst, con));
6289 
6290   ins_pipe(ialu_imm);
6291 %}
6292 
6293 // Load Byte Map Base Constant
6294 
6295 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6296 %{
6297   match(Set dst con);
6298 
6299   ins_cost(INSN_COST);
6300   format %{ "adr  $dst, $con\t# Byte Map Base" %}
6301 
6302   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6303 
6304   ins_pipe(ialu_imm);
6305 %}
6306 
6307 // Load Narrow Pointer Constant
6308 
6309 instruct loadConN(iRegNNoSp dst, immN con)
6310 %{
6311   match(Set dst con);
6312 
6313   ins_cost(INSN_COST * 4);
6314   format %{ "mov  $dst, $con\t# compressed ptr" %}
6315 
6316   ins_encode(aarch64_enc_mov_n(dst, con));
6317 
6318   ins_pipe(ialu_imm);
6319 %}
6320 
6321 // Load Narrow Null Pointer Constant
6322 
6323 instruct loadConN0(iRegNNoSp dst, immN0 con)
6324 %{
6325   match(Set dst con);
6326 
6327   ins_cost(INSN_COST);
6328   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
6329 
6330   ins_encode(aarch64_enc_mov_n0(dst, con));
6331 
6332   ins_pipe(ialu_imm);
6333 %}
6334 
6335 // Load Narrow Klass Constant
6336 
6337 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6338 %{
6339   match(Set dst con);
6340 
6341   ins_cost(INSN_COST);
6342   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
6343 
6344   ins_encode(aarch64_enc_mov_nk(dst, con));
6345 
6346   ins_pipe(ialu_imm);
6347 %}
6348 
6349 // Load Packed Float Constant
6350 
6351 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6352   match(Set dst con);
6353   ins_cost(INSN_COST * 4);
6354   format %{ "fmovs  $dst, $con"%}
6355   ins_encode %{
6356     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6357   %}
6358 
6359   ins_pipe(pipe_class_default);
6360 %}
6361 
6362 // Load Float Constant
6363 
6364 instruct loadConF(vRegF dst, immF con) %{
6365   match(Set dst con);
6366 
6367   ins_cost(INSN_COST * 4);
6368 
6369   format %{
6370     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6371   %}
6372 
6373   ins_encode %{
6374     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6375   %}
6376 
6377   ins_pipe(pipe_class_default);
6378 %}
6379 
6380 // Load Packed Double Constant
6381 
6382 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6383   match(Set dst con);
6384   ins_cost(INSN_COST);
6385   format %{ "fmovd  $dst, $con"%}
6386   ins_encode %{
6387     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6388   %}
6389 
6390   ins_pipe(pipe_class_default);
6391 %}
6392 
6393 // Load Double Constant
6394 
6395 instruct loadConD(vRegD dst, immD con) %{
6396   match(Set dst con);
6397 
6398   ins_cost(INSN_COST * 5);
6399   format %{
6400     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6401   %}
6402 
6403   ins_encode %{
6404     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6405   %}
6406 
6407   ins_pipe(pipe_class_default);
6408 %}
6409 
6410 // Store Instructions
6411 
6412 // Store CMS card-mark Immediate
6413 instruct storeimmCM0(immI0 zero, memory mem)
6414 %{
6415   match(Set mem (StoreCM mem zero));
6416 
6417   ins_cost(INSN_COST);
6418   format %{ "strb zr, $mem\t# byte" %}
6419 
6420   ins_encode(aarch64_enc_strb0(mem));
6421 
6422   ins_pipe(istore_mem);
6423 %}
6424 
6425 // Store Byte
6426 instruct storeB(iRegIorL2I src, memory mem)
6427 %{
6428   match(Set mem (StoreB mem src));
6429   predicate(!needs_releasing_store(n));
6430 
6431   ins_cost(INSN_COST);
6432   format %{ "strb  $src, $mem\t# byte" %}
6433 
6434   ins_encode(aarch64_enc_strb(src, mem));
6435 
6436   ins_pipe(istore_reg_mem);
6437 %}
6438 
6439 
6440 instruct storeimmB0(immI0 zero, memory mem)
6441 %{
6442   match(Set mem (StoreB mem zero));
6443   predicate(!needs_releasing_store(n));
6444 
6445   ins_cost(INSN_COST);
6446   format %{ "strb zr, $mem\t# byte" %}
6447 
6448   ins_encode(aarch64_enc_strb0(mem));
6449 
6450   ins_pipe(istore_mem);
6451 %}
6452 
6453 // Store Char/Short
6454 instruct storeC(iRegIorL2I src, memory mem)
6455 %{
6456   match(Set mem (StoreC mem src));
6457   predicate(!needs_releasing_store(n));
6458 
6459   ins_cost(INSN_COST);
6460   format %{ "strh  $src, $mem\t# short" %}
6461 
6462   ins_encode(aarch64_enc_strh(src, mem));
6463 
6464   ins_pipe(istore_reg_mem);
6465 %}
6466 
6467 instruct storeimmC0(immI0 zero, memory mem)
6468 %{
6469   match(Set mem (StoreC mem zero));
6470   predicate(!needs_releasing_store(n));
6471 
6472   ins_cost(INSN_COST);
6473   format %{ "strh  zr, $mem\t# short" %}
6474 
6475   ins_encode(aarch64_enc_strh0(mem));
6476 
6477   ins_pipe(istore_mem);
6478 %}
6479 
6480 // Store Integer
6481 
6482 instruct storeI(iRegIorL2I src, memory mem)
6483 %{
6484   match(Set mem(StoreI mem src));
6485   predicate(!needs_releasing_store(n));
6486 
6487   ins_cost(INSN_COST);
6488   format %{ "strw  $src, $mem\t# int" %}
6489 
6490   ins_encode(aarch64_enc_strw(src, mem));
6491 
6492   ins_pipe(istore_reg_mem);
6493 %}
6494 
6495 instruct storeimmI0(immI0 zero, memory mem)
6496 %{
6497   match(Set mem(StoreI mem zero));
6498   predicate(!needs_releasing_store(n));
6499 
6500   ins_cost(INSN_COST);
6501   format %{ "strw  zr, $mem\t# int" %}
6502 
6503   ins_encode(aarch64_enc_strw0(mem));
6504 
6505   ins_pipe(istore_mem);
6506 %}
6507 
6508 // Store Long (64 bit signed)
6509 instruct storeL(iRegL src, memory mem)
6510 %{
6511   match(Set mem (StoreL mem src));
6512   predicate(!needs_releasing_store(n));
6513 
6514   ins_cost(INSN_COST);
6515   format %{ "str  $src, $mem\t# int" %}
6516 
6517   ins_encode(aarch64_enc_str(src, mem));
6518 
6519   ins_pipe(istore_reg_mem);
6520 %}
6521 
6522 // Store Long (64 bit signed)
6523 instruct storeimmL0(immL0 zero, memory mem)
6524 %{
6525   match(Set mem (StoreL mem zero));
6526   predicate(!needs_releasing_store(n));
6527 
6528   ins_cost(INSN_COST);
6529   format %{ "str  zr, $mem\t# int" %}
6530 
6531   ins_encode(aarch64_enc_str0(mem));
6532 
6533   ins_pipe(istore_mem);
6534 %}
6535 
6536 // Store Pointer
6537 instruct storeP(iRegP src, memory mem)
6538 %{
6539   match(Set mem (StoreP mem src));
6540   predicate(!needs_releasing_store(n));
6541 
6542   ins_cost(INSN_COST);
6543   format %{ "str  $src, $mem\t# ptr" %}
6544 
6545   ins_encode(aarch64_enc_str(src, mem));
6546 
6547   ins_pipe(istore_reg_mem);
6548 %}
6549 
6550 // Store Pointer
6551 instruct storeimmP0(immP0 zero, memory mem)
6552 %{
6553   match(Set mem (StoreP mem zero));
6554   predicate(!needs_releasing_store(n));
6555 
6556   ins_cost(INSN_COST);
6557   format %{ "str zr, $mem\t# ptr" %}
6558 
6559   ins_encode(aarch64_enc_str0(mem));
6560 
6561   ins_pipe(istore_mem);
6562 %}
6563 
6564 // Store Compressed Pointer
6565 instruct storeN(iRegN src, memory mem)
6566 %{
6567   match(Set mem (StoreN mem src));
6568   predicate(!needs_releasing_store(n));
6569 
6570   ins_cost(INSN_COST);
6571   format %{ "strw  $src, $mem\t# compressed ptr" %}
6572 
6573   ins_encode(aarch64_enc_strw(src, mem));
6574 
6575   ins_pipe(istore_reg_mem);
6576 %}
6577 
6578 instruct storeImmN0(iRegIHeapbase heapbase, immN0 zero, memory mem)
6579 %{
6580   match(Set mem (StoreN mem zero));
6581   predicate(Universe::narrow_oop_base() == NULL &&
6582             Universe::narrow_klass_base() == NULL &&
6583             (!needs_releasing_store(n)));
6584 
6585   ins_cost(INSN_COST);
6586   format %{ "strw  rheapbase, $mem\t# compressed ptr (rheapbase==0)" %}
6587 
6588   ins_encode(aarch64_enc_strw(heapbase, mem));
6589 
6590   ins_pipe(istore_reg_mem);
6591 %}
6592 
6593 // Store Float
6594 instruct storeF(vRegF src, memory mem)
6595 %{
6596   match(Set mem (StoreF mem src));
6597   predicate(!needs_releasing_store(n));
6598 
6599   ins_cost(INSN_COST);
6600   format %{ "strs  $src, $mem\t# float" %}
6601 
6602   ins_encode( aarch64_enc_strs(src, mem) );
6603 
6604   ins_pipe(pipe_class_memory);
6605 %}
6606 
6607 // TODO
6608 // implement storeImmF0 and storeFImmPacked
6609 
6610 // Store Double
6611 instruct storeD(vRegD src, memory mem)
6612 %{
6613   match(Set mem (StoreD mem src));
6614   predicate(!needs_releasing_store(n));
6615 
6616   ins_cost(INSN_COST);
6617   format %{ "strd  $src, $mem\t# double" %}
6618 
6619   ins_encode( aarch64_enc_strd(src, mem) );
6620 
6621   ins_pipe(pipe_class_memory);
6622 %}
6623 
6624 // Store Compressed Klass Pointer
6625 instruct storeNKlass(iRegN src, memory mem)
6626 %{
6627   predicate(!needs_releasing_store(n));
6628   match(Set mem (StoreNKlass mem src));
6629 
6630   ins_cost(INSN_COST);
6631   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
6632 
6633   ins_encode(aarch64_enc_strw(src, mem));
6634 
6635   ins_pipe(istore_reg_mem);
6636 %}
6637 
6638 // TODO
6639 // implement storeImmD0 and storeDImmPacked
6640 
6641 // prefetch instructions
6642 // Must be safe to execute with invalid address (cannot fault).
6643 
6644 instruct prefetchalloc( memory mem ) %{
6645   match(PrefetchAllocation mem);
6646 
6647   ins_cost(INSN_COST);
6648   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
6649 
6650   ins_encode( aarch64_enc_prefetchw(mem) );
6651 
6652   ins_pipe(iload_prefetch);
6653 %}
6654 
6655 //  ---------------- volatile loads and stores ----------------
6656 
6657 // Load Byte (8 bit signed)
6658 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
6659 %{
6660   match(Set dst (LoadB mem));
6661 
6662   ins_cost(VOLATILE_REF_COST);
6663   format %{ "ldarsb  $dst, $mem\t# byte" %}
6664 
6665   ins_encode(aarch64_enc_ldarsb(dst, mem));
6666 
6667   ins_pipe(pipe_serial);
6668 %}
6669 
6670 // Load Byte (8 bit signed) into long
6671 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
6672 %{
6673   match(Set dst (ConvI2L (LoadB mem)));
6674 
6675   ins_cost(VOLATILE_REF_COST);
6676   format %{ "ldarsb  $dst, $mem\t# byte" %}
6677 
6678   ins_encode(aarch64_enc_ldarsb(dst, mem));
6679 
6680   ins_pipe(pipe_serial);
6681 %}
6682 
6683 // Load Byte (8 bit unsigned)
6684 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
6685 %{
6686   match(Set dst (LoadUB mem));
6687 
6688   ins_cost(VOLATILE_REF_COST);
6689   format %{ "ldarb  $dst, $mem\t# byte" %}
6690 
6691   ins_encode(aarch64_enc_ldarb(dst, mem));
6692 
6693   ins_pipe(pipe_serial);
6694 %}
6695 
6696 // Load Byte (8 bit unsigned) into long
6697 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
6698 %{
6699   match(Set dst (ConvI2L (LoadUB mem)));
6700 
6701   ins_cost(VOLATILE_REF_COST);
6702   format %{ "ldarb  $dst, $mem\t# byte" %}
6703 
6704   ins_encode(aarch64_enc_ldarb(dst, mem));
6705 
6706   ins_pipe(pipe_serial);
6707 %}
6708 
6709 // Load Short (16 bit signed)
6710 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
6711 %{
6712   match(Set dst (LoadS mem));
6713 
6714   ins_cost(VOLATILE_REF_COST);
6715   format %{ "ldarshw  $dst, $mem\t# short" %}
6716 
6717   ins_encode(aarch64_enc_ldarshw(dst, mem));
6718 
6719   ins_pipe(pipe_serial);
6720 %}
6721 
6722 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
6723 %{
6724   match(Set dst (LoadUS mem));
6725 
6726   ins_cost(VOLATILE_REF_COST);
6727   format %{ "ldarhw  $dst, $mem\t# short" %}
6728 
6729   ins_encode(aarch64_enc_ldarhw(dst, mem));
6730 
6731   ins_pipe(pipe_serial);
6732 %}
6733 
6734 // Load Short/Char (16 bit unsigned) into long
6735 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
6736 %{
6737   match(Set dst (ConvI2L (LoadUS mem)));
6738 
6739   ins_cost(VOLATILE_REF_COST);
6740   format %{ "ldarh  $dst, $mem\t# short" %}
6741 
6742   ins_encode(aarch64_enc_ldarh(dst, mem));
6743 
6744   ins_pipe(pipe_serial);
6745 %}
6746 
6747 // Load Short/Char (16 bit signed) into long
6748 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
6749 %{
6750   match(Set dst (ConvI2L (LoadS mem)));
6751 
6752   ins_cost(VOLATILE_REF_COST);
6753   format %{ "ldarh  $dst, $mem\t# short" %}
6754 
6755   ins_encode(aarch64_enc_ldarsh(dst, mem));
6756 
6757   ins_pipe(pipe_serial);
6758 %}
6759 
6760 // Load Integer (32 bit signed)
6761 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
6762 %{
6763   match(Set dst (LoadI mem));
6764 
6765   ins_cost(VOLATILE_REF_COST);
6766   format %{ "ldarw  $dst, $mem\t# int" %}
6767 
6768   ins_encode(aarch64_enc_ldarw(dst, mem));
6769 
6770   ins_pipe(pipe_serial);
6771 %}
6772 
6773 // Load Integer (32 bit unsigned) into long
6774 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
6775 %{
6776   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6777 
6778   ins_cost(VOLATILE_REF_COST);
6779   format %{ "ldarw  $dst, $mem\t# int" %}
6780 
6781   ins_encode(aarch64_enc_ldarw(dst, mem));
6782 
6783   ins_pipe(pipe_serial);
6784 %}
6785 
6786 // Load Long (64 bit signed)
6787 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
6788 %{
6789   match(Set dst (LoadL mem));
6790 
6791   ins_cost(VOLATILE_REF_COST);
6792   format %{ "ldar  $dst, $mem\t# int" %}
6793 
6794   ins_encode(aarch64_enc_ldar(dst, mem));
6795 
6796   ins_pipe(pipe_serial);
6797 %}
6798 
6799 // Load Pointer
6800 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
6801 %{
6802   match(Set dst (LoadP mem));
6803 
6804   ins_cost(VOLATILE_REF_COST);
6805   format %{ "ldar  $dst, $mem\t# ptr" %}
6806 
6807   ins_encode(aarch64_enc_ldar(dst, mem));
6808 
6809   ins_pipe(pipe_serial);
6810 %}
6811 
6812 // Load Compressed Pointer
6813 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
6814 %{
6815   match(Set dst (LoadN mem));
6816 
6817   ins_cost(VOLATILE_REF_COST);
6818   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
6819 
6820   ins_encode(aarch64_enc_ldarw(dst, mem));
6821 
6822   ins_pipe(pipe_serial);
6823 %}
6824 
6825 // Load Float
6826 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
6827 %{
6828   match(Set dst (LoadF mem));
6829 
6830   ins_cost(VOLATILE_REF_COST);
6831   format %{ "ldars  $dst, $mem\t# float" %}
6832 
6833   ins_encode( aarch64_enc_fldars(dst, mem) );
6834 
6835   ins_pipe(pipe_serial);
6836 %}
6837 
6838 // Load Double
6839 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
6840 %{
6841   match(Set dst (LoadD mem));
6842 
6843   ins_cost(VOLATILE_REF_COST);
6844   format %{ "ldard  $dst, $mem\t# double" %}
6845 
6846   ins_encode( aarch64_enc_fldard(dst, mem) );
6847 
6848   ins_pipe(pipe_serial);
6849 %}
6850 
6851 // Store Byte
6852 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
6853 %{
6854   match(Set mem (StoreB mem src));
6855 
6856   ins_cost(VOLATILE_REF_COST);
6857   format %{ "stlrb  $src, $mem\t# byte" %}
6858 
6859   ins_encode(aarch64_enc_stlrb(src, mem));
6860 
6861   ins_pipe(pipe_class_memory);
6862 %}
6863 
6864 // Store Char/Short
6865 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
6866 %{
6867   match(Set mem (StoreC mem src));
6868 
6869   ins_cost(VOLATILE_REF_COST);
6870   format %{ "stlrh  $src, $mem\t# short" %}
6871 
6872   ins_encode(aarch64_enc_stlrh(src, mem));
6873 
6874   ins_pipe(pipe_class_memory);
6875 %}
6876 
6877 // Store Integer
6878 
6879 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
6880 %{
6881   match(Set mem(StoreI mem src));
6882 
6883   ins_cost(VOLATILE_REF_COST);
6884   format %{ "stlrw  $src, $mem\t# int" %}
6885 
6886   ins_encode(aarch64_enc_stlrw(src, mem));
6887 
6888   ins_pipe(pipe_class_memory);
6889 %}
6890 
6891 // Store Long (64 bit signed)
6892 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
6893 %{
6894   match(Set mem (StoreL mem src));
6895 
6896   ins_cost(VOLATILE_REF_COST);
6897   format %{ "stlr  $src, $mem\t# int" %}
6898 
6899   ins_encode(aarch64_enc_stlr(src, mem));
6900 
6901   ins_pipe(pipe_class_memory);
6902 %}
6903 
6904 // Store Pointer
6905 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
6906 %{
6907   match(Set mem (StoreP mem src));
6908 
6909   ins_cost(VOLATILE_REF_COST);
6910   format %{ "stlr  $src, $mem\t# ptr" %}
6911 
6912   ins_encode(aarch64_enc_stlr(src, mem));
6913 
6914   ins_pipe(pipe_class_memory);
6915 %}
6916 
6917 // Store Compressed Pointer
6918 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
6919 %{
6920   match(Set mem (StoreN mem src));
6921 
6922   ins_cost(VOLATILE_REF_COST);
6923   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
6924 
6925   ins_encode(aarch64_enc_stlrw(src, mem));
6926 
6927   ins_pipe(pipe_class_memory);
6928 %}
6929 
6930 // Store Float
6931 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
6932 %{
6933   match(Set mem (StoreF mem src));
6934 
6935   ins_cost(VOLATILE_REF_COST);
6936   format %{ "stlrs  $src, $mem\t# float" %}
6937 
6938   ins_encode( aarch64_enc_fstlrs(src, mem) );
6939 
6940   ins_pipe(pipe_class_memory);
6941 %}
6942 
6943 // TODO
6944 // implement storeImmF0 and storeFImmPacked
6945 
6946 // Store Double
6947 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
6948 %{
6949   match(Set mem (StoreD mem src));
6950 
6951   ins_cost(VOLATILE_REF_COST);
6952   format %{ "stlrd  $src, $mem\t# double" %}
6953 
6954   ins_encode( aarch64_enc_fstlrd(src, mem) );
6955 
6956   ins_pipe(pipe_class_memory);
6957 %}
6958 
6959 //  ---------------- end of volatile loads and stores ----------------
6960 
6961 // ============================================================================
6962 // BSWAP Instructions
6963 
6964 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
6965   match(Set dst (ReverseBytesI src));
6966 
6967   ins_cost(INSN_COST);
6968   format %{ "revw  $dst, $src" %}
6969 
6970   ins_encode %{
6971     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
6972   %}
6973 
6974   ins_pipe(ialu_reg);
6975 %}
6976 
6977 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
6978   match(Set dst (ReverseBytesL src));
6979 
6980   ins_cost(INSN_COST);
6981   format %{ "rev  $dst, $src" %}
6982 
6983   ins_encode %{
6984     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
6985   %}
6986 
6987   ins_pipe(ialu_reg);
6988 %}
6989 
6990 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
6991   match(Set dst (ReverseBytesUS src));
6992 
6993   ins_cost(INSN_COST);
6994   format %{ "rev16w  $dst, $src" %}
6995 
6996   ins_encode %{
6997     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
6998   %}
6999 
7000   ins_pipe(ialu_reg);
7001 %}
7002 
7003 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7004   match(Set dst (ReverseBytesS src));
7005 
7006   ins_cost(INSN_COST);
7007   format %{ "rev16w  $dst, $src\n\t"
7008             "sbfmw $dst, $dst, #0, #15" %}
7009 
7010   ins_encode %{
7011     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7012     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7013   %}
7014 
7015   ins_pipe(ialu_reg);
7016 %}
7017 
7018 // ============================================================================
7019 // Zero Count Instructions
7020 
7021 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7022   match(Set dst (CountLeadingZerosI src));
7023 
7024   ins_cost(INSN_COST);
7025   format %{ "clzw  $dst, $src" %}
7026   ins_encode %{
7027     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7028   %}
7029 
7030   ins_pipe(ialu_reg);
7031 %}
7032 
7033 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7034   match(Set dst (CountLeadingZerosL src));
7035 
7036   ins_cost(INSN_COST);
7037   format %{ "clz   $dst, $src" %}
7038   ins_encode %{
7039     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7040   %}
7041 
7042   ins_pipe(ialu_reg);
7043 %}
7044 
7045 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7046   match(Set dst (CountTrailingZerosI src));
7047 
7048   ins_cost(INSN_COST * 2);
7049   format %{ "rbitw  $dst, $src\n\t"
7050             "clzw   $dst, $dst" %}
7051   ins_encode %{
7052     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7053     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7054   %}
7055 
7056   ins_pipe(ialu_reg);
7057 %}
7058 
7059 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7060   match(Set dst (CountTrailingZerosL src));
7061 
7062   ins_cost(INSN_COST * 2);
7063   format %{ "rbit   $dst, $src\n\t"
7064             "clz    $dst, $dst" %}
7065   ins_encode %{
7066     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7067     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7068   %}
7069 
7070   ins_pipe(ialu_reg);
7071 %}
7072 
7073 // ============================================================================
7074 // MemBar Instruction
7075 
7076 instruct load_fence() %{
7077   match(LoadFence);
7078   ins_cost(VOLATILE_REF_COST);
7079 
7080   format %{ "load_fence" %}
7081 
7082   ins_encode %{
7083     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7084   %}
7085   ins_pipe(pipe_serial);
7086 %}
7087 
7088 instruct unnecessary_membar_acquire() %{
7089   predicate(unnecessary_acquire(n));
7090   match(MemBarAcquire);
7091   ins_cost(0);
7092 
7093   format %{ "membar_acquire (elided)" %}
7094 
7095   ins_encode %{
7096     __ block_comment("membar_acquire (elided)");
7097   %}
7098 
7099   ins_pipe(pipe_class_empty);
7100 %}
7101 
7102 instruct membar_acquire() %{
7103   match(MemBarAcquire);
7104   ins_cost(VOLATILE_REF_COST);
7105 
7106   format %{ "membar_acquire" %}
7107 
7108   ins_encode %{
7109     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7110   %}
7111 
7112   ins_pipe(pipe_serial);
7113 %}
7114 
7115 
7116 instruct membar_acquire_lock() %{
7117   match(MemBarAcquireLock);
7118   ins_cost(VOLATILE_REF_COST);
7119 
7120   format %{ "membar_acquire_lock" %}
7121 
7122   ins_encode %{
7123     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7124   %}
7125 
7126   ins_pipe(pipe_serial);
7127 %}
7128 
7129 instruct store_fence() %{
7130   match(StoreFence);
7131   ins_cost(VOLATILE_REF_COST);
7132 
7133   format %{ "store_fence" %}
7134 
7135   ins_encode %{
7136     __ membar(Assembler::LoadStore|Assembler::StoreStore);
7137   %}
7138   ins_pipe(pipe_serial);
7139 %}
7140 
7141 instruct unnecessary_membar_release() %{
7142   predicate(unnecessary_release(n));
7143   match(MemBarRelease);
7144   ins_cost(0);
7145 
7146   format %{ "membar_release (elided)" %}
7147 
7148   ins_encode %{
7149     __ block_comment("membar_release (elided)");
7150   %}
7151   ins_pipe(pipe_serial);
7152 %}
7153 
7154 instruct membar_release() %{
7155   match(MemBarRelease);
7156   ins_cost(VOLATILE_REF_COST);
7157 
7158   format %{ "membar_release" %}
7159 
7160   ins_encode %{
7161     __ membar(Assembler::LoadStore|Assembler::StoreStore);
7162   %}
7163   ins_pipe(pipe_serial);
7164 %}
7165 
7166 instruct membar_storestore() %{
7167   match(MemBarStoreStore);
7168   ins_cost(VOLATILE_REF_COST);
7169 
7170   format %{ "MEMBAR-store-store" %}
7171 
7172   ins_encode %{
7173     __ membar(Assembler::StoreStore);
7174   %}
7175   ins_pipe(pipe_serial);
7176 %}
7177 
7178 instruct membar_release_lock() %{
7179   match(MemBarReleaseLock);
7180   ins_cost(VOLATILE_REF_COST);
7181 
7182   format %{ "membar_release_lock" %}
7183 
7184   ins_encode %{
7185     __ membar(Assembler::LoadStore|Assembler::StoreStore);
7186   %}
7187 
7188   ins_pipe(pipe_serial);
7189 %}
7190 
7191 instruct unnecessary_membar_volatile() %{
7192   predicate(unnecessary_volatile(n));
7193   match(MemBarVolatile);
7194   ins_cost(0);
7195 
7196   format %{ "membar_volatile (elided)" %}
7197 
7198   ins_encode %{
7199     __ block_comment("membar_volatile (elided)");
7200   %}
7201 
7202   ins_pipe(pipe_serial);
7203 %}
7204 
7205 instruct membar_volatile() %{
7206   match(MemBarVolatile);
7207   ins_cost(VOLATILE_REF_COST*100);
7208 
7209   format %{ "membar_volatile" %}
7210 
7211   ins_encode %{
7212     __ membar(Assembler::StoreLoad);
7213   %}
7214 
7215   ins_pipe(pipe_serial);
7216 %}
7217 
7218 // ============================================================================
7219 // Cast/Convert Instructions
7220 
7221 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7222   match(Set dst (CastX2P src));
7223 
7224   ins_cost(INSN_COST);
7225   format %{ "mov $dst, $src\t# long -> ptr" %}
7226 
7227   ins_encode %{
7228     if ($dst$$reg != $src$$reg) {
7229       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7230     }
7231   %}
7232 
7233   ins_pipe(ialu_reg);
7234 %}
7235 
7236 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7237   match(Set dst (CastP2X src));
7238 
7239   ins_cost(INSN_COST);
7240   format %{ "mov $dst, $src\t# ptr -> long" %}
7241 
7242   ins_encode %{
7243     if ($dst$$reg != $src$$reg) {
7244       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7245     }
7246   %}
7247 
7248   ins_pipe(ialu_reg);
7249 %}
7250 
7251 // Convert oop into int for vectors alignment masking
7252 instruct convP2I(iRegINoSp dst, iRegP src) %{
7253   match(Set dst (ConvL2I (CastP2X src)));
7254 
7255   ins_cost(INSN_COST);
7256   format %{ "movw $dst, $src\t# ptr -> int" %}
7257   ins_encode %{
7258     __ movw($dst$$Register, $src$$Register);
7259   %}
7260 
7261   ins_pipe(ialu_reg);
7262 %}
7263 
7264 // Convert compressed oop into int for vectors alignment masking
7265 // in case of 32bit oops (heap < 4Gb).
7266 instruct convN2I(iRegINoSp dst, iRegN src)
7267 %{
7268   predicate(Universe::narrow_oop_shift() == 0);
7269   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
7270 
7271   ins_cost(INSN_COST);
7272   format %{ "mov dst, $src\t# compressed ptr -> int" %}
7273   ins_encode %{
7274     __ movw($dst$$Register, $src$$Register);
7275   %}
7276 
7277   ins_pipe(ialu_reg);
7278 %}
7279 
7280 
7281 // Convert oop pointer into compressed form
7282 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7283   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
7284   match(Set dst (EncodeP src));
7285   effect(KILL cr);
7286   ins_cost(INSN_COST * 3);
7287   format %{ "encode_heap_oop $dst, $src" %}
7288   ins_encode %{
7289     Register s = $src$$Register;
7290     Register d = $dst$$Register;
7291     __ encode_heap_oop(d, s);
7292   %}
7293   ins_pipe(ialu_reg);
7294 %}
7295 
7296 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7297   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
7298   match(Set dst (EncodeP src));
7299   ins_cost(INSN_COST * 3);
7300   format %{ "encode_heap_oop_not_null $dst, $src" %}
7301   ins_encode %{
7302     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
7303   %}
7304   ins_pipe(ialu_reg);
7305 %}
7306 
7307 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7308   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
7309             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
7310   match(Set dst (DecodeN src));
7311   ins_cost(INSN_COST * 3);
7312   format %{ "decode_heap_oop $dst, $src" %}
7313   ins_encode %{
7314     Register s = $src$$Register;
7315     Register d = $dst$$Register;
7316     __ decode_heap_oop(d, s);
7317   %}
7318   ins_pipe(ialu_reg);
7319 %}
7320 
7321 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7322   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
7323             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
7324   match(Set dst (DecodeN src));
7325   ins_cost(INSN_COST * 3);
7326   format %{ "decode_heap_oop_not_null $dst, $src" %}
7327   ins_encode %{
7328     Register s = $src$$Register;
7329     Register d = $dst$$Register;
7330     __ decode_heap_oop_not_null(d, s);
7331   %}
7332   ins_pipe(ialu_reg);
7333 %}
7334 
7335 // n.b. AArch64 implementations of encode_klass_not_null and
7336 // decode_klass_not_null do not modify the flags register so, unlike
7337 // Intel, we don't kill CR as a side effect here
7338 
7339 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
7340   match(Set dst (EncodePKlass src));
7341 
7342   ins_cost(INSN_COST * 3);
7343   format %{ "encode_klass_not_null $dst,$src" %}
7344 
7345   ins_encode %{
7346     Register src_reg = as_Register($src$$reg);
7347     Register dst_reg = as_Register($dst$$reg);
7348     __ encode_klass_not_null(dst_reg, src_reg);
7349   %}
7350 
7351    ins_pipe(ialu_reg);
7352 %}
7353 
7354 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
7355   match(Set dst (DecodeNKlass src));
7356 
7357   ins_cost(INSN_COST * 3);
7358   format %{ "decode_klass_not_null $dst,$src" %}
7359 
7360   ins_encode %{
7361     Register src_reg = as_Register($src$$reg);
7362     Register dst_reg = as_Register($dst$$reg);
7363     if (dst_reg != src_reg) {
7364       __ decode_klass_not_null(dst_reg, src_reg);
7365     } else {
7366       __ decode_klass_not_null(dst_reg);
7367     }
7368   %}
7369 
7370    ins_pipe(ialu_reg);
7371 %}
7372 
7373 instruct checkCastPP(iRegPNoSp dst)
7374 %{
7375   match(Set dst (CheckCastPP dst));
7376 
7377   size(0);
7378   format %{ "# checkcastPP of $dst" %}
7379   ins_encode(/* empty encoding */);
7380   ins_pipe(pipe_class_empty);
7381 %}
7382 
7383 instruct castPP(iRegPNoSp dst)
7384 %{
7385   match(Set dst (CastPP dst));
7386 
7387   size(0);
7388   format %{ "# castPP of $dst" %}
7389   ins_encode(/* empty encoding */);
7390   ins_pipe(pipe_class_empty);
7391 %}
7392 
7393 instruct castII(iRegI dst)
7394 %{
7395   match(Set dst (CastII dst));
7396 
7397   size(0);
7398   format %{ "# castII of $dst" %}
7399   ins_encode(/* empty encoding */);
7400   ins_cost(0);
7401   ins_pipe(pipe_class_empty);
7402 %}
7403 
7404 // ============================================================================
7405 // Atomic operation instructions
7406 //
7407 // Intel and SPARC both implement Ideal Node LoadPLocked and
7408 // Store{PIL}Conditional instructions using a normal load for the
7409 // LoadPLocked and a CAS for the Store{PIL}Conditional.
7410 //
7411 // The ideal code appears only to use LoadPLocked/StorePLocked as a
7412 // pair to lock object allocations from Eden space when not using
7413 // TLABs.
7414 //
7415 // There does not appear to be a Load{IL}Locked Ideal Node and the
7416 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
7417 // and to use StoreIConditional only for 32-bit and StoreLConditional
7418 // only for 64-bit.
7419 //
7420 // We implement LoadPLocked and StorePLocked instructions using,
7421 // respectively the AArch64 hw load-exclusive and store-conditional
7422 // instructions. Whereas we must implement each of
7423 // Store{IL}Conditional using a CAS which employs a pair of
7424 // instructions comprising a load-exclusive followed by a
7425 // store-conditional.
7426 
7427 
7428 // Locked-load (linked load) of the current heap-top
7429 // used when updating the eden heap top
7430 // implemented using ldaxr on AArch64
7431 
7432 instruct loadPLocked(iRegPNoSp dst, indirect mem)
7433 %{
7434   match(Set dst (LoadPLocked mem));
7435 
7436   ins_cost(VOLATILE_REF_COST);
7437 
7438   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
7439 
7440   ins_encode(aarch64_enc_ldaxr(dst, mem));
7441 
7442   ins_pipe(pipe_serial);
7443 %}
7444 
7445 // Conditional-store of the updated heap-top.
7446 // Used during allocation of the shared heap.
7447 // Sets flag (EQ) on success.
7448 // implemented using stlxr on AArch64.
7449 
7450 instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
7451 %{
7452   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7453 
7454   ins_cost(VOLATILE_REF_COST);
7455 
7456  // TODO
7457  // do we need to do a store-conditional release or can we just use a
7458  // plain store-conditional?
7459 
7460   format %{
7461     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
7462     "cmpw rscratch1, zr\t# EQ on successful write"
7463   %}
7464 
7465   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
7466 
7467   ins_pipe(pipe_serial);
7468 %}
7469 
7470 // this has to be implemented as a CAS
7471 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
7472 %{
7473   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7474 
7475   ins_cost(VOLATILE_REF_COST);
7476 
7477   format %{
7478     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
7479     "cmpw rscratch1, zr\t# EQ on successful write"
7480   %}
7481 
7482   ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval));
7483 
7484   ins_pipe(pipe_slow);
7485 %}
7486 
7487 // this has to be implemented as a CAS
7488 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
7489 %{
7490   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7491 
7492   ins_cost(VOLATILE_REF_COST);
7493 
7494   format %{
7495     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
7496     "cmpw rscratch1, zr\t# EQ on successful write"
7497   %}
7498 
7499   ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval));
7500 
7501   ins_pipe(pipe_slow);
7502 %}
7503 
7504 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
7505 // can't match them
7506 
7507 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
7508 
7509   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
7510 
7511   effect(KILL cr);
7512 
7513  format %{
7514     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
7515     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
7516  %}
7517 
7518  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
7519             aarch64_enc_cset_eq(res));
7520 
7521   ins_pipe(pipe_slow);
7522 %}
7523 
7524 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
7525 
7526   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
7527 
7528   effect(KILL cr);
7529 
7530  format %{
7531     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
7532     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
7533  %}
7534 
7535  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
7536             aarch64_enc_cset_eq(res));
7537 
7538   ins_pipe(pipe_slow);
7539 %}
7540 
7541 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
7542 
7543   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
7544 
7545   effect(KILL cr);
7546 
7547  format %{
7548     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
7549     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
7550  %}
7551 
7552  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
7553             aarch64_enc_cset_eq(res));
7554 
7555   ins_pipe(pipe_slow);
7556 %}
7557 
7558 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
7559 
7560   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
7561 
7562   effect(KILL cr);
7563 
7564  format %{
7565     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
7566     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
7567  %}
7568 
7569  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
7570             aarch64_enc_cset_eq(res));
7571 
7572   ins_pipe(pipe_slow);
7573 %}
7574 
7575 
7576 instruct get_and_setI(indirect mem, iRegINoSp newv, iRegI prev) %{
7577   match(Set prev (GetAndSetI mem newv));
7578   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
7579   ins_encode %{
7580     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
7581   %}
7582   ins_pipe(pipe_serial);
7583 %}
7584 
7585 instruct get_and_setL(indirect mem, iRegLNoSp newv, iRegL prev) %{
7586   match(Set prev (GetAndSetL mem newv));
7587   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
7588   ins_encode %{
7589     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
7590   %}
7591   ins_pipe(pipe_serial);
7592 %}
7593 
7594 instruct get_and_setN(indirect mem, iRegNNoSp newv, iRegI prev) %{
7595   match(Set prev (GetAndSetN mem newv));
7596   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
7597   ins_encode %{
7598     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
7599   %}
7600   ins_pipe(pipe_serial);
7601 %}
7602 
7603 instruct get_and_setP(indirect mem, iRegPNoSp newv, iRegP prev) %{
7604   match(Set prev (GetAndSetP mem newv));
7605   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
7606   ins_encode %{
7607     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
7608   %}
7609   ins_pipe(pipe_serial);
7610 %}
7611 
7612 
7613 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
7614   match(Set newval (GetAndAddL mem incr));
7615   ins_cost(INSN_COST * 10);
7616   format %{ "get_and_addL $newval, [$mem], $incr" %}
7617   ins_encode %{
7618     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
7619   %}
7620   ins_pipe(pipe_serial);
7621 %}
7622 
7623 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
7624   predicate(n->as_LoadStore()->result_not_used());
7625   match(Set dummy (GetAndAddL mem incr));
7626   ins_cost(INSN_COST * 9);
7627   format %{ "get_and_addL [$mem], $incr" %}
7628   ins_encode %{
7629     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
7630   %}
7631   ins_pipe(pipe_serial);
7632 %}
7633 
7634 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
7635   match(Set newval (GetAndAddL mem incr));
7636   ins_cost(INSN_COST * 10);
7637   format %{ "get_and_addL $newval, [$mem], $incr" %}
7638   ins_encode %{
7639     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
7640   %}
7641   ins_pipe(pipe_serial);
7642 %}
7643 
7644 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
7645   predicate(n->as_LoadStore()->result_not_used());
7646   match(Set dummy (GetAndAddL mem incr));
7647   ins_cost(INSN_COST * 9);
7648   format %{ "get_and_addL [$mem], $incr" %}
7649   ins_encode %{
7650     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
7651   %}
7652   ins_pipe(pipe_serial);
7653 %}
7654 
7655 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
7656   match(Set newval (GetAndAddI mem incr));
7657   ins_cost(INSN_COST * 10);
7658   format %{ "get_and_addI $newval, [$mem], $incr" %}
7659   ins_encode %{
7660     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
7661   %}
7662   ins_pipe(pipe_serial);
7663 %}
7664 
7665 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
7666   predicate(n->as_LoadStore()->result_not_used());
7667   match(Set dummy (GetAndAddI mem incr));
7668   ins_cost(INSN_COST * 9);
7669   format %{ "get_and_addI [$mem], $incr" %}
7670   ins_encode %{
7671     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
7672   %}
7673   ins_pipe(pipe_serial);
7674 %}
7675 
7676 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
7677   match(Set newval (GetAndAddI mem incr));
7678   ins_cost(INSN_COST * 10);
7679   format %{ "get_and_addI $newval, [$mem], $incr" %}
7680   ins_encode %{
7681     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
7682   %}
7683   ins_pipe(pipe_serial);
7684 %}
7685 
7686 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
7687   predicate(n->as_LoadStore()->result_not_used());
7688   match(Set dummy (GetAndAddI mem incr));
7689   ins_cost(INSN_COST * 9);
7690   format %{ "get_and_addI [$mem], $incr" %}
7691   ins_encode %{
7692     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
7693   %}
7694   ins_pipe(pipe_serial);
7695 %}
7696 
7697 // Manifest a CmpL result in an integer register.
7698 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
7699 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
7700 %{
7701   match(Set dst (CmpL3 src1 src2));
7702   effect(KILL flags);
7703 
7704   ins_cost(INSN_COST * 6);
7705   format %{
7706       "cmp $src1, $src2"
7707       "csetw $dst, ne"
7708       "cnegw $dst, lt"
7709   %}
7710   // format %{ "CmpL3 $dst, $src1, $src2" %}
7711   ins_encode %{
7712     __ cmp($src1$$Register, $src2$$Register);
7713     __ csetw($dst$$Register, Assembler::NE);
7714     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
7715   %}
7716 
7717   ins_pipe(pipe_class_default);
7718 %}
7719 
7720 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
7721 %{
7722   match(Set dst (CmpL3 src1 src2));
7723   effect(KILL flags);
7724 
7725   ins_cost(INSN_COST * 6);
7726   format %{
7727       "cmp $src1, $src2"
7728       "csetw $dst, ne"
7729       "cnegw $dst, lt"
7730   %}
7731   ins_encode %{
7732     int32_t con = (int32_t)$src2$$constant;
7733      if (con < 0) {
7734       __ adds(zr, $src1$$Register, -con);
7735     } else {
7736       __ subs(zr, $src1$$Register, con);
7737     }
7738     __ csetw($dst$$Register, Assembler::NE);
7739     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
7740   %}
7741 
7742   ins_pipe(pipe_class_default);
7743 %}
7744 
7745 // ============================================================================
7746 // Conditional Move Instructions
7747 
7748 // n.b. we have identical rules for both a signed compare op (cmpOp)
7749 // and an unsigned compare op (cmpOpU). it would be nice if we could
7750 // define an op class which merged both inputs and use it to type the
7751 // argument to a single rule. unfortunatelyt his fails because the
7752 // opclass does not live up to the COND_INTER interface of its
7753 // component operands. When the generic code tries to negate the
7754 // operand it ends up running the generci Machoper::negate method
7755 // which throws a ShouldNotHappen. So, we have to provide two flavours
7756 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
7757 
7758 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7759   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
7760 
7761   ins_cost(INSN_COST * 2);
7762   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
7763 
7764   ins_encode %{
7765     __ cselw(as_Register($dst$$reg),
7766              as_Register($src2$$reg),
7767              as_Register($src1$$reg),
7768              (Assembler::Condition)$cmp$$cmpcode);
7769   %}
7770 
7771   ins_pipe(icond_reg_reg);
7772 %}
7773 
7774 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7775   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
7776 
7777   ins_cost(INSN_COST * 2);
7778   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
7779 
7780   ins_encode %{
7781     __ cselw(as_Register($dst$$reg),
7782              as_Register($src2$$reg),
7783              as_Register($src1$$reg),
7784              (Assembler::Condition)$cmp$$cmpcode);
7785   %}
7786 
7787   ins_pipe(icond_reg_reg);
7788 %}
7789 
7790 // special cases where one arg is zero
7791 
7792 // n.b. this is selected in preference to the rule above because it
7793 // avoids loading constant 0 into a source register
7794 
7795 // TODO
7796 // we ought only to be able to cull one of these variants as the ideal
7797 // transforms ought always to order the zero consistently (to left/right?)
7798 
7799 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
7800   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
7801 
7802   ins_cost(INSN_COST * 2);
7803   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
7804 
7805   ins_encode %{
7806     __ cselw(as_Register($dst$$reg),
7807              as_Register($src$$reg),
7808              zr,
7809              (Assembler::Condition)$cmp$$cmpcode);
7810   %}
7811 
7812   ins_pipe(icond_reg);
7813 %}
7814 
7815 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
7816   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
7817 
7818   ins_cost(INSN_COST * 2);
7819   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
7820 
7821   ins_encode %{
7822     __ cselw(as_Register($dst$$reg),
7823              as_Register($src$$reg),
7824              zr,
7825              (Assembler::Condition)$cmp$$cmpcode);
7826   %}
7827 
7828   ins_pipe(icond_reg);
7829 %}
7830 
7831 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
7832   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
7833 
7834   ins_cost(INSN_COST * 2);
7835   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
7836 
7837   ins_encode %{
7838     __ cselw(as_Register($dst$$reg),
7839              zr,
7840              as_Register($src$$reg),
7841              (Assembler::Condition)$cmp$$cmpcode);
7842   %}
7843 
7844   ins_pipe(icond_reg);
7845 %}
7846 
7847 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
7848   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
7849 
7850   ins_cost(INSN_COST * 2);
7851   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
7852 
7853   ins_encode %{
7854     __ cselw(as_Register($dst$$reg),
7855              zr,
7856              as_Register($src$$reg),
7857              (Assembler::Condition)$cmp$$cmpcode);
7858   %}
7859 
7860   ins_pipe(icond_reg);
7861 %}
7862 
7863 // special case for creating a boolean 0 or 1
7864 
7865 // n.b. this is selected in preference to the rule above because it
7866 // avoids loading constants 0 and 1 into a source register
7867 
7868 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
7869   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
7870 
7871   ins_cost(INSN_COST * 2);
7872   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
7873 
7874   ins_encode %{
7875     // equivalently
7876     // cset(as_Register($dst$$reg),
7877     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
7878     __ csincw(as_Register($dst$$reg),
7879              zr,
7880              zr,
7881              (Assembler::Condition)$cmp$$cmpcode);
7882   %}
7883 
7884   ins_pipe(icond_none);
7885 %}
7886 
7887 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
7888   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
7889 
7890   ins_cost(INSN_COST * 2);
7891   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
7892 
7893   ins_encode %{
7894     // equivalently
7895     // cset(as_Register($dst$$reg),
7896     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
7897     __ csincw(as_Register($dst$$reg),
7898              zr,
7899              zr,
7900              (Assembler::Condition)$cmp$$cmpcode);
7901   %}
7902 
7903   ins_pipe(icond_none);
7904 %}
7905 
7906 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
7907   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
7908 
7909   ins_cost(INSN_COST * 2);
7910   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
7911 
7912   ins_encode %{
7913     __ csel(as_Register($dst$$reg),
7914             as_Register($src2$$reg),
7915             as_Register($src1$$reg),
7916             (Assembler::Condition)$cmp$$cmpcode);
7917   %}
7918 
7919   ins_pipe(icond_reg_reg);
7920 %}
7921 
7922 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
7923   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
7924 
7925   ins_cost(INSN_COST * 2);
7926   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
7927 
7928   ins_encode %{
7929     __ csel(as_Register($dst$$reg),
7930             as_Register($src2$$reg),
7931             as_Register($src1$$reg),
7932             (Assembler::Condition)$cmp$$cmpcode);
7933   %}
7934 
7935   ins_pipe(icond_reg_reg);
7936 %}
7937 
7938 // special cases where one arg is zero
7939 
7940 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
7941   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
7942 
7943   ins_cost(INSN_COST * 2);
7944   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
7945 
7946   ins_encode %{
7947     __ csel(as_Register($dst$$reg),
7948             zr,
7949             as_Register($src$$reg),
7950             (Assembler::Condition)$cmp$$cmpcode);
7951   %}
7952 
7953   ins_pipe(icond_reg);
7954 %}
7955 
7956 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
7957   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
7958 
7959   ins_cost(INSN_COST * 2);
7960   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
7961 
7962   ins_encode %{
7963     __ csel(as_Register($dst$$reg),
7964             zr,
7965             as_Register($src$$reg),
7966             (Assembler::Condition)$cmp$$cmpcode);
7967   %}
7968 
7969   ins_pipe(icond_reg);
7970 %}
7971 
7972 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
7973   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
7974 
7975   ins_cost(INSN_COST * 2);
7976   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
7977 
7978   ins_encode %{
7979     __ csel(as_Register($dst$$reg),
7980             as_Register($src$$reg),
7981             zr,
7982             (Assembler::Condition)$cmp$$cmpcode);
7983   %}
7984 
7985   ins_pipe(icond_reg);
7986 %}
7987 
7988 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
7989   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
7990 
7991   ins_cost(INSN_COST * 2);
7992   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
7993 
7994   ins_encode %{
7995     __ csel(as_Register($dst$$reg),
7996             as_Register($src$$reg),
7997             zr,
7998             (Assembler::Condition)$cmp$$cmpcode);
7999   %}
8000 
8001   ins_pipe(icond_reg);
8002 %}
8003 
8004 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8005   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8006 
8007   ins_cost(INSN_COST * 2);
8008   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
8009 
8010   ins_encode %{
8011     __ csel(as_Register($dst$$reg),
8012             as_Register($src2$$reg),
8013             as_Register($src1$$reg),
8014             (Assembler::Condition)$cmp$$cmpcode);
8015   %}
8016 
8017   ins_pipe(icond_reg_reg);
8018 %}
8019 
8020 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8021   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8022 
8023   ins_cost(INSN_COST * 2);
8024   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
8025 
8026   ins_encode %{
8027     __ csel(as_Register($dst$$reg),
8028             as_Register($src2$$reg),
8029             as_Register($src1$$reg),
8030             (Assembler::Condition)$cmp$$cmpcode);
8031   %}
8032 
8033   ins_pipe(icond_reg_reg);
8034 %}
8035 
8036 // special cases where one arg is zero
8037 
8038 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8039   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8040 
8041   ins_cost(INSN_COST * 2);
8042   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
8043 
8044   ins_encode %{
8045     __ csel(as_Register($dst$$reg),
8046             zr,
8047             as_Register($src$$reg),
8048             (Assembler::Condition)$cmp$$cmpcode);
8049   %}
8050 
8051   ins_pipe(icond_reg);
8052 %}
8053 
8054 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8055   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8056 
8057   ins_cost(INSN_COST * 2);
8058   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
8059 
8060   ins_encode %{
8061     __ csel(as_Register($dst$$reg),
8062             zr,
8063             as_Register($src$$reg),
8064             (Assembler::Condition)$cmp$$cmpcode);
8065   %}
8066 
8067   ins_pipe(icond_reg);
8068 %}
8069 
8070 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8071   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8072 
8073   ins_cost(INSN_COST * 2);
8074   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
8075 
8076   ins_encode %{
8077     __ csel(as_Register($dst$$reg),
8078             as_Register($src$$reg),
8079             zr,
8080             (Assembler::Condition)$cmp$$cmpcode);
8081   %}
8082 
8083   ins_pipe(icond_reg);
8084 %}
8085 
8086 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8087   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8088 
8089   ins_cost(INSN_COST * 2);
8090   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
8091 
8092   ins_encode %{
8093     __ csel(as_Register($dst$$reg),
8094             as_Register($src$$reg),
8095             zr,
8096             (Assembler::Condition)$cmp$$cmpcode);
8097   %}
8098 
8099   ins_pipe(icond_reg);
8100 %}
8101 
8102 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8103   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8104 
8105   ins_cost(INSN_COST * 2);
8106   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
8107 
8108   ins_encode %{
8109     __ cselw(as_Register($dst$$reg),
8110              as_Register($src2$$reg),
8111              as_Register($src1$$reg),
8112              (Assembler::Condition)$cmp$$cmpcode);
8113   %}
8114 
8115   ins_pipe(icond_reg_reg);
8116 %}
8117 
8118 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8119   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8120 
8121   ins_cost(INSN_COST * 2);
8122   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
8123 
8124   ins_encode %{
8125     __ cselw(as_Register($dst$$reg),
8126              as_Register($src2$$reg),
8127              as_Register($src1$$reg),
8128              (Assembler::Condition)$cmp$$cmpcode);
8129   %}
8130 
8131   ins_pipe(icond_reg_reg);
8132 %}
8133 
8134 // special cases where one arg is zero
8135 
8136 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8137   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8138 
8139   ins_cost(INSN_COST * 2);
8140   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
8141 
8142   ins_encode %{
8143     __ cselw(as_Register($dst$$reg),
8144              zr,
8145              as_Register($src$$reg),
8146              (Assembler::Condition)$cmp$$cmpcode);
8147   %}
8148 
8149   ins_pipe(icond_reg);
8150 %}
8151 
8152 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8153   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8154 
8155   ins_cost(INSN_COST * 2);
8156   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
8157 
8158   ins_encode %{
8159     __ cselw(as_Register($dst$$reg),
8160              zr,
8161              as_Register($src$$reg),
8162              (Assembler::Condition)$cmp$$cmpcode);
8163   %}
8164 
8165   ins_pipe(icond_reg);
8166 %}
8167 
8168 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8169   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8170 
8171   ins_cost(INSN_COST * 2);
8172   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
8173 
8174   ins_encode %{
8175     __ cselw(as_Register($dst$$reg),
8176              as_Register($src$$reg),
8177              zr,
8178              (Assembler::Condition)$cmp$$cmpcode);
8179   %}
8180 
8181   ins_pipe(icond_reg);
8182 %}
8183 
8184 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8185   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8186 
8187   ins_cost(INSN_COST * 2);
8188   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
8189 
8190   ins_encode %{
8191     __ cselw(as_Register($dst$$reg),
8192              as_Register($src$$reg),
8193              zr,
8194              (Assembler::Condition)$cmp$$cmpcode);
8195   %}
8196 
8197   ins_pipe(icond_reg);
8198 %}
8199 
8200 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
8201 %{
8202   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8203 
8204   ins_cost(INSN_COST * 3);
8205 
8206   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8207   ins_encode %{
8208     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8209     __ fcsels(as_FloatRegister($dst$$reg),
8210               as_FloatRegister($src2$$reg),
8211               as_FloatRegister($src1$$reg),
8212               cond);
8213   %}
8214 
8215   ins_pipe(pipe_class_default);
8216 %}
8217 
8218 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
8219 %{
8220   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8221 
8222   ins_cost(INSN_COST * 3);
8223 
8224   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8225   ins_encode %{
8226     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8227     __ fcsels(as_FloatRegister($dst$$reg),
8228               as_FloatRegister($src2$$reg),
8229               as_FloatRegister($src1$$reg),
8230               cond);
8231   %}
8232 
8233   ins_pipe(pipe_class_default);
8234 %}
8235 
8236 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
8237 %{
8238   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8239 
8240   ins_cost(INSN_COST * 3);
8241 
8242   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8243   ins_encode %{
8244     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8245     __ fcseld(as_FloatRegister($dst$$reg),
8246               as_FloatRegister($src2$$reg),
8247               as_FloatRegister($src1$$reg),
8248               cond);
8249   %}
8250 
8251   ins_pipe(pipe_class_default);
8252 %}
8253 
8254 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
8255 %{
8256   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8257 
8258   ins_cost(INSN_COST * 3);
8259 
8260   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8261   ins_encode %{
8262     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8263     __ fcseld(as_FloatRegister($dst$$reg),
8264               as_FloatRegister($src2$$reg),
8265               as_FloatRegister($src1$$reg),
8266               cond);
8267   %}
8268 
8269   ins_pipe(pipe_class_default);
8270 %}
8271 
8272 // ============================================================================
8273 // Arithmetic Instructions
8274 //
8275 
8276 // Integer Addition
8277 
8278 // TODO
8279 // these currently employ operations which do not set CR and hence are
8280 // not flagged as killing CR but we would like to isolate the cases
8281 // where we want to set flags from those where we don't. need to work
8282 // out how to do that.
8283 
8284 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8285   match(Set dst (AddI src1 src2));
8286 
8287   ins_cost(INSN_COST);
8288   format %{ "addw  $dst, $src1, $src2" %}
8289 
8290   ins_encode %{
8291     __ addw(as_Register($dst$$reg),
8292             as_Register($src1$$reg),
8293             as_Register($src2$$reg));
8294   %}
8295 
8296   ins_pipe(ialu_reg_reg);
8297 %}
8298 
8299 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
8300   match(Set dst (AddI src1 src2));
8301 
8302   ins_cost(INSN_COST);
8303   format %{ "addw $dst, $src1, $src2" %}
8304 
8305   // use opcode to indicate that this is an add not a sub
8306   opcode(0x0);
8307 
8308   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
8309 
8310   ins_pipe(ialu_reg_imm);
8311 %}
8312 
8313 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
8314   match(Set dst (AddI (ConvL2I src1) src2));
8315 
8316   ins_cost(INSN_COST);
8317   format %{ "addw $dst, $src1, $src2" %}
8318 
8319   // use opcode to indicate that this is an add not a sub
8320   opcode(0x0);
8321 
8322   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
8323 
8324   ins_pipe(ialu_reg_imm);
8325 %}
8326 
8327 // Pointer Addition
8328 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
8329   match(Set dst (AddP src1 src2));
8330 
8331   ins_cost(INSN_COST);
8332   format %{ "add $dst, $src1, $src2\t# ptr" %}
8333 
8334   ins_encode %{
8335     __ add(as_Register($dst$$reg),
8336            as_Register($src1$$reg),
8337            as_Register($src2$$reg));
8338   %}
8339 
8340   ins_pipe(ialu_reg_reg);
8341 %}
8342 
8343 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
8344   match(Set dst (AddP src1 (ConvI2L src2)));
8345 
8346   ins_cost(INSN_COST);
8347   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
8348 
8349   ins_encode %{
8350     __ add(as_Register($dst$$reg),
8351            as_Register($src1$$reg),
8352            as_Register($src2$$reg), ext::sxtw);
8353   %}
8354 
8355   ins_pipe(ialu_reg_reg);
8356 %}
8357 
8358 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
8359   match(Set dst (AddP src1 (LShiftL src2 scale)));
8360 
8361   ins_cost(1.9 * INSN_COST);
8362   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
8363 
8364   ins_encode %{
8365     __ lea(as_Register($dst$$reg),
8366            Address(as_Register($src1$$reg), as_Register($src2$$reg),
8367                    Address::lsl($scale$$constant)));
8368   %}
8369 
8370   ins_pipe(ialu_reg_reg_shift);
8371 %}
8372 
8373 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
8374   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
8375 
8376   ins_cost(1.9 * INSN_COST);
8377   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
8378 
8379   ins_encode %{
8380     __ lea(as_Register($dst$$reg),
8381            Address(as_Register($src1$$reg), as_Register($src2$$reg),
8382                    Address::sxtw($scale$$constant)));
8383   %}
8384 
8385   ins_pipe(ialu_reg_reg_shift);
8386 %}
8387 
8388 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
8389   match(Set dst (LShiftL (ConvI2L src) scale));
8390 
8391   ins_cost(INSN_COST);
8392   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
8393 
8394   ins_encode %{
8395     __ sbfiz(as_Register($dst$$reg),
8396           as_Register($src$$reg),
8397           $scale$$constant & 63, MIN(32, (-$scale$$constant) & 63));
8398   %}
8399 
8400   ins_pipe(ialu_reg_shift);
8401 %}
8402 
8403 // Pointer Immediate Addition
8404 // n.b. this needs to be more expensive than using an indirect memory
8405 // operand
8406 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
8407   match(Set dst (AddP src1 src2));
8408 
8409   ins_cost(INSN_COST);
8410   format %{ "add $dst, $src1, $src2\t# ptr" %}
8411 
8412   // use opcode to indicate that this is an add not a sub
8413   opcode(0x0);
8414 
8415   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
8416 
8417   ins_pipe(ialu_reg_imm);
8418 %}
8419 
8420 // Long Addition
8421 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
8422 
8423   match(Set dst (AddL src1 src2));
8424 
8425   ins_cost(INSN_COST);
8426   format %{ "add  $dst, $src1, $src2" %}
8427 
8428   ins_encode %{
8429     __ add(as_Register($dst$$reg),
8430            as_Register($src1$$reg),
8431            as_Register($src2$$reg));
8432   %}
8433 
8434   ins_pipe(ialu_reg_reg);
8435 %}
8436 
8437 // No constant pool entries requiredLong Immediate Addition.
8438 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
8439   match(Set dst (AddL src1 src2));
8440 
8441   ins_cost(INSN_COST);
8442   format %{ "add $dst, $src1, $src2" %}
8443 
8444   // use opcode to indicate that this is an add not a sub
8445   opcode(0x0);
8446 
8447   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
8448 
8449   ins_pipe(ialu_reg_imm);
8450 %}
8451 
8452 // Integer Subtraction
8453 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8454   match(Set dst (SubI src1 src2));
8455 
8456   ins_cost(INSN_COST);
8457   format %{ "subw  $dst, $src1, $src2" %}
8458 
8459   ins_encode %{
8460     __ subw(as_Register($dst$$reg),
8461             as_Register($src1$$reg),
8462             as_Register($src2$$reg));
8463   %}
8464 
8465   ins_pipe(ialu_reg_reg);
8466 %}
8467 
8468 // Immediate Subtraction
8469 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
8470   match(Set dst (SubI src1 src2));
8471 
8472   ins_cost(INSN_COST);
8473   format %{ "subw $dst, $src1, $src2" %}
8474 
8475   // use opcode to indicate that this is a sub not an add
8476   opcode(0x1);
8477 
8478   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
8479 
8480   ins_pipe(ialu_reg_imm);
8481 %}
8482 
8483 // Long Subtraction
8484 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
8485 
8486   match(Set dst (SubL src1 src2));
8487 
8488   ins_cost(INSN_COST);
8489   format %{ "sub  $dst, $src1, $src2" %}
8490 
8491   ins_encode %{
8492     __ sub(as_Register($dst$$reg),
8493            as_Register($src1$$reg),
8494            as_Register($src2$$reg));
8495   %}
8496 
8497   ins_pipe(ialu_reg_reg);
8498 %}
8499 
8500 // No constant pool entries requiredLong Immediate Subtraction.
8501 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
8502   match(Set dst (SubL src1 src2));
8503 
8504   ins_cost(INSN_COST);
8505   format %{ "sub$dst, $src1, $src2" %}
8506 
8507   // use opcode to indicate that this is a sub not an add
8508   opcode(0x1);
8509 
8510   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
8511 
8512   ins_pipe(ialu_reg_imm);
8513 %}
8514 
8515 // Integer Negation (special case for sub)
8516 
8517 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
8518   match(Set dst (SubI zero src));
8519 
8520   ins_cost(INSN_COST);
8521   format %{ "negw $dst, $src\t# int" %}
8522 
8523   ins_encode %{
8524     __ negw(as_Register($dst$$reg),
8525             as_Register($src$$reg));
8526   %}
8527 
8528   ins_pipe(ialu_reg);
8529 %}
8530 
8531 // Long Negation
8532 
8533 instruct negL_reg(iRegLNoSp dst, iRegIorL2I src, immL0 zero, rFlagsReg cr) %{
8534   match(Set dst (SubL zero src));
8535 
8536   ins_cost(INSN_COST);
8537   format %{ "neg $dst, $src\t# long" %}
8538 
8539   ins_encode %{
8540     __ neg(as_Register($dst$$reg),
8541            as_Register($src$$reg));
8542   %}
8543 
8544   ins_pipe(ialu_reg);
8545 %}
8546 
8547 // Integer Multiply
8548 
8549 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8550   match(Set dst (MulI src1 src2));
8551 
8552   ins_cost(INSN_COST * 3);
8553   format %{ "mulw  $dst, $src1, $src2" %}
8554 
8555   ins_encode %{
8556     __ mulw(as_Register($dst$$reg),
8557             as_Register($src1$$reg),
8558             as_Register($src2$$reg));
8559   %}
8560 
8561   ins_pipe(imul_reg_reg);
8562 %}
8563 
8564 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8565   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
8566 
8567   ins_cost(INSN_COST * 3);
8568   format %{ "smull  $dst, $src1, $src2" %}
8569 
8570   ins_encode %{
8571     __ smull(as_Register($dst$$reg),
8572              as_Register($src1$$reg),
8573              as_Register($src2$$reg));
8574   %}
8575 
8576   ins_pipe(imul_reg_reg);
8577 %}
8578 
8579 // Long Multiply
8580 
8581 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
8582   match(Set dst (MulL src1 src2));
8583 
8584   ins_cost(INSN_COST * 5);
8585   format %{ "mul  $dst, $src1, $src2" %}
8586 
8587   ins_encode %{
8588     __ mul(as_Register($dst$$reg),
8589            as_Register($src1$$reg),
8590            as_Register($src2$$reg));
8591   %}
8592 
8593   ins_pipe(lmul_reg_reg);
8594 %}
8595 
8596 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
8597 %{
8598   match(Set dst (MulHiL src1 src2));
8599 
8600   ins_cost(INSN_COST * 7);
8601   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
8602 
8603   ins_encode %{
8604     __ smulh(as_Register($dst$$reg),
8605              as_Register($src1$$reg),
8606              as_Register($src2$$reg));
8607   %}
8608 
8609   ins_pipe(lmul_reg_reg);
8610 %}
8611 
8612 // Combined Integer Multiply & Add/Sub
8613 
8614 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
8615   match(Set dst (AddI src3 (MulI src1 src2)));
8616 
8617   ins_cost(INSN_COST * 3);
8618   format %{ "madd  $dst, $src1, $src2, $src3" %}
8619 
8620   ins_encode %{
8621     __ maddw(as_Register($dst$$reg),
8622              as_Register($src1$$reg),
8623              as_Register($src2$$reg),
8624              as_Register($src3$$reg));
8625   %}
8626 
8627   ins_pipe(imac_reg_reg);
8628 %}
8629 
8630 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
8631   match(Set dst (SubI src3 (MulI src1 src2)));
8632 
8633   ins_cost(INSN_COST * 3);
8634   format %{ "msub  $dst, $src1, $src2, $src3" %}
8635 
8636   ins_encode %{
8637     __ msubw(as_Register($dst$$reg),
8638              as_Register($src1$$reg),
8639              as_Register($src2$$reg),
8640              as_Register($src3$$reg));
8641   %}
8642 
8643   ins_pipe(imac_reg_reg);
8644 %}
8645 
8646 // Combined Long Multiply & Add/Sub
8647 
8648 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
8649   match(Set dst (AddL src3 (MulL src1 src2)));
8650 
8651   ins_cost(INSN_COST * 5);
8652   format %{ "madd  $dst, $src1, $src2, $src3" %}
8653 
8654   ins_encode %{
8655     __ madd(as_Register($dst$$reg),
8656             as_Register($src1$$reg),
8657             as_Register($src2$$reg),
8658             as_Register($src3$$reg));
8659   %}
8660 
8661   ins_pipe(lmac_reg_reg);
8662 %}
8663 
8664 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
8665   match(Set dst (SubL src3 (MulL src1 src2)));
8666 
8667   ins_cost(INSN_COST * 5);
8668   format %{ "msub  $dst, $src1, $src2, $src3" %}
8669 
8670   ins_encode %{
8671     __ msub(as_Register($dst$$reg),
8672             as_Register($src1$$reg),
8673             as_Register($src2$$reg),
8674             as_Register($src3$$reg));
8675   %}
8676 
8677   ins_pipe(lmac_reg_reg);
8678 %}
8679 
8680 // Integer Divide
8681 
8682 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8683   match(Set dst (DivI src1 src2));
8684 
8685   ins_cost(INSN_COST * 19);
8686   format %{ "sdivw  $dst, $src1, $src2" %}
8687 
8688   ins_encode(aarch64_enc_divw(dst, src1, src2));
8689   ins_pipe(idiv_reg_reg);
8690 %}
8691 
8692 instruct signExtract(iRegINoSp dst, iRegIorL2I src1, immI_31 div1, immI_31 div2) %{
8693   match(Set dst (URShiftI (RShiftI src1 div1) div2));
8694   ins_cost(INSN_COST);
8695   format %{ "lsrw $dst, $src1, $div1" %}
8696   ins_encode %{
8697     __ lsrw(as_Register($dst$$reg), as_Register($src1$$reg), 31);
8698   %}
8699   ins_pipe(ialu_reg_shift);
8700 %}
8701 
8702 instruct div2Round(iRegINoSp dst, iRegIorL2I src, immI_31 div1, immI_31 div2) %{
8703   match(Set dst (AddI src (URShiftI (RShiftI src div1) div2)));
8704   ins_cost(INSN_COST);
8705   format %{ "addw $dst, $src, LSR $div1" %}
8706 
8707   ins_encode %{
8708     __ addw(as_Register($dst$$reg),
8709               as_Register($src$$reg),
8710               as_Register($src$$reg),
8711               Assembler::LSR, 31);
8712   %}
8713   ins_pipe(ialu_reg);
8714 %}
8715 
8716 // Long Divide
8717 
8718 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
8719   match(Set dst (DivL src1 src2));
8720 
8721   ins_cost(INSN_COST * 35);
8722   format %{ "sdiv   $dst, $src1, $src2" %}
8723 
8724   ins_encode(aarch64_enc_div(dst, src1, src2));
8725   ins_pipe(ldiv_reg_reg);
8726 %}
8727 
8728 instruct signExtractL(iRegLNoSp dst, iRegL src1, immL_63 div1, immL_63 div2) %{
8729   match(Set dst (URShiftL (RShiftL src1 div1) div2));
8730   ins_cost(INSN_COST);
8731   format %{ "lsr $dst, $src1, $div1" %}
8732   ins_encode %{
8733     __ lsr(as_Register($dst$$reg), as_Register($src1$$reg), 63);
8734   %}
8735   ins_pipe(ialu_reg_shift);
8736 %}
8737 
8738 instruct div2RoundL(iRegLNoSp dst, iRegL src, immL_63 div1, immL_63 div2) %{
8739   match(Set dst (AddL src (URShiftL (RShiftL src div1) div2)));
8740   ins_cost(INSN_COST);
8741   format %{ "add $dst, $src, $div1" %}
8742 
8743   ins_encode %{
8744     __ add(as_Register($dst$$reg),
8745               as_Register($src$$reg),
8746               as_Register($src$$reg),
8747               Assembler::LSR, 63);
8748   %}
8749   ins_pipe(ialu_reg);
8750 %}
8751 
8752 // Integer Remainder
8753 
8754 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8755   match(Set dst (ModI src1 src2));
8756 
8757   ins_cost(INSN_COST * 22);
8758   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
8759             "msubw($dst, rscratch1, $src2, $src1" %}
8760 
8761   ins_encode(aarch64_enc_modw(dst, src1, src2));
8762   ins_pipe(idiv_reg_reg);
8763 %}
8764 
8765 // Long Remainder
8766 
8767 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
8768   match(Set dst (ModL src1 src2));
8769 
8770   ins_cost(INSN_COST * 38);
8771   format %{ "sdiv   rscratch1, $src1, $src2\n"
8772             "msub($dst, rscratch1, $src2, $src1" %}
8773 
8774   ins_encode(aarch64_enc_mod(dst, src1, src2));
8775   ins_pipe(ldiv_reg_reg);
8776 %}
8777 
8778 // Integer Shifts
8779 
8780 // Shift Left Register
8781 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8782   match(Set dst (LShiftI src1 src2));
8783 
8784   ins_cost(INSN_COST * 2);
8785   format %{ "lslvw  $dst, $src1, $src2" %}
8786 
8787   ins_encode %{
8788     __ lslvw(as_Register($dst$$reg),
8789              as_Register($src1$$reg),
8790              as_Register($src2$$reg));
8791   %}
8792 
8793   ins_pipe(ialu_reg_reg_vshift);
8794 %}
8795 
8796 // Shift Left Immediate
8797 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
8798   match(Set dst (LShiftI src1 src2));
8799 
8800   ins_cost(INSN_COST);
8801   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
8802 
8803   ins_encode %{
8804     __ lslw(as_Register($dst$$reg),
8805             as_Register($src1$$reg),
8806             $src2$$constant & 0x1f);
8807   %}
8808 
8809   ins_pipe(ialu_reg_shift);
8810 %}
8811 
8812 // Shift Right Logical Register
8813 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8814   match(Set dst (URShiftI src1 src2));
8815 
8816   ins_cost(INSN_COST * 2);
8817   format %{ "lsrvw  $dst, $src1, $src2" %}
8818 
8819   ins_encode %{
8820     __ lsrvw(as_Register($dst$$reg),
8821              as_Register($src1$$reg),
8822              as_Register($src2$$reg));
8823   %}
8824 
8825   ins_pipe(ialu_reg_reg_vshift);
8826 %}
8827 
8828 // Shift Right Logical Immediate
8829 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
8830   match(Set dst (URShiftI src1 src2));
8831 
8832   ins_cost(INSN_COST);
8833   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
8834 
8835   ins_encode %{
8836     __ lsrw(as_Register($dst$$reg),
8837             as_Register($src1$$reg),
8838             $src2$$constant & 0x1f);
8839   %}
8840 
8841   ins_pipe(ialu_reg_shift);
8842 %}
8843 
8844 // Shift Right Arithmetic Register
8845 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8846   match(Set dst (RShiftI src1 src2));
8847 
8848   ins_cost(INSN_COST * 2);
8849   format %{ "asrvw  $dst, $src1, $src2" %}
8850 
8851   ins_encode %{
8852     __ asrvw(as_Register($dst$$reg),
8853              as_Register($src1$$reg),
8854              as_Register($src2$$reg));
8855   %}
8856 
8857   ins_pipe(ialu_reg_reg_vshift);
8858 %}
8859 
8860 // Shift Right Arithmetic Immediate
8861 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
8862   match(Set dst (RShiftI src1 src2));
8863 
8864   ins_cost(INSN_COST);
8865   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
8866 
8867   ins_encode %{
8868     __ asrw(as_Register($dst$$reg),
8869             as_Register($src1$$reg),
8870             $src2$$constant & 0x1f);
8871   %}
8872 
8873   ins_pipe(ialu_reg_shift);
8874 %}
8875 
8876 // Combined Int Mask and Right Shift (using UBFM)
8877 // TODO
8878 
8879 // Long Shifts
8880 
8881 // Shift Left Register
8882 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
8883   match(Set dst (LShiftL src1 src2));
8884 
8885   ins_cost(INSN_COST * 2);
8886   format %{ "lslv  $dst, $src1, $src2" %}
8887 
8888   ins_encode %{
8889     __ lslv(as_Register($dst$$reg),
8890             as_Register($src1$$reg),
8891             as_Register($src2$$reg));
8892   %}
8893 
8894   ins_pipe(ialu_reg_reg_vshift);
8895 %}
8896 
8897 // Shift Left Immediate
8898 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
8899   match(Set dst (LShiftL src1 src2));
8900 
8901   ins_cost(INSN_COST);
8902   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
8903 
8904   ins_encode %{
8905     __ lsl(as_Register($dst$$reg),
8906             as_Register($src1$$reg),
8907             $src2$$constant & 0x3f);
8908   %}
8909 
8910   ins_pipe(ialu_reg_shift);
8911 %}
8912 
8913 // Shift Right Logical Register
8914 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
8915   match(Set dst (URShiftL src1 src2));
8916 
8917   ins_cost(INSN_COST * 2);
8918   format %{ "lsrv  $dst, $src1, $src2" %}
8919 
8920   ins_encode %{
8921     __ lsrv(as_Register($dst$$reg),
8922             as_Register($src1$$reg),
8923             as_Register($src2$$reg));
8924   %}
8925 
8926   ins_pipe(ialu_reg_reg_vshift);
8927 %}
8928 
8929 // Shift Right Logical Immediate
8930 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
8931   match(Set dst (URShiftL src1 src2));
8932 
8933   ins_cost(INSN_COST);
8934   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
8935 
8936   ins_encode %{
8937     __ lsr(as_Register($dst$$reg),
8938            as_Register($src1$$reg),
8939            $src2$$constant & 0x3f);
8940   %}
8941 
8942   ins_pipe(ialu_reg_shift);
8943 %}
8944 
8945 // A special-case pattern for card table stores.
8946 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
8947   match(Set dst (URShiftL (CastP2X src1) src2));
8948 
8949   ins_cost(INSN_COST);
8950   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
8951 
8952   ins_encode %{
8953     __ lsr(as_Register($dst$$reg),
8954            as_Register($src1$$reg),
8955            $src2$$constant & 0x3f);
8956   %}
8957 
8958   ins_pipe(ialu_reg_shift);
8959 %}
8960 
8961 // Shift Right Arithmetic Register
8962 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
8963   match(Set dst (RShiftL src1 src2));
8964 
8965   ins_cost(INSN_COST * 2);
8966   format %{ "asrv  $dst, $src1, $src2" %}
8967 
8968   ins_encode %{
8969     __ asrv(as_Register($dst$$reg),
8970             as_Register($src1$$reg),
8971             as_Register($src2$$reg));
8972   %}
8973 
8974   ins_pipe(ialu_reg_reg_vshift);
8975 %}
8976 
8977 // Shift Right Arithmetic Immediate
8978 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
8979   match(Set dst (RShiftL src1 src2));
8980 
8981   ins_cost(INSN_COST);
8982   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
8983 
8984   ins_encode %{
8985     __ asr(as_Register($dst$$reg),
8986            as_Register($src1$$reg),
8987            $src2$$constant & 0x3f);
8988   %}
8989 
8990   ins_pipe(ialu_reg_shift);
8991 %}
8992 
8993 // BEGIN This section of the file is automatically generated. Do not edit --------------
8994 
8995 instruct regL_not_reg(iRegLNoSp dst,
8996                          iRegL src1, immL_M1 m1,
8997                          rFlagsReg cr) %{
8998   match(Set dst (XorL src1 m1));
8999   ins_cost(INSN_COST);
9000   format %{ "eon  $dst, $src1, zr" %}
9001 
9002   ins_encode %{
9003     __ eon(as_Register($dst$$reg),
9004               as_Register($src1$$reg),
9005               zr,
9006               Assembler::LSL, 0);
9007   %}
9008 
9009   ins_pipe(ialu_reg);
9010 %}
9011 instruct regI_not_reg(iRegINoSp dst,
9012                          iRegIorL2I src1, immI_M1 m1,
9013                          rFlagsReg cr) %{
9014   match(Set dst (XorI src1 m1));
9015   ins_cost(INSN_COST);
9016   format %{ "eonw  $dst, $src1, zr" %}
9017 
9018   ins_encode %{
9019     __ eonw(as_Register($dst$$reg),
9020               as_Register($src1$$reg),
9021               zr,
9022               Assembler::LSL, 0);
9023   %}
9024 
9025   ins_pipe(ialu_reg);
9026 %}
9027 
9028 instruct AndI_reg_not_reg(iRegINoSp dst,
9029                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
9030                          rFlagsReg cr) %{
9031   match(Set dst (AndI src1 (XorI src2 m1)));
9032   ins_cost(INSN_COST);
9033   format %{ "bicw  $dst, $src1, $src2" %}
9034 
9035   ins_encode %{
9036     __ bicw(as_Register($dst$$reg),
9037               as_Register($src1$$reg),
9038               as_Register($src2$$reg),
9039               Assembler::LSL, 0);
9040   %}
9041 
9042   ins_pipe(ialu_reg_reg);
9043 %}
9044 
9045 instruct AndL_reg_not_reg(iRegLNoSp dst,
9046                          iRegL src1, iRegL src2, immL_M1 m1,
9047                          rFlagsReg cr) %{
9048   match(Set dst (AndL src1 (XorL src2 m1)));
9049   ins_cost(INSN_COST);
9050   format %{ "bic  $dst, $src1, $src2" %}
9051 
9052   ins_encode %{
9053     __ bic(as_Register($dst$$reg),
9054               as_Register($src1$$reg),
9055               as_Register($src2$$reg),
9056               Assembler::LSL, 0);
9057   %}
9058 
9059   ins_pipe(ialu_reg_reg);
9060 %}
9061 
9062 instruct OrI_reg_not_reg(iRegINoSp dst,
9063                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
9064                          rFlagsReg cr) %{
9065   match(Set dst (OrI src1 (XorI src2 m1)));
9066   ins_cost(INSN_COST);
9067   format %{ "ornw  $dst, $src1, $src2" %}
9068 
9069   ins_encode %{
9070     __ ornw(as_Register($dst$$reg),
9071               as_Register($src1$$reg),
9072               as_Register($src2$$reg),
9073               Assembler::LSL, 0);
9074   %}
9075 
9076   ins_pipe(ialu_reg_reg);
9077 %}
9078 
9079 instruct OrL_reg_not_reg(iRegLNoSp dst,
9080                          iRegL src1, iRegL src2, immL_M1 m1,
9081                          rFlagsReg cr) %{
9082   match(Set dst (OrL src1 (XorL src2 m1)));
9083   ins_cost(INSN_COST);
9084   format %{ "orn  $dst, $src1, $src2" %}
9085 
9086   ins_encode %{
9087     __ orn(as_Register($dst$$reg),
9088               as_Register($src1$$reg),
9089               as_Register($src2$$reg),
9090               Assembler::LSL, 0);
9091   %}
9092 
9093   ins_pipe(ialu_reg_reg);
9094 %}
9095 
9096 instruct XorI_reg_not_reg(iRegINoSp dst,
9097                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
9098                          rFlagsReg cr) %{
9099   match(Set dst (XorI m1 (XorI src2 src1)));
9100   ins_cost(INSN_COST);
9101   format %{ "eonw  $dst, $src1, $src2" %}
9102 
9103   ins_encode %{
9104     __ eonw(as_Register($dst$$reg),
9105               as_Register($src1$$reg),
9106               as_Register($src2$$reg),
9107               Assembler::LSL, 0);
9108   %}
9109 
9110   ins_pipe(ialu_reg_reg);
9111 %}
9112 
9113 instruct XorL_reg_not_reg(iRegLNoSp dst,
9114                          iRegL src1, iRegL src2, immL_M1 m1,
9115                          rFlagsReg cr) %{
9116   match(Set dst (XorL m1 (XorL src2 src1)));
9117   ins_cost(INSN_COST);
9118   format %{ "eon  $dst, $src1, $src2" %}
9119 
9120   ins_encode %{
9121     __ eon(as_Register($dst$$reg),
9122               as_Register($src1$$reg),
9123               as_Register($src2$$reg),
9124               Assembler::LSL, 0);
9125   %}
9126 
9127   ins_pipe(ialu_reg_reg);
9128 %}
9129 
9130 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
9131                          iRegIorL2I src1, iRegIorL2I src2,
9132                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9133   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
9134   ins_cost(1.9 * INSN_COST);
9135   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
9136 
9137   ins_encode %{
9138     __ bicw(as_Register($dst$$reg),
9139               as_Register($src1$$reg),
9140               as_Register($src2$$reg),
9141               Assembler::LSR,
9142               $src3$$constant & 0x3f);
9143   %}
9144 
9145   ins_pipe(ialu_reg_reg_shift);
9146 %}
9147 
9148 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
9149                          iRegL src1, iRegL src2,
9150                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9151   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
9152   ins_cost(1.9 * INSN_COST);
9153   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
9154 
9155   ins_encode %{
9156     __ bic(as_Register($dst$$reg),
9157               as_Register($src1$$reg),
9158               as_Register($src2$$reg),
9159               Assembler::LSR,
9160               $src3$$constant & 0x3f);
9161   %}
9162 
9163   ins_pipe(ialu_reg_reg_shift);
9164 %}
9165 
9166 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
9167                          iRegIorL2I src1, iRegIorL2I src2,
9168                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9169   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
9170   ins_cost(1.9 * INSN_COST);
9171   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
9172 
9173   ins_encode %{
9174     __ bicw(as_Register($dst$$reg),
9175               as_Register($src1$$reg),
9176               as_Register($src2$$reg),
9177               Assembler::ASR,
9178               $src3$$constant & 0x3f);
9179   %}
9180 
9181   ins_pipe(ialu_reg_reg_shift);
9182 %}
9183 
9184 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
9185                          iRegL src1, iRegL src2,
9186                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9187   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
9188   ins_cost(1.9 * INSN_COST);
9189   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
9190 
9191   ins_encode %{
9192     __ bic(as_Register($dst$$reg),
9193               as_Register($src1$$reg),
9194               as_Register($src2$$reg),
9195               Assembler::ASR,
9196               $src3$$constant & 0x3f);
9197   %}
9198 
9199   ins_pipe(ialu_reg_reg_shift);
9200 %}
9201 
9202 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
9203                          iRegIorL2I src1, iRegIorL2I src2,
9204                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9205   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
9206   ins_cost(1.9 * INSN_COST);
9207   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
9208 
9209   ins_encode %{
9210     __ bicw(as_Register($dst$$reg),
9211               as_Register($src1$$reg),
9212               as_Register($src2$$reg),
9213               Assembler::LSL,
9214               $src3$$constant & 0x3f);
9215   %}
9216 
9217   ins_pipe(ialu_reg_reg_shift);
9218 %}
9219 
9220 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
9221                          iRegL src1, iRegL src2,
9222                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9223   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
9224   ins_cost(1.9 * INSN_COST);
9225   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
9226 
9227   ins_encode %{
9228     __ bic(as_Register($dst$$reg),
9229               as_Register($src1$$reg),
9230               as_Register($src2$$reg),
9231               Assembler::LSL,
9232               $src3$$constant & 0x3f);
9233   %}
9234 
9235   ins_pipe(ialu_reg_reg_shift);
9236 %}
9237 
9238 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
9239                          iRegIorL2I src1, iRegIorL2I src2,
9240                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9241   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
9242   ins_cost(1.9 * INSN_COST);
9243   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
9244 
9245   ins_encode %{
9246     __ eonw(as_Register($dst$$reg),
9247               as_Register($src1$$reg),
9248               as_Register($src2$$reg),
9249               Assembler::LSR,
9250               $src3$$constant & 0x3f);
9251   %}
9252 
9253   ins_pipe(ialu_reg_reg_shift);
9254 %}
9255 
9256 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
9257                          iRegL src1, iRegL src2,
9258                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9259   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
9260   ins_cost(1.9 * INSN_COST);
9261   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
9262 
9263   ins_encode %{
9264     __ eon(as_Register($dst$$reg),
9265               as_Register($src1$$reg),
9266               as_Register($src2$$reg),
9267               Assembler::LSR,
9268               $src3$$constant & 0x3f);
9269   %}
9270 
9271   ins_pipe(ialu_reg_reg_shift);
9272 %}
9273 
9274 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
9275                          iRegIorL2I src1, iRegIorL2I src2,
9276                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9277   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
9278   ins_cost(1.9 * INSN_COST);
9279   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
9280 
9281   ins_encode %{
9282     __ eonw(as_Register($dst$$reg),
9283               as_Register($src1$$reg),
9284               as_Register($src2$$reg),
9285               Assembler::ASR,
9286               $src3$$constant & 0x3f);
9287   %}
9288 
9289   ins_pipe(ialu_reg_reg_shift);
9290 %}
9291 
9292 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
9293                          iRegL src1, iRegL src2,
9294                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9295   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
9296   ins_cost(1.9 * INSN_COST);
9297   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
9298 
9299   ins_encode %{
9300     __ eon(as_Register($dst$$reg),
9301               as_Register($src1$$reg),
9302               as_Register($src2$$reg),
9303               Assembler::ASR,
9304               $src3$$constant & 0x3f);
9305   %}
9306 
9307   ins_pipe(ialu_reg_reg_shift);
9308 %}
9309 
9310 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
9311                          iRegIorL2I src1, iRegIorL2I src2,
9312                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9313   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
9314   ins_cost(1.9 * INSN_COST);
9315   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
9316 
9317   ins_encode %{
9318     __ eonw(as_Register($dst$$reg),
9319               as_Register($src1$$reg),
9320               as_Register($src2$$reg),
9321               Assembler::LSL,
9322               $src3$$constant & 0x3f);
9323   %}
9324 
9325   ins_pipe(ialu_reg_reg_shift);
9326 %}
9327 
9328 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
9329                          iRegL src1, iRegL src2,
9330                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9331   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
9332   ins_cost(1.9 * INSN_COST);
9333   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
9334 
9335   ins_encode %{
9336     __ eon(as_Register($dst$$reg),
9337               as_Register($src1$$reg),
9338               as_Register($src2$$reg),
9339               Assembler::LSL,
9340               $src3$$constant & 0x3f);
9341   %}
9342 
9343   ins_pipe(ialu_reg_reg_shift);
9344 %}
9345 
9346 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
9347                          iRegIorL2I src1, iRegIorL2I src2,
9348                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9349   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
9350   ins_cost(1.9 * INSN_COST);
9351   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
9352 
9353   ins_encode %{
9354     __ ornw(as_Register($dst$$reg),
9355               as_Register($src1$$reg),
9356               as_Register($src2$$reg),
9357               Assembler::LSR,
9358               $src3$$constant & 0x3f);
9359   %}
9360 
9361   ins_pipe(ialu_reg_reg_shift);
9362 %}
9363 
9364 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
9365                          iRegL src1, iRegL src2,
9366                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9367   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
9368   ins_cost(1.9 * INSN_COST);
9369   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
9370 
9371   ins_encode %{
9372     __ orn(as_Register($dst$$reg),
9373               as_Register($src1$$reg),
9374               as_Register($src2$$reg),
9375               Assembler::LSR,
9376               $src3$$constant & 0x3f);
9377   %}
9378 
9379   ins_pipe(ialu_reg_reg_shift);
9380 %}
9381 
9382 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
9383                          iRegIorL2I src1, iRegIorL2I src2,
9384                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9385   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
9386   ins_cost(1.9 * INSN_COST);
9387   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
9388 
9389   ins_encode %{
9390     __ ornw(as_Register($dst$$reg),
9391               as_Register($src1$$reg),
9392               as_Register($src2$$reg),
9393               Assembler::ASR,
9394               $src3$$constant & 0x3f);
9395   %}
9396 
9397   ins_pipe(ialu_reg_reg_shift);
9398 %}
9399 
9400 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
9401                          iRegL src1, iRegL src2,
9402                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9403   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
9404   ins_cost(1.9 * INSN_COST);
9405   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
9406 
9407   ins_encode %{
9408     __ orn(as_Register($dst$$reg),
9409               as_Register($src1$$reg),
9410               as_Register($src2$$reg),
9411               Assembler::ASR,
9412               $src3$$constant & 0x3f);
9413   %}
9414 
9415   ins_pipe(ialu_reg_reg_shift);
9416 %}
9417 
9418 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
9419                          iRegIorL2I src1, iRegIorL2I src2,
9420                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9421   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
9422   ins_cost(1.9 * INSN_COST);
9423   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
9424 
9425   ins_encode %{
9426     __ ornw(as_Register($dst$$reg),
9427               as_Register($src1$$reg),
9428               as_Register($src2$$reg),
9429               Assembler::LSL,
9430               $src3$$constant & 0x3f);
9431   %}
9432 
9433   ins_pipe(ialu_reg_reg_shift);
9434 %}
9435 
9436 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
9437                          iRegL src1, iRegL src2,
9438                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9439   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
9440   ins_cost(1.9 * INSN_COST);
9441   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
9442 
9443   ins_encode %{
9444     __ orn(as_Register($dst$$reg),
9445               as_Register($src1$$reg),
9446               as_Register($src2$$reg),
9447               Assembler::LSL,
9448               $src3$$constant & 0x3f);
9449   %}
9450 
9451   ins_pipe(ialu_reg_reg_shift);
9452 %}
9453 
9454 instruct AndI_reg_URShift_reg(iRegINoSp dst,
9455                          iRegIorL2I src1, iRegIorL2I src2,
9456                          immI src3, rFlagsReg cr) %{
9457   match(Set dst (AndI src1 (URShiftI src2 src3)));
9458 
9459   ins_cost(1.9 * INSN_COST);
9460   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
9461 
9462   ins_encode %{
9463     __ andw(as_Register($dst$$reg),
9464               as_Register($src1$$reg),
9465               as_Register($src2$$reg),
9466               Assembler::LSR,
9467               $src3$$constant & 0x3f);
9468   %}
9469 
9470   ins_pipe(ialu_reg_reg_shift);
9471 %}
9472 
9473 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
9474                          iRegL src1, iRegL src2,
9475                          immI src3, rFlagsReg cr) %{
9476   match(Set dst (AndL src1 (URShiftL src2 src3)));
9477 
9478   ins_cost(1.9 * INSN_COST);
9479   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
9480 
9481   ins_encode %{
9482     __ andr(as_Register($dst$$reg),
9483               as_Register($src1$$reg),
9484               as_Register($src2$$reg),
9485               Assembler::LSR,
9486               $src3$$constant & 0x3f);
9487   %}
9488 
9489   ins_pipe(ialu_reg_reg_shift);
9490 %}
9491 
9492 instruct AndI_reg_RShift_reg(iRegINoSp dst,
9493                          iRegIorL2I src1, iRegIorL2I src2,
9494                          immI src3, rFlagsReg cr) %{
9495   match(Set dst (AndI src1 (RShiftI src2 src3)));
9496 
9497   ins_cost(1.9 * INSN_COST);
9498   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
9499 
9500   ins_encode %{
9501     __ andw(as_Register($dst$$reg),
9502               as_Register($src1$$reg),
9503               as_Register($src2$$reg),
9504               Assembler::ASR,
9505               $src3$$constant & 0x3f);
9506   %}
9507 
9508   ins_pipe(ialu_reg_reg_shift);
9509 %}
9510 
9511 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
9512                          iRegL src1, iRegL src2,
9513                          immI src3, rFlagsReg cr) %{
9514   match(Set dst (AndL src1 (RShiftL src2 src3)));
9515 
9516   ins_cost(1.9 * INSN_COST);
9517   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
9518 
9519   ins_encode %{
9520     __ andr(as_Register($dst$$reg),
9521               as_Register($src1$$reg),
9522               as_Register($src2$$reg),
9523               Assembler::ASR,
9524               $src3$$constant & 0x3f);
9525   %}
9526 
9527   ins_pipe(ialu_reg_reg_shift);
9528 %}
9529 
9530 instruct AndI_reg_LShift_reg(iRegINoSp dst,
9531                          iRegIorL2I src1, iRegIorL2I src2,
9532                          immI src3, rFlagsReg cr) %{
9533   match(Set dst (AndI src1 (LShiftI src2 src3)));
9534 
9535   ins_cost(1.9 * INSN_COST);
9536   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
9537 
9538   ins_encode %{
9539     __ andw(as_Register($dst$$reg),
9540               as_Register($src1$$reg),
9541               as_Register($src2$$reg),
9542               Assembler::LSL,
9543               $src3$$constant & 0x3f);
9544   %}
9545 
9546   ins_pipe(ialu_reg_reg_shift);
9547 %}
9548 
9549 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
9550                          iRegL src1, iRegL src2,
9551                          immI src3, rFlagsReg cr) %{
9552   match(Set dst (AndL src1 (LShiftL src2 src3)));
9553 
9554   ins_cost(1.9 * INSN_COST);
9555   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
9556 
9557   ins_encode %{
9558     __ andr(as_Register($dst$$reg),
9559               as_Register($src1$$reg),
9560               as_Register($src2$$reg),
9561               Assembler::LSL,
9562               $src3$$constant & 0x3f);
9563   %}
9564 
9565   ins_pipe(ialu_reg_reg_shift);
9566 %}
9567 
9568 instruct XorI_reg_URShift_reg(iRegINoSp dst,
9569                          iRegIorL2I src1, iRegIorL2I src2,
9570                          immI src3, rFlagsReg cr) %{
9571   match(Set dst (XorI src1 (URShiftI src2 src3)));
9572 
9573   ins_cost(1.9 * INSN_COST);
9574   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
9575 
9576   ins_encode %{
9577     __ eorw(as_Register($dst$$reg),
9578               as_Register($src1$$reg),
9579               as_Register($src2$$reg),
9580               Assembler::LSR,
9581               $src3$$constant & 0x3f);
9582   %}
9583 
9584   ins_pipe(ialu_reg_reg_shift);
9585 %}
9586 
9587 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
9588                          iRegL src1, iRegL src2,
9589                          immI src3, rFlagsReg cr) %{
9590   match(Set dst (XorL src1 (URShiftL src2 src3)));
9591 
9592   ins_cost(1.9 * INSN_COST);
9593   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
9594 
9595   ins_encode %{
9596     __ eor(as_Register($dst$$reg),
9597               as_Register($src1$$reg),
9598               as_Register($src2$$reg),
9599               Assembler::LSR,
9600               $src3$$constant & 0x3f);
9601   %}
9602 
9603   ins_pipe(ialu_reg_reg_shift);
9604 %}
9605 
9606 instruct XorI_reg_RShift_reg(iRegINoSp dst,
9607                          iRegIorL2I src1, iRegIorL2I src2,
9608                          immI src3, rFlagsReg cr) %{
9609   match(Set dst (XorI src1 (RShiftI src2 src3)));
9610 
9611   ins_cost(1.9 * INSN_COST);
9612   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
9613 
9614   ins_encode %{
9615     __ eorw(as_Register($dst$$reg),
9616               as_Register($src1$$reg),
9617               as_Register($src2$$reg),
9618               Assembler::ASR,
9619               $src3$$constant & 0x3f);
9620   %}
9621 
9622   ins_pipe(ialu_reg_reg_shift);
9623 %}
9624 
9625 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
9626                          iRegL src1, iRegL src2,
9627                          immI src3, rFlagsReg cr) %{
9628   match(Set dst (XorL src1 (RShiftL src2 src3)));
9629 
9630   ins_cost(1.9 * INSN_COST);
9631   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
9632 
9633   ins_encode %{
9634     __ eor(as_Register($dst$$reg),
9635               as_Register($src1$$reg),
9636               as_Register($src2$$reg),
9637               Assembler::ASR,
9638               $src3$$constant & 0x3f);
9639   %}
9640 
9641   ins_pipe(ialu_reg_reg_shift);
9642 %}
9643 
9644 instruct XorI_reg_LShift_reg(iRegINoSp dst,
9645                          iRegIorL2I src1, iRegIorL2I src2,
9646                          immI src3, rFlagsReg cr) %{
9647   match(Set dst (XorI src1 (LShiftI src2 src3)));
9648 
9649   ins_cost(1.9 * INSN_COST);
9650   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
9651 
9652   ins_encode %{
9653     __ eorw(as_Register($dst$$reg),
9654               as_Register($src1$$reg),
9655               as_Register($src2$$reg),
9656               Assembler::LSL,
9657               $src3$$constant & 0x3f);
9658   %}
9659 
9660   ins_pipe(ialu_reg_reg_shift);
9661 %}
9662 
9663 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
9664                          iRegL src1, iRegL src2,
9665                          immI src3, rFlagsReg cr) %{
9666   match(Set dst (XorL src1 (LShiftL src2 src3)));
9667 
9668   ins_cost(1.9 * INSN_COST);
9669   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
9670 
9671   ins_encode %{
9672     __ eor(as_Register($dst$$reg),
9673               as_Register($src1$$reg),
9674               as_Register($src2$$reg),
9675               Assembler::LSL,
9676               $src3$$constant & 0x3f);
9677   %}
9678 
9679   ins_pipe(ialu_reg_reg_shift);
9680 %}
9681 
9682 instruct OrI_reg_URShift_reg(iRegINoSp dst,
9683                          iRegIorL2I src1, iRegIorL2I src2,
9684                          immI src3, rFlagsReg cr) %{
9685   match(Set dst (OrI src1 (URShiftI src2 src3)));
9686 
9687   ins_cost(1.9 * INSN_COST);
9688   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
9689 
9690   ins_encode %{
9691     __ orrw(as_Register($dst$$reg),
9692               as_Register($src1$$reg),
9693               as_Register($src2$$reg),
9694               Assembler::LSR,
9695               $src3$$constant & 0x3f);
9696   %}
9697 
9698   ins_pipe(ialu_reg_reg_shift);
9699 %}
9700 
9701 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
9702                          iRegL src1, iRegL src2,
9703                          immI src3, rFlagsReg cr) %{
9704   match(Set dst (OrL src1 (URShiftL src2 src3)));
9705 
9706   ins_cost(1.9 * INSN_COST);
9707   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
9708 
9709   ins_encode %{
9710     __ orr(as_Register($dst$$reg),
9711               as_Register($src1$$reg),
9712               as_Register($src2$$reg),
9713               Assembler::LSR,
9714               $src3$$constant & 0x3f);
9715   %}
9716 
9717   ins_pipe(ialu_reg_reg_shift);
9718 %}
9719 
9720 instruct OrI_reg_RShift_reg(iRegINoSp dst,
9721                          iRegIorL2I src1, iRegIorL2I src2,
9722                          immI src3, rFlagsReg cr) %{
9723   match(Set dst (OrI src1 (RShiftI src2 src3)));
9724 
9725   ins_cost(1.9 * INSN_COST);
9726   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
9727 
9728   ins_encode %{
9729     __ orrw(as_Register($dst$$reg),
9730               as_Register($src1$$reg),
9731               as_Register($src2$$reg),
9732               Assembler::ASR,
9733               $src3$$constant & 0x3f);
9734   %}
9735 
9736   ins_pipe(ialu_reg_reg_shift);
9737 %}
9738 
9739 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
9740                          iRegL src1, iRegL src2,
9741                          immI src3, rFlagsReg cr) %{
9742   match(Set dst (OrL src1 (RShiftL src2 src3)));
9743 
9744   ins_cost(1.9 * INSN_COST);
9745   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
9746 
9747   ins_encode %{
9748     __ orr(as_Register($dst$$reg),
9749               as_Register($src1$$reg),
9750               as_Register($src2$$reg),
9751               Assembler::ASR,
9752               $src3$$constant & 0x3f);
9753   %}
9754 
9755   ins_pipe(ialu_reg_reg_shift);
9756 %}
9757 
9758 instruct OrI_reg_LShift_reg(iRegINoSp dst,
9759                          iRegIorL2I src1, iRegIorL2I src2,
9760                          immI src3, rFlagsReg cr) %{
9761   match(Set dst (OrI src1 (LShiftI src2 src3)));
9762 
9763   ins_cost(1.9 * INSN_COST);
9764   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
9765 
9766   ins_encode %{
9767     __ orrw(as_Register($dst$$reg),
9768               as_Register($src1$$reg),
9769               as_Register($src2$$reg),
9770               Assembler::LSL,
9771               $src3$$constant & 0x3f);
9772   %}
9773 
9774   ins_pipe(ialu_reg_reg_shift);
9775 %}
9776 
9777 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
9778                          iRegL src1, iRegL src2,
9779                          immI src3, rFlagsReg cr) %{
9780   match(Set dst (OrL src1 (LShiftL src2 src3)));
9781 
9782   ins_cost(1.9 * INSN_COST);
9783   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
9784 
9785   ins_encode %{
9786     __ orr(as_Register($dst$$reg),
9787               as_Register($src1$$reg),
9788               as_Register($src2$$reg),
9789               Assembler::LSL,
9790               $src3$$constant & 0x3f);
9791   %}
9792 
9793   ins_pipe(ialu_reg_reg_shift);
9794 %}
9795 
9796 instruct AddI_reg_URShift_reg(iRegINoSp dst,
9797                          iRegIorL2I src1, iRegIorL2I src2,
9798                          immI src3, rFlagsReg cr) %{
9799   match(Set dst (AddI src1 (URShiftI src2 src3)));
9800 
9801   ins_cost(1.9 * INSN_COST);
9802   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
9803 
9804   ins_encode %{
9805     __ addw(as_Register($dst$$reg),
9806               as_Register($src1$$reg),
9807               as_Register($src2$$reg),
9808               Assembler::LSR,
9809               $src3$$constant & 0x3f);
9810   %}
9811 
9812   ins_pipe(ialu_reg_reg_shift);
9813 %}
9814 
9815 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
9816                          iRegL src1, iRegL src2,
9817                          immI src3, rFlagsReg cr) %{
9818   match(Set dst (AddL src1 (URShiftL src2 src3)));
9819 
9820   ins_cost(1.9 * INSN_COST);
9821   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
9822 
9823   ins_encode %{
9824     __ add(as_Register($dst$$reg),
9825               as_Register($src1$$reg),
9826               as_Register($src2$$reg),
9827               Assembler::LSR,
9828               $src3$$constant & 0x3f);
9829   %}
9830 
9831   ins_pipe(ialu_reg_reg_shift);
9832 %}
9833 
9834 instruct AddI_reg_RShift_reg(iRegINoSp dst,
9835                          iRegIorL2I src1, iRegIorL2I src2,
9836                          immI src3, rFlagsReg cr) %{
9837   match(Set dst (AddI src1 (RShiftI src2 src3)));
9838 
9839   ins_cost(1.9 * INSN_COST);
9840   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
9841 
9842   ins_encode %{
9843     __ addw(as_Register($dst$$reg),
9844               as_Register($src1$$reg),
9845               as_Register($src2$$reg),
9846               Assembler::ASR,
9847               $src3$$constant & 0x3f);
9848   %}
9849 
9850   ins_pipe(ialu_reg_reg_shift);
9851 %}
9852 
9853 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
9854                          iRegL src1, iRegL src2,
9855                          immI src3, rFlagsReg cr) %{
9856   match(Set dst (AddL src1 (RShiftL src2 src3)));
9857 
9858   ins_cost(1.9 * INSN_COST);
9859   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
9860 
9861   ins_encode %{
9862     __ add(as_Register($dst$$reg),
9863               as_Register($src1$$reg),
9864               as_Register($src2$$reg),
9865               Assembler::ASR,
9866               $src3$$constant & 0x3f);
9867   %}
9868 
9869   ins_pipe(ialu_reg_reg_shift);
9870 %}
9871 
9872 instruct AddI_reg_LShift_reg(iRegINoSp dst,
9873                          iRegIorL2I src1, iRegIorL2I src2,
9874                          immI src3, rFlagsReg cr) %{
9875   match(Set dst (AddI src1 (LShiftI src2 src3)));
9876 
9877   ins_cost(1.9 * INSN_COST);
9878   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
9879 
9880   ins_encode %{
9881     __ addw(as_Register($dst$$reg),
9882               as_Register($src1$$reg),
9883               as_Register($src2$$reg),
9884               Assembler::LSL,
9885               $src3$$constant & 0x3f);
9886   %}
9887 
9888   ins_pipe(ialu_reg_reg_shift);
9889 %}
9890 
9891 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
9892                          iRegL src1, iRegL src2,
9893                          immI src3, rFlagsReg cr) %{
9894   match(Set dst (AddL src1 (LShiftL src2 src3)));
9895 
9896   ins_cost(1.9 * INSN_COST);
9897   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
9898 
9899   ins_encode %{
9900     __ add(as_Register($dst$$reg),
9901               as_Register($src1$$reg),
9902               as_Register($src2$$reg),
9903               Assembler::LSL,
9904               $src3$$constant & 0x3f);
9905   %}
9906 
9907   ins_pipe(ialu_reg_reg_shift);
9908 %}
9909 
9910 instruct SubI_reg_URShift_reg(iRegINoSp dst,
9911                          iRegIorL2I src1, iRegIorL2I src2,
9912                          immI src3, rFlagsReg cr) %{
9913   match(Set dst (SubI src1 (URShiftI src2 src3)));
9914 
9915   ins_cost(1.9 * INSN_COST);
9916   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
9917 
9918   ins_encode %{
9919     __ subw(as_Register($dst$$reg),
9920               as_Register($src1$$reg),
9921               as_Register($src2$$reg),
9922               Assembler::LSR,
9923               $src3$$constant & 0x3f);
9924   %}
9925 
9926   ins_pipe(ialu_reg_reg_shift);
9927 %}
9928 
9929 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
9930                          iRegL src1, iRegL src2,
9931                          immI src3, rFlagsReg cr) %{
9932   match(Set dst (SubL src1 (URShiftL src2 src3)));
9933 
9934   ins_cost(1.9 * INSN_COST);
9935   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
9936 
9937   ins_encode %{
9938     __ sub(as_Register($dst$$reg),
9939               as_Register($src1$$reg),
9940               as_Register($src2$$reg),
9941               Assembler::LSR,
9942               $src3$$constant & 0x3f);
9943   %}
9944 
9945   ins_pipe(ialu_reg_reg_shift);
9946 %}
9947 
9948 instruct SubI_reg_RShift_reg(iRegINoSp dst,
9949                          iRegIorL2I src1, iRegIorL2I src2,
9950                          immI src3, rFlagsReg cr) %{
9951   match(Set dst (SubI src1 (RShiftI src2 src3)));
9952 
9953   ins_cost(1.9 * INSN_COST);
9954   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
9955 
9956   ins_encode %{
9957     __ subw(as_Register($dst$$reg),
9958               as_Register($src1$$reg),
9959               as_Register($src2$$reg),
9960               Assembler::ASR,
9961               $src3$$constant & 0x3f);
9962   %}
9963 
9964   ins_pipe(ialu_reg_reg_shift);
9965 %}
9966 
9967 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
9968                          iRegL src1, iRegL src2,
9969                          immI src3, rFlagsReg cr) %{
9970   match(Set dst (SubL src1 (RShiftL src2 src3)));
9971 
9972   ins_cost(1.9 * INSN_COST);
9973   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
9974 
9975   ins_encode %{
9976     __ sub(as_Register($dst$$reg),
9977               as_Register($src1$$reg),
9978               as_Register($src2$$reg),
9979               Assembler::ASR,
9980               $src3$$constant & 0x3f);
9981   %}
9982 
9983   ins_pipe(ialu_reg_reg_shift);
9984 %}
9985 
9986 instruct SubI_reg_LShift_reg(iRegINoSp dst,
9987                          iRegIorL2I src1, iRegIorL2I src2,
9988                          immI src3, rFlagsReg cr) %{
9989   match(Set dst (SubI src1 (LShiftI src2 src3)));
9990 
9991   ins_cost(1.9 * INSN_COST);
9992   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
9993 
9994   ins_encode %{
9995     __ subw(as_Register($dst$$reg),
9996               as_Register($src1$$reg),
9997               as_Register($src2$$reg),
9998               Assembler::LSL,
9999               $src3$$constant & 0x3f);
10000   %}
10001 
10002   ins_pipe(ialu_reg_reg_shift);
10003 %}
10004 
10005 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
10006                          iRegL src1, iRegL src2,
10007                          immI src3, rFlagsReg cr) %{
10008   match(Set dst (SubL src1 (LShiftL src2 src3)));
10009 
10010   ins_cost(1.9 * INSN_COST);
10011   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
10012 
10013   ins_encode %{
10014     __ sub(as_Register($dst$$reg),
10015               as_Register($src1$$reg),
10016               as_Register($src2$$reg),
10017               Assembler::LSL,
10018               $src3$$constant & 0x3f);
10019   %}
10020 
10021   ins_pipe(ialu_reg_reg_shift);
10022 %}
10023 
10024 
10025 
10026 // Shift Left followed by Shift Right.
10027 // This idiom is used by the compiler for the i2b bytecode etc.
10028 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
10029 %{
10030   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
10031   // Make sure we are not going to exceed what sbfm can do.
10032   predicate((unsigned int)n->in(2)->get_int() <= 63
10033             && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
10034 
10035   ins_cost(INSN_COST * 2);
10036   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
10037   ins_encode %{
10038     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10039     int s = 63 - lshift;
10040     int r = (rshift - lshift) & 63;
10041     __ sbfm(as_Register($dst$$reg),
10042             as_Register($src$$reg),
10043             r, s);
10044   %}
10045 
10046   ins_pipe(ialu_reg_shift);
10047 %}
10048 
10049 // Shift Left followed by Shift Right.
10050 // This idiom is used by the compiler for the i2b bytecode etc.
10051 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
10052 %{
10053   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
10054   // Make sure we are not going to exceed what sbfmw can do.
10055   predicate((unsigned int)n->in(2)->get_int() <= 31
10056             && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
10057 
10058   ins_cost(INSN_COST * 2);
10059   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
10060   ins_encode %{
10061     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10062     int s = 31 - lshift;
10063     int r = (rshift - lshift) & 31;
10064     __ sbfmw(as_Register($dst$$reg),
10065             as_Register($src$$reg),
10066             r, s);
10067   %}
10068 
10069   ins_pipe(ialu_reg_shift);
10070 %}
10071 
10072 // Shift Left followed by Shift Right.
10073 // This idiom is used by the compiler for the i2b bytecode etc.
10074 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
10075 %{
10076   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
10077   // Make sure we are not going to exceed what ubfm can do.
10078   predicate((unsigned int)n->in(2)->get_int() <= 63
10079             && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
10080 
10081   ins_cost(INSN_COST * 2);
10082   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
10083   ins_encode %{
10084     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10085     int s = 63 - lshift;
10086     int r = (rshift - lshift) & 63;
10087     __ ubfm(as_Register($dst$$reg),
10088             as_Register($src$$reg),
10089             r, s);
10090   %}
10091 
10092   ins_pipe(ialu_reg_shift);
10093 %}
10094 
10095 // Shift Left followed by Shift Right.
10096 // This idiom is used by the compiler for the i2b bytecode etc.
10097 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
10098 %{
10099   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
10100   // Make sure we are not going to exceed what ubfmw can do.
10101   predicate((unsigned int)n->in(2)->get_int() <= 31
10102             && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
10103 
10104   ins_cost(INSN_COST * 2);
10105   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
10106   ins_encode %{
10107     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10108     int s = 31 - lshift;
10109     int r = (rshift - lshift) & 31;
10110     __ ubfmw(as_Register($dst$$reg),
10111             as_Register($src$$reg),
10112             r, s);
10113   %}
10114 
10115   ins_pipe(ialu_reg_shift);
10116 %}
10117 // Bitfield extract with shift & mask
10118 
10119 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
10120 %{
10121   match(Set dst (AndI (URShiftI src rshift) mask));
10122 
10123   ins_cost(INSN_COST);
10124   format %{ "ubfxw $dst, $src, $mask" %}
10125   ins_encode %{
10126     int rshift = $rshift$$constant;
10127     long mask = $mask$$constant;
10128     int width = exact_log2(mask+1);
10129     __ ubfxw(as_Register($dst$$reg),
10130             as_Register($src$$reg), rshift, width);
10131   %}
10132   ins_pipe(ialu_reg_shift);
10133 %}
10134 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
10135 %{
10136   match(Set dst (AndL (URShiftL src rshift) mask));
10137 
10138   ins_cost(INSN_COST);
10139   format %{ "ubfx $dst, $src, $mask" %}
10140   ins_encode %{
10141     int rshift = $rshift$$constant;
10142     long mask = $mask$$constant;
10143     int width = exact_log2(mask+1);
10144     __ ubfx(as_Register($dst$$reg),
10145             as_Register($src$$reg), rshift, width);
10146   %}
10147   ins_pipe(ialu_reg_shift);
10148 %}
10149 
10150 // We can use ubfx when extending an And with a mask when we know mask
10151 // is positive.  We know that because immI_bitmask guarantees it.
10152 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
10153 %{
10154   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
10155 
10156   ins_cost(INSN_COST * 2);
10157   format %{ "ubfx $dst, $src, $mask" %}
10158   ins_encode %{
10159     int rshift = $rshift$$constant;
10160     long mask = $mask$$constant;
10161     int width = exact_log2(mask+1);
10162     __ ubfx(as_Register($dst$$reg),
10163             as_Register($src$$reg), rshift, width);
10164   %}
10165   ins_pipe(ialu_reg_shift);
10166 %}
10167 
10168 // Rotations
10169 
10170 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
10171 %{
10172   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
10173   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
10174 
10175   ins_cost(INSN_COST);
10176   format %{ "extr $dst, $src1, $src2, #$rshift" %}
10177 
10178   ins_encode %{
10179     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
10180             $rshift$$constant & 63);
10181   %}
10182   ins_pipe(ialu_reg_reg_extr);
10183 %}
10184 
10185 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
10186 %{
10187   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
10188   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
10189 
10190   ins_cost(INSN_COST);
10191   format %{ "extr $dst, $src1, $src2, #$rshift" %}
10192 
10193   ins_encode %{
10194     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
10195             $rshift$$constant & 31);
10196   %}
10197   ins_pipe(ialu_reg_reg_extr);
10198 %}
10199 
10200 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
10201 %{
10202   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
10203   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
10204 
10205   ins_cost(INSN_COST);
10206   format %{ "extr $dst, $src1, $src2, #$rshift" %}
10207 
10208   ins_encode %{
10209     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
10210             $rshift$$constant & 63);
10211   %}
10212   ins_pipe(ialu_reg_reg_extr);
10213 %}
10214 
10215 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
10216 %{
10217   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
10218   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
10219 
10220   ins_cost(INSN_COST);
10221   format %{ "extr $dst, $src1, $src2, #$rshift" %}
10222 
10223   ins_encode %{
10224     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
10225             $rshift$$constant & 31);
10226   %}
10227   ins_pipe(ialu_reg_reg_extr);
10228 %}
10229 
10230 
10231 // rol expander
10232 
10233 instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
10234 %{
10235   effect(DEF dst, USE src, USE shift);
10236 
10237   format %{ "rol    $dst, $src, $shift" %}
10238   ins_cost(INSN_COST * 3);
10239   ins_encode %{
10240     __ subw(rscratch1, zr, as_Register($shift$$reg));
10241     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
10242             rscratch1);
10243     %}
10244   ins_pipe(ialu_reg_reg_vshift);
10245 %}
10246 
10247 // rol expander
10248 
10249 instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
10250 %{
10251   effect(DEF dst, USE src, USE shift);
10252 
10253   format %{ "rol    $dst, $src, $shift" %}
10254   ins_cost(INSN_COST * 3);
10255   ins_encode %{
10256     __ subw(rscratch1, zr, as_Register($shift$$reg));
10257     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
10258             rscratch1);
10259     %}
10260   ins_pipe(ialu_reg_reg_vshift);
10261 %}
10262 
10263 instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
10264 %{
10265   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
10266 
10267   expand %{
10268     rolL_rReg(dst, src, shift, cr);
10269   %}
10270 %}
10271 
10272 instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
10273 %{
10274   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
10275 
10276   expand %{
10277     rolL_rReg(dst, src, shift, cr);
10278   %}
10279 %}
10280 
10281 instruct rolI_rReg_Var_C_32(iRegLNoSp dst, iRegL src, iRegI shift, immI_32 c_32, rFlagsReg cr)
10282 %{
10283   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
10284 
10285   expand %{
10286     rolL_rReg(dst, src, shift, cr);
10287   %}
10288 %}
10289 
10290 instruct rolI_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
10291 %{
10292   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
10293 
10294   expand %{
10295     rolL_rReg(dst, src, shift, cr);
10296   %}
10297 %}
10298 
10299 // ror expander
10300 
10301 instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
10302 %{
10303   effect(DEF dst, USE src, USE shift);
10304 
10305   format %{ "ror    $dst, $src, $shift" %}
10306   ins_cost(INSN_COST);
10307   ins_encode %{
10308     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
10309             as_Register($shift$$reg));
10310     %}
10311   ins_pipe(ialu_reg_reg_vshift);
10312 %}
10313 
10314 // ror expander
10315 
10316 instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
10317 %{
10318   effect(DEF dst, USE src, USE shift);
10319 
10320   format %{ "ror    $dst, $src, $shift" %}
10321   ins_cost(INSN_COST);
10322   ins_encode %{
10323     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
10324             as_Register($shift$$reg));
10325     %}
10326   ins_pipe(ialu_reg_reg_vshift);
10327 %}
10328 
10329 instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
10330 %{
10331   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
10332 
10333   expand %{
10334     rorL_rReg(dst, src, shift, cr);
10335   %}
10336 %}
10337 
10338 instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
10339 %{
10340   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
10341 
10342   expand %{
10343     rorL_rReg(dst, src, shift, cr);
10344   %}
10345 %}
10346 
10347 instruct rorI_rReg_Var_C_32(iRegLNoSp dst, iRegL src, iRegI shift, immI_32 c_32, rFlagsReg cr)
10348 %{
10349   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
10350 
10351   expand %{
10352     rorL_rReg(dst, src, shift, cr);
10353   %}
10354 %}
10355 
10356 instruct rorI_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
10357 %{
10358   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
10359 
10360   expand %{
10361     rorL_rReg(dst, src, shift, cr);
10362   %}
10363 %}
10364 
10365 // Add/subtract (extended)
10366 
10367 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
10368 %{
10369   match(Set dst (AddL src1 (ConvI2L src2)));
10370   ins_cost(INSN_COST);
10371   format %{ "add  $dst, $src1, sxtw $src2" %}
10372 
10373    ins_encode %{
10374      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10375             as_Register($src2$$reg), ext::sxtw);
10376    %}
10377   ins_pipe(ialu_reg_reg);
10378 %};
10379 
10380 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
10381 %{
10382   match(Set dst (SubL src1 (ConvI2L src2)));
10383   ins_cost(INSN_COST);
10384   format %{ "sub  $dst, $src1, sxtw $src2" %}
10385 
10386    ins_encode %{
10387      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
10388             as_Register($src2$$reg), ext::sxtw);
10389    %}
10390   ins_pipe(ialu_reg_reg);
10391 %};
10392 
10393 
10394 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
10395 %{
10396   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
10397   ins_cost(INSN_COST);
10398   format %{ "add  $dst, $src1, sxth $src2" %}
10399 
10400    ins_encode %{
10401      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10402             as_Register($src2$$reg), ext::sxth);
10403    %}
10404   ins_pipe(ialu_reg_reg);
10405 %}
10406 
10407 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
10408 %{
10409   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
10410   ins_cost(INSN_COST);
10411   format %{ "add  $dst, $src1, sxtb $src2" %}
10412 
10413    ins_encode %{
10414      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10415             as_Register($src2$$reg), ext::sxtb);
10416    %}
10417   ins_pipe(ialu_reg_reg);
10418 %}
10419 
10420 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
10421 %{
10422   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
10423   ins_cost(INSN_COST);
10424   format %{ "add  $dst, $src1, uxtb $src2" %}
10425 
10426    ins_encode %{
10427      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10428             as_Register($src2$$reg), ext::uxtb);
10429    %}
10430   ins_pipe(ialu_reg_reg);
10431 %}
10432 
10433 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
10434 %{
10435   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
10436   ins_cost(INSN_COST);
10437   format %{ "add  $dst, $src1, sxth $src2" %}
10438 
10439    ins_encode %{
10440      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10441             as_Register($src2$$reg), ext::sxth);
10442    %}
10443   ins_pipe(ialu_reg_reg);
10444 %}
10445 
10446 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
10447 %{
10448   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
10449   ins_cost(INSN_COST);
10450   format %{ "add  $dst, $src1, sxtw $src2" %}
10451 
10452    ins_encode %{
10453      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10454             as_Register($src2$$reg), ext::sxtw);
10455    %}
10456   ins_pipe(ialu_reg_reg);
10457 %}
10458 
10459 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
10460 %{
10461   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
10462   ins_cost(INSN_COST);
10463   format %{ "add  $dst, $src1, sxtb $src2" %}
10464 
10465    ins_encode %{
10466      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10467             as_Register($src2$$reg), ext::sxtb);
10468    %}
10469   ins_pipe(ialu_reg_reg);
10470 %}
10471 
10472 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
10473 %{
10474   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
10475   ins_cost(INSN_COST);
10476   format %{ "add  $dst, $src1, uxtb $src2" %}
10477 
10478    ins_encode %{
10479      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10480             as_Register($src2$$reg), ext::uxtb);
10481    %}
10482   ins_pipe(ialu_reg_reg);
10483 %}
10484 
10485 
10486 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
10487 %{
10488   match(Set dst (AddI src1 (AndI src2 mask)));
10489   ins_cost(INSN_COST);
10490   format %{ "addw  $dst, $src1, $src2, uxtb" %}
10491 
10492    ins_encode %{
10493      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
10494             as_Register($src2$$reg), ext::uxtb);
10495    %}
10496   ins_pipe(ialu_reg_reg);
10497 %}
10498 
10499 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
10500 %{
10501   match(Set dst (AddI src1 (AndI src2 mask)));
10502   ins_cost(INSN_COST);
10503   format %{ "addw  $dst, $src1, $src2, uxth" %}
10504 
10505    ins_encode %{
10506      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
10507             as_Register($src2$$reg), ext::uxth);
10508    %}
10509   ins_pipe(ialu_reg_reg);
10510 %}
10511 
10512 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
10513 %{
10514   match(Set dst (AddL src1 (AndL src2 mask)));
10515   ins_cost(INSN_COST);
10516   format %{ "add  $dst, $src1, $src2, uxtb" %}
10517 
10518    ins_encode %{
10519      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10520             as_Register($src2$$reg), ext::uxtb);
10521    %}
10522   ins_pipe(ialu_reg_reg);
10523 %}
10524 
10525 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
10526 %{
10527   match(Set dst (AddL src1 (AndL src2 mask)));
10528   ins_cost(INSN_COST);
10529   format %{ "add  $dst, $src1, $src2, uxth" %}
10530 
10531    ins_encode %{
10532      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10533             as_Register($src2$$reg), ext::uxth);
10534    %}
10535   ins_pipe(ialu_reg_reg);
10536 %}
10537 
10538 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
10539 %{
10540   match(Set dst (AddL src1 (AndL src2 mask)));
10541   ins_cost(INSN_COST);
10542   format %{ "add  $dst, $src1, $src2, uxtw" %}
10543 
10544    ins_encode %{
10545      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10546             as_Register($src2$$reg), ext::uxtw);
10547    %}
10548   ins_pipe(ialu_reg_reg);
10549 %}
10550 
10551 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
10552 %{
10553   match(Set dst (SubI src1 (AndI src2 mask)));
10554   ins_cost(INSN_COST);
10555   format %{ "subw  $dst, $src1, $src2, uxtb" %}
10556 
10557    ins_encode %{
10558      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
10559             as_Register($src2$$reg), ext::uxtb);
10560    %}
10561   ins_pipe(ialu_reg_reg);
10562 %}
10563 
10564 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
10565 %{
10566   match(Set dst (SubI src1 (AndI src2 mask)));
10567   ins_cost(INSN_COST);
10568   format %{ "subw  $dst, $src1, $src2, uxth" %}
10569 
10570    ins_encode %{
10571      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
10572             as_Register($src2$$reg), ext::uxth);
10573    %}
10574   ins_pipe(ialu_reg_reg);
10575 %}
10576 
10577 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
10578 %{
10579   match(Set dst (SubL src1 (AndL src2 mask)));
10580   ins_cost(INSN_COST);
10581   format %{ "sub  $dst, $src1, $src2, uxtb" %}
10582 
10583    ins_encode %{
10584      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
10585             as_Register($src2$$reg), ext::uxtb);
10586    %}
10587   ins_pipe(ialu_reg_reg);
10588 %}
10589 
10590 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
10591 %{
10592   match(Set dst (SubL src1 (AndL src2 mask)));
10593   ins_cost(INSN_COST);
10594   format %{ "sub  $dst, $src1, $src2, uxth" %}
10595 
10596    ins_encode %{
10597      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
10598             as_Register($src2$$reg), ext::uxth);
10599    %}
10600   ins_pipe(ialu_reg_reg);
10601 %}
10602 
10603 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
10604 %{
10605   match(Set dst (SubL src1 (AndL src2 mask)));
10606   ins_cost(INSN_COST);
10607   format %{ "sub  $dst, $src1, $src2, uxtw" %}
10608 
10609    ins_encode %{
10610      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
10611             as_Register($src2$$reg), ext::uxtw);
10612    %}
10613   ins_pipe(ialu_reg_reg);
10614 %}
10615 
10616 // END This section of the file is automatically generated. Do not edit --------------
10617 
10618 // ============================================================================
10619 // Floating Point Arithmetic Instructions
10620 
10621 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
10622   match(Set dst (AddF src1 src2));
10623 
10624   ins_cost(INSN_COST * 5);
10625   format %{ "fadds   $dst, $src1, $src2" %}
10626 
10627   ins_encode %{
10628     __ fadds(as_FloatRegister($dst$$reg),
10629              as_FloatRegister($src1$$reg),
10630              as_FloatRegister($src2$$reg));
10631   %}
10632 
10633   ins_pipe(pipe_class_default);
10634 %}
10635 
10636 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
10637   match(Set dst (AddD src1 src2));
10638 
10639   ins_cost(INSN_COST * 5);
10640   format %{ "faddd   $dst, $src1, $src2" %}
10641 
10642   ins_encode %{
10643     __ faddd(as_FloatRegister($dst$$reg),
10644              as_FloatRegister($src1$$reg),
10645              as_FloatRegister($src2$$reg));
10646   %}
10647 
10648   ins_pipe(pipe_class_default);
10649 %}
10650 
10651 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
10652   match(Set dst (SubF src1 src2));
10653 
10654   ins_cost(INSN_COST * 5);
10655   format %{ "fsubs   $dst, $src1, $src2" %}
10656 
10657   ins_encode %{
10658     __ fsubs(as_FloatRegister($dst$$reg),
10659              as_FloatRegister($src1$$reg),
10660              as_FloatRegister($src2$$reg));
10661   %}
10662 
10663   ins_pipe(pipe_class_default);
10664 %}
10665 
10666 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
10667   match(Set dst (SubD src1 src2));
10668 
10669   ins_cost(INSN_COST * 5);
10670   format %{ "fsubd   $dst, $src1, $src2" %}
10671 
10672   ins_encode %{
10673     __ fsubd(as_FloatRegister($dst$$reg),
10674              as_FloatRegister($src1$$reg),
10675              as_FloatRegister($src2$$reg));
10676   %}
10677 
10678   ins_pipe(pipe_class_default);
10679 %}
10680 
10681 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
10682   match(Set dst (MulF src1 src2));
10683 
10684   ins_cost(INSN_COST * 6);
10685   format %{ "fmuls   $dst, $src1, $src2" %}
10686 
10687   ins_encode %{
10688     __ fmuls(as_FloatRegister($dst$$reg),
10689              as_FloatRegister($src1$$reg),
10690              as_FloatRegister($src2$$reg));
10691   %}
10692 
10693   ins_pipe(pipe_class_default);
10694 %}
10695 
10696 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
10697   match(Set dst (MulD src1 src2));
10698 
10699   ins_cost(INSN_COST * 6);
10700   format %{ "fmuld   $dst, $src1, $src2" %}
10701 
10702   ins_encode %{
10703     __ fmuld(as_FloatRegister($dst$$reg),
10704              as_FloatRegister($src1$$reg),
10705              as_FloatRegister($src2$$reg));
10706   %}
10707 
10708   ins_pipe(pipe_class_default);
10709 %}
10710 
10711 // We cannot use these fused mul w add/sub ops because they don't
10712 // produce the same result as the equivalent separated ops
10713 // (essentially they don't round the intermediate result). that's a
10714 // shame. leaving them here in case we can idenitfy cases where it is
10715 // legitimate to use them
10716 
10717 
10718 // instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
10719 //   match(Set dst (AddF (MulF src1 src2) src3));
10720 
10721 //   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
10722 
10723 //   ins_encode %{
10724 //     __ fmadds(as_FloatRegister($dst$$reg),
10725 //              as_FloatRegister($src1$$reg),
10726 //              as_FloatRegister($src2$$reg),
10727 //              as_FloatRegister($src3$$reg));
10728 //   %}
10729 
10730 //   ins_pipe(pipe_class_default);
10731 // %}
10732 
10733 // instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
10734 //   match(Set dst (AddD (MulD src1 src2) src3));
10735 
10736 //   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
10737 
10738 //   ins_encode %{
10739 //     __ fmaddd(as_FloatRegister($dst$$reg),
10740 //              as_FloatRegister($src1$$reg),
10741 //              as_FloatRegister($src2$$reg),
10742 //              as_FloatRegister($src3$$reg));
10743 //   %}
10744 
10745 //   ins_pipe(pipe_class_default);
10746 // %}
10747 
10748 // instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
10749 //   match(Set dst (AddF (MulF (NegF src1) src2) src3));
10750 //   match(Set dst (AddF (NegF (MulF src1 src2)) src3));
10751 
10752 //   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
10753 
10754 //   ins_encode %{
10755 //     __ fmsubs(as_FloatRegister($dst$$reg),
10756 //               as_FloatRegister($src1$$reg),
10757 //               as_FloatRegister($src2$$reg),
10758 //              as_FloatRegister($src3$$reg));
10759 //   %}
10760 
10761 //   ins_pipe(pipe_class_default);
10762 // %}
10763 
10764 // instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
10765 //   match(Set dst (AddD (MulD (NegD src1) src2) src3));
10766 //   match(Set dst (AddD (NegD (MulD src1 src2)) src3));
10767 
10768 //   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
10769 
10770 //   ins_encode %{
10771 //     __ fmsubd(as_FloatRegister($dst$$reg),
10772 //               as_FloatRegister($src1$$reg),
10773 //               as_FloatRegister($src2$$reg),
10774 //               as_FloatRegister($src3$$reg));
10775 //   %}
10776 
10777 //   ins_pipe(pipe_class_default);
10778 // %}
10779 
10780 // instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
10781 //   match(Set dst (SubF (MulF (NegF src1) src2) src3));
10782 //   match(Set dst (SubF (NegF (MulF src1 src2)) src3));
10783 
10784 //   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
10785 
10786 //   ins_encode %{
10787 //     __ fnmadds(as_FloatRegister($dst$$reg),
10788 //                as_FloatRegister($src1$$reg),
10789 //                as_FloatRegister($src2$$reg),
10790 //                as_FloatRegister($src3$$reg));
10791 //   %}
10792 
10793 //   ins_pipe(pipe_class_default);
10794 // %}
10795 
10796 // instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
10797 //   match(Set dst (SubD (MulD (NegD src1) src2) src3));
10798 //   match(Set dst (SubD (NegD (MulD src1 src2)) src3));
10799 
10800 //   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
10801 
10802 //   ins_encode %{
10803 //     __ fnmaddd(as_FloatRegister($dst$$reg),
10804 //                as_FloatRegister($src1$$reg),
10805 //                as_FloatRegister($src2$$reg),
10806 //                as_FloatRegister($src3$$reg));
10807 //   %}
10808 
10809 //   ins_pipe(pipe_class_default);
10810 // %}
10811 
10812 // instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
10813 //   match(Set dst (SubF (MulF src1 src2) src3));
10814 
10815 //   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
10816 
10817 //   ins_encode %{
10818 //     __ fnmsubs(as_FloatRegister($dst$$reg),
10819 //                as_FloatRegister($src1$$reg),
10820 //                as_FloatRegister($src2$$reg),
10821 //                as_FloatRegister($src3$$reg));
10822 //   %}
10823 
10824 //   ins_pipe(pipe_class_default);
10825 // %}
10826 
10827 // instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
10828 //   match(Set dst (SubD (MulD src1 src2) src3));
10829 
10830 //   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
10831 
10832 //   ins_encode %{
10833 //   // n.b. insn name should be fnmsubd
10834 //     __ fnmsub(as_FloatRegister($dst$$reg),
10835 //                as_FloatRegister($src1$$reg),
10836 //                as_FloatRegister($src2$$reg),
10837 //                as_FloatRegister($src3$$reg));
10838 //   %}
10839 
10840 //   ins_pipe(pipe_class_default);
10841 // %}
10842 
10843 
10844 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
10845   match(Set dst (DivF src1  src2));
10846 
10847   ins_cost(INSN_COST * 18);
10848   format %{ "fdivs   $dst, $src1, $src2" %}
10849 
10850   ins_encode %{
10851     __ fdivs(as_FloatRegister($dst$$reg),
10852              as_FloatRegister($src1$$reg),
10853              as_FloatRegister($src2$$reg));
10854   %}
10855 
10856   ins_pipe(pipe_class_default);
10857 %}
10858 
10859 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
10860   match(Set dst (DivD src1  src2));
10861 
10862   ins_cost(INSN_COST * 32);
10863   format %{ "fdivd   $dst, $src1, $src2" %}
10864 
10865   ins_encode %{
10866     __ fdivd(as_FloatRegister($dst$$reg),
10867              as_FloatRegister($src1$$reg),
10868              as_FloatRegister($src2$$reg));
10869   %}
10870 
10871   ins_pipe(pipe_class_default);
10872 %}
10873 
10874 instruct negF_reg_reg(vRegF dst, vRegF src) %{
10875   match(Set dst (NegF src));
10876 
10877   ins_cost(INSN_COST * 3);
10878   format %{ "fneg   $dst, $src" %}
10879 
10880   ins_encode %{
10881     __ fnegs(as_FloatRegister($dst$$reg),
10882              as_FloatRegister($src$$reg));
10883   %}
10884 
10885   ins_pipe(pipe_class_default);
10886 %}
10887 
10888 instruct negD_reg_reg(vRegD dst, vRegD src) %{
10889   match(Set dst (NegD src));
10890 
10891   ins_cost(INSN_COST * 3);
10892   format %{ "fnegd   $dst, $src" %}
10893 
10894   ins_encode %{
10895     __ fnegd(as_FloatRegister($dst$$reg),
10896              as_FloatRegister($src$$reg));
10897   %}
10898 
10899   ins_pipe(pipe_class_default);
10900 %}
10901 
10902 instruct absF_reg(vRegF dst, vRegF src) %{
10903   match(Set dst (AbsF src));
10904 
10905   ins_cost(INSN_COST * 3);
10906   format %{ "fabss   $dst, $src" %}
10907   ins_encode %{
10908     __ fabss(as_FloatRegister($dst$$reg),
10909              as_FloatRegister($src$$reg));
10910   %}
10911 
10912   ins_pipe(pipe_class_default);
10913 %}
10914 
10915 instruct absD_reg(vRegD dst, vRegD src) %{
10916   match(Set dst (AbsD src));
10917 
10918   ins_cost(INSN_COST * 3);
10919   format %{ "fabsd   $dst, $src" %}
10920   ins_encode %{
10921     __ fabsd(as_FloatRegister($dst$$reg),
10922              as_FloatRegister($src$$reg));
10923   %}
10924 
10925   ins_pipe(pipe_class_default);
10926 %}
10927 
10928 instruct sqrtD_reg(vRegD dst, vRegD src) %{
10929   match(Set dst (SqrtD src));
10930 
10931   ins_cost(INSN_COST * 50);
10932   format %{ "fsqrtd  $dst, $src" %}
10933   ins_encode %{
10934     __ fsqrtd(as_FloatRegister($dst$$reg),
10935              as_FloatRegister($src$$reg));
10936   %}
10937 
10938   ins_pipe(pipe_class_default);
10939 %}
10940 
10941 instruct sqrtF_reg(vRegF dst, vRegF src) %{
10942   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
10943 
10944   ins_cost(INSN_COST * 50);
10945   format %{ "fsqrts  $dst, $src" %}
10946   ins_encode %{
10947     __ fsqrts(as_FloatRegister($dst$$reg),
10948              as_FloatRegister($src$$reg));
10949   %}
10950 
10951   ins_pipe(pipe_class_default);
10952 %}
10953 
10954 // ============================================================================
10955 // Logical Instructions
10956 
10957 // Integer Logical Instructions
10958 
10959 // And Instructions
10960 
10961 
10962 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
10963   match(Set dst (AndI src1 src2));
10964 
10965   format %{ "andw  $dst, $src1, $src2\t# int" %}
10966 
10967   ins_cost(INSN_COST);
10968   ins_encode %{
10969     __ andw(as_Register($dst$$reg),
10970             as_Register($src1$$reg),
10971             as_Register($src2$$reg));
10972   %}
10973 
10974   ins_pipe(ialu_reg_reg);
10975 %}
10976 
10977 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
10978   match(Set dst (AndI src1 src2));
10979 
10980   format %{ "andsw  $dst, $src1, $src2\t# int" %}
10981 
10982   ins_cost(INSN_COST);
10983   ins_encode %{
10984     __ andw(as_Register($dst$$reg),
10985             as_Register($src1$$reg),
10986             (unsigned long)($src2$$constant));
10987   %}
10988 
10989   ins_pipe(ialu_reg_imm);
10990 %}
10991 
10992 // Or Instructions
10993 
10994 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10995   match(Set dst (OrI src1 src2));
10996 
10997   format %{ "orrw  $dst, $src1, $src2\t# int" %}
10998 
10999   ins_cost(INSN_COST);
11000   ins_encode %{
11001     __ orrw(as_Register($dst$$reg),
11002             as_Register($src1$$reg),
11003             as_Register($src2$$reg));
11004   %}
11005 
11006   ins_pipe(ialu_reg_reg);
11007 %}
11008 
11009 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
11010   match(Set dst (OrI src1 src2));
11011 
11012   format %{ "orrw  $dst, $src1, $src2\t# int" %}
11013 
11014   ins_cost(INSN_COST);
11015   ins_encode %{
11016     __ orrw(as_Register($dst$$reg),
11017             as_Register($src1$$reg),
11018             (unsigned long)($src2$$constant));
11019   %}
11020 
11021   ins_pipe(ialu_reg_imm);
11022 %}
11023 
11024 // Xor Instructions
11025 
11026 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11027   match(Set dst (XorI src1 src2));
11028 
11029   format %{ "eorw  $dst, $src1, $src2\t# int" %}
11030 
11031   ins_cost(INSN_COST);
11032   ins_encode %{
11033     __ eorw(as_Register($dst$$reg),
11034             as_Register($src1$$reg),
11035             as_Register($src2$$reg));
11036   %}
11037 
11038   ins_pipe(ialu_reg_reg);
11039 %}
11040 
11041 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
11042   match(Set dst (XorI src1 src2));
11043 
11044   format %{ "eorw  $dst, $src1, $src2\t# int" %}
11045 
11046   ins_cost(INSN_COST);
11047   ins_encode %{
11048     __ eorw(as_Register($dst$$reg),
11049             as_Register($src1$$reg),
11050             (unsigned long)($src2$$constant));
11051   %}
11052 
11053   ins_pipe(ialu_reg_imm);
11054 %}
11055 
11056 // Long Logical Instructions
11057 // TODO
11058 
11059 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
11060   match(Set dst (AndL src1 src2));
11061 
11062   format %{ "and  $dst, $src1, $src2\t# int" %}
11063 
11064   ins_cost(INSN_COST);
11065   ins_encode %{
11066     __ andr(as_Register($dst$$reg),
11067             as_Register($src1$$reg),
11068             as_Register($src2$$reg));
11069   %}
11070 
11071   ins_pipe(ialu_reg_reg);
11072 %}
11073 
11074 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
11075   match(Set dst (AndL src1 src2));
11076 
11077   format %{ "and  $dst, $src1, $src2\t# int" %}
11078 
11079   ins_cost(INSN_COST);
11080   ins_encode %{
11081     __ andr(as_Register($dst$$reg),
11082             as_Register($src1$$reg),
11083             (unsigned long)($src2$$constant));
11084   %}
11085 
11086   ins_pipe(ialu_reg_imm);
11087 %}
11088 
11089 // Or Instructions
11090 
11091 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11092   match(Set dst (OrL src1 src2));
11093 
11094   format %{ "orr  $dst, $src1, $src2\t# int" %}
11095 
11096   ins_cost(INSN_COST);
11097   ins_encode %{
11098     __ orr(as_Register($dst$$reg),
11099            as_Register($src1$$reg),
11100            as_Register($src2$$reg));
11101   %}
11102 
11103   ins_pipe(ialu_reg_reg);
11104 %}
11105 
11106 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
11107   match(Set dst (OrL src1 src2));
11108 
11109   format %{ "orr  $dst, $src1, $src2\t# int" %}
11110 
11111   ins_cost(INSN_COST);
11112   ins_encode %{
11113     __ orr(as_Register($dst$$reg),
11114            as_Register($src1$$reg),
11115            (unsigned long)($src2$$constant));
11116   %}
11117 
11118   ins_pipe(ialu_reg_imm);
11119 %}
11120 
11121 // Xor Instructions
11122 
11123 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11124   match(Set dst (XorL src1 src2));
11125 
11126   format %{ "eor  $dst, $src1, $src2\t# int" %}
11127 
11128   ins_cost(INSN_COST);
11129   ins_encode %{
11130     __ eor(as_Register($dst$$reg),
11131            as_Register($src1$$reg),
11132            as_Register($src2$$reg));
11133   %}
11134 
11135   ins_pipe(ialu_reg_reg);
11136 %}
11137 
11138 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
11139   match(Set dst (XorL src1 src2));
11140 
11141   ins_cost(INSN_COST);
11142   format %{ "eor  $dst, $src1, $src2\t# int" %}
11143 
11144   ins_encode %{
11145     __ eor(as_Register($dst$$reg),
11146            as_Register($src1$$reg),
11147            (unsigned long)($src2$$constant));
11148   %}
11149 
11150   ins_pipe(ialu_reg_imm);
11151 %}
11152 
11153 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
11154 %{
11155   match(Set dst (ConvI2L src));
11156 
11157   ins_cost(INSN_COST);
11158   format %{ "sxtw  $dst, $src\t# i2l" %}
11159   ins_encode %{
11160     __ sbfm($dst$$Register, $src$$Register, 0, 31);
11161   %}
11162   ins_pipe(ialu_reg_shift);
11163 %}
11164 
11165 // this pattern occurs in bigmath arithmetic
11166 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
11167 %{
11168   match(Set dst (AndL (ConvI2L src) mask));
11169 
11170   ins_cost(INSN_COST);
11171   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
11172   ins_encode %{
11173     __ ubfm($dst$$Register, $src$$Register, 0, 31);
11174   %}
11175 
11176   ins_pipe(ialu_reg_shift);
11177 %}
11178 
11179 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
11180   match(Set dst (ConvL2I src));
11181 
11182   ins_cost(INSN_COST);
11183   format %{ "movw  $dst, $src \t// l2i" %}
11184 
11185   ins_encode %{
11186     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
11187   %}
11188 
11189   ins_pipe(ialu_reg);
11190 %}
11191 
11192 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
11193 %{
11194   match(Set dst (Conv2B src));
11195   effect(KILL cr);
11196 
11197   format %{
11198     "cmpw $src, zr\n\t"
11199     "cset $dst, ne"
11200   %}
11201 
11202   ins_encode %{
11203     __ cmpw(as_Register($src$$reg), zr);
11204     __ cset(as_Register($dst$$reg), Assembler::NE);
11205   %}
11206 
11207   ins_pipe(ialu_reg);
11208 %}
11209 
11210 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
11211 %{
11212   match(Set dst (Conv2B src));
11213   effect(KILL cr);
11214 
11215   format %{
11216     "cmp  $src, zr\n\t"
11217     "cset $dst, ne"
11218   %}
11219 
11220   ins_encode %{
11221     __ cmp(as_Register($src$$reg), zr);
11222     __ cset(as_Register($dst$$reg), Assembler::NE);
11223   %}
11224 
11225   ins_pipe(ialu_reg);
11226 %}
11227 
11228 instruct convD2F_reg(vRegF dst, vRegD src) %{
11229   match(Set dst (ConvD2F src));
11230 
11231   ins_cost(INSN_COST * 5);
11232   format %{ "fcvtd  $dst, $src \t// d2f" %}
11233 
11234   ins_encode %{
11235     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
11236   %}
11237 
11238   ins_pipe(pipe_class_default);
11239 %}
11240 
11241 instruct convF2D_reg(vRegD dst, vRegF src) %{
11242   match(Set dst (ConvF2D src));
11243 
11244   ins_cost(INSN_COST * 5);
11245   format %{ "fcvts  $dst, $src \t// f2d" %}
11246 
11247   ins_encode %{
11248     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
11249   %}
11250 
11251   ins_pipe(pipe_class_default);
11252 %}
11253 
11254 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
11255   match(Set dst (ConvF2I src));
11256 
11257   ins_cost(INSN_COST * 5);
11258   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
11259 
11260   ins_encode %{
11261     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
11262   %}
11263 
11264   ins_pipe(pipe_class_default);
11265 %}
11266 
11267 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
11268   match(Set dst (ConvF2L src));
11269 
11270   ins_cost(INSN_COST * 5);
11271   format %{ "fcvtzs  $dst, $src \t// f2l" %}
11272 
11273   ins_encode %{
11274     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
11275   %}
11276 
11277   ins_pipe(pipe_class_default);
11278 %}
11279 
11280 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
11281   match(Set dst (ConvI2F src));
11282 
11283   ins_cost(INSN_COST * 5);
11284   format %{ "scvtfws  $dst, $src \t// i2f" %}
11285 
11286   ins_encode %{
11287     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
11288   %}
11289 
11290   ins_pipe(pipe_class_default);
11291 %}
11292 
11293 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
11294   match(Set dst (ConvL2F src));
11295 
11296   ins_cost(INSN_COST * 5);
11297   format %{ "scvtfs  $dst, $src \t// l2f" %}
11298 
11299   ins_encode %{
11300     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
11301   %}
11302 
11303   ins_pipe(pipe_class_default);
11304 %}
11305 
11306 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
11307   match(Set dst (ConvD2I src));
11308 
11309   ins_cost(INSN_COST * 5);
11310   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
11311 
11312   ins_encode %{
11313     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
11314   %}
11315 
11316   ins_pipe(pipe_class_default);
11317 %}
11318 
11319 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
11320   match(Set dst (ConvD2L src));
11321 
11322   ins_cost(INSN_COST * 5);
11323   format %{ "fcvtzd  $dst, $src \t// d2l" %}
11324 
11325   ins_encode %{
11326     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
11327   %}
11328 
11329   ins_pipe(pipe_class_default);
11330 %}
11331 
11332 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
11333   match(Set dst (ConvI2D src));
11334 
11335   ins_cost(INSN_COST * 5);
11336   format %{ "scvtfwd  $dst, $src \t// i2d" %}
11337 
11338   ins_encode %{
11339     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
11340   %}
11341 
11342   ins_pipe(pipe_class_default);
11343 %}
11344 
11345 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
11346   match(Set dst (ConvL2D src));
11347 
11348   ins_cost(INSN_COST * 5);
11349   format %{ "scvtfd  $dst, $src \t// l2d" %}
11350 
11351   ins_encode %{
11352     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
11353   %}
11354 
11355   ins_pipe(pipe_class_default);
11356 %}
11357 
11358 // stack <-> reg and reg <-> reg shuffles with no conversion
11359 
11360 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
11361 
11362   match(Set dst (MoveF2I src));
11363 
11364   effect(DEF dst, USE src);
11365 
11366   ins_cost(4 * INSN_COST);
11367 
11368   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
11369 
11370   ins_encode %{
11371     __ ldrw($dst$$Register, Address(sp, $src$$disp));
11372   %}
11373 
11374   ins_pipe(iload_reg_reg);
11375 
11376 %}
11377 
11378 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
11379 
11380   match(Set dst (MoveI2F src));
11381 
11382   effect(DEF dst, USE src);
11383 
11384   ins_cost(4 * INSN_COST);
11385 
11386   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
11387 
11388   ins_encode %{
11389     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
11390   %}
11391 
11392   ins_pipe(pipe_class_memory);
11393 
11394 %}
11395 
11396 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
11397 
11398   match(Set dst (MoveD2L src));
11399 
11400   effect(DEF dst, USE src);
11401 
11402   ins_cost(4 * INSN_COST);
11403 
11404   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
11405 
11406   ins_encode %{
11407     __ ldr($dst$$Register, Address(sp, $src$$disp));
11408   %}
11409 
11410   ins_pipe(iload_reg_reg);
11411 
11412 %}
11413 
11414 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
11415 
11416   match(Set dst (MoveL2D src));
11417 
11418   effect(DEF dst, USE src);
11419 
11420   ins_cost(4 * INSN_COST);
11421 
11422   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
11423 
11424   ins_encode %{
11425     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
11426   %}
11427 
11428   ins_pipe(pipe_class_memory);
11429 
11430 %}
11431 
11432 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
11433 
11434   match(Set dst (MoveF2I src));
11435 
11436   effect(DEF dst, USE src);
11437 
11438   ins_cost(INSN_COST);
11439 
11440   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
11441 
11442   ins_encode %{
11443     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
11444   %}
11445 
11446   ins_pipe(pipe_class_memory);
11447 
11448 %}
11449 
11450 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
11451 
11452   match(Set dst (MoveI2F src));
11453 
11454   effect(DEF dst, USE src);
11455 
11456   ins_cost(INSN_COST);
11457 
11458   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
11459 
11460   ins_encode %{
11461     __ strw($src$$Register, Address(sp, $dst$$disp));
11462   %}
11463 
11464   ins_pipe(istore_reg_reg);
11465 
11466 %}
11467 
11468 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
11469 
11470   match(Set dst (MoveD2L src));
11471 
11472   effect(DEF dst, USE src);
11473 
11474   ins_cost(INSN_COST);
11475 
11476   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
11477 
11478   ins_encode %{
11479     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
11480   %}
11481 
11482   ins_pipe(pipe_class_memory);
11483 
11484 %}
11485 
11486 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
11487 
11488   match(Set dst (MoveL2D src));
11489 
11490   effect(DEF dst, USE src);
11491 
11492   ins_cost(INSN_COST);
11493 
11494   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
11495 
11496   ins_encode %{
11497     __ str($src$$Register, Address(sp, $dst$$disp));
11498   %}
11499 
11500   ins_pipe(istore_reg_reg);
11501 
11502 %}
11503 
11504 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
11505 
11506   match(Set dst (MoveF2I src));
11507 
11508   effect(DEF dst, USE src);
11509 
11510   ins_cost(INSN_COST);
11511 
11512   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
11513 
11514   ins_encode %{
11515     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
11516   %}
11517 
11518   ins_pipe(pipe_class_memory);
11519 
11520 %}
11521 
11522 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
11523 
11524   match(Set dst (MoveI2F src));
11525 
11526   effect(DEF dst, USE src);
11527 
11528   ins_cost(INSN_COST);
11529 
11530   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
11531 
11532   ins_encode %{
11533     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
11534   %}
11535 
11536   ins_pipe(pipe_class_memory);
11537 
11538 %}
11539 
11540 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
11541 
11542   match(Set dst (MoveD2L src));
11543 
11544   effect(DEF dst, USE src);
11545 
11546   ins_cost(INSN_COST);
11547 
11548   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
11549 
11550   ins_encode %{
11551     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
11552   %}
11553 
11554   ins_pipe(pipe_class_memory);
11555 
11556 %}
11557 
11558 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
11559 
11560   match(Set dst (MoveL2D src));
11561 
11562   effect(DEF dst, USE src);
11563 
11564   ins_cost(INSN_COST);
11565 
11566   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
11567 
11568   ins_encode %{
11569     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
11570   %}
11571 
11572   ins_pipe(pipe_class_memory);
11573 
11574 %}
11575 
11576 // ============================================================================
11577 // clearing of an array
11578 
11579 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
11580 %{
11581   match(Set dummy (ClearArray cnt base));
11582   effect(USE_KILL cnt, USE_KILL base);
11583 
11584   ins_cost(4 * INSN_COST);
11585   format %{ "ClearArray $cnt, $base" %}
11586 
11587   ins_encode(aarch64_enc_clear_array_reg_reg(cnt, base));
11588 
11589   ins_pipe(pipe_class_memory);
11590 %}
11591 
11592 // ============================================================================
11593 // Overflow Math Instructions
11594 
11595 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
11596 %{
11597   match(Set cr (OverflowAddI op1 op2));
11598 
11599   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
11600   ins_cost(INSN_COST);
11601   ins_encode %{
11602     __ cmnw($op1$$Register, $op2$$Register);
11603   %}
11604 
11605   ins_pipe(icmp_reg_reg);
11606 %}
11607 
11608 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
11609 %{
11610   match(Set cr (OverflowAddI op1 op2));
11611 
11612   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
11613   ins_cost(INSN_COST);
11614   ins_encode %{
11615     __ cmnw($op1$$Register, $op2$$constant);
11616   %}
11617 
11618   ins_pipe(icmp_reg_imm);
11619 %}
11620 
11621 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
11622 %{
11623   match(Set cr (OverflowAddL op1 op2));
11624 
11625   format %{ "cmn   $op1, $op2\t# overflow check long" %}
11626   ins_cost(INSN_COST);
11627   ins_encode %{
11628     __ cmn($op1$$Register, $op2$$Register);
11629   %}
11630 
11631   ins_pipe(icmp_reg_reg);
11632 %}
11633 
11634 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
11635 %{
11636   match(Set cr (OverflowAddL op1 op2));
11637 
11638   format %{ "cmn   $op1, $op2\t# overflow check long" %}
11639   ins_cost(INSN_COST);
11640   ins_encode %{
11641     __ cmn($op1$$Register, $op2$$constant);
11642   %}
11643 
11644   ins_pipe(icmp_reg_imm);
11645 %}
11646 
11647 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
11648 %{
11649   match(Set cr (OverflowSubI op1 op2));
11650 
11651   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
11652   ins_cost(INSN_COST);
11653   ins_encode %{
11654     __ cmpw($op1$$Register, $op2$$Register);
11655   %}
11656 
11657   ins_pipe(icmp_reg_reg);
11658 %}
11659 
11660 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
11661 %{
11662   match(Set cr (OverflowSubI op1 op2));
11663 
11664   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
11665   ins_cost(INSN_COST);
11666   ins_encode %{
11667     __ cmpw($op1$$Register, $op2$$constant);
11668   %}
11669 
11670   ins_pipe(icmp_reg_imm);
11671 %}
11672 
11673 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
11674 %{
11675   match(Set cr (OverflowSubL op1 op2));
11676 
11677   format %{ "cmp   $op1, $op2\t# overflow check long" %}
11678   ins_cost(INSN_COST);
11679   ins_encode %{
11680     __ cmp($op1$$Register, $op2$$Register);
11681   %}
11682 
11683   ins_pipe(icmp_reg_reg);
11684 %}
11685 
11686 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
11687 %{
11688   match(Set cr (OverflowSubL op1 op2));
11689 
11690   format %{ "cmp   $op1, $op2\t# overflow check long" %}
11691   ins_cost(INSN_COST);
11692   ins_encode %{
11693     __ cmp($op1$$Register, $op2$$constant);
11694   %}
11695 
11696   ins_pipe(icmp_reg_imm);
11697 %}
11698 
11699 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
11700 %{
11701   match(Set cr (OverflowSubI zero op1));
11702 
11703   format %{ "cmpw  zr, $op1\t# overflow check int" %}
11704   ins_cost(INSN_COST);
11705   ins_encode %{
11706     __ cmpw(zr, $op1$$Register);
11707   %}
11708 
11709   ins_pipe(icmp_reg_imm);
11710 %}
11711 
11712 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
11713 %{
11714   match(Set cr (OverflowSubL zero op1));
11715 
11716   format %{ "cmp   zr, $op1\t# overflow check long" %}
11717   ins_cost(INSN_COST);
11718   ins_encode %{
11719     __ cmp(zr, $op1$$Register);
11720   %}
11721 
11722   ins_pipe(icmp_reg_imm);
11723 %}
11724 
11725 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
11726 %{
11727   match(Set cr (OverflowMulI op1 op2));
11728 
11729   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
11730             "cmp   rscratch1, rscratch1, sxtw\n\t"
11731             "movw  rscratch1, #0x80000000\n\t"
11732             "cselw rscratch1, rscratch1, zr, NE\n\t"
11733             "cmpw  rscratch1, #1" %}
11734   ins_cost(5 * INSN_COST);
11735   ins_encode %{
11736     __ smull(rscratch1, $op1$$Register, $op2$$Register);
11737     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
11738     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
11739     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
11740     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
11741   %}
11742 
11743   ins_pipe(pipe_slow);
11744 %}
11745 
11746 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
11747 %{
11748   match(If cmp (OverflowMulI op1 op2));
11749   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
11750             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
11751   effect(USE labl, KILL cr);
11752 
11753   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
11754             "cmp   rscratch1, rscratch1, sxtw\n\t"
11755             "b$cmp   $labl" %}
11756   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
11757   ins_encode %{
11758     Label* L = $labl$$label;
11759     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
11760     __ smull(rscratch1, $op1$$Register, $op2$$Register);
11761     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
11762     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
11763   %}
11764 
11765   ins_pipe(pipe_serial);
11766 %}
11767 
11768 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
11769 %{
11770   match(Set cr (OverflowMulL op1 op2));
11771 
11772   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
11773             "smulh rscratch2, $op1, $op2\n\t"
11774             "cmp   rscratch2, rscratch1, ASR #31\n\t"
11775             "movw  rscratch1, #0x80000000\n\t"
11776             "cselw rscratch1, rscratch1, zr, NE\n\t"
11777             "cmpw  rscratch1, #1" %}
11778   ins_cost(6 * INSN_COST);
11779   ins_encode %{
11780     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
11781     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
11782     __ cmp(rscratch2, rscratch1, Assembler::ASR, 31);    // Top is pure sign ext
11783     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
11784     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
11785     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
11786   %}
11787 
11788   ins_pipe(pipe_slow);
11789 %}
11790 
11791 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
11792 %{
11793   match(If cmp (OverflowMulL op1 op2));
11794   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
11795             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
11796   effect(USE labl, KILL cr);
11797 
11798   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
11799             "smulh rscratch2, $op1, $op2\n\t"
11800             "cmp   rscratch2, rscratch1, ASR #31\n\t"
11801             "b$cmp $labl" %}
11802   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
11803   ins_encode %{
11804     Label* L = $labl$$label;
11805     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
11806     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
11807     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
11808     __ cmp(rscratch2, rscratch1, Assembler::ASR, 31);    // Top is pure sign ext
11809     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
11810   %}
11811 
11812   ins_pipe(pipe_serial);
11813 %}
11814 
11815 // ============================================================================
11816 // Compare Instructions
11817 
11818 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
11819 %{
11820   match(Set cr (CmpI op1 op2));
11821 
11822   effect(DEF cr, USE op1, USE op2);
11823 
11824   ins_cost(INSN_COST);
11825   format %{ "cmpw  $op1, $op2" %}
11826 
11827   ins_encode(aarch64_enc_cmpw(op1, op2));
11828 
11829   ins_pipe(icmp_reg_reg);
11830 %}
11831 
11832 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
11833 %{
11834   match(Set cr (CmpI op1 zero));
11835 
11836   effect(DEF cr, USE op1);
11837 
11838   ins_cost(INSN_COST);
11839   format %{ "cmpw $op1, 0" %}
11840 
11841   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
11842 
11843   ins_pipe(icmp_reg_imm);
11844 %}
11845 
11846 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
11847 %{
11848   match(Set cr (CmpI op1 op2));
11849 
11850   effect(DEF cr, USE op1);
11851 
11852   ins_cost(INSN_COST);
11853   format %{ "cmpw  $op1, $op2" %}
11854 
11855   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
11856 
11857   ins_pipe(icmp_reg_imm);
11858 %}
11859 
11860 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
11861 %{
11862   match(Set cr (CmpI op1 op2));
11863 
11864   effect(DEF cr, USE op1);
11865 
11866   ins_cost(INSN_COST * 2);
11867   format %{ "cmpw  $op1, $op2" %}
11868 
11869   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
11870 
11871   ins_pipe(icmp_reg_imm);
11872 %}
11873 
11874 // Unsigned compare Instructions; really, same as signed compare
11875 // except it should only be used to feed an If or a CMovI which takes a
11876 // cmpOpU.
11877 
11878 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
11879 %{
11880   match(Set cr (CmpU op1 op2));
11881 
11882   effect(DEF cr, USE op1, USE op2);
11883 
11884   ins_cost(INSN_COST);
11885   format %{ "cmpw  $op1, $op2\t# unsigned" %}
11886 
11887   ins_encode(aarch64_enc_cmpw(op1, op2));
11888 
11889   ins_pipe(icmp_reg_reg);
11890 %}
11891 
11892 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
11893 %{
11894   match(Set cr (CmpU op1 zero));
11895 
11896   effect(DEF cr, USE op1);
11897 
11898   ins_cost(INSN_COST);
11899   format %{ "cmpw $op1, #0\t# unsigned" %}
11900 
11901   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
11902 
11903   ins_pipe(icmp_reg_imm);
11904 %}
11905 
11906 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
11907 %{
11908   match(Set cr (CmpU op1 op2));
11909 
11910   effect(DEF cr, USE op1);
11911 
11912   ins_cost(INSN_COST);
11913   format %{ "cmpw  $op1, $op2\t# unsigned" %}
11914 
11915   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
11916 
11917   ins_pipe(icmp_reg_imm);
11918 %}
11919 
11920 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
11921 %{
11922   match(Set cr (CmpU op1 op2));
11923 
11924   effect(DEF cr, USE op1);
11925 
11926   ins_cost(INSN_COST * 2);
11927   format %{ "cmpw  $op1, $op2\t# unsigned" %}
11928 
11929   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
11930 
11931   ins_pipe(icmp_reg_imm);
11932 %}
11933 
11934 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
11935 %{
11936   match(Set cr (CmpL op1 op2));
11937 
11938   effect(DEF cr, USE op1, USE op2);
11939 
11940   ins_cost(INSN_COST);
11941   format %{ "cmp  $op1, $op2" %}
11942 
11943   ins_encode(aarch64_enc_cmp(op1, op2));
11944 
11945   ins_pipe(icmp_reg_reg);
11946 %}
11947 
11948 instruct compL_reg_immI0(rFlagsReg cr, iRegL op1, immI0 zero)
11949 %{
11950   match(Set cr (CmpL op1 zero));
11951 
11952   effect(DEF cr, USE op1);
11953 
11954   ins_cost(INSN_COST);
11955   format %{ "tst  $op1" %}
11956 
11957   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
11958 
11959   ins_pipe(icmp_reg_imm);
11960 %}
11961 
11962 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
11963 %{
11964   match(Set cr (CmpL op1 op2));
11965 
11966   effect(DEF cr, USE op1);
11967 
11968   ins_cost(INSN_COST);
11969   format %{ "cmp  $op1, $op2" %}
11970 
11971   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
11972 
11973   ins_pipe(icmp_reg_imm);
11974 %}
11975 
11976 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
11977 %{
11978   match(Set cr (CmpL op1 op2));
11979 
11980   effect(DEF cr, USE op1);
11981 
11982   ins_cost(INSN_COST * 2);
11983   format %{ "cmp  $op1, $op2" %}
11984 
11985   ins_encode(aarch64_enc_cmp_imm(op1, op2));
11986 
11987   ins_pipe(icmp_reg_imm);
11988 %}
11989 
11990 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
11991 %{
11992   match(Set cr (CmpP op1 op2));
11993 
11994   effect(DEF cr, USE op1, USE op2);
11995 
11996   ins_cost(INSN_COST);
11997   format %{ "cmp  $op1, $op2\t // ptr" %}
11998 
11999   ins_encode(aarch64_enc_cmpp(op1, op2));
12000 
12001   ins_pipe(icmp_reg_reg);
12002 %}
12003 
12004 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
12005 %{
12006   match(Set cr (CmpN op1 op2));
12007 
12008   effect(DEF cr, USE op1, USE op2);
12009 
12010   ins_cost(INSN_COST);
12011   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
12012 
12013   ins_encode(aarch64_enc_cmpn(op1, op2));
12014 
12015   ins_pipe(icmp_reg_reg);
12016 %}
12017 
12018 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
12019 %{
12020   match(Set cr (CmpP op1 zero));
12021 
12022   effect(DEF cr, USE op1, USE zero);
12023 
12024   ins_cost(INSN_COST);
12025   format %{ "cmp  $op1, 0\t // ptr" %}
12026 
12027   ins_encode(aarch64_enc_testp(op1));
12028 
12029   ins_pipe(icmp_reg_imm);
12030 %}
12031 
12032 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
12033 %{
12034   match(Set cr (CmpN op1 zero));
12035 
12036   effect(DEF cr, USE op1, USE zero);
12037 
12038   ins_cost(INSN_COST);
12039   format %{ "cmp  $op1, 0\t // compressed ptr" %}
12040 
12041   ins_encode(aarch64_enc_testn(op1));
12042 
12043   ins_pipe(icmp_reg_imm);
12044 %}
12045 
12046 // FP comparisons
12047 //
12048 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
12049 // using normal cmpOp. See declaration of rFlagsReg for details.
12050 
12051 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
12052 %{
12053   match(Set cr (CmpF src1 src2));
12054 
12055   ins_cost(3 * INSN_COST);
12056   format %{ "fcmps $src1, $src2" %}
12057 
12058   ins_encode %{
12059     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
12060   %}
12061 
12062   ins_pipe(pipe_class_compare);
12063 %}
12064 
12065 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
12066 %{
12067   match(Set cr (CmpF src1 src2));
12068 
12069   ins_cost(3 * INSN_COST);
12070   format %{ "fcmps $src1, 0.0" %}
12071 
12072   ins_encode %{
12073     __ fcmps(as_FloatRegister($src1$$reg), 0.0D);
12074   %}
12075 
12076   ins_pipe(pipe_class_compare);
12077 %}
12078 // FROM HERE
12079 
12080 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
12081 %{
12082   match(Set cr (CmpD src1 src2));
12083 
12084   ins_cost(3 * INSN_COST);
12085   format %{ "fcmpd $src1, $src2" %}
12086 
12087   ins_encode %{
12088     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
12089   %}
12090 
12091   ins_pipe(pipe_class_compare);
12092 %}
12093 
12094 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
12095 %{
12096   match(Set cr (CmpD src1 src2));
12097 
12098   ins_cost(3 * INSN_COST);
12099   format %{ "fcmpd $src1, 0.0" %}
12100 
12101   ins_encode %{
12102     __ fcmpd(as_FloatRegister($src1$$reg), 0.0D);
12103   %}
12104 
12105   ins_pipe(pipe_class_compare);
12106 %}
12107 
12108 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
12109 %{
12110   match(Set dst (CmpF3 src1 src2));
12111   effect(KILL cr);
12112 
12113   ins_cost(5 * INSN_COST);
12114   format %{ "fcmps $src1, $src2\n\t"
12115             "csinvw($dst, zr, zr, eq\n\t"
12116             "csnegw($dst, $dst, $dst, lt)"
12117   %}
12118 
12119   ins_encode %{
12120     Label done;
12121     FloatRegister s1 = as_FloatRegister($src1$$reg);
12122     FloatRegister s2 = as_FloatRegister($src2$$reg);
12123     Register d = as_Register($dst$$reg);
12124     __ fcmps(s1, s2);
12125     // installs 0 if EQ else -1
12126     __ csinvw(d, zr, zr, Assembler::EQ);
12127     // keeps -1 if less or unordered else installs 1
12128     __ csnegw(d, d, d, Assembler::LT);
12129     __ bind(done);
12130   %}
12131 
12132   ins_pipe(pipe_class_default);
12133 
12134 %}
12135 
12136 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
12137 %{
12138   match(Set dst (CmpD3 src1 src2));
12139   effect(KILL cr);
12140 
12141   ins_cost(5 * INSN_COST);
12142   format %{ "fcmpd $src1, $src2\n\t"
12143             "csinvw($dst, zr, zr, eq\n\t"
12144             "csnegw($dst, $dst, $dst, lt)"
12145   %}
12146 
12147   ins_encode %{
12148     Label done;
12149     FloatRegister s1 = as_FloatRegister($src1$$reg);
12150     FloatRegister s2 = as_FloatRegister($src2$$reg);
12151     Register d = as_Register($dst$$reg);
12152     __ fcmpd(s1, s2);
12153     // installs 0 if EQ else -1
12154     __ csinvw(d, zr, zr, Assembler::EQ);
12155     // keeps -1 if less or unordered else installs 1
12156     __ csnegw(d, d, d, Assembler::LT);
12157     __ bind(done);
12158   %}
12159   ins_pipe(pipe_class_default);
12160 
12161 %}
12162 
12163 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
12164 %{
12165   match(Set dst (CmpF3 src1 zero));
12166   effect(KILL cr);
12167 
12168   ins_cost(5 * INSN_COST);
12169   format %{ "fcmps $src1, 0.0\n\t"
12170             "csinvw($dst, zr, zr, eq\n\t"
12171             "csnegw($dst, $dst, $dst, lt)"
12172   %}
12173 
12174   ins_encode %{
12175     Label done;
12176     FloatRegister s1 = as_FloatRegister($src1$$reg);
12177     Register d = as_Register($dst$$reg);
12178     __ fcmps(s1, 0.0D);
12179     // installs 0 if EQ else -1
12180     __ csinvw(d, zr, zr, Assembler::EQ);
12181     // keeps -1 if less or unordered else installs 1
12182     __ csnegw(d, d, d, Assembler::LT);
12183     __ bind(done);
12184   %}
12185 
12186   ins_pipe(pipe_class_default);
12187 
12188 %}
12189 
12190 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
12191 %{
12192   match(Set dst (CmpD3 src1 zero));
12193   effect(KILL cr);
12194 
12195   ins_cost(5 * INSN_COST);
12196   format %{ "fcmpd $src1, 0.0\n\t"
12197             "csinvw($dst, zr, zr, eq\n\t"
12198             "csnegw($dst, $dst, $dst, lt)"
12199   %}
12200 
12201   ins_encode %{
12202     Label done;
12203     FloatRegister s1 = as_FloatRegister($src1$$reg);
12204     Register d = as_Register($dst$$reg);
12205     __ fcmpd(s1, 0.0D);
12206     // installs 0 if EQ else -1
12207     __ csinvw(d, zr, zr, Assembler::EQ);
12208     // keeps -1 if less or unordered else installs 1
12209     __ csnegw(d, d, d, Assembler::LT);
12210     __ bind(done);
12211   %}
12212   ins_pipe(pipe_class_default);
12213 
12214 %}
12215 
12216 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
12217 %{
12218   match(Set dst (CmpLTMask p q));
12219   effect(KILL cr);
12220 
12221   ins_cost(3 * INSN_COST);
12222 
12223   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
12224             "csetw $dst, lt\n\t"
12225             "subw $dst, zr, $dst"
12226   %}
12227 
12228   ins_encode %{
12229     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
12230     __ csetw(as_Register($dst$$reg), Assembler::LT);
12231     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
12232   %}
12233 
12234   ins_pipe(ialu_reg_reg);
12235 %}
12236 
12237 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
12238 %{
12239   match(Set dst (CmpLTMask src zero));
12240   effect(KILL cr);
12241 
12242   ins_cost(INSN_COST);
12243 
12244   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
12245 
12246   ins_encode %{
12247     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
12248   %}
12249 
12250   ins_pipe(ialu_reg_shift);
12251 %}
12252 
12253 // ============================================================================
12254 // Max and Min
12255 
12256 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12257 %{
12258   match(Set dst (MinI src1 src2));
12259 
12260   effect(DEF dst, USE src1, USE src2, KILL cr);
12261   size(8);
12262 
12263   ins_cost(INSN_COST * 3);
12264   format %{
12265     "cmpw $src1 $src2\t signed int\n\t"
12266     "cselw $dst, $src1, $src2 lt\t"
12267   %}
12268 
12269   ins_encode %{
12270     __ cmpw(as_Register($src1$$reg),
12271             as_Register($src2$$reg));
12272     __ cselw(as_Register($dst$$reg),
12273              as_Register($src1$$reg),
12274              as_Register($src2$$reg),
12275              Assembler::LT);
12276   %}
12277 
12278   ins_pipe(ialu_reg_reg);
12279 %}
12280 // FROM HERE
12281 
12282 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12283 %{
12284   match(Set dst (MaxI src1 src2));
12285 
12286   effect(DEF dst, USE src1, USE src2, KILL cr);
12287   size(8);
12288 
12289   ins_cost(INSN_COST * 3);
12290   format %{
12291     "cmpw $src1 $src2\t signed int\n\t"
12292     "cselw $dst, $src1, $src2 gt\t"
12293   %}
12294 
12295   ins_encode %{
12296     __ cmpw(as_Register($src1$$reg),
12297             as_Register($src2$$reg));
12298     __ cselw(as_Register($dst$$reg),
12299              as_Register($src1$$reg),
12300              as_Register($src2$$reg),
12301              Assembler::GT);
12302   %}
12303 
12304   ins_pipe(ialu_reg_reg);
12305 %}
12306 
12307 // ============================================================================
12308 // Branch Instructions
12309 
12310 // Direct Branch.
12311 instruct branch(label lbl)
12312 %{
12313   match(Goto);
12314 
12315   effect(USE lbl);
12316 
12317   ins_cost(BRANCH_COST);
12318   format %{ "b  $lbl" %}
12319 
12320   ins_encode(aarch64_enc_b(lbl));
12321 
12322   ins_pipe(pipe_branch);
12323 %}
12324 
12325 // Conditional Near Branch
12326 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
12327 %{
12328   // Same match rule as `branchConFar'.
12329   match(If cmp cr);
12330 
12331   effect(USE lbl);
12332 
12333   ins_cost(BRANCH_COST);
12334   // If set to 1 this indicates that the current instruction is a
12335   // short variant of a long branch. This avoids using this
12336   // instruction in first-pass matching. It will then only be used in
12337   // the `Shorten_branches' pass.
12338   // ins_short_branch(1);
12339   format %{ "b$cmp  $lbl" %}
12340 
12341   ins_encode(aarch64_enc_br_con(cmp, lbl));
12342 
12343   ins_pipe(pipe_branch_cond);
12344 %}
12345 
12346 // Conditional Near Branch Unsigned
12347 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
12348 %{
12349   // Same match rule as `branchConFar'.
12350   match(If cmp cr);
12351 
12352   effect(USE lbl);
12353 
12354   ins_cost(BRANCH_COST);
12355   // If set to 1 this indicates that the current instruction is a
12356   // short variant of a long branch. This avoids using this
12357   // instruction in first-pass matching. It will then only be used in
12358   // the `Shorten_branches' pass.
12359   // ins_short_branch(1);
12360   format %{ "b$cmp  $lbl\t# unsigned" %}
12361 
12362   ins_encode(aarch64_enc_br_conU(cmp, lbl));
12363 
12364   ins_pipe(pipe_branch_cond);
12365 %}
12366 
12367 // Make use of CBZ and CBNZ.  These instructions, as well as being
12368 // shorter than (cmp; branch), have the additional benefit of not
12369 // killing the flags.
12370 
12371 instruct cmpI_imm0_branch(cmpOp cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
12372   match(If cmp (CmpI op1 op2));
12373   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
12374             || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
12375   effect(USE labl);
12376 
12377   ins_cost(BRANCH_COST);
12378   format %{ "cbw$cmp   $op1, $labl" %}
12379   ins_encode %{
12380     Label* L = $labl$$label;
12381     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
12382     if (cond == Assembler::EQ)
12383       __ cbzw($op1$$Register, *L);
12384     else
12385       __ cbnzw($op1$$Register, *L);
12386   %}
12387   ins_pipe(pipe_cmp_branch);
12388 %}
12389 
12390 instruct cmpL_imm0_branch(cmpOp cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
12391   match(If cmp (CmpL op1 op2));
12392   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
12393             || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
12394   effect(USE labl);
12395 
12396   ins_cost(BRANCH_COST);
12397   format %{ "cb$cmp   $op1, $labl" %}
12398   ins_encode %{
12399     Label* L = $labl$$label;
12400     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
12401     if (cond == Assembler::EQ)
12402       __ cbz($op1$$Register, *L);
12403     else
12404       __ cbnz($op1$$Register, *L);
12405   %}
12406   ins_pipe(pipe_cmp_branch);
12407 %}
12408 
12409 instruct cmpP_imm0_branch(cmpOp cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
12410   match(If cmp (CmpP op1 op2));
12411   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
12412             || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
12413   effect(USE labl);
12414 
12415   ins_cost(BRANCH_COST);
12416   format %{ "cb$cmp   $op1, $labl" %}
12417   ins_encode %{
12418     Label* L = $labl$$label;
12419     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
12420     if (cond == Assembler::EQ)
12421       __ cbz($op1$$Register, *L);
12422     else
12423       __ cbnz($op1$$Register, *L);
12424   %}
12425   ins_pipe(pipe_cmp_branch);
12426 %}
12427 
12428 // Conditional Far Branch
12429 // Conditional Far Branch Unsigned
12430 // TODO: fixme
12431 
12432 // counted loop end branch near
12433 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
12434 %{
12435   match(CountedLoopEnd cmp cr);
12436 
12437   effect(USE lbl);
12438 
12439   ins_cost(BRANCH_COST);
12440   // short variant.
12441   // ins_short_branch(1);
12442   format %{ "b$cmp $lbl \t// counted loop end" %}
12443 
12444   ins_encode(aarch64_enc_br_con(cmp, lbl));
12445 
12446   ins_pipe(pipe_branch);
12447 %}
12448 
12449 // counted loop end branch near Unsigned
12450 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
12451 %{
12452   match(CountedLoopEnd cmp cr);
12453 
12454   effect(USE lbl);
12455 
12456   ins_cost(BRANCH_COST);
12457   // short variant.
12458   // ins_short_branch(1);
12459   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
12460 
12461   ins_encode(aarch64_enc_br_conU(cmp, lbl));
12462 
12463   ins_pipe(pipe_branch);
12464 %}
12465 
12466 // counted loop end branch far
12467 // counted loop end branch far unsigned
12468 // TODO: fixme
12469 
12470 // ============================================================================
12471 // inlined locking and unlocking
12472 
12473 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
12474 %{
12475   match(Set cr (FastLock object box));
12476   effect(TEMP tmp, TEMP tmp2);
12477 
12478   // TODO
12479   // identify correct cost
12480   ins_cost(5 * INSN_COST);
12481   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
12482 
12483   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
12484 
12485   ins_pipe(pipe_serial);
12486 %}
12487 
12488 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
12489 %{
12490   match(Set cr (FastUnlock object box));
12491   effect(TEMP tmp, TEMP tmp2);
12492 
12493   ins_cost(5 * INSN_COST);
12494   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
12495 
12496   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
12497 
12498   ins_pipe(pipe_serial);
12499 %}
12500 
12501 
12502 // ============================================================================
12503 // Safepoint Instructions
12504 
12505 // TODO
12506 // provide a near and far version of this code
12507 
12508 instruct safePoint(iRegP poll)
12509 %{
12510   match(SafePoint poll);
12511 
12512   format %{
12513     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
12514   %}
12515   ins_encode %{
12516     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
12517   %}
12518   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
12519 %}
12520 
12521 
12522 // ============================================================================
12523 // Procedure Call/Return Instructions
12524 
12525 // Call Java Static Instruction
12526 
12527 instruct CallStaticJavaDirect(method meth)
12528 %{
12529   match(CallStaticJava);
12530 
12531   effect(USE meth);
12532 
12533   predicate(!((CallStaticJavaNode*)n)->is_method_handle_invoke());
12534 
12535   ins_cost(CALL_COST);
12536 
12537   format %{ "call,static $meth \t// ==> " %}
12538 
12539   ins_encode( aarch64_enc_java_static_call(meth),
12540               aarch64_enc_call_epilog );
12541 
12542   ins_pipe(pipe_class_call);
12543 %}
12544 
12545 // TO HERE
12546 
12547 // Call Java Static Instruction (method handle version)
12548 
12549 instruct CallStaticJavaDirectHandle(method meth, iRegP_FP reg_mh_save)
12550 %{
12551   match(CallStaticJava);
12552 
12553   effect(USE meth);
12554 
12555   predicate(((CallStaticJavaNode*)n)->is_method_handle_invoke());
12556 
12557   ins_cost(CALL_COST);
12558 
12559   format %{ "call,static $meth \t// (methodhandle) ==> " %}
12560 
12561   ins_encode( aarch64_enc_java_handle_call(meth),
12562               aarch64_enc_call_epilog );
12563 
12564   ins_pipe(pipe_class_call);
12565 %}
12566 
12567 // Call Java Dynamic Instruction
12568 instruct CallDynamicJavaDirect(method meth)
12569 %{
12570   match(CallDynamicJava);
12571 
12572   effect(USE meth);
12573 
12574   ins_cost(CALL_COST);
12575 
12576   format %{ "CALL,dynamic $meth \t// ==> " %}
12577 
12578   ins_encode( aarch64_enc_java_dynamic_call(meth),
12579                aarch64_enc_call_epilog );
12580 
12581   ins_pipe(pipe_class_call);
12582 %}
12583 
12584 // Call Runtime Instruction
12585 
12586 instruct CallRuntimeDirect(method meth)
12587 %{
12588   match(CallRuntime);
12589 
12590   effect(USE meth);
12591 
12592   ins_cost(CALL_COST);
12593 
12594   format %{ "CALL, runtime $meth" %}
12595 
12596   ins_encode( aarch64_enc_java_to_runtime(meth) );
12597 
12598   ins_pipe(pipe_class_call);
12599 %}
12600 
12601 // Call Runtime Instruction
12602 
12603 instruct CallLeafDirect(method meth)
12604 %{
12605   match(CallLeaf);
12606 
12607   effect(USE meth);
12608 
12609   ins_cost(CALL_COST);
12610 
12611   format %{ "CALL, runtime leaf $meth" %}
12612 
12613   ins_encode( aarch64_enc_java_to_runtime(meth) );
12614 
12615   ins_pipe(pipe_class_call);
12616 %}
12617 
12618 // Call Runtime Instruction
12619 
12620 instruct CallLeafNoFPDirect(method meth)
12621 %{
12622   match(CallLeafNoFP);
12623 
12624   effect(USE meth);
12625 
12626   ins_cost(CALL_COST);
12627 
12628   format %{ "CALL, runtime leaf nofp $meth" %}
12629 
12630   ins_encode( aarch64_enc_java_to_runtime(meth) );
12631 
12632   ins_pipe(pipe_class_call);
12633 %}
12634 
12635 // Tail Call; Jump from runtime stub to Java code.
12636 // Also known as an 'interprocedural jump'.
12637 // Target of jump will eventually return to caller.
12638 // TailJump below removes the return address.
12639 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop)
12640 %{
12641   match(TailCall jump_target method_oop);
12642 
12643   ins_cost(CALL_COST);
12644 
12645   format %{ "br $jump_target\t# $method_oop holds method oop" %}
12646 
12647   ins_encode(aarch64_enc_tail_call(jump_target));
12648 
12649   ins_pipe(pipe_class_call);
12650 %}
12651 
12652 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
12653 %{
12654   match(TailJump jump_target ex_oop);
12655 
12656   ins_cost(CALL_COST);
12657 
12658   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
12659 
12660   ins_encode(aarch64_enc_tail_jmp(jump_target));
12661 
12662   ins_pipe(pipe_class_call);
12663 %}
12664 
12665 // Create exception oop: created by stack-crawling runtime code.
12666 // Created exception is now available to this handler, and is setup
12667 // just prior to jumping to this handler. No code emitted.
12668 // TODO check
12669 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
12670 instruct CreateException(iRegP_R0 ex_oop)
12671 %{
12672   match(Set ex_oop (CreateEx));
12673 
12674   format %{ " -- \t// exception oop; no code emitted" %}
12675 
12676   size(0);
12677 
12678   ins_encode( /*empty*/ );
12679 
12680   ins_pipe(pipe_class_empty);
12681 %}
12682 
12683 // Rethrow exception: The exception oop will come in the first
12684 // argument position. Then JUMP (not call) to the rethrow stub code.
12685 instruct RethrowException() %{
12686   match(Rethrow);
12687   ins_cost(CALL_COST);
12688 
12689   format %{ "b rethrow_stub" %}
12690 
12691   ins_encode( aarch64_enc_rethrow() );
12692 
12693   ins_pipe(pipe_class_call);
12694 %}
12695 
12696 
12697 // Return Instruction
12698 // epilog node loads ret address into lr as part of frame pop
12699 instruct Ret()
12700 %{
12701   match(Return);
12702 
12703   format %{ "ret\t// return register" %}
12704 
12705   ins_encode( aarch64_enc_ret() );
12706 
12707   ins_pipe(pipe_branch);
12708 %}
12709 
12710 // Die now.
12711 instruct ShouldNotReachHere() %{
12712   match(Halt);
12713 
12714   ins_cost(CALL_COST);
12715   format %{ "ShouldNotReachHere" %}
12716 
12717   ins_encode %{
12718     // TODO
12719     // implement proper trap call here
12720     __ brk(999);
12721   %}
12722 
12723   ins_pipe(pipe_class_default);
12724 %}
12725 
12726 // ============================================================================
12727 // Partial Subtype Check
12728 //
12729 // superklass array for an instance of the superklass.  Set a hidden
12730 // internal cache on a hit (cache is checked with exposed code in
12731 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12732 // encoding ALSO sets flags.
12733 
12734 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
12735 %{
12736   match(Set result (PartialSubtypeCheck sub super));
12737   effect(KILL cr, KILL temp);
12738 
12739   ins_cost(1100);  // slightly larger than the next version
12740   format %{ "partialSubtypeCheck $result, $sub, $super" %}
12741 
12742   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
12743 
12744   opcode(0x1); // Force zero of result reg on hit
12745 
12746   ins_pipe(pipe_class_memory);
12747 %}
12748 
12749 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
12750 %{
12751   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12752   effect(KILL temp, KILL result);
12753 
12754   ins_cost(1100);  // slightly larger than the next version
12755   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
12756 
12757   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
12758 
12759   opcode(0x0); // Don't zero result reg on hit
12760 
12761   ins_pipe(pipe_class_memory);
12762 %}
12763 
12764 instruct string_compare(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
12765                         iRegI_R0 result, iRegP_R10 tmp1, rFlagsReg cr)
12766 %{
12767   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12768   effect(KILL tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12769 
12770   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
12771   ins_encode %{
12772     __ string_compare($str1$$Register, $str2$$Register,
12773                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12774                       $tmp1$$Register);
12775   %}
12776   ins_pipe(pipe_class_memory);
12777 %}
12778 
12779 instruct string_indexof(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
12780        iRegI_R0 result, iRegI tmp1, iRegI tmp2, iRegI tmp3, iRegI tmp4, rFlagsReg cr)
12781 %{
12782   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12783   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
12784          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
12785   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result" %}
12786 
12787   ins_encode %{
12788     __ string_indexof($str1$$Register, $str2$$Register,
12789                       $cnt1$$Register, $cnt2$$Register,
12790                       $tmp1$$Register, $tmp2$$Register,
12791                       $tmp3$$Register, $tmp4$$Register,
12792                       -1, $result$$Register);
12793   %}
12794   ins_pipe(pipe_class_memory);
12795 %}
12796 
12797 instruct string_indexof_con(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
12798                  immI_le_4 int_cnt2, iRegI_R0 result, iRegI tmp1, iRegI tmp2,
12799                  iRegI tmp3, iRegI tmp4, rFlagsReg cr)
12800 %{
12801   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12802   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
12803          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
12804   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result" %}
12805 
12806   ins_encode %{
12807     int icnt2 = (int)$int_cnt2$$constant;
12808     __ string_indexof($str1$$Register, $str2$$Register,
12809                       $cnt1$$Register, zr,
12810                       $tmp1$$Register, $tmp2$$Register,
12811                       $tmp3$$Register, $tmp4$$Register,
12812                       icnt2, $result$$Register);
12813   %}
12814   ins_pipe(pipe_class_memory);
12815 %}
12816 
12817 instruct string_equals(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
12818                         iRegI_R0 result, iRegP_R10 tmp, rFlagsReg cr)
12819 %{
12820   match(Set result (StrEquals (Binary str1 str2) cnt));
12821   effect(KILL tmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
12822 
12823   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp" %}
12824   ins_encode %{
12825     __ string_equals($str1$$Register, $str2$$Register,
12826                       $cnt$$Register, $result$$Register,
12827                       $tmp$$Register);
12828   %}
12829   ins_pipe(pipe_class_memory);
12830 %}
12831 
12832 instruct array_equals(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
12833                       iRegP_R10 tmp, rFlagsReg cr)
12834 %{
12835   match(Set result (AryEq ary1 ary2));
12836   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, KILL cr);
12837 
12838   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
12839   ins_encode %{
12840     __ char_arrays_equals($ary1$$Register, $ary2$$Register,
12841                           $result$$Register, $tmp$$Register);
12842   %}
12843   ins_pipe(pipe_class_memory);
12844 %}
12845 
12846 // encode char[] to byte[] in ISO_8859_1
12847 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
12848                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
12849                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
12850                           iRegI_R0 result, rFlagsReg cr)
12851 %{
12852   match(Set result (EncodeISOArray src (Binary dst len)));
12853   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
12854          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
12855 
12856   format %{ "Encode array $src,$dst,$len -> $result" %}
12857   ins_encode %{
12858     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12859          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
12860          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
12861   %}
12862   ins_pipe( pipe_class_memory );
12863 %}
12864 
12865 // ============================================================================
12866 // This name is KNOWN by the ADLC and cannot be changed.
12867 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12868 // for this guy.
12869 instruct tlsLoadP(thread_RegP dst)
12870 %{
12871   match(Set dst (ThreadLocal));
12872 
12873   ins_cost(0);
12874 
12875   format %{ " -- \t// $dst=Thread::current(), empty" %}
12876 
12877   size(0);
12878 
12879   ins_encode( /*empty*/ );
12880 
12881   ins_pipe(pipe_class_empty);
12882 %}
12883 
12884 
12885 
12886 //----------PEEPHOLE RULES-----------------------------------------------------
12887 // These must follow all instruction definitions as they use the names
12888 // defined in the instructions definitions.
12889 //
12890 // peepmatch ( root_instr_name [preceding_instruction]* );
12891 //
12892 // peepconstraint %{
12893 // (instruction_number.operand_name relational_op instruction_number.operand_name
12894 //  [, ...] );
12895 // // instruction numbers are zero-based using left to right order in peepmatch
12896 //
12897 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12898 // // provide an instruction_number.operand_name for each operand that appears
12899 // // in the replacement instruction's match rule
12900 //
12901 // ---------VM FLAGS---------------------------------------------------------
12902 //
12903 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12904 //
12905 // Each peephole rule is given an identifying number starting with zero and
12906 // increasing by one in the order seen by the parser.  An individual peephole
12907 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12908 // on the command-line.
12909 //
12910 // ---------CURRENT LIMITATIONS----------------------------------------------
12911 //
12912 // Only match adjacent instructions in same basic block
12913 // Only equality constraints
12914 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12915 // Only one replacement instruction
12916 //
12917 // ---------EXAMPLE----------------------------------------------------------
12918 //
12919 // // pertinent parts of existing instructions in architecture description
12920 // instruct movI(iRegINoSp dst, iRegI src)
12921 // %{
12922 //   match(Set dst (CopyI src));
12923 // %}
12924 //
12925 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
12926 // %{
12927 //   match(Set dst (AddI dst src));
12928 //   effect(KILL cr);
12929 // %}
12930 //
12931 // // Change (inc mov) to lea
12932 // peephole %{
12933 //   // increment preceeded by register-register move
12934 //   peepmatch ( incI_iReg movI );
12935 //   // require that the destination register of the increment
12936 //   // match the destination register of the move
12937 //   peepconstraint ( 0.dst == 1.dst );
12938 //   // construct a replacement instruction that sets
12939 //   // the destination to ( move's source register + one )
12940 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
12941 // %}
12942 //
12943 
12944 // Implementation no longer uses movX instructions since
12945 // machine-independent system no longer uses CopyX nodes.
12946 //
12947 // peephole
12948 // %{
12949 //   peepmatch (incI_iReg movI);
12950 //   peepconstraint (0.dst == 1.dst);
12951 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
12952 // %}
12953 
12954 // peephole
12955 // %{
12956 //   peepmatch (decI_iReg movI);
12957 //   peepconstraint (0.dst == 1.dst);
12958 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
12959 // %}
12960 
12961 // peephole
12962 // %{
12963 //   peepmatch (addI_iReg_imm movI);
12964 //   peepconstraint (0.dst == 1.dst);
12965 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
12966 // %}
12967 
12968 // peephole
12969 // %{
12970 //   peepmatch (incL_iReg movL);
12971 //   peepconstraint (0.dst == 1.dst);
12972 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
12973 // %}
12974 
12975 // peephole
12976 // %{
12977 //   peepmatch (decL_iReg movL);
12978 //   peepconstraint (0.dst == 1.dst);
12979 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
12980 // %}
12981 
12982 // peephole
12983 // %{
12984 //   peepmatch (addL_iReg_imm movL);
12985 //   peepconstraint (0.dst == 1.dst);
12986 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
12987 // %}
12988 
12989 // peephole
12990 // %{
12991 //   peepmatch (addP_iReg_imm movP);
12992 //   peepconstraint (0.dst == 1.dst);
12993 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
12994 // %}
12995 
12996 // // Change load of spilled value to only a spill
12997 // instruct storeI(memory mem, iRegI src)
12998 // %{
12999 //   match(Set mem (StoreI mem src));
13000 // %}
13001 //
13002 // instruct loadI(iRegINoSp dst, memory mem)
13003 // %{
13004 //   match(Set dst (LoadI mem));
13005 // %}
13006 //
13007 
13008 //----------SMARTSPILL RULES---------------------------------------------------
13009 // These must follow all instruction definitions as they use the names
13010 // defined in the instructions definitions.
13011 
13012 // Local Variables:
13013 // mode: c++
13014 // End: