1 //
   2 // Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 // AArch64 Architecture Description File
  27 
  28 //----------REGISTER DEFINITION BLOCK------------------------------------------
  29 // This information is used by the matcher and the register allocator to
  30 // describe individual registers and classes of registers within the target
  31 // archtecture.
  32 
  33 register %{
  34 //----------Architecture Description Register Definitions----------------------
  35 // General Registers
  36 // "reg_def"  name ( register save type, C convention save type,
  37 //                   ideal register type, encoding );
  38 // Register Save Types:
  39 //
  40 // NS  = No-Save:       The register allocator assumes that these registers
  41 //                      can be used without saving upon entry to the method, &
  42 //                      that they do not need to be saved at call sites.
  43 //
  44 // SOC = Save-On-Call:  The register allocator assumes that these registers
  45 //                      can be used without saving upon entry to the method,
  46 //                      but that they must be saved at call sites.
  47 //
  48 // SOE = Save-On-Entry: The register allocator assumes that these registers
  49 //                      must be saved before using them upon entry to the
  50 //                      method, but they do not need to be saved at call
  51 //                      sites.
  52 //
  53 // AS  = Always-Save:   The register allocator assumes that these registers
  54 //                      must be saved before using them upon entry to the
  55 //                      method, & that they must be saved at call sites.
  56 //
  57 // Ideal Register Type is used to determine how to save & restore a
  58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  60 //
  61 // The encoding number is the actual bit-pattern placed into the opcodes.
  62 
  63 // We must define the 64 bit int registers in two 32 bit halves, the
  64 // real lower register and a virtual upper half register. upper halves
  65 // are used by the register allocator but are not actually supplied as
  66 // operands to memory ops.
  67 //
  68 // follow the C1 compiler in making registers
  69 //
  70 //   r0-r7,r10-r26 volatile (caller save)
  71 //   r27-r32 system (no save, no allocate)
  72 //   r8-r9 invisible to the allocator (so we can use them as scratch regs)
  73 //
  74 // as regards Java usage. we don't use any callee save registers
  75 // because this makes it difficult to de-optimise a frame (see comment
  76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
  77 //
  78 
  79 // General Registers
  80 
  81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
  82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
  83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
  84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
  85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
  86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
  87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
  88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
  89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
  90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
  91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
  92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
  93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
  94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
  95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
  96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
  97 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  98 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  99 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
 100 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 101 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
 102 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
 103 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
 104 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
 105 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
 106 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
 107 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
 108 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
 109 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
 110 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
 111 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
 112 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
 113 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()        );
 114 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
 115 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
 116 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
 117 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
 118 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
 119 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
 120 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
 121 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
 122 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
 123 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
 124 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
 125 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
 126 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
 127 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
 128 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
 129 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
 130 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
 131 reg_def R27     (  NS, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
 132 reg_def R27_H   (  NS, SOE, Op_RegI, 27, r27->as_VMReg()->next());
 133 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
 134 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
 135 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
 136 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
 137 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
 138 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
 139 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
 140 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
 141 
 142 // ----------------------------
 143 // Float/Double Registers
 144 // ----------------------------
 145 
 146 // Double Registers
 147 
 148 // The rules of ADL require that double registers be defined in pairs.
 149 // Each pair must be two 32-bit values, but not necessarily a pair of
 150 // single float registers. In each pair, ADLC-assigned register numbers
 151 // must be adjacent, with the lower number even. Finally, when the
 152 // CPU stores such a register pair to memory, the word associated with
 153 // the lower ADLC-assigned number must be stored to the lower address.
 154 
 155 // AArch64 has 32 floating-point registers. Each can store a vector of
 156 // single or double precision floating-point values up to 8 * 32
 157 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
 158 // use the first float or double element of the vector.
 159 
 160 // for Java use float registers v0-v15 are always save on call whereas
 161 // the platform ABI treats v8-v15 as callee save). float registers
 162 // v16-v31 are SOC as per the platform spec
 163 
 164   reg_def V0   ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()         );
 165   reg_def V0_H ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next() );
 166   reg_def V1   ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()         );
 167   reg_def V1_H ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next() );
 168   reg_def V2   ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()         );
 169   reg_def V2_H ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next() );
 170   reg_def V3   ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()         );
 171   reg_def V3_H ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next() );
 172   reg_def V4   ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()         );
 173   reg_def V4_H ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next() );
 174   reg_def V5   ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()         );
 175   reg_def V5_H ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next() );
 176   reg_def V6   ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()         );
 177   reg_def V6_H ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next() );
 178   reg_def V7   ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()         );
 179   reg_def V7_H ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next() );
 180   reg_def V8   ( SOC, SOE, Op_RegF,  8, v8->as_VMReg()         );
 181   reg_def V8_H ( SOC, SOE, Op_RegF,  8, v8->as_VMReg()->next() );
 182   reg_def V9   ( SOC, SOE, Op_RegF,  9, v9->as_VMReg()         );
 183   reg_def V9_H ( SOC, SOE, Op_RegF,  9, v9->as_VMReg()->next() );
 184   reg_def V10  ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()        );
 185   reg_def V10_H( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next());
 186   reg_def V11  ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()        );
 187   reg_def V11_H( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next());
 188   reg_def V12  ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()        );
 189   reg_def V12_H( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next());
 190   reg_def V13  ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()        );
 191   reg_def V13_H( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next());
 192   reg_def V14  ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()        );
 193   reg_def V14_H( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next());
 194   reg_def V15  ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()        );
 195   reg_def V15_H( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next());
 196   reg_def V16  ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()        );
 197   reg_def V16_H( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next());
 198   reg_def V17  ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()        );
 199   reg_def V17_H( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next());
 200   reg_def V18  ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()        );
 201   reg_def V18_H( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next());
 202   reg_def V19  ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()        );
 203   reg_def V19_H( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next());
 204   reg_def V20  ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()        );
 205   reg_def V20_H( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next());
 206   reg_def V21  ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()        );
 207   reg_def V21_H( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next());
 208   reg_def V22  ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()        );
 209   reg_def V22_H( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next());
 210   reg_def V23  ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()        );
 211   reg_def V23_H( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next());
 212   reg_def V24  ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()        );
 213   reg_def V24_H( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next());
 214   reg_def V25  ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()        );
 215   reg_def V25_H( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next());
 216   reg_def V26  ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()        );
 217   reg_def V26_H( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next());
 218   reg_def V27  ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()        );
 219   reg_def V27_H( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next());
 220   reg_def V28  ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()        );
 221   reg_def V28_H( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next());
 222   reg_def V29  ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()        );
 223   reg_def V29_H( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next());
 224   reg_def V30  ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()        );
 225   reg_def V30_H( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next());
 226   reg_def V31  ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()        );
 227   reg_def V31_H( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next());
 228 
 229 // ----------------------------
 230 // Special Registers
 231 // ----------------------------
 232 
 233 // the AArch64 CSPR status flag register is not directly acessible as
 234 // instruction operand. the FPSR status flag register is a system
 235 // register which can be written/read using MSR/MRS but again does not
 236 // appear as an operand (a code identifying the FSPR occurs as an
 237 // immediate value in the instruction).
 238 
 239 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
 240 
 241 
 242 // Specify priority of register selection within phases of register
 243 // allocation.  Highest priority is first.  A useful heuristic is to
 244 // give registers a low priority when they are required by machine
 245 // instructions, like EAX and EDX on I486, and choose no-save registers
 246 // before save-on-call, & save-on-call before save-on-entry.  Registers
 247 // which participate in fixed calling sequences should come last.
 248 // Registers which are used as pairs must fall on an even boundary.
 249 
 250 alloc_class chunk0(
 251     // volatiles
 252     R10, R10_H,
 253     R11, R11_H,
 254     R12, R12_H,
 255     R13, R13_H,
 256     R14, R14_H,
 257     R15, R15_H,
 258     R16, R16_H,
 259     R17, R17_H,
 260     R18, R18_H,
 261 
 262     // arg registers
 263     R0, R0_H,
 264     R1, R1_H,
 265     R2, R2_H,
 266     R3, R3_H,
 267     R4, R4_H,
 268     R5, R5_H,
 269     R6, R6_H,
 270     R7, R7_H,
 271 
 272     // non-volatiles
 273     R19, R19_H,
 274     R20, R20_H,
 275     R21, R21_H,
 276     R22, R22_H,
 277     R23, R23_H,
 278     R24, R24_H,
 279     R25, R25_H,
 280     R26, R26_H,
 281 
 282     // non-allocatable registers
 283 
 284     R27, R27_H, // heapbase
 285     R28, R28_H, // thread
 286     R29, R29_H, // fp
 287     R30, R30_H, // lr
 288     R31, R31_H, // sp
 289 );
 290 
 291 alloc_class chunk1(
 292 
 293     // no save
 294     V16, V16_H,
 295     V17, V17_H,
 296     V18, V18_H,
 297     V19, V19_H,
 298     V20, V20_H,
 299     V21, V21_H,
 300     V22, V22_H,
 301     V23, V23_H,
 302     V24, V24_H,
 303     V25, V25_H,
 304     V26, V26_H,
 305     V27, V27_H,
 306     V28, V28_H,
 307     V29, V29_H,
 308     V30, V30_H,
 309     V31, V31_H,
 310 
 311     // arg registers
 312     V0, V0_H,
 313     V1, V1_H,
 314     V2, V2_H,
 315     V3, V3_H,
 316     V4, V4_H,
 317     V5, V5_H,
 318     V6, V6_H,
 319     V7, V7_H,
 320 
 321     // non-volatiles
 322     V8, V8_H,
 323     V9, V9_H,
 324     V10, V10_H,
 325     V11, V11_H,
 326     V12, V12_H,
 327     V13, V13_H,
 328     V14, V14_H,
 329     V15, V15_H,
 330 );
 331 
 332 alloc_class chunk2(RFLAGS);
 333 
 334 //----------Architecture Description Register Classes--------------------------
 335 // Several register classes are automatically defined based upon information in
 336 // this architecture description.
 337 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 338 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 339 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 340 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 341 //
 342 
 343 // Class for all 32 bit integer registers -- excludes SP which will
 344 // never be used as an integer register
 345 reg_class any_reg32(
 346     R0,
 347     R1,
 348     R2,
 349     R3,
 350     R4,
 351     R5,
 352     R6,
 353     R7,
 354     R10,
 355     R11,
 356     R12,
 357     R13,
 358     R14,
 359     R15,
 360     R16,
 361     R17,
 362     R18,
 363     R19,
 364     R20,
 365     R21,
 366     R22,
 367     R23,
 368     R24,
 369     R25,
 370     R26,
 371     R27,
 372     R28,
 373     R29,
 374     R30
 375 );
 376 
 377 // Singleton class for R0 int register
 378 reg_class int_r0_reg(R0);
 379 
 380 // Singleton class for R2 int register
 381 reg_class int_r2_reg(R2);
 382 
 383 // Singleton class for R3 int register
 384 reg_class int_r3_reg(R3);
 385 
 386 // Singleton class for R4 int register
 387 reg_class int_r4_reg(R4);
 388 
 389 // Class for all long integer registers (including RSP)
 390 reg_class any_reg(
 391     R0, R0_H,
 392     R1, R1_H,
 393     R2, R2_H,
 394     R3, R3_H,
 395     R4, R4_H,
 396     R5, R5_H,
 397     R6, R6_H,
 398     R7, R7_H,
 399     R10, R10_H,
 400     R11, R11_H,
 401     R12, R12_H,
 402     R13, R13_H,
 403     R14, R14_H,
 404     R15, R15_H,
 405     R16, R16_H,
 406     R17, R17_H,
 407     R18, R18_H,
 408     R19, R19_H,
 409     R20, R20_H,
 410     R21, R21_H,
 411     R22, R22_H,
 412     R23, R23_H,
 413     R24, R24_H,
 414     R25, R25_H,
 415     R26, R26_H,
 416     R27, R27_H,
 417     R28, R28_H,
 418     R29, R29_H,
 419     R30, R30_H,
 420     R31, R31_H
 421 );
 422 
 423 // Class for all non-special integer registers
 424 reg_class no_special_reg32_no_fp(
 425     R0,
 426     R1,
 427     R2,
 428     R3,
 429     R4,
 430     R5,
 431     R6,
 432     R7,
 433     R10,
 434     R11,
 435     R12,                        // rmethod
 436     R13,
 437     R14,
 438     R15,
 439     R16,
 440     R17,
 441     R18,
 442     R19,
 443     R20,
 444     R21,
 445     R22,
 446     R23,
 447     R24,
 448     R25,
 449     R26
 450  /* R27, */                     // heapbase
 451  /* R28, */                     // thread
 452  /* R29, */                     // fp
 453  /* R30, */                     // lr
 454  /* R31 */                      // sp
 455 );
 456 
 457 reg_class no_special_reg32_with_fp(
 458     R0,
 459     R1,
 460     R2,
 461     R3,
 462     R4,
 463     R5,
 464     R6,
 465     R7,
 466     R10,
 467     R11,
 468     R12,                        // rmethod
 469     R13,
 470     R14,
 471     R15,
 472     R16,
 473     R17,
 474     R18,
 475     R19,
 476     R20,
 477     R21,
 478     R22,
 479     R23,
 480     R24,
 481     R25,
 482     R26
 483  /* R27, */                     // heapbase
 484  /* R28, */                     // thread
 485     R29,                        // fp
 486  /* R30, */                     // lr
 487  /* R31 */                      // sp
 488 );
 489 
 490 reg_class_dynamic no_special_reg32(no_special_reg32_no_fp, no_special_reg32_with_fp, %{ PreserveFramePointer %});
 491 
 492 // Class for all non-special long integer registers
 493 reg_class no_special_reg_no_fp(
 494     R0, R0_H,
 495     R1, R1_H,
 496     R2, R2_H,
 497     R3, R3_H,
 498     R4, R4_H,
 499     R5, R5_H,
 500     R6, R6_H,
 501     R7, R7_H,
 502     R10, R10_H,
 503     R11, R11_H,
 504     R12, R12_H,                 // rmethod
 505     R13, R13_H,
 506     R14, R14_H,
 507     R15, R15_H,
 508     R16, R16_H,
 509     R17, R17_H,
 510     R18, R18_H,
 511     R19, R19_H,
 512     R20, R20_H,
 513     R21, R21_H,
 514     R22, R22_H,
 515     R23, R23_H,
 516     R24, R24_H,
 517     R25, R25_H,
 518     R26, R26_H,
 519  /* R27, R27_H, */              // heapbase
 520  /* R28, R28_H, */              // thread
 521  /* R29, R29_H, */              // fp
 522  /* R30, R30_H, */              // lr
 523  /* R31, R31_H */               // sp
 524 );
 525 
 526 reg_class no_special_reg_with_fp(
 527     R0, R0_H,
 528     R1, R1_H,
 529     R2, R2_H,
 530     R3, R3_H,
 531     R4, R4_H,
 532     R5, R5_H,
 533     R6, R6_H,
 534     R7, R7_H,
 535     R10, R10_H,
 536     R11, R11_H,
 537     R12, R12_H,                 // rmethod
 538     R13, R13_H,
 539     R14, R14_H,
 540     R15, R15_H,
 541     R16, R16_H,
 542     R17, R17_H,
 543     R18, R18_H,
 544     R19, R19_H,
 545     R20, R20_H,
 546     R21, R21_H,
 547     R22, R22_H,
 548     R23, R23_H,
 549     R24, R24_H,
 550     R25, R25_H,
 551     R26, R26_H,
 552  /* R27, R27_H, */              // heapbase
 553  /* R28, R28_H, */              // thread
 554     R29, R29_H,                 // fp
 555  /* R30, R30_H, */              // lr
 556  /* R31, R31_H */               // sp
 557 );
 558 
 559 reg_class_dynamic no_special_reg(no_special_reg_no_fp, no_special_reg_with_fp, %{ PreserveFramePointer %});
 560 
 561 // Class for 64 bit register r0
 562 reg_class r0_reg(
 563     R0, R0_H
 564 );
 565 
 566 // Class for 64 bit register r1
 567 reg_class r1_reg(
 568     R1, R1_H
 569 );
 570 
 571 // Class for 64 bit register r2
 572 reg_class r2_reg(
 573     R2, R2_H
 574 );
 575 
 576 // Class for 64 bit register r3
 577 reg_class r3_reg(
 578     R3, R3_H
 579 );
 580 
 581 // Class for 64 bit register r4
 582 reg_class r4_reg(
 583     R4, R4_H
 584 );
 585 
 586 // Class for 64 bit register r5
 587 reg_class r5_reg(
 588     R5, R5_H
 589 );
 590 
 591 // Class for 64 bit register r10
 592 reg_class r10_reg(
 593     R10, R10_H
 594 );
 595 
 596 // Class for 64 bit register r11
 597 reg_class r11_reg(
 598     R11, R11_H
 599 );
 600 
 601 // Class for method register
 602 reg_class method_reg(
 603     R12, R12_H
 604 );
 605 
 606 // Class for heapbase register
 607 reg_class heapbase_reg(
 608     R27, R27_H
 609 );
 610 
 611 // Class for thread register
 612 reg_class thread_reg(
 613     R28, R28_H
 614 );
 615 
 616 // Class for frame pointer register
 617 reg_class fp_reg(
 618     R29, R29_H
 619 );
 620 
 621 // Class for link register
 622 reg_class lr_reg(
 623     R30, R30_H
 624 );
 625 
 626 // Class for long sp register
 627 reg_class sp_reg(
 628   R31, R31_H
 629 );
 630 
 631 // Class for all pointer registers
 632 reg_class ptr_reg(
 633     R0, R0_H,
 634     R1, R1_H,
 635     R2, R2_H,
 636     R3, R3_H,
 637     R4, R4_H,
 638     R5, R5_H,
 639     R6, R6_H,
 640     R7, R7_H,
 641     R10, R10_H,
 642     R11, R11_H,
 643     R12, R12_H,
 644     R13, R13_H,
 645     R14, R14_H,
 646     R15, R15_H,
 647     R16, R16_H,
 648     R17, R17_H,
 649     R18, R18_H,
 650     R19, R19_H,
 651     R20, R20_H,
 652     R21, R21_H,
 653     R22, R22_H,
 654     R23, R23_H,
 655     R24, R24_H,
 656     R25, R25_H,
 657     R26, R26_H,
 658     R27, R27_H,
 659     R28, R28_H,
 660     R29, R29_H,
 661     R30, R30_H,
 662     R31, R31_H
 663 );
 664 
 665 // Class for all non_special pointer registers
 666 reg_class no_special_ptr_reg(
 667     R0, R0_H,
 668     R1, R1_H,
 669     R2, R2_H,
 670     R3, R3_H,
 671     R4, R4_H,
 672     R5, R5_H,
 673     R6, R6_H,
 674     R7, R7_H,
 675     R10, R10_H,
 676     R11, R11_H,
 677     R12, R12_H,
 678     R13, R13_H,
 679     R14, R14_H,
 680     R15, R15_H,
 681     R16, R16_H,
 682     R17, R17_H,
 683     R18, R18_H,
 684     R19, R19_H,
 685     R20, R20_H,
 686     R21, R21_H,
 687     R22, R22_H,
 688     R23, R23_H,
 689     R24, R24_H,
 690     R25, R25_H,
 691     R26, R26_H,
 692  /* R27, R27_H, */              // heapbase
 693  /* R28, R28_H, */              // thread
 694  /* R29, R29_H, */              // fp
 695  /* R30, R30_H, */              // lr
 696  /* R31, R31_H */               // sp
 697 );
 698 
 699 // Class for all float registers
 700 reg_class float_reg(
 701     V0,
 702     V1,
 703     V2,
 704     V3,
 705     V4,
 706     V5,
 707     V6,
 708     V7,
 709     V8,
 710     V9,
 711     V10,
 712     V11,
 713     V12,
 714     V13,
 715     V14,
 716     V15,
 717     V16,
 718     V17,
 719     V18,
 720     V19,
 721     V20,
 722     V21,
 723     V22,
 724     V23,
 725     V24,
 726     V25,
 727     V26,
 728     V27,
 729     V28,
 730     V29,
 731     V30,
 732     V31
 733 );
 734 
 735 // Double precision float registers have virtual `high halves' that
 736 // are needed by the allocator.
 737 // Class for all double registers
 738 reg_class double_reg(
 739     V0, V0_H,
 740     V1, V1_H,
 741     V2, V2_H,
 742     V3, V3_H,
 743     V4, V4_H,
 744     V5, V5_H,
 745     V6, V6_H,
 746     V7, V7_H,
 747     V8, V8_H,
 748     V9, V9_H,
 749     V10, V10_H,
 750     V11, V11_H,
 751     V12, V12_H,
 752     V13, V13_H,
 753     V14, V14_H,
 754     V15, V15_H,
 755     V16, V16_H,
 756     V17, V17_H,
 757     V18, V18_H,
 758     V19, V19_H,
 759     V20, V20_H,
 760     V21, V21_H,
 761     V22, V22_H,
 762     V23, V23_H,
 763     V24, V24_H,
 764     V25, V25_H,
 765     V26, V26_H,
 766     V27, V27_H,
 767     V28, V28_H,
 768     V29, V29_H,
 769     V30, V30_H,
 770     V31, V31_H
 771 );
 772 
 773 // Class for 128 bit register v0
 774 reg_class v0_reg(
 775     V0, V0_H
 776 );
 777 
 778 // Class for 128 bit register v1
 779 reg_class v1_reg(
 780     V1, V1_H
 781 );
 782 
 783 // Class for 128 bit register v2
 784 reg_class v2_reg(
 785     V2, V2_H
 786 );
 787 
 788 // Class for 128 bit register v3
 789 reg_class v3_reg(
 790     V3, V3_H
 791 );
 792 
 793 // Singleton class for condition codes
 794 reg_class int_flags(RFLAGS);
 795 
 796 %}
 797 
 798 //----------DEFINITION BLOCK---------------------------------------------------
 799 // Define name --> value mappings to inform the ADLC of an integer valued name
 800 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 801 // Format:
 802 //        int_def  <name>         ( <int_value>, <expression>);
 803 // Generated Code in ad_<arch>.hpp
 804 //        #define  <name>   (<expression>)
 805 //        // value == <int_value>
 806 // Generated code in ad_<arch>.cpp adlc_verification()
 807 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 808 //
 809 
 810 // we follow the ppc-aix port in using a simple cost model which ranks
 811 // register operations as cheap, memory ops as more expensive and
 812 // branches as most expensive. the first two have a low as well as a
 813 // normal cost. huge cost appears to be a way of saying don't do
 814 // something
 815 
 816 definitions %{
 817   // The default cost (of a register move instruction).
 818   int_def INSN_COST            (    100,     100);
 819   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 820   int_def CALL_COST            (    200,     2 * INSN_COST);
 821   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 822 %}
 823 
 824 
 825 //----------SOURCE BLOCK-------------------------------------------------------
 826 // This is a block of C++ code which provides values, functions, and
 827 // definitions necessary in the rest of the architecture description
 828 
 829 source_hpp %{
 830 
 831 #include "memory/cardTableModRefBS.hpp"
 832 
 833 class CallStubImpl {
 834 
 835   //--------------------------------------------------------------
 836   //---<  Used for optimization in Compile::shorten_branches  >---
 837   //--------------------------------------------------------------
 838 
 839  public:
 840   // Size of call trampoline stub.
 841   static uint size_call_trampoline() {
 842     return 0; // no call trampolines on this platform
 843   }
 844 
 845   // number of relocations needed by a call trampoline stub
 846   static uint reloc_call_trampoline() {
 847     return 0; // no call trampolines on this platform
 848   }
 849 };
 850 
 851 class HandlerImpl {
 852 
 853  public:
 854 
 855   static int emit_exception_handler(CodeBuffer &cbuf);
 856   static int emit_deopt_handler(CodeBuffer& cbuf);
 857 
 858   static uint size_exception_handler() {
 859     return MacroAssembler::far_branch_size();
 860   }
 861 
 862   static uint size_deopt_handler() {
 863     // count one adr and one far branch instruction
 864     return 4 * NativeInstruction::instruction_size;
 865   }
 866 };
 867 
 868   // graph traversal helpers
 869   MemBarNode *has_parent_membar(const Node *n,
 870                                 ProjNode *&ctl, ProjNode *&mem);
 871   MemBarNode *has_child_membar(const MemBarNode *n,
 872                                ProjNode *&ctl, ProjNode *&mem);
 873 
 874   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 875   bool unnecessary_acquire(const Node *barrier);
 876   bool needs_acquiring_load(const Node *load);
 877 
 878   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 879   bool unnecessary_release(const Node *barrier);
 880   bool unnecessary_volatile(const Node *barrier);
 881   bool needs_releasing_store(const Node *store);
 882 
 883   // Use barrier instructions rather than load acquire / store
 884   // release.
 885   const bool UseBarriersForVolatile = false;
 886   // Use barrier instructions for unsafe volatile gets rather than
 887   // trying to identify an exact signature for them
 888   const bool UseBarriersForUnsafeVolatileGet = false;
 889 %}
 890 
 891 source %{
 892 
 893   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 894   // use to implement volatile reads and writes. For a volatile read
 895   // we simply need
 896   //
 897   //   ldar<x>
 898   //
 899   // and for a volatile write we need
 900   //
 901   //   stlr<x>
 902   // 
 903   // Alternatively, we can implement them by pairing a normal
 904   // load/store with a memory barrier. For a volatile read we need
 905   // 
 906   //   ldr<x>
 907   //   dmb ishld
 908   //
 909   // for a volatile write
 910   //
 911   //   dmb ish
 912   //   str<x>
 913   //   dmb ish
 914   //
 915   // In order to generate the desired instruction sequence we need to
 916   // be able to identify specific 'signature' ideal graph node
 917   // sequences which i) occur as a translation of a volatile reads or
 918   // writes and ii) do not occur through any other translation or
 919   // graph transformation. We can then provide alternative aldc
 920   // matching rules which translate these node sequences to the
 921   // desired machine code sequences. Selection of the alternative
 922   // rules can be implemented by predicates which identify the
 923   // relevant node sequences.
 924   //
 925   // The ideal graph generator translates a volatile read to the node
 926   // sequence
 927   //
 928   //   LoadX[mo_acquire]
 929   //   MemBarAcquire
 930   //
 931   // As a special case when using the compressed oops optimization we
 932   // may also see this variant
 933   //
 934   //   LoadN[mo_acquire]
 935   //   DecodeN
 936   //   MemBarAcquire
 937   //
 938   // A volatile write is translated to the node sequence
 939   //
 940   //   MemBarRelease
 941   //   StoreX[mo_release]
 942   //   MemBarVolatile
 943   //
 944   // n.b. the above node patterns are generated with a strict
 945   // 'signature' configuration of input and output dependencies (see
 946   // the predicates below for exact details). The two signatures are
 947   // unique to translated volatile reads/stores -- they will not
 948   // appear as a result of any other bytecode translation or inlining
 949   // nor as a consequence of optimizing transforms.
 950   //
 951   // We also want to catch inlined unsafe volatile gets and puts and
 952   // be able to implement them using either ldar<x>/stlr<x> or some
 953   // combination of ldr<x>/stlr<x> and dmb instructions.
 954   //
 955   // Inlined unsafe volatiles puts manifest as a minor variant of the
 956   // normal volatile put node sequence containing an extra cpuorder
 957   // membar
 958   //
 959   //   MemBarRelease
 960   //   MemBarCPUOrder
 961   //   StoreX[mo_release]
 962   //   MemBarVolatile
 963   //
 964   // n.b. as an aside, the cpuorder membar is not itself subject to
 965   // matching and translation by adlc rules.  However, the rule
 966   // predicates need to detect its presence in order to correctly
 967   // select the desired adlc rules.
 968   //
 969   // Inlined unsafe volatiles gets manifest as a somewhat different
 970   // node sequence to a normal volatile get
 971   //
 972   //   MemBarCPUOrder
 973   //        ||       \\
 974   //   MemBarAcquire LoadX[mo_acquire]
 975   //        ||
 976   //   MemBarCPUOrder
 977   //
 978   // In this case the acquire membar does not directly depend on the
 979   // load. However, we can be sure that the load is generated from an
 980   // inlined unsafe volatile get if we see it dependent on this unique
 981   // sequence of membar nodes. Similarly, given an acquire membar we
 982   // can know that it was added because of an inlined unsafe volatile
 983   // get if it is fed and feeds a cpuorder membar and if its feed
 984   // membar also feeds an acquiring load.
 985   //
 986   // So, where we can identify these volatile read and write
 987   // signatures we can choose to plant either of the above two code
 988   // sequences. For a volatile read we can simply plant a normal
 989   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 990   // also choose to inhibit translation of the MemBarAcquire and
 991   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 992   //
 993   // When we recognise a volatile store signature we can choose to
 994   // plant at a dmb ish as a translation for the MemBarRelease, a
 995   // normal str<x> and then a dmb ish for the MemBarVolatile.
 996   // Alternatively, we can inhibit translation of the MemBarRelease
 997   // and MemBarVolatile and instead plant a simple stlr<x>
 998   // instruction.
 999   //
1000   // Of course, the above only applies when we see these signature
1001   // configurations. We still want to plant dmb instructions in any
1002   // other cases where we may see a MemBarAcquire, MemBarRelease or
1003   // MemBarVolatile. For example, at the end of a constructor which
1004   // writes final/volatile fields we will see a MemBarRelease
1005   // instruction and this needs a 'dmb ish' lest we risk the
1006   // constructed object being visible without making the
1007   // final/volatile field writes visible.
1008   //
1009   // n.b. the translation rules below which rely on detection of the
1010   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1011   // If we see anything other than the signature configurations we
1012   // always just translate the loads and stors to ldr<x> and str<x>
1013   // and translate acquire, release and volatile membars to the
1014   // relevant dmb instructions.
1015   //
1016   // n.b.b as a case in point for the above comment, the current
1017   // predicates don't detect the precise signature for certain types
1018   // of volatile object stores (where the heap_base input type is not
1019   // known at compile-time to be non-NULL). In those cases the
1020   // MemBarRelease and MemBarVolatile bracket an if-then-else sequence
1021   // with a store in each branch (we need a different store depending
1022   // on whether heap_base is actually NULL). In such a case we will
1023   // just plant a dmb both before and after the branch/merge. The
1024   // predicate could (and probably should) be fixed later to also
1025   // detect this case.
1026 
1027   // graph traversal helpers
1028 
1029   // if node n is linked to a parent MemBarNode by an intervening
1030   // Control or Memory ProjNode return the MemBarNode otherwise return
1031   // NULL.
1032   //
1033   // n may only be a Load or a MemBar.
1034   //
1035   // The ProjNode* references c and m are used to return the relevant
1036   // nodes.
1037 
1038   MemBarNode *has_parent_membar(const Node *n, ProjNode *&c, ProjNode *&m)
1039   {
1040     Node *ctl = NULL;
1041     Node *mem = NULL;
1042     Node *membar = NULL;
1043 
1044     if (n->is_Load()) {
1045       ctl = n->lookup(LoadNode::Control);
1046       mem = n->lookup(LoadNode::Memory);
1047     } else if (n->is_MemBar()) {
1048       ctl = n->lookup(TypeFunc::Control);
1049       mem = n->lookup(TypeFunc::Memory);
1050     } else {
1051         return NULL;
1052     }
1053 
1054     if (!ctl || !mem || !ctl->is_Proj() || !mem->is_Proj())
1055       return NULL;
1056 
1057     c = ctl->as_Proj();
1058 
1059     membar = ctl->lookup(0);
1060 
1061     if (!membar || !membar->is_MemBar())
1062       return NULL;
1063 
1064     m = mem->as_Proj();
1065 
1066     if (mem->lookup(0) != membar)
1067       return NULL;
1068 
1069     return membar->as_MemBar();
1070   }
1071 
1072   // if n is linked to a child MemBarNode by intervening Control and
1073   // Memory ProjNodes return the MemBarNode otherwise return NULL.
1074   //
1075   // The ProjNode** arguments c and m are used to return pointers to
1076   // the relevant nodes. A null argument means don't don't return a
1077   // value.
1078 
1079   MemBarNode *has_child_membar(const MemBarNode *n, ProjNode *&c, ProjNode *&m)
1080   {
1081     ProjNode *ctl = n->proj_out(TypeFunc::Control);
1082     ProjNode *mem = n->proj_out(TypeFunc::Memory);
1083 
1084     // MemBar needs to have both a Ctl and Mem projection
1085     if (! ctl || ! mem)
1086       return NULL;
1087 
1088     c = ctl;
1089     m = mem;
1090 
1091     MemBarNode *child = NULL;
1092     Node *x;
1093 
1094     for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {
1095       x = ctl->fast_out(i);
1096       // if we see a membar we keep hold of it. we may also see a new
1097       // arena copy of the original but it will appear later
1098       if (x->is_MemBar()) {
1099           child = x->as_MemBar();
1100           break;
1101       }
1102     }
1103 
1104     if (child == NULL)
1105       return NULL;
1106 
1107     for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
1108       x = mem->fast_out(i);
1109       // if we see a membar we keep hold of it. we may also see a new
1110       // arena copy of the original but it will appear later
1111       if (x == child) {
1112         return child;
1113       }
1114     }
1115     return NULL;
1116   }
1117 
1118   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1119 
1120 bool unnecessary_acquire(const Node *barrier) {
1121   // assert barrier->is_MemBar();
1122   if (UseBarriersForVolatile)
1123     // we need to plant a dmb
1124     return false;
1125 
1126   // a volatile read derived from bytecode (or also from an inlined
1127   // SHA field read via LibraryCallKit::load_field_from_object)
1128   // manifests as a LoadX[mo_acquire] followed by an acquire membar
1129   // with a bogus read dependency on it's preceding load. so in those
1130   // cases we will find the load node at the PARMS offset of the
1131   // acquire membar.  n.b. there may be an intervening DecodeN node.
1132   //
1133   // a volatile load derived from an inlined unsafe field access
1134   // manifests as a cpuorder membar with Ctl and Mem projections
1135   // feeding both an acquire membar and a LoadX[mo_acquire]. The
1136   // acquire then feeds another cpuorder membar via Ctl and Mem
1137   // projections. The load has no output dependency on these trailing
1138   // membars because subsequent nodes inserted into the graph take
1139   // their control feed from the final membar cpuorder meaning they
1140   // are all ordered after the load.
1141 
1142   Node *x = barrier->lookup(TypeFunc::Parms);
1143   if (x) {
1144     // we are starting from an acquire and it has a fake dependency
1145     //
1146     // need to check for
1147     //
1148     //   LoadX[mo_acquire]
1149     //   {  |1   }
1150     //   {DecodeN}
1151     //      |Parms
1152     //   MemBarAcquire*
1153     //
1154     // where * tags node we were passed
1155     // and |k means input k
1156     if (x->is_DecodeNarrowPtr())
1157       x = x->in(1);
1158 
1159     return (x->is_Load() && x->as_Load()->is_acquire());
1160   }
1161   
1162   // only continue if we want to try to match unsafe volatile gets
1163   if (UseBarriersForUnsafeVolatileGet)
1164     return false;
1165 
1166   // need to check for
1167   //
1168   //     MemBarCPUOrder
1169   //        ||       \\
1170   //   MemBarAcquire* LoadX[mo_acquire]
1171   //        ||
1172   //   MemBarCPUOrder
1173   //
1174   // where * tags node we were passed
1175   // and || or \\ are Ctl+Mem feeds via intermediate Proj Nodes
1176 
1177   // check for a parent MemBarCPUOrder
1178   ProjNode *ctl;
1179   ProjNode *mem;
1180   MemBarNode *parent = has_parent_membar(barrier, ctl, mem);
1181   if (!parent || parent->Opcode() != Op_MemBarCPUOrder)
1182     return false;
1183   // ensure the proj nodes both feed a LoadX[mo_acquire]
1184   LoadNode *ld = NULL;
1185   for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {
1186     x = ctl->fast_out(i);
1187     // if we see a load we keep hold of it and stop searching
1188     if (x->is_Load()) {
1189       ld = x->as_Load();
1190       break;
1191     }
1192   }
1193   // it must be an acquiring load
1194   if (! ld || ! ld->is_acquire())
1195     return false;
1196   for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
1197     x = mem->fast_out(i);
1198     // if we see the same load we drop it and stop searching
1199     if (x == ld) {
1200       ld = NULL;
1201       break;
1202     }
1203   }
1204   // we must have dropped the load
1205   if (ld)
1206     return false;
1207   // check for a child cpuorder membar
1208   MemBarNode *child  = has_child_membar(barrier->as_MemBar(), ctl, mem);
1209   if (!child || child->Opcode() != Op_MemBarCPUOrder)
1210     return false;
1211 
1212   return true;
1213 }
1214 
1215 bool needs_acquiring_load(const Node *n)
1216 {
1217   // assert n->is_Load();
1218   if (UseBarriersForVolatile)
1219     // we use a normal load and a dmb
1220     return false;
1221 
1222   LoadNode *ld = n->as_Load();
1223 
1224   if (!ld->is_acquire())
1225     return false;
1226 
1227   // check if this load is feeding an acquire membar
1228   //
1229   //   LoadX[mo_acquire]
1230   //   {  |1   }
1231   //   {DecodeN}
1232   //      |Parms
1233   //   MemBarAcquire*
1234   //
1235   // where * tags node we were passed
1236   // and |k means input k
1237 
1238   Node *start = ld;
1239   Node *mbacq = NULL;
1240 
1241   // if we hit a DecodeNarrowPtr we reset the start node and restart
1242   // the search through the outputs
1243  restart:
1244 
1245   for (DUIterator_Fast imax, i = start->fast_outs(imax); i < imax; i++) {
1246     Node *x = start->fast_out(i);
1247     if (x->is_MemBar() && x->Opcode() == Op_MemBarAcquire) {
1248       mbacq = x;
1249     } else if (!mbacq &&
1250                (x->is_DecodeNarrowPtr() ||
1251                 (x->is_Mach() && x->Opcode() == Op_DecodeN))) {
1252       start = x;
1253       goto restart;
1254     }
1255   }
1256 
1257   if (mbacq) {
1258     return true;
1259   }
1260 
1261   // only continue if we want to try to match unsafe volatile gets
1262   if (UseBarriersForUnsafeVolatileGet)
1263     return false;
1264 
1265   // check if Ctl and Proj feed comes from a MemBarCPUOrder
1266   //
1267   //     MemBarCPUOrder
1268   //        ||       \\
1269   //   MemBarAcquire* LoadX[mo_acquire]
1270   //        ||
1271   //   MemBarCPUOrder
1272 
1273   MemBarNode *membar;
1274   ProjNode *ctl;
1275   ProjNode *mem;
1276 
1277   membar = has_parent_membar(ld, ctl, mem);
1278 
1279   if (!membar || !membar->Opcode() == Op_MemBarCPUOrder)
1280     return false;
1281 
1282   // ensure that there is a CPUOrder->Acquire->CPUOrder membar chain
1283 
1284   membar = has_child_membar(membar, ctl, mem);
1285 
1286   if (!membar || !membar->Opcode() == Op_MemBarAcquire)
1287     return false;
1288 
1289   membar = has_child_membar(membar, ctl, mem);
1290   
1291   if (!membar || !membar->Opcode() == Op_MemBarCPUOrder)
1292     return false;
1293 
1294   return true;
1295 }
1296 
1297 bool unnecessary_release(const Node *n) {
1298   // assert n->is_MemBar();
1299   if (UseBarriersForVolatile)
1300     // we need to plant a dmb
1301     return false;
1302 
1303   // ok, so we can omit this release barrier if it has been inserted
1304   // as part of a volatile store sequence
1305   //
1306   //   MemBarRelease
1307   //  {      ||      }
1308   //  {MemBarCPUOrder} -- optional
1309   //         ||     \\
1310   //         ||     StoreX[mo_release]
1311   //         | \     /
1312   //         | MergeMem
1313   //         | /
1314   //   MemBarVolatile
1315   //
1316   // where
1317   //  || and \\ represent Ctl and Mem feeds via Proj nodes
1318   //  | \ and / indicate further routing of the Ctl and Mem feeds
1319   // 
1320   // so we need to check that
1321   //
1322   // ia) the release membar (or its dependent cpuorder membar) feeds
1323   // control to a store node (via a Control project node)
1324   //
1325   // ii) the store is ordered release
1326   //
1327   // iii) the release membar (or its dependent cpuorder membar) feeds
1328   // control to a volatile membar (via the same Control project node)
1329   //
1330   // iv) the release membar feeds memory to a merge mem and to the
1331   // same store (both via a single Memory proj node)
1332   //
1333   // v) the store outputs to the merge mem
1334   //
1335   // vi) the merge mem outputs to the same volatile membar
1336   //
1337   // n.b. if this is an inlined unsafe node then the release membar
1338   // may feed its control and memory links via an intervening cpuorder
1339   // membar. this case can be dealt with when we check the release
1340   // membar projections. if they both feed a single cpuorder membar
1341   // node continue to make the same checks as above but with the
1342   // cpuorder membar substituted for the release membar. if they don't
1343   // both feed a cpuorder membar then the check fails.
1344   //
1345   // n.b.b. for an inlined unsafe store of an object in the case where
1346   // !TypePtr::NULL_PTR->higher_equal(type(heap_base_oop)) we may see
1347   // an embedded if then else where we expect the store. this is
1348   // needed to do the right type of store depending on whether
1349   // heap_base is NULL. We could check for that but for now we can
1350   // just take the hit of on inserting a redundant dmb for this
1351   // redundant volatile membar
1352 
1353   MemBarNode *barrier = n->as_MemBar();
1354   ProjNode *ctl;
1355   ProjNode *mem;
1356   // check for an intervening cpuorder membar
1357   MemBarNode *b = has_child_membar(barrier, ctl, mem);
1358   if (b && b->Opcode() == Op_MemBarCPUOrder) {
1359     // ok, so start form the dependent cpuorder barrier
1360     barrier = b;
1361   }
1362   // check the ctl and mem flow
1363   ctl = barrier->proj_out(TypeFunc::Control);
1364   mem = barrier->proj_out(TypeFunc::Memory);
1365 
1366   // the barrier needs to have both a Ctl and Mem projection
1367   if (! ctl || ! mem)
1368     return false;
1369 
1370   Node *x = NULL;
1371   Node *mbvol = NULL;
1372   StoreNode * st = NULL;
1373 
1374   // For a normal volatile write the Ctl ProjNode should have output
1375   // to a MemBarVolatile and a Store marked as releasing
1376   //
1377   // n.b. for an inlined unsafe store of an object in the case where
1378   // !TypePtr::NULL_PTR->higher_equal(type(heap_base_oop)) we may see
1379   // an embedded if then else where we expect the store. this is
1380   // needed to do the right type of store depending on whether
1381   // heap_base is NULL. We could check for that case too but for now
1382   // we can just take the hit of inserting a dmb and a non-volatile
1383   // store to implement the volatile store
1384 
1385   for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {
1386     x = ctl->fast_out(i);
1387     if (x->is_MemBar() && x->Opcode() == Op_MemBarVolatile) {
1388       if (mbvol) {
1389         return false;
1390       }
1391       mbvol = x;
1392     } else if (x->is_Store()) {
1393       st = x->as_Store();
1394       if (! st->is_release()) {
1395         return false;
1396       }
1397     } else if (!x->is_Mach()) {
1398       // we may see mach nodes added during matching but nothing else
1399       return false;
1400     }
1401   }
1402 
1403   if (!mbvol || !st)
1404     return false;
1405 
1406   // the Mem ProjNode should output to a MergeMem and the same Store
1407   Node *mm = NULL;
1408   for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
1409     x = mem->fast_out(i);
1410     if (!mm && x->is_MergeMem()) {
1411       mm = x;
1412     } else if (x != st && !x->is_Mach()) {
1413       // we may see mach nodes added during matching but nothing else
1414       return false;
1415     }
1416   }
1417 
1418   if (!mm)
1419     return false;
1420 
1421   // the MergeMem should output to the MemBarVolatile
1422   for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {
1423     x = mm->fast_out(i);
1424     if (x != mbvol && !x->is_Mach()) {
1425       // we may see mach nodes added during matching but nothing else
1426       return false;
1427     }
1428   }
1429 
1430   return true;
1431 }
1432 
1433 bool unnecessary_volatile(const Node *n) {
1434   // assert n->is_MemBar();
1435   if (UseBarriersForVolatile)
1436     // we need to plant a dmb
1437     return false;
1438 
1439   // ok, so we can omit this volatile barrier if it has been inserted
1440   // as part of a volatile store sequence
1441   //
1442   //   MemBarRelease
1443   //  {      ||      }
1444   //  {MemBarCPUOrder} -- optional
1445   //         ||     \\
1446   //         ||     StoreX[mo_release]
1447   //         | \     /
1448   //         | MergeMem
1449   //         | /
1450   //   MemBarVolatile
1451   //
1452   // where
1453   //  || and \\ represent Ctl and Mem feeds via Proj nodes
1454   //  | \ and / indicate further routing of the Ctl and Mem feeds
1455   // 
1456   // we need to check that
1457   //
1458   // i) the volatile membar gets its control feed from a release
1459   // membar (or its dependent cpuorder membar) via a Control project
1460   // node
1461   //
1462   // ii) the release membar (or its dependent cpuorder membar) also
1463   // feeds control to a store node via the same proj node
1464   //
1465   // iii) the store is ordered release
1466   //
1467   // iv) the release membar (or its dependent cpuorder membar) feeds
1468   // memory to a merge mem and to the same store (both via a single
1469   // Memory proj node)
1470   //
1471   // v) the store outputs to the merge mem
1472   //
1473   // vi) the merge mem outputs to the volatile membar
1474   //
1475   // n.b. for an inlined unsafe store of an object in the case where
1476   // !TypePtr::NULL_PTR->higher_equal(type(heap_base_oop)) we may see
1477   // an embedded if then else where we expect the store. this is
1478   // needed to do the right type of store depending on whether
1479   // heap_base is NULL. We could check for that but for now we can
1480   // just take the hit of on inserting a redundant dmb for this
1481   // redundant volatile membar
1482 
1483   MemBarNode *mbvol = n->as_MemBar();
1484   Node *x = n->lookup(TypeFunc::Control);
1485 
1486   if (! x || !x->is_Proj())
1487     return false;
1488 
1489   ProjNode *proj = x->as_Proj();
1490 
1491   x = proj->lookup(0);
1492 
1493   if (!x || !x->is_MemBar())
1494     return false;
1495 
1496   MemBarNode *barrier = x->as_MemBar();
1497 
1498   // if the barrier is a release membar we have what we want. if it is
1499   // a cpuorder membar then we need to ensure that it is fed by a
1500   // release membar in which case we proceed to check the graph below
1501   // this cpuorder membar as the feed
1502 
1503   if (x->Opcode() != Op_MemBarRelease) {
1504     if (x->Opcode() != Op_MemBarCPUOrder)
1505       return false;
1506     ProjNode *ctl;
1507     ProjNode *mem;
1508     MemBarNode *b = has_parent_membar(x, ctl, mem);
1509     if (!b || !b->Opcode() == Op_MemBarRelease)
1510       return false;
1511   }
1512 
1513   ProjNode *ctl = barrier->proj_out(TypeFunc::Control);
1514   ProjNode *mem = barrier->proj_out(TypeFunc::Memory);
1515 
1516   // barrier needs to have both a Ctl and Mem projection
1517   // and we need to have reached it via the Ctl projection
1518   if (! ctl || ! mem || ctl != proj)
1519     return false;
1520 
1521   StoreNode * st = NULL;
1522 
1523   // The Ctl ProjNode should have output to a MemBarVolatile and
1524   // a Store marked as releasing
1525   for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {
1526     x = ctl->fast_out(i);
1527     if (x->is_MemBar() && x->Opcode() == Op_MemBarVolatile) {
1528       if (x != mbvol) {
1529         return false;
1530       }
1531     } else if (x->is_Store()) {
1532       st = x->as_Store();
1533       if (! st->is_release()) {
1534         return false;
1535       }
1536     } else if (!x->is_Mach()){
1537       // we may see mach nodes added during matching but nothing else
1538       return false;
1539     }
1540   }
1541 
1542   if (!st)
1543     return false;
1544 
1545   // the Mem ProjNode should output to a MergeMem and the same Store
1546   Node *mm = NULL;
1547   for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
1548     x = mem->fast_out(i);
1549     if (!mm && x->is_MergeMem()) {
1550       mm = x;
1551     } else if (x != st && !x->is_Mach()) {
1552       // we may see mach nodes added during matching but nothing else
1553       return false;
1554     }
1555   }
1556 
1557   if (!mm)
1558     return false;
1559 
1560   // the MergeMem should output to the MemBarVolatile
1561   for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {
1562     x = mm->fast_out(i);
1563     if (x != mbvol && !x->is_Mach()) {
1564       // we may see mach nodes added during matching but nothing else
1565       return false;
1566     }
1567   }
1568 
1569   return true;
1570 }
1571 
1572 
1573 
1574 bool needs_releasing_store(const Node *n)
1575 {
1576   // assert n->is_Store();
1577   if (UseBarriersForVolatile)
1578     // we use a normal store and dmb combination
1579     return false;
1580 
1581   StoreNode *st = n->as_Store();
1582 
1583   if (!st->is_release())
1584     return false;
1585 
1586   // check if this store is bracketed by a release (or its dependent
1587   // cpuorder membar) and a volatile membar
1588   //
1589   //   MemBarRelease
1590   //  {      ||      }
1591   //  {MemBarCPUOrder} -- optional
1592   //         ||     \\
1593   //         ||     StoreX[mo_release]
1594   //         | \     /
1595   //         | MergeMem
1596   //         | /
1597   //   MemBarVolatile
1598   //
1599   // where
1600   //  || and \\ represent Ctl and Mem feeds via Proj nodes
1601   //  | \ and / indicate further routing of the Ctl and Mem feeds
1602   // 
1603 
1604 
1605   Node *x = st->lookup(TypeFunc::Control);
1606 
1607   if (! x || !x->is_Proj())
1608     return false;
1609 
1610   ProjNode *proj = x->as_Proj();
1611 
1612   x = proj->lookup(0);
1613 
1614   if (!x || !x->is_MemBar())
1615     return false;
1616 
1617   MemBarNode *barrier = x->as_MemBar();
1618 
1619   // if the barrier is a release membar we have what we want. if it is
1620   // a cpuorder membar then we need to ensure that it is fed by a
1621   // release membar in which case we proceed to check the graph below
1622   // this cpuorder membar as the feed
1623 
1624   if (x->Opcode() != Op_MemBarRelease) {
1625     if (x->Opcode() != Op_MemBarCPUOrder)
1626       return false;
1627     Node *ctl = x->lookup(TypeFunc::Control);
1628     Node *mem = x->lookup(TypeFunc::Memory);
1629     if (!ctl || !ctl->is_Proj() || !mem || !mem->is_Proj())
1630       return false;
1631     x = ctl->lookup(0);
1632     if (!x || !x->is_MemBar() || !x->Opcode() == Op_MemBarRelease)
1633       return false;
1634     Node *y = mem->lookup(0);
1635     if (!y || y != x)
1636       return false;
1637   }
1638 
1639   ProjNode *ctl = barrier->proj_out(TypeFunc::Control);
1640   ProjNode *mem = barrier->proj_out(TypeFunc::Memory);
1641 
1642   // MemBarRelease needs to have both a Ctl and Mem projection
1643   // and we need to have reached it via the Ctl projection
1644   if (! ctl || ! mem || ctl != proj)
1645     return false;
1646 
1647   MemBarNode *mbvol = NULL;
1648 
1649   // The Ctl ProjNode should have output to a MemBarVolatile and
1650   // a Store marked as releasing
1651   for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {
1652     x = ctl->fast_out(i);
1653     if (x->is_MemBar() && x->Opcode() == Op_MemBarVolatile) {
1654       mbvol = x->as_MemBar();
1655     } else if (x->is_Store()) {
1656       if (x != st) {
1657         return false;
1658       }
1659     } else if (!x->is_Mach()){
1660       return false;
1661     }
1662   }
1663 
1664   if (!mbvol)
1665     return false;
1666 
1667   // the Mem ProjNode should output to a MergeMem and the same Store
1668   Node *mm = NULL;
1669   for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
1670     x = mem->fast_out(i);
1671     if (!mm && x->is_MergeMem()) {
1672       mm = x;
1673     } else if (x != st && !x->is_Mach()) {
1674       return false;
1675     }
1676   }
1677 
1678   if (!mm)
1679     return false;
1680 
1681   // the MergeMem should output to the MemBarVolatile
1682   for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {
1683     x = mm->fast_out(i);
1684     if (x != mbvol && !x->is_Mach()) {
1685       return false;
1686     }
1687   }
1688 
1689   return true;
1690 }
1691 
1692 
1693 
1694 #define __ _masm.
1695 
1696 // advance declarations for helper functions to convert register
1697 // indices to register objects
1698 
1699 // the ad file has to provide implementations of certain methods
1700 // expected by the generic code
1701 //
1702 // REQUIRED FUNCTIONALITY
1703 
1704 //=============================================================================
1705 
1706 // !!!!! Special hack to get all types of calls to specify the byte offset
1707 //       from the start of the call to the point where the return address
1708 //       will point.
1709 
1710 int MachCallStaticJavaNode::ret_addr_offset()
1711 {
1712   // call should be a simple bl
1713   int off = 4;
1714   return off;
1715 }
1716 
1717 int MachCallDynamicJavaNode::ret_addr_offset()
1718 {
1719   return 16; // movz, movk, movk, bl
1720 }
1721 
1722 int MachCallRuntimeNode::ret_addr_offset() {
1723   // for generated stubs the call will be
1724   //   far_call(addr)
1725   // for real runtime callouts it will be six instructions
1726   // see aarch64_enc_java_to_runtime
1727   //   adr(rscratch2, retaddr)
1728   //   lea(rscratch1, RuntimeAddress(addr)
1729   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1730   //   blrt rscratch1
1731   CodeBlob *cb = CodeCache::find_blob(_entry_point);
1732   if (cb) {
1733     return MacroAssembler::far_branch_size();
1734   } else {
1735     return 6 * NativeInstruction::instruction_size;
1736   }
1737 }
1738 
1739 // Indicate if the safepoint node needs the polling page as an input
1740 
1741 // the shared code plants the oop data at the start of the generated
1742 // code for the safepoint node and that needs ot be at the load
1743 // instruction itself. so we cannot plant a mov of the safepoint poll
1744 // address followed by a load. setting this to true means the mov is
1745 // scheduled as a prior instruction. that's better for scheduling
1746 // anyway.
1747 
1748 bool SafePointNode::needs_polling_address_input()
1749 {
1750   return true;
1751 }
1752 
1753 //=============================================================================
1754 
1755 #ifndef PRODUCT
1756 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1757   st->print("BREAKPOINT");
1758 }
1759 #endif
1760 
1761 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1762   MacroAssembler _masm(&cbuf);
1763   __ brk(0);
1764 }
1765 
1766 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1767   return MachNode::size(ra_);
1768 }
1769 
1770 //=============================================================================
1771 
1772 #ifndef PRODUCT
1773   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1774     st->print("nop \t# %d bytes pad for loops and calls", _count);
1775   }
1776 #endif
1777 
1778   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1779     MacroAssembler _masm(&cbuf);
1780     for (int i = 0; i < _count; i++) {
1781       __ nop();
1782     }
1783   }
1784 
1785   uint MachNopNode::size(PhaseRegAlloc*) const {
1786     return _count * NativeInstruction::instruction_size;
1787   }
1788 
1789 //=============================================================================
1790 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1791 
1792 int Compile::ConstantTable::calculate_table_base_offset() const {
1793   return 0;  // absolute addressing, no offset
1794 }
1795 
1796 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1797 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1798   ShouldNotReachHere();
1799 }
1800 
1801 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1802   // Empty encoding
1803 }
1804 
1805 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1806   return 0;
1807 }
1808 
1809 #ifndef PRODUCT
1810 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1811   st->print("-- \t// MachConstantBaseNode (empty encoding)");
1812 }
1813 #endif
1814 
1815 #ifndef PRODUCT
1816 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1817   Compile* C = ra_->C;
1818 
1819   int framesize = C->frame_slots() << LogBytesPerInt;
1820 
1821   if (C->need_stack_bang(framesize))
1822     st->print("# stack bang size=%d\n\t", framesize);
1823 
1824   if (framesize < ((1 << 9) + 2 * wordSize)) {
1825     st->print("sub  sp, sp, #%d\n\t", framesize);
1826     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1827     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
1828   } else {
1829     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1830     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
1831     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1832     st->print("sub  sp, sp, rscratch1");
1833   }
1834 }
1835 #endif
1836 
1837 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1838   Compile* C = ra_->C;
1839   MacroAssembler _masm(&cbuf);
1840 
1841   // n.b. frame size includes space for return pc and rfp
1842   const long framesize = C->frame_size_in_bytes();
1843   assert(framesize%(2*wordSize) == 0, "must preserve 2*wordSize alignment");
1844 
1845   // insert a nop at the start of the prolog so we can patch in a
1846   // branch if we need to invalidate the method later
1847   __ nop();
1848 
1849   int bangsize = C->bang_size_in_bytes();
1850   if (C->need_stack_bang(bangsize) && UseStackBanging)
1851     __ generate_stack_overflow_check(bangsize);
1852 
1853   __ build_frame(framesize);
1854 
1855   if (NotifySimulator) {
1856     __ notify(Assembler::method_entry);
1857   }
1858 
1859   if (VerifyStackAtCalls) {
1860     Unimplemented();
1861   }
1862 
1863   C->set_frame_complete(cbuf.insts_size());
1864 
1865   if (C->has_mach_constant_base_node()) {
1866     // NOTE: We set the table base offset here because users might be
1867     // emitted before MachConstantBaseNode.
1868     Compile::ConstantTable& constant_table = C->constant_table();
1869     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1870   }
1871 }
1872 
1873 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1874 {
1875   return MachNode::size(ra_); // too many variables; just compute it
1876                               // the hard way
1877 }
1878 
1879 int MachPrologNode::reloc() const
1880 {
1881   return 0;
1882 }
1883 
1884 //=============================================================================
1885 
1886 #ifndef PRODUCT
1887 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1888   Compile* C = ra_->C;
1889   int framesize = C->frame_slots() << LogBytesPerInt;
1890 
1891   st->print("# pop frame %d\n\t",framesize);
1892 
1893   if (framesize == 0) {
1894     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1895   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1896     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1897     st->print("add  sp, sp, #%d\n\t", framesize);
1898   } else {
1899     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1900     st->print("add  sp, sp, rscratch1\n\t");
1901     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1902   }
1903 
1904   if (do_polling() && C->is_method_compilation()) {
1905     st->print("# touch polling page\n\t");
1906     st->print("mov  rscratch1, #0x%lx\n\t", p2i(os::get_polling_page()));
1907     st->print("ldr zr, [rscratch1]");
1908   }
1909 }
1910 #endif
1911 
1912 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1913   Compile* C = ra_->C;
1914   MacroAssembler _masm(&cbuf);
1915   int framesize = C->frame_slots() << LogBytesPerInt;
1916 
1917   __ remove_frame(framesize);
1918 
1919   if (NotifySimulator) {
1920     __ notify(Assembler::method_reentry);
1921   }
1922 
1923   if (do_polling() && C->is_method_compilation()) {
1924     __ read_polling_page(rscratch1, os::get_polling_page(), relocInfo::poll_return_type);
1925   }
1926 }
1927 
1928 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1929   // Variable size. Determine dynamically.
1930   return MachNode::size(ra_);
1931 }
1932 
1933 int MachEpilogNode::reloc() const {
1934   // Return number of relocatable values contained in this instruction.
1935   return 1; // 1 for polling page.
1936 }
1937 
1938 const Pipeline * MachEpilogNode::pipeline() const {
1939   return MachNode::pipeline_class();
1940 }
1941 
1942 // This method seems to be obsolete. It is declared in machnode.hpp
1943 // and defined in all *.ad files, but it is never called. Should we
1944 // get rid of it?
1945 int MachEpilogNode::safepoint_offset() const {
1946   assert(do_polling(), "no return for this epilog node");
1947   return 4;
1948 }
1949 
1950 //=============================================================================
1951 
1952 // Figure out which register class each belongs in: rc_int, rc_float or
1953 // rc_stack.
1954 enum RC { rc_bad, rc_int, rc_float, rc_stack };
1955 
1956 static enum RC rc_class(OptoReg::Name reg) {
1957 
1958   if (reg == OptoReg::Bad) {
1959     return rc_bad;
1960   }
1961 
1962   // we have 30 int registers * 2 halves
1963   // (rscratch1 and rscratch2 are omitted)
1964 
1965   if (reg < 60) {
1966     return rc_int;
1967   }
1968 
1969   // we have 32 float register * 2 halves
1970   if (reg < 60 + 64) {
1971     return rc_float;
1972   }
1973 
1974   // Between float regs & stack is the flags regs.
1975   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1976 
1977   return rc_stack;
1978 }
1979 
1980 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1981   Compile* C = ra_->C;
1982 
1983   // Get registers to move.
1984   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1985   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1986   OptoReg::Name dst_hi = ra_->get_reg_second(this);
1987   OptoReg::Name dst_lo = ra_->get_reg_first(this);
1988 
1989   enum RC src_hi_rc = rc_class(src_hi);
1990   enum RC src_lo_rc = rc_class(src_lo);
1991   enum RC dst_hi_rc = rc_class(dst_hi);
1992   enum RC dst_lo_rc = rc_class(dst_lo);
1993 
1994   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1995 
1996   if (src_hi != OptoReg::Bad) {
1997     assert((src_lo&1)==0 && src_lo+1==src_hi &&
1998            (dst_lo&1)==0 && dst_lo+1==dst_hi,
1999            "expected aligned-adjacent pairs");
2000   }
2001 
2002   if (src_lo == dst_lo && src_hi == dst_hi) {
2003     return 0;            // Self copy, no move.
2004   }
2005 
2006   switch (src_lo_rc) {
2007   case rc_int:
2008     if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
2009       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2010           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2011           // 64 bit
2012         if (cbuf) {
2013           MacroAssembler _masm(cbuf);
2014           __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2015                  as_Register(Matcher::_regEncode[src_lo]));
2016         } else if (st) {
2017           st->print("mov  %s, %s\t# shuffle",
2018                     Matcher::regName[dst_lo],
2019                     Matcher::regName[src_lo]);
2020         }
2021       } else {
2022         // 32 bit
2023         if (cbuf) {
2024           MacroAssembler _masm(cbuf);
2025           __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2026                   as_Register(Matcher::_regEncode[src_lo]));
2027         } else if (st) {
2028           st->print("movw  %s, %s\t# shuffle",
2029                     Matcher::regName[dst_lo],
2030                     Matcher::regName[src_lo]);
2031         }
2032       }
2033     } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2034       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2035           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2036           // 64 bit
2037         if (cbuf) {
2038           MacroAssembler _masm(cbuf);
2039           __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2040                    as_Register(Matcher::_regEncode[src_lo]));
2041         } else if (st) {
2042           st->print("fmovd  %s, %s\t# shuffle",
2043                     Matcher::regName[dst_lo],
2044                     Matcher::regName[src_lo]);
2045         }
2046       } else {
2047         // 32 bit
2048         if (cbuf) {
2049           MacroAssembler _masm(cbuf);
2050           __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2051                    as_Register(Matcher::_regEncode[src_lo]));
2052         } else if (st) {
2053           st->print("fmovs  %s, %s\t# shuffle",
2054                     Matcher::regName[dst_lo],
2055                     Matcher::regName[src_lo]);
2056         }
2057       }
2058     } else {                    // gpr --> stack spill
2059       assert(dst_lo_rc == rc_stack, "spill to bad register class");
2060       int dst_offset = ra_->reg2offset(dst_lo);
2061       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2062           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2063           // 64 bit
2064         if (cbuf) {
2065           MacroAssembler _masm(cbuf);
2066           __ str(as_Register(Matcher::_regEncode[src_lo]),
2067                  Address(sp, dst_offset));
2068         } else if (st) {
2069           st->print("str  %s, [sp, #%d]\t# spill",
2070                     Matcher::regName[src_lo],
2071                     dst_offset);
2072         }
2073       } else {
2074         // 32 bit
2075         if (cbuf) {
2076           MacroAssembler _masm(cbuf);
2077           __ strw(as_Register(Matcher::_regEncode[src_lo]),
2078                  Address(sp, dst_offset));
2079         } else if (st) {
2080           st->print("strw  %s, [sp, #%d]\t# spill",
2081                     Matcher::regName[src_lo],
2082                     dst_offset);
2083         }
2084       }
2085     }
2086     return 4;
2087   case rc_float:
2088     if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
2089       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2090           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2091           // 64 bit
2092         if (cbuf) {
2093           MacroAssembler _masm(cbuf);
2094           __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2095                    as_FloatRegister(Matcher::_regEncode[src_lo]));
2096         } else if (st) {
2097           st->print("fmovd  %s, %s\t# shuffle",
2098                     Matcher::regName[dst_lo],
2099                     Matcher::regName[src_lo]);
2100         }
2101       } else {
2102         // 32 bit
2103         if (cbuf) {
2104           MacroAssembler _masm(cbuf);
2105           __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2106                    as_FloatRegister(Matcher::_regEncode[src_lo]));
2107         } else if (st) {
2108           st->print("fmovs  %s, %s\t# shuffle",
2109                     Matcher::regName[dst_lo],
2110                     Matcher::regName[src_lo]);
2111         }
2112       }
2113     } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2114       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2115           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2116           // 64 bit
2117         if (cbuf) {
2118           MacroAssembler _masm(cbuf);
2119           __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2120                    as_FloatRegister(Matcher::_regEncode[src_lo]));
2121         } else if (st) {
2122           st->print("fmovd  %s, %s\t# shuffle",
2123                     Matcher::regName[dst_lo],
2124                     Matcher::regName[src_lo]);
2125         }
2126       } else {
2127         // 32 bit
2128         if (cbuf) {
2129           MacroAssembler _masm(cbuf);
2130           __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2131                    as_FloatRegister(Matcher::_regEncode[src_lo]));
2132         } else if (st) {
2133           st->print("fmovs  %s, %s\t# shuffle",
2134                     Matcher::regName[dst_lo],
2135                     Matcher::regName[src_lo]);
2136         }
2137       }
2138     } else {                    // fpr --> stack spill
2139       assert(dst_lo_rc == rc_stack, "spill to bad register class");
2140       int dst_offset = ra_->reg2offset(dst_lo);
2141       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2142           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2143           // 64 bit
2144         if (cbuf) {
2145           MacroAssembler _masm(cbuf);
2146           __ strd(as_FloatRegister(Matcher::_regEncode[src_lo]),
2147                  Address(sp, dst_offset));
2148         } else if (st) {
2149           st->print("strd  %s, [sp, #%d]\t# spill",
2150                     Matcher::regName[src_lo],
2151                     dst_offset);
2152         }
2153       } else {
2154         // 32 bit
2155         if (cbuf) {
2156           MacroAssembler _masm(cbuf);
2157           __ strs(as_FloatRegister(Matcher::_regEncode[src_lo]),
2158                  Address(sp, dst_offset));
2159         } else if (st) {
2160           st->print("strs  %s, [sp, #%d]\t# spill",
2161                     Matcher::regName[src_lo],
2162                     dst_offset);
2163         }
2164       }
2165     }
2166     return 4;
2167   case rc_stack:
2168     int src_offset = ra_->reg2offset(src_lo);
2169     if (dst_lo_rc == rc_int) {  // stack --> gpr load
2170       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2171           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2172           // 64 bit
2173         if (cbuf) {
2174           MacroAssembler _masm(cbuf);
2175           __ ldr(as_Register(Matcher::_regEncode[dst_lo]),
2176                  Address(sp, src_offset));
2177         } else if (st) {
2178           st->print("ldr  %s, [sp, %d]\t# restore",
2179                     Matcher::regName[dst_lo],
2180                     src_offset);
2181         }
2182       } else {
2183         // 32 bit
2184         if (cbuf) {
2185           MacroAssembler _masm(cbuf);
2186           __ ldrw(as_Register(Matcher::_regEncode[dst_lo]),
2187                   Address(sp, src_offset));
2188         } else if (st) {
2189           st->print("ldr  %s, [sp, %d]\t# restore",
2190                     Matcher::regName[dst_lo],
2191                    src_offset);
2192         }
2193       }
2194       return 4;
2195     } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2196       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2197           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2198           // 64 bit
2199         if (cbuf) {
2200           MacroAssembler _masm(cbuf);
2201           __ ldrd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2202                  Address(sp, src_offset));
2203         } else if (st) {
2204           st->print("ldrd  %s, [sp, %d]\t# restore",
2205                     Matcher::regName[dst_lo],
2206                     src_offset);
2207         }
2208       } else {
2209         // 32 bit
2210         if (cbuf) {
2211           MacroAssembler _masm(cbuf);
2212           __ ldrs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2213                   Address(sp, src_offset));
2214         } else if (st) {
2215           st->print("ldrs  %s, [sp, %d]\t# restore",
2216                     Matcher::regName[dst_lo],
2217                    src_offset);
2218         }
2219       }
2220       return 4;
2221     } else {                    // stack --> stack copy
2222       assert(dst_lo_rc == rc_stack, "spill to bad register class");
2223       int dst_offset = ra_->reg2offset(dst_lo);
2224       if (((src_lo & 1) == 0 && src_lo + 1 == src_hi) &&
2225           (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi) {
2226           // 64 bit
2227         if (cbuf) {
2228           MacroAssembler _masm(cbuf);
2229           __ ldr(rscratch1, Address(sp, src_offset));
2230           __ str(rscratch1, Address(sp, dst_offset));
2231         } else if (st) {
2232           st->print("ldr  rscratch1, [sp, %d]\t# mem-mem spill",
2233                     src_offset);
2234           st->print("\n\t");
2235           st->print("str  rscratch1, [sp, %d]",
2236                     dst_offset);
2237         }
2238       } else {
2239         // 32 bit
2240         if (cbuf) {
2241           MacroAssembler _masm(cbuf);
2242           __ ldrw(rscratch1, Address(sp, src_offset));
2243           __ strw(rscratch1, Address(sp, dst_offset));
2244         } else if (st) {
2245           st->print("ldrw  rscratch1, [sp, %d]\t# mem-mem spill",
2246                     src_offset);
2247           st->print("\n\t");
2248           st->print("strw  rscratch1, [sp, %d]",
2249                     dst_offset);
2250         }
2251       }
2252       return 8;
2253     }
2254   }
2255 
2256   assert(false," bad rc_class for spill ");
2257   Unimplemented();
2258   return 0;
2259 
2260 }
2261 
2262 #ifndef PRODUCT
2263 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2264   if (!ra_)
2265     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2266   else
2267     implementation(NULL, ra_, false, st);
2268 }
2269 #endif
2270 
2271 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2272   implementation(&cbuf, ra_, false, NULL);
2273 }
2274 
2275 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2276   return implementation(NULL, ra_, true, NULL);
2277 }
2278 
2279 //=============================================================================
2280 
2281 #ifndef PRODUCT
2282 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2283   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2284   int reg = ra_->get_reg_first(this);
2285   st->print("add %s, rsp, #%d]\t# box lock",
2286             Matcher::regName[reg], offset);
2287 }
2288 #endif
2289 
2290 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2291   MacroAssembler _masm(&cbuf);
2292 
2293   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2294   int reg    = ra_->get_encode(this);
2295 
2296   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2297     __ add(as_Register(reg), sp, offset);
2298   } else {
2299     ShouldNotReachHere();
2300   }
2301 }
2302 
2303 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2304   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2305   return 4;
2306 }
2307 
2308 //=============================================================================
2309 
2310 #ifndef PRODUCT
2311 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2312 {
2313   st->print_cr("# MachUEPNode");
2314   if (UseCompressedClassPointers) {
2315     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2316     if (Universe::narrow_klass_shift() != 0) {
2317       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2318     }
2319   } else {
2320    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2321   }
2322   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2323   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2324 }
2325 #endif
2326 
2327 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2328 {
2329   // This is the unverified entry point.
2330   MacroAssembler _masm(&cbuf);
2331 
2332   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2333   Label skip;
2334   // TODO
2335   // can we avoid this skip and still use a reloc?
2336   __ br(Assembler::EQ, skip);
2337   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2338   __ bind(skip);
2339 }
2340 
2341 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2342 {
2343   return MachNode::size(ra_);
2344 }
2345 
2346 // REQUIRED EMIT CODE
2347 
2348 //=============================================================================
2349 
2350 // Emit exception handler code.
2351 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2352 {
2353   // mov rscratch1 #exception_blob_entry_point
2354   // br rscratch1
2355   // Note that the code buffer's insts_mark is always relative to insts.
2356   // That's why we must use the macroassembler to generate a handler.
2357   MacroAssembler _masm(&cbuf);
2358   address base =
2359   __ start_a_stub(size_exception_handler());
2360   if (base == NULL)  return 0;  // CodeBuffer::expand failed
2361   int offset = __ offset();
2362   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2363   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2364   __ end_a_stub();
2365   return offset;
2366 }
2367 
2368 // Emit deopt handler code.
2369 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2370 {
2371   // Note that the code buffer's insts_mark is always relative to insts.
2372   // That's why we must use the macroassembler to generate a handler.
2373   MacroAssembler _masm(&cbuf);
2374   address base =
2375   __ start_a_stub(size_deopt_handler());
2376   if (base == NULL)  return 0;  // CodeBuffer::expand failed
2377   int offset = __ offset();
2378 
2379   __ adr(lr, __ pc());
2380   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2381 
2382   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
2383   __ end_a_stub();
2384   return offset;
2385 }
2386 
2387 // REQUIRED MATCHER CODE
2388 
2389 //=============================================================================
2390 
2391 const bool Matcher::match_rule_supported(int opcode) {
2392 
2393   // TODO
2394   // identify extra cases that we might want to provide match rules for
2395   // e.g. Op_StrEquals and other intrinsics
2396   if (!has_match_rule(opcode)) {
2397     return false;
2398   }
2399 
2400   return true;  // Per default match rules are supported.
2401 }
2402 
2403 int Matcher::regnum_to_fpu_offset(int regnum)
2404 {
2405   Unimplemented();
2406   return 0;
2407 }
2408 
2409 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset)
2410 {
2411   Unimplemented();
2412   return false;
2413 }
2414 
2415 const bool Matcher::isSimpleConstant64(jlong value) {
2416   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
2417   // Probably always true, even if a temp register is required.
2418   return true;
2419 }
2420 
2421 // true just means we have fast l2f conversion
2422 const bool Matcher::convL2FSupported(void) {
2423   return true;
2424 }
2425 
2426 // Vector width in bytes.
2427 const int Matcher::vector_width_in_bytes(BasicType bt) {
2428   // TODO fixme
2429   return 0;
2430 }
2431 
2432 // Limits on vector size (number of elements) loaded into vector.
2433 const int Matcher::max_vector_size(const BasicType bt) {
2434   return vector_width_in_bytes(bt)/type2aelembytes(bt);
2435 }
2436 const int Matcher::min_vector_size(const BasicType bt) {
2437   int max_size = max_vector_size(bt);
2438   // Min size which can be loaded into vector is 4 bytes.
2439   int size = (type2aelembytes(bt) == 1) ? 4 : 2;
2440   return MIN2(size,max_size);
2441 }
2442 
2443 // Vector ideal reg.
2444 const int Matcher::vector_ideal_reg(int len) {
2445   // TODO fixme
2446   return Op_RegD;
2447 }
2448 
2449 // Only lowest bits of xmm reg are used for vector shift count.
2450 const int Matcher::vector_shift_count_ideal_reg(int size) {
2451   // TODO fixme
2452   return Op_RegL;
2453 }
2454 
2455 // AES support not yet implemented
2456 const bool Matcher::pass_original_key_for_aes() {
2457   return false;
2458 }
2459 
2460 // x86 supports misaligned vectors store/load.
2461 const bool Matcher::misaligned_vectors_ok() {
2462   // TODO fixme
2463   // return !AlignVector; // can be changed by flag
2464   return false;
2465 }
2466 
2467 // false => size gets scaled to BytesPerLong, ok.
2468 const bool Matcher::init_array_count_is_in_bytes = false;
2469 
2470 // Threshold size for cleararray.
2471 const int Matcher::init_array_short_size = 18 * BytesPerLong;
2472 
2473 // Use conditional move (CMOVL)
2474 const int Matcher::long_cmove_cost() {
2475   // long cmoves are no more expensive than int cmoves
2476   return 0;
2477 }
2478 
2479 const int Matcher::float_cmove_cost() {
2480   // float cmoves are no more expensive than int cmoves
2481   return 0;
2482 }
2483 
2484 // Does the CPU require late expand (see block.cpp for description of late expand)?
2485 const bool Matcher::require_postalloc_expand = false;
2486 
2487 // Should the Matcher clone shifts on addressing modes, expecting them
2488 // to be subsumed into complex addressing expressions or compute them
2489 // into registers?  True for Intel but false for most RISCs
2490 const bool Matcher::clone_shift_expressions = false;
2491 
2492 // Do we need to mask the count passed to shift instructions or does
2493 // the cpu only look at the lower 5/6 bits anyway?
2494 const bool Matcher::need_masked_shift_count = false;
2495 
2496 // This affects two different things:
2497 //  - how Decode nodes are matched
2498 //  - how ImplicitNullCheck opportunities are recognized
2499 // If true, the matcher will try to remove all Decodes and match them
2500 // (as operands) into nodes. NullChecks are not prepared to deal with
2501 // Decodes by final_graph_reshaping().
2502 // If false, final_graph_reshaping() forces the decode behind the Cmp
2503 // for a NullCheck. The matcher matches the Decode node into a register.
2504 // Implicit_null_check optimization moves the Decode along with the
2505 // memory operation back up before the NullCheck.
2506 bool Matcher::narrow_oop_use_complex_address() {
2507   return Universe::narrow_oop_shift() == 0;
2508 }
2509 
2510 bool Matcher::narrow_klass_use_complex_address() {
2511 // TODO
2512 // decide whether we need to set this to true
2513   return false;
2514 }
2515 
2516 // Is it better to copy float constants, or load them directly from
2517 // memory?  Intel can load a float constant from a direct address,
2518 // requiring no extra registers.  Most RISCs will have to materialize
2519 // an address into a register first, so they would do better to copy
2520 // the constant from stack.
2521 const bool Matcher::rematerialize_float_constants = false;
2522 
2523 // If CPU can load and store mis-aligned doubles directly then no
2524 // fixup is needed.  Else we split the double into 2 integer pieces
2525 // and move it piece-by-piece.  Only happens when passing doubles into
2526 // C code as the Java calling convention forces doubles to be aligned.
2527 const bool Matcher::misaligned_doubles_ok = true;
2528 
2529 // No-op on amd64
2530 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
2531   Unimplemented();
2532 }
2533 
2534 // Advertise here if the CPU requires explicit rounding operations to
2535 // implement the UseStrictFP mode.
2536 const bool Matcher::strict_fp_requires_explicit_rounding = false;
2537 
2538 // Are floats converted to double when stored to stack during
2539 // deoptimization?
2540 bool Matcher::float_in_double() { return true; }
2541 
2542 // Do ints take an entire long register or just half?
2543 // The relevant question is how the int is callee-saved:
2544 // the whole long is written but de-opt'ing will have to extract
2545 // the relevant 32 bits.
2546 const bool Matcher::int_in_long = true;
2547 
2548 // Return whether or not this register is ever used as an argument.
2549 // This function is used on startup to build the trampoline stubs in
2550 // generateOptoStub.  Registers not mentioned will be killed by the VM
2551 // call in the trampoline, and arguments in those registers not be
2552 // available to the callee.
2553 bool Matcher::can_be_java_arg(int reg)
2554 {
2555   return
2556     reg ==  R0_num || reg == R0_H_num ||
2557     reg ==  R1_num || reg == R1_H_num ||
2558     reg ==  R2_num || reg == R2_H_num ||
2559     reg ==  R3_num || reg == R3_H_num ||
2560     reg ==  R4_num || reg == R4_H_num ||
2561     reg ==  R5_num || reg == R5_H_num ||
2562     reg ==  R6_num || reg == R6_H_num ||
2563     reg ==  R7_num || reg == R7_H_num ||
2564     reg ==  V0_num || reg == V0_H_num ||
2565     reg ==  V1_num || reg == V1_H_num ||
2566     reg ==  V2_num || reg == V2_H_num ||
2567     reg ==  V3_num || reg == V3_H_num ||
2568     reg ==  V4_num || reg == V4_H_num ||
2569     reg ==  V5_num || reg == V5_H_num ||
2570     reg ==  V6_num || reg == V6_H_num ||
2571     reg ==  V7_num || reg == V7_H_num;
2572 }
2573 
2574 bool Matcher::is_spillable_arg(int reg)
2575 {
2576   return can_be_java_arg(reg);
2577 }
2578 
2579 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2580   return false;
2581 }
2582 
2583 RegMask Matcher::divI_proj_mask() {
2584   ShouldNotReachHere();
2585   return RegMask();
2586 }
2587 
2588 // Register for MODI projection of divmodI.
2589 RegMask Matcher::modI_proj_mask() {
2590   ShouldNotReachHere();
2591   return RegMask();
2592 }
2593 
2594 // Register for DIVL projection of divmodL.
2595 RegMask Matcher::divL_proj_mask() {
2596   ShouldNotReachHere();
2597   return RegMask();
2598 }
2599 
2600 // Register for MODL projection of divmodL.
2601 RegMask Matcher::modL_proj_mask() {
2602   ShouldNotReachHere();
2603   return RegMask();
2604 }
2605 
2606 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2607   return FP_REG_mask();
2608 }
2609 
2610 // helper for encoding java_to_runtime calls on sim
2611 //
2612 // this is needed to compute the extra arguments required when
2613 // planting a call to the simulator blrt instruction. the TypeFunc
2614 // can be queried to identify the counts for integral, and floating
2615 // arguments and the return type
2616 
2617 static void getCallInfo(const TypeFunc *tf, int &gpcnt, int &fpcnt, int &rtype)
2618 {
2619   int gps = 0;
2620   int fps = 0;
2621   const TypeTuple *domain = tf->domain();
2622   int max = domain->cnt();
2623   for (int i = TypeFunc::Parms; i < max; i++) {
2624     const Type *t = domain->field_at(i);
2625     switch(t->basic_type()) {
2626     case T_FLOAT:
2627     case T_DOUBLE:
2628       fps++;
2629     default:
2630       gps++;
2631     }
2632   }
2633   gpcnt = gps;
2634   fpcnt = fps;
2635   BasicType rt = tf->return_type();
2636   switch (rt) {
2637   case T_VOID:
2638     rtype = MacroAssembler::ret_type_void;
2639     break;
2640   default:
2641     rtype = MacroAssembler::ret_type_integral;
2642     break;
2643   case T_FLOAT:
2644     rtype = MacroAssembler::ret_type_float;
2645     break;
2646   case T_DOUBLE:
2647     rtype = MacroAssembler::ret_type_double;
2648     break;
2649   }
2650 }
2651 
2652 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
2653   MacroAssembler _masm(&cbuf);                                          \
2654   {                                                                     \
2655     guarantee(INDEX == -1, "mode not permitted for volatile");          \
2656     guarantee(DISP == 0, "mode not permitted for volatile");            \
2657     guarantee(SCALE == 0, "mode not permitted for volatile");           \
2658     __ INSN(REG, as_Register(BASE));                                    \
2659   }
2660 
2661 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2662 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2663 
2664   // Used for all non-volatile memory accesses.  The use of
2665   // $mem->opcode() to discover whether this pattern uses sign-extended
2666   // offsets is something of a kludge.
2667   static void loadStore(MacroAssembler masm, mem_insn insn,
2668                          Register reg, int opcode,
2669                          Register base, int index, int size, int disp)
2670   {
2671     Address::extend scale;
2672 
2673     // Hooboy, this is fugly.  We need a way to communicate to the
2674     // encoder that the index needs to be sign extended, so we have to
2675     // enumerate all the cases.
2676     switch (opcode) {
2677     case INDINDEXSCALEDOFFSETI2L:
2678     case INDINDEXSCALEDI2L:
2679     case INDINDEXSCALEDOFFSETI2LN:
2680     case INDINDEXSCALEDI2LN:
2681     case INDINDEXOFFSETI2L:
2682     case INDINDEXOFFSETI2LN:
2683       scale = Address::sxtw(size);
2684       break;
2685     default:
2686       scale = Address::lsl(size);
2687     }
2688 
2689     if (index == -1) {
2690       (masm.*insn)(reg, Address(base, disp));
2691     } else {
2692       if (disp == 0) {
2693         (masm.*insn)(reg, Address(base, as_Register(index), scale));
2694       } else {
2695         masm.lea(rscratch1, Address(base, disp));
2696         (masm.*insn)(reg, Address(rscratch1, as_Register(index), scale));
2697       }
2698     }
2699   }
2700 
2701   static void loadStore(MacroAssembler masm, mem_float_insn insn,
2702                          FloatRegister reg, int opcode,
2703                          Register base, int index, int size, int disp)
2704   {
2705     Address::extend scale;
2706 
2707     switch (opcode) {
2708     case INDINDEXSCALEDOFFSETI2L:
2709     case INDINDEXSCALEDI2L:
2710     case INDINDEXSCALEDOFFSETI2LN:
2711     case INDINDEXSCALEDI2LN:
2712       scale = Address::sxtw(size);
2713       break;
2714     default:
2715       scale = Address::lsl(size);
2716     }
2717 
2718      if (index == -1) {
2719       (masm.*insn)(reg, Address(base, disp));
2720     } else {
2721       if (disp == 0) {
2722         (masm.*insn)(reg, Address(base, as_Register(index), scale));
2723       } else {
2724         masm.lea(rscratch1, Address(base, disp));
2725         (masm.*insn)(reg, Address(rscratch1, as_Register(index), scale));
2726       }
2727     }
2728   }
2729 
2730 %}
2731 
2732 
2733 
2734 //----------ENCODING BLOCK-----------------------------------------------------
2735 // This block specifies the encoding classes used by the compiler to
2736 // output byte streams.  Encoding classes are parameterized macros
2737 // used by Machine Instruction Nodes in order to generate the bit
2738 // encoding of the instruction.  Operands specify their base encoding
2739 // interface with the interface keyword.  There are currently
2740 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2741 // COND_INTER.  REG_INTER causes an operand to generate a function
2742 // which returns its register number when queried.  CONST_INTER causes
2743 // an operand to generate a function which returns the value of the
2744 // constant when queried.  MEMORY_INTER causes an operand to generate
2745 // four functions which return the Base Register, the Index Register,
2746 // the Scale Value, and the Offset Value of the operand when queried.
2747 // COND_INTER causes an operand to generate six functions which return
2748 // the encoding code (ie - encoding bits for the instruction)
2749 // associated with each basic boolean condition for a conditional
2750 // instruction.
2751 //
2752 // Instructions specify two basic values for encoding.  Again, a
2753 // function is available to check if the constant displacement is an
2754 // oop. They use the ins_encode keyword to specify their encoding
2755 // classes (which must be a sequence of enc_class names, and their
2756 // parameters, specified in the encoding block), and they use the
2757 // opcode keyword to specify, in order, their primary, secondary, and
2758 // tertiary opcode.  Only the opcode sections which a particular
2759 // instruction needs for encoding need to be specified.
2760 encode %{
2761   // Build emit functions for each basic byte or larger field in the
2762   // intel encoding scheme (opcode, rm, sib, immediate), and call them
2763   // from C++ code in the enc_class source block.  Emit functions will
2764   // live in the main source block for now.  In future, we can
2765   // generalize this by adding a syntax that specifies the sizes of
2766   // fields in an order, so that the adlc can build the emit functions
2767   // automagically
2768 
2769   // catch all for unimplemented encodings
2770   enc_class enc_unimplemented %{
2771     MacroAssembler _masm(&cbuf);
2772     __ unimplemented("C2 catch all");
2773   %}
2774 
2775   // BEGIN Non-volatile memory access
2776 
2777   enc_class aarch64_enc_ldrsbw(iRegI dst, memory mem) %{
2778     Register dst_reg = as_Register($dst$$reg);
2779     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2780                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2781   %}
2782 
2783   enc_class aarch64_enc_ldrsb(iRegI dst, memory mem) %{
2784     Register dst_reg = as_Register($dst$$reg);
2785     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2786                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2787   %}
2788 
2789   enc_class aarch64_enc_ldrb(iRegI dst, memory mem) %{
2790     Register dst_reg = as_Register($dst$$reg);
2791     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2792                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2793   %}
2794 
2795   enc_class aarch64_enc_ldrb(iRegL dst, memory mem) %{
2796     Register dst_reg = as_Register($dst$$reg);
2797     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2798                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2799   %}
2800 
2801   enc_class aarch64_enc_ldrshw(iRegI dst, memory mem) %{
2802     Register dst_reg = as_Register($dst$$reg);
2803     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2804                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2805   %}
2806 
2807   enc_class aarch64_enc_ldrsh(iRegI dst, memory mem) %{
2808     Register dst_reg = as_Register($dst$$reg);
2809     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2810                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2811   %}
2812 
2813   enc_class aarch64_enc_ldrh(iRegI dst, memory mem) %{
2814     Register dst_reg = as_Register($dst$$reg);
2815     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2816                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2817   %}
2818 
2819   enc_class aarch64_enc_ldrh(iRegL dst, memory mem) %{
2820     Register dst_reg = as_Register($dst$$reg);
2821     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2822                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2823   %}
2824 
2825   enc_class aarch64_enc_ldrw(iRegI dst, memory mem) %{
2826     Register dst_reg = as_Register($dst$$reg);
2827     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2828                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2829   %}
2830 
2831   enc_class aarch64_enc_ldrw(iRegL dst, memory mem) %{
2832     Register dst_reg = as_Register($dst$$reg);
2833     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2834                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2835   %}
2836 
2837   enc_class aarch64_enc_ldrsw(iRegL dst, memory mem) %{
2838     Register dst_reg = as_Register($dst$$reg);
2839     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2840                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2841   %}
2842 
2843   enc_class aarch64_enc_ldr(iRegL dst, memory mem) %{
2844     Register dst_reg = as_Register($dst$$reg);
2845     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2846                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2847   %}
2848 
2849   enc_class aarch64_enc_ldrs(vRegF dst, memory mem) %{
2850     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2851     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2852                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2853   %}
2854 
2855   enc_class aarch64_enc_ldrd(vRegD dst, memory mem) %{
2856     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2857     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2858                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2859   %}
2860 
2861   enc_class aarch64_enc_strb(iRegI src, memory mem) %{
2862     Register src_reg = as_Register($src$$reg);
2863     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2864                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2865   %}
2866 
2867   enc_class aarch64_enc_strb0(memory mem) %{
2868     MacroAssembler _masm(&cbuf);
2869     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2870                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2871   %}
2872 
2873   enc_class aarch64_enc_strh(iRegI src, memory mem) %{
2874     Register src_reg = as_Register($src$$reg);
2875     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2876                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2877   %}
2878 
2879   enc_class aarch64_enc_strh0(memory mem) %{
2880     MacroAssembler _masm(&cbuf);
2881     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2882                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2883   %}
2884 
2885   enc_class aarch64_enc_strw(iRegI src, memory mem) %{
2886     Register src_reg = as_Register($src$$reg);
2887     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2888                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2889   %}
2890 
2891   enc_class aarch64_enc_strw0(memory mem) %{
2892     MacroAssembler _masm(&cbuf);
2893     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2894                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2895   %}
2896 
2897   enc_class aarch64_enc_str(iRegL src, memory mem) %{
2898     Register src_reg = as_Register($src$$reg);
2899     // we sometimes get asked to store the stack pointer into the
2900     // current thread -- we cannot do that directly on AArch64
2901     if (src_reg == r31_sp) {
2902       MacroAssembler _masm(&cbuf);
2903       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2904       __ mov(rscratch2, sp);
2905       src_reg = rscratch2;
2906     }
2907     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2908                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2909   %}
2910 
2911   enc_class aarch64_enc_str0(memory mem) %{
2912     MacroAssembler _masm(&cbuf);
2913     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2914                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2915   %}
2916 
2917   enc_class aarch64_enc_strs(vRegF src, memory mem) %{
2918     FloatRegister src_reg = as_FloatRegister($src$$reg);
2919     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2920                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2921   %}
2922 
2923   enc_class aarch64_enc_strd(vRegD src, memory mem) %{
2924     FloatRegister src_reg = as_FloatRegister($src$$reg);
2925     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2926                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2927   %}
2928 
2929   // END Non-volatile memory access
2930 
2931   // volatile loads and stores
2932 
2933   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
2934     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2935                  rscratch1, stlrb);
2936   %}
2937 
2938   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
2939     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2940                  rscratch1, stlrh);
2941   %}
2942 
2943   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
2944     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2945                  rscratch1, stlrw);
2946   %}
2947 
2948 
2949   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
2950     Register dst_reg = as_Register($dst$$reg);
2951     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2952              rscratch1, ldarb);
2953     __ sxtbw(dst_reg, dst_reg);
2954   %}
2955 
2956   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
2957     Register dst_reg = as_Register($dst$$reg);
2958     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2959              rscratch1, ldarb);
2960     __ sxtb(dst_reg, dst_reg);
2961   %}
2962 
2963   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
2964     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2965              rscratch1, ldarb);
2966   %}
2967 
2968   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
2969     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2970              rscratch1, ldarb);
2971   %}
2972 
2973   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
2974     Register dst_reg = as_Register($dst$$reg);
2975     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2976              rscratch1, ldarh);
2977     __ sxthw(dst_reg, dst_reg);
2978   %}
2979 
2980   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
2981     Register dst_reg = as_Register($dst$$reg);
2982     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2983              rscratch1, ldarh);
2984     __ sxth(dst_reg, dst_reg);
2985   %}
2986 
2987   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
2988     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2989              rscratch1, ldarh);
2990   %}
2991 
2992   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
2993     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2994              rscratch1, ldarh);
2995   %}
2996 
2997   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
2998     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2999              rscratch1, ldarw);
3000   %}
3001 
3002   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3003     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3004              rscratch1, ldarw);
3005   %}
3006 
3007   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3008     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3009              rscratch1, ldar);
3010   %}
3011 
3012   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3013     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3014              rscratch1, ldarw);
3015     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3016   %}
3017 
3018   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3019     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3020              rscratch1, ldar);
3021     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3022   %}
3023 
3024   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3025     Register src_reg = as_Register($src$$reg);
3026     // we sometimes get asked to store the stack pointer into the
3027     // current thread -- we cannot do that directly on AArch64
3028     if (src_reg == r31_sp) {
3029         MacroAssembler _masm(&cbuf);
3030       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3031       __ mov(rscratch2, sp);
3032       src_reg = rscratch2;
3033     }
3034     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3035                  rscratch1, stlr);
3036   %}
3037 
3038   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3039     {
3040       MacroAssembler _masm(&cbuf);
3041       FloatRegister src_reg = as_FloatRegister($src$$reg);
3042       __ fmovs(rscratch2, src_reg);
3043     }
3044     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3045                  rscratch1, stlrw);
3046   %}
3047 
3048   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3049     {
3050       MacroAssembler _masm(&cbuf);
3051       FloatRegister src_reg = as_FloatRegister($src$$reg);
3052       __ fmovd(rscratch2, src_reg);
3053     }
3054     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3055                  rscratch1, stlr);
3056   %}
3057 
3058   // synchronized read/update encodings
3059 
3060   enc_class aarch64_enc_ldaxr(iRegL dst, memory mem) %{
3061     MacroAssembler _masm(&cbuf);
3062     Register dst_reg = as_Register($dst$$reg);
3063     Register base = as_Register($mem$$base);
3064     int index = $mem$$index;
3065     int scale = $mem$$scale;
3066     int disp = $mem$$disp;
3067     if (index == -1) {
3068        if (disp != 0) {
3069         __ lea(rscratch1, Address(base, disp));
3070         __ ldaxr(dst_reg, rscratch1);
3071       } else {
3072         // TODO
3073         // should we ever get anything other than this case?
3074         __ ldaxr(dst_reg, base);
3075       }
3076     } else {
3077       Register index_reg = as_Register(index);
3078       if (disp == 0) {
3079         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3080         __ ldaxr(dst_reg, rscratch1);
3081       } else {
3082         __ lea(rscratch1, Address(base, disp));
3083         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3084         __ ldaxr(dst_reg, rscratch1);
3085       }
3086     }
3087   %}
3088 
3089   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory mem) %{
3090     MacroAssembler _masm(&cbuf);
3091     Register src_reg = as_Register($src$$reg);
3092     Register base = as_Register($mem$$base);
3093     int index = $mem$$index;
3094     int scale = $mem$$scale;
3095     int disp = $mem$$disp;
3096     if (index == -1) {
3097        if (disp != 0) {
3098         __ lea(rscratch2, Address(base, disp));
3099         __ stlxr(rscratch1, src_reg, rscratch2);
3100       } else {
3101         // TODO
3102         // should we ever get anything other than this case?
3103         __ stlxr(rscratch1, src_reg, base);
3104       }
3105     } else {
3106       Register index_reg = as_Register(index);
3107       if (disp == 0) {
3108         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3109         __ stlxr(rscratch1, src_reg, rscratch2);
3110       } else {
3111         __ lea(rscratch2, Address(base, disp));
3112         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3113         __ stlxr(rscratch1, src_reg, rscratch2);
3114       }
3115     }
3116     __ cmpw(rscratch1, zr);
3117   %}
3118 
3119   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3120     MacroAssembler _masm(&cbuf);
3121     Register old_reg = as_Register($oldval$$reg);
3122     Register new_reg = as_Register($newval$$reg);
3123     Register base = as_Register($mem$$base);
3124     Register addr_reg;
3125     int index = $mem$$index;
3126     int scale = $mem$$scale;
3127     int disp = $mem$$disp;
3128     if (index == -1) {
3129        if (disp != 0) {
3130         __ lea(rscratch2, Address(base, disp));
3131         addr_reg = rscratch2;
3132       } else {
3133         // TODO
3134         // should we ever get anything other than this case?
3135         addr_reg = base;
3136       }
3137     } else {
3138       Register index_reg = as_Register(index);
3139       if (disp == 0) {
3140         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3141         addr_reg = rscratch2;
3142       } else {
3143         __ lea(rscratch2, Address(base, disp));
3144         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3145         addr_reg = rscratch2;
3146       }
3147     }
3148     Label retry_load, done;
3149     __ bind(retry_load);
3150     __ ldxr(rscratch1, addr_reg);
3151     __ cmp(rscratch1, old_reg);
3152     __ br(Assembler::NE, done);
3153     __ stlxr(rscratch1, new_reg, addr_reg);
3154     __ cbnzw(rscratch1, retry_load);
3155     __ bind(done);
3156   %}
3157 
3158   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3159     MacroAssembler _masm(&cbuf);
3160     Register old_reg = as_Register($oldval$$reg);
3161     Register new_reg = as_Register($newval$$reg);
3162     Register base = as_Register($mem$$base);
3163     Register addr_reg;
3164     int index = $mem$$index;
3165     int scale = $mem$$scale;
3166     int disp = $mem$$disp;
3167     if (index == -1) {
3168        if (disp != 0) {
3169         __ lea(rscratch2, Address(base, disp));
3170         addr_reg = rscratch2;
3171       } else {
3172         // TODO
3173         // should we ever get anything other than this case?
3174         addr_reg = base;
3175       }
3176     } else {
3177       Register index_reg = as_Register(index);
3178       if (disp == 0) {
3179         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3180         addr_reg = rscratch2;
3181       } else {
3182         __ lea(rscratch2, Address(base, disp));
3183         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3184         addr_reg = rscratch2;
3185       }
3186     }
3187     Label retry_load, done;
3188     __ bind(retry_load);
3189     __ ldxrw(rscratch1, addr_reg);
3190     __ cmpw(rscratch1, old_reg);
3191     __ br(Assembler::NE, done);
3192     __ stlxrw(rscratch1, new_reg, addr_reg);
3193     __ cbnzw(rscratch1, retry_load);
3194     __ bind(done);
3195   %}
3196 
3197   // auxiliary used for CompareAndSwapX to set result register
3198   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3199     MacroAssembler _masm(&cbuf);
3200     Register res_reg = as_Register($res$$reg);
3201     __ cset(res_reg, Assembler::EQ);
3202   %}
3203 
3204   // prefetch encodings
3205 
3206   enc_class aarch64_enc_prefetchw(memory mem) %{
3207     MacroAssembler _masm(&cbuf);
3208     Register base = as_Register($mem$$base);
3209     int index = $mem$$index;
3210     int scale = $mem$$scale;
3211     int disp = $mem$$disp;
3212     if (index == -1) {
3213       __ prfm(Address(base, disp), PSTL1KEEP);
3214       __ nop();
3215     } else {
3216       Register index_reg = as_Register(index);
3217       if (disp == 0) {
3218         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3219       } else {
3220         __ lea(rscratch1, Address(base, disp));
3221         __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3222       }
3223     }
3224   %}
3225 
3226   enc_class aarch64_enc_clear_array_reg_reg(iRegL_R11 cnt, iRegP_R10 base) %{
3227     MacroAssembler _masm(&cbuf);
3228     Register cnt_reg = as_Register($cnt$$reg);
3229     Register base_reg = as_Register($base$$reg);
3230     // base is word aligned
3231     // cnt is count of words
3232 
3233     Label loop;
3234     Label entry;
3235 
3236 //  Algorithm:
3237 //
3238 //    scratch1 = cnt & 7;
3239 //    cnt -= scratch1;
3240 //    p += scratch1;
3241 //    switch (scratch1) {
3242 //      do {
3243 //        cnt -= 8;
3244 //          p[-8] = 0;
3245 //        case 7:
3246 //          p[-7] = 0;
3247 //        case 6:
3248 //          p[-6] = 0;
3249 //          // ...
3250 //        case 1:
3251 //          p[-1] = 0;
3252 //        case 0:
3253 //          p += 8;
3254 //      } while (cnt);
3255 //    }
3256 
3257     const int unroll = 8; // Number of str(zr) instructions we'll unroll
3258 
3259     __ andr(rscratch1, cnt_reg, unroll - 1);  // tmp1 = cnt % unroll
3260     __ sub(cnt_reg, cnt_reg, rscratch1);      // cnt -= unroll
3261     // base_reg always points to the end of the region we're about to zero
3262     __ add(base_reg, base_reg, rscratch1, Assembler::LSL, exact_log2(wordSize));
3263     __ adr(rscratch2, entry);
3264     __ sub(rscratch2, rscratch2, rscratch1, Assembler::LSL, 2);
3265     __ br(rscratch2);
3266     __ bind(loop);
3267     __ sub(cnt_reg, cnt_reg, unroll);
3268     for (int i = -unroll; i < 0; i++)
3269       __ str(zr, Address(base_reg, i * wordSize));
3270     __ bind(entry);
3271     __ add(base_reg, base_reg, unroll * wordSize);
3272     __ cbnz(cnt_reg, loop);
3273   %}
3274 
3275   /// mov envcodings
3276 
3277   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3278     MacroAssembler _masm(&cbuf);
3279     u_int32_t con = (u_int32_t)$src$$constant;
3280     Register dst_reg = as_Register($dst$$reg);
3281     if (con == 0) {
3282       __ movw(dst_reg, zr);
3283     } else {
3284       __ movw(dst_reg, con);
3285     }
3286   %}
3287 
3288   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3289     MacroAssembler _masm(&cbuf);
3290     Register dst_reg = as_Register($dst$$reg);
3291     u_int64_t con = (u_int64_t)$src$$constant;
3292     if (con == 0) {
3293       __ mov(dst_reg, zr);
3294     } else {
3295       __ mov(dst_reg, con);
3296     }
3297   %}
3298 
3299   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3300     MacroAssembler _masm(&cbuf);
3301     Register dst_reg = as_Register($dst$$reg);
3302     address con = (address)$src$$constant;
3303     if (con == NULL || con == (address)1) {
3304       ShouldNotReachHere();
3305     } else {
3306       relocInfo::relocType rtype = $src->constant_reloc();
3307       if (rtype == relocInfo::oop_type) {
3308         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
3309       } else if (rtype == relocInfo::metadata_type) {
3310         __ mov_metadata(dst_reg, (Metadata*)con);
3311       } else {
3312         assert(rtype == relocInfo::none, "unexpected reloc type");
3313         if (con < (address)(uintptr_t)os::vm_page_size()) {
3314           __ mov(dst_reg, con);
3315         } else {
3316           unsigned long offset;
3317           __ adrp(dst_reg, con, offset);
3318           __ add(dst_reg, dst_reg, offset);
3319         }
3320       }
3321     }
3322   %}
3323 
3324   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3325     MacroAssembler _masm(&cbuf);
3326     Register dst_reg = as_Register($dst$$reg);
3327     __ mov(dst_reg, zr);
3328   %}
3329 
3330   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3331     MacroAssembler _masm(&cbuf);
3332     Register dst_reg = as_Register($dst$$reg);
3333     __ mov(dst_reg, (u_int64_t)1);
3334   %}
3335 
3336   enc_class aarch64_enc_mov_poll_page(iRegP dst, immPollPage src) %{
3337     MacroAssembler _masm(&cbuf);
3338     address page = (address)$src$$constant;
3339     Register dst_reg = as_Register($dst$$reg);
3340     unsigned long off;
3341     __ adrp(dst_reg, Address(page, relocInfo::poll_type), off);
3342     assert(off == 0, "assumed offset == 0");
3343   %}
3344 
3345   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3346     MacroAssembler _masm(&cbuf);
3347     address page = (address)$src$$constant;
3348     Register dst_reg = as_Register($dst$$reg);
3349     unsigned long off;
3350     __ adrp(dst_reg, ExternalAddress(page), off);
3351     assert(off == 0, "assumed offset == 0");
3352   %}
3353 
3354   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3355     MacroAssembler _masm(&cbuf);
3356     Register dst_reg = as_Register($dst$$reg);
3357     address con = (address)$src$$constant;
3358     if (con == NULL) {
3359       ShouldNotReachHere();
3360     } else {
3361       relocInfo::relocType rtype = $src->constant_reloc();
3362       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3363       __ set_narrow_oop(dst_reg, (jobject)con);
3364     }
3365   %}
3366 
3367   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3368     MacroAssembler _masm(&cbuf);
3369     Register dst_reg = as_Register($dst$$reg);
3370     __ mov(dst_reg, zr);
3371   %}
3372 
3373   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3374     MacroAssembler _masm(&cbuf);
3375     Register dst_reg = as_Register($dst$$reg);
3376     address con = (address)$src$$constant;
3377     if (con == NULL) {
3378       ShouldNotReachHere();
3379     } else {
3380       relocInfo::relocType rtype = $src->constant_reloc();
3381       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3382       __ set_narrow_klass(dst_reg, (Klass *)con);
3383     }
3384   %}
3385 
3386   // arithmetic encodings
3387 
3388   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3389     MacroAssembler _masm(&cbuf);
3390     Register dst_reg = as_Register($dst$$reg);
3391     Register src_reg = as_Register($src1$$reg);
3392     int32_t con = (int32_t)$src2$$constant;
3393     // add has primary == 0, subtract has primary == 1
3394     if ($primary) { con = -con; }
3395     if (con < 0) {
3396       __ subw(dst_reg, src_reg, -con);
3397     } else {
3398       __ addw(dst_reg, src_reg, con);
3399     }
3400   %}
3401 
3402   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3403     MacroAssembler _masm(&cbuf);
3404     Register dst_reg = as_Register($dst$$reg);
3405     Register src_reg = as_Register($src1$$reg);
3406     int32_t con = (int32_t)$src2$$constant;
3407     // add has primary == 0, subtract has primary == 1
3408     if ($primary) { con = -con; }
3409     if (con < 0) {
3410       __ sub(dst_reg, src_reg, -con);
3411     } else {
3412       __ add(dst_reg, src_reg, con);
3413     }
3414   %}
3415 
3416   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3417     MacroAssembler _masm(&cbuf);
3418    Register dst_reg = as_Register($dst$$reg);
3419    Register src1_reg = as_Register($src1$$reg);
3420    Register src2_reg = as_Register($src2$$reg);
3421     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3422   %}
3423 
3424   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3425     MacroAssembler _masm(&cbuf);
3426    Register dst_reg = as_Register($dst$$reg);
3427    Register src1_reg = as_Register($src1$$reg);
3428    Register src2_reg = as_Register($src2$$reg);
3429     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3430   %}
3431 
3432   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3433     MacroAssembler _masm(&cbuf);
3434    Register dst_reg = as_Register($dst$$reg);
3435    Register src1_reg = as_Register($src1$$reg);
3436    Register src2_reg = as_Register($src2$$reg);
3437     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3438   %}
3439 
3440   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3441     MacroAssembler _masm(&cbuf);
3442    Register dst_reg = as_Register($dst$$reg);
3443    Register src1_reg = as_Register($src1$$reg);
3444    Register src2_reg = as_Register($src2$$reg);
3445     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3446   %}
3447 
3448   // compare instruction encodings
3449 
3450   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3451     MacroAssembler _masm(&cbuf);
3452     Register reg1 = as_Register($src1$$reg);
3453     Register reg2 = as_Register($src2$$reg);
3454     __ cmpw(reg1, reg2);
3455   %}
3456 
3457   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3458     MacroAssembler _masm(&cbuf);
3459     Register reg = as_Register($src1$$reg);
3460     int32_t val = $src2$$constant;
3461     if (val >= 0) {
3462       __ subsw(zr, reg, val);
3463     } else {
3464       __ addsw(zr, reg, -val);
3465     }
3466   %}
3467 
3468   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3469     MacroAssembler _masm(&cbuf);
3470     Register reg1 = as_Register($src1$$reg);
3471     u_int32_t val = (u_int32_t)$src2$$constant;
3472     __ movw(rscratch1, val);
3473     __ cmpw(reg1, rscratch1);
3474   %}
3475 
3476   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3477     MacroAssembler _masm(&cbuf);
3478     Register reg1 = as_Register($src1$$reg);
3479     Register reg2 = as_Register($src2$$reg);
3480     __ cmp(reg1, reg2);
3481   %}
3482 
3483   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3484     MacroAssembler _masm(&cbuf);
3485     Register reg = as_Register($src1$$reg);
3486     int64_t val = $src2$$constant;
3487     if (val >= 0) {
3488       __ subs(zr, reg, val);
3489     } else if (val != -val) {
3490       __ adds(zr, reg, -val);
3491     } else {
3492     // aargh, Long.MIN_VALUE is a special case
3493       __ orr(rscratch1, zr, (u_int64_t)val);
3494       __ subs(zr, reg, rscratch1);
3495     }
3496   %}
3497 
3498   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3499     MacroAssembler _masm(&cbuf);
3500     Register reg1 = as_Register($src1$$reg);
3501     u_int64_t val = (u_int64_t)$src2$$constant;
3502     __ mov(rscratch1, val);
3503     __ cmp(reg1, rscratch1);
3504   %}
3505 
3506   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3507     MacroAssembler _masm(&cbuf);
3508     Register reg1 = as_Register($src1$$reg);
3509     Register reg2 = as_Register($src2$$reg);
3510     __ cmp(reg1, reg2);
3511   %}
3512 
3513   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3514     MacroAssembler _masm(&cbuf);
3515     Register reg1 = as_Register($src1$$reg);
3516     Register reg2 = as_Register($src2$$reg);
3517     __ cmpw(reg1, reg2);
3518   %}
3519 
3520   enc_class aarch64_enc_testp(iRegP src) %{
3521     MacroAssembler _masm(&cbuf);
3522     Register reg = as_Register($src$$reg);
3523     __ cmp(reg, zr);
3524   %}
3525 
3526   enc_class aarch64_enc_testn(iRegN src) %{
3527     MacroAssembler _masm(&cbuf);
3528     Register reg = as_Register($src$$reg);
3529     __ cmpw(reg, zr);
3530   %}
3531 
3532   enc_class aarch64_enc_b(label lbl) %{
3533     MacroAssembler _masm(&cbuf);
3534     Label *L = $lbl$$label;
3535     __ b(*L);
3536   %}
3537 
3538   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3539     MacroAssembler _masm(&cbuf);
3540     Label *L = $lbl$$label;
3541     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3542   %}
3543 
3544   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3545     MacroAssembler _masm(&cbuf);
3546     Label *L = $lbl$$label;
3547     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3548   %}
3549 
3550   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3551   %{
3552      Register sub_reg = as_Register($sub$$reg);
3553      Register super_reg = as_Register($super$$reg);
3554      Register temp_reg = as_Register($temp$$reg);
3555      Register result_reg = as_Register($result$$reg);
3556 
3557      Label miss;
3558      MacroAssembler _masm(&cbuf);
3559      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3560                                      NULL, &miss,
3561                                      /*set_cond_codes:*/ true);
3562      if ($primary) {
3563        __ mov(result_reg, zr);
3564      }
3565      __ bind(miss);
3566   %}
3567 
3568   enc_class aarch64_enc_java_static_call(method meth) %{
3569     MacroAssembler _masm(&cbuf);
3570 
3571     address addr = (address)$meth$$method;
3572     if (!_method) {
3573       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3574       __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3575     } else if (_optimized_virtual) {
3576       __ trampoline_call(Address(addr, relocInfo::opt_virtual_call_type), &cbuf);
3577     } else {
3578       __ trampoline_call(Address(addr, relocInfo::static_call_type), &cbuf);
3579     }
3580 
3581     if (_method) {
3582       // Emit stub for static call
3583       CompiledStaticCall::emit_to_interp_stub(cbuf);
3584     }
3585   %}
3586 
3587   enc_class aarch64_enc_java_dynamic_call(method meth) %{
3588     MacroAssembler _masm(&cbuf);
3589     __ ic_call((address)$meth$$method);
3590   %}
3591 
3592   enc_class aarch64_enc_call_epilog() %{
3593     MacroAssembler _masm(&cbuf);
3594     if (VerifyStackAtCalls) {
3595       // Check that stack depth is unchanged: find majik cookie on stack
3596       __ call_Unimplemented();
3597     }
3598   %}
3599 
3600   enc_class aarch64_enc_java_to_runtime(method meth) %{
3601     MacroAssembler _masm(&cbuf);
3602 
3603     // some calls to generated routines (arraycopy code) are scheduled
3604     // by C2 as runtime calls. if so we can call them using a br (they
3605     // will be in a reachable segment) otherwise we have to use a blrt
3606     // which loads the absolute address into a register.
3607     address entry = (address)$meth$$method;
3608     CodeBlob *cb = CodeCache::find_blob(entry);
3609     if (cb) {
3610       __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3611     } else {
3612       int gpcnt;
3613       int fpcnt;
3614       int rtype;
3615       getCallInfo(tf(), gpcnt, fpcnt, rtype);
3616       Label retaddr;
3617       __ adr(rscratch2, retaddr);
3618       __ lea(rscratch1, RuntimeAddress(entry));
3619       // Leave a breadcrumb for JavaThread::pd_last_frame().
3620       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3621       __ blrt(rscratch1, gpcnt, fpcnt, rtype);
3622       __ bind(retaddr);
3623       __ add(sp, sp, 2 * wordSize);
3624     }
3625   %}
3626 
3627   enc_class aarch64_enc_rethrow() %{
3628     MacroAssembler _masm(&cbuf);
3629     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3630   %}
3631 
3632   enc_class aarch64_enc_ret() %{
3633     MacroAssembler _masm(&cbuf);
3634     __ ret(lr);
3635   %}
3636 
3637   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3638     MacroAssembler _masm(&cbuf);
3639     Register target_reg = as_Register($jump_target$$reg);
3640     __ br(target_reg);
3641   %}
3642 
3643   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3644     MacroAssembler _masm(&cbuf);
3645     Register target_reg = as_Register($jump_target$$reg);
3646     // exception oop should be in r0
3647     // ret addr has been popped into lr
3648     // callee expects it in r3
3649     __ mov(r3, lr);
3650     __ br(target_reg);
3651   %}
3652 
3653   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3654     MacroAssembler _masm(&cbuf);
3655     Register oop = as_Register($object$$reg);
3656     Register box = as_Register($box$$reg);
3657     Register disp_hdr = as_Register($tmp$$reg);
3658     Register tmp = as_Register($tmp2$$reg);
3659     Label cont;
3660     Label object_has_monitor;
3661     Label cas_failed;
3662 
3663     assert_different_registers(oop, box, tmp, disp_hdr);
3664 
3665     // Load markOop from object into displaced_header.
3666     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3667 
3668     // Always do locking in runtime.
3669     if (EmitSync & 0x01) {
3670       __ cmp(oop, zr);
3671       return;
3672     }
3673 
3674     if (UseBiasedLocking) {
3675       __ biased_locking_enter(disp_hdr, oop, box, tmp, true, cont);
3676     }
3677 
3678     // Handle existing monitor
3679     if (EmitSync & 0x02) {
3680       // we can use AArch64's bit test and branch here but
3681       // markoopDesc does not define a bit index just the bit value
3682       // so assert in case the bit pos changes
3683 #     define __monitor_value_log2 1
3684       assert(markOopDesc::monitor_value == (1 << __monitor_value_log2), "incorrect bit position");
3685       __ tbnz(disp_hdr, __monitor_value_log2, object_has_monitor);
3686 #     undef __monitor_value_log2
3687     }
3688 
3689     // Set displaced_header to be (markOop of object | UNLOCK_VALUE).
3690     __ orr(disp_hdr, disp_hdr, markOopDesc::unlocked_value);
3691 
3692     // Load Compare Value application register.
3693 
3694     // Initialize the box. (Must happen before we update the object mark!)
3695     __ str(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3696 
3697     // Compare object markOop with mark and if equal exchange scratch1
3698     // with object markOop.
3699     // Note that this is simply a CAS: it does not generate any
3700     // barriers.  These are separately generated by
3701     // membar_acquire_lock().
3702     {
3703       Label retry_load;
3704       __ bind(retry_load);
3705       __ ldxr(tmp, oop);
3706       __ cmp(tmp, disp_hdr);
3707       __ br(Assembler::NE, cas_failed);
3708       // use stlxr to ensure update is immediately visible
3709       __ stlxr(tmp, box, oop);
3710       __ cbzw(tmp, cont);
3711       __ b(retry_load);
3712     }
3713 
3714     // Formerly:
3715     // __ cmpxchgptr(/*oldv=*/disp_hdr,
3716     //               /*newv=*/box,
3717     //               /*addr=*/oop,
3718     //               /*tmp=*/tmp,
3719     //               cont,
3720     //               /*fail*/NULL);
3721 
3722     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3723 
3724     // If the compare-and-exchange succeeded, then we found an unlocked
3725     // object, will have now locked it will continue at label cont
3726 
3727     __ bind(cas_failed);
3728     // We did not see an unlocked object so try the fast recursive case.
3729 
3730     // Check if the owner is self by comparing the value in the
3731     // markOop of object (disp_hdr) with the stack pointer.
3732     __ mov(rscratch1, sp);
3733     __ sub(disp_hdr, disp_hdr, rscratch1);
3734     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place));
3735     // If condition is true we are cont and hence we can store 0 as the
3736     // displaced header in the box, which indicates that it is a recursive lock.
3737     __ ands(tmp/*==0?*/, disp_hdr, tmp);
3738     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3739 
3740     // Handle existing monitor.
3741     if ((EmitSync & 0x02) == 0) {
3742       __ b(cont);
3743 
3744       __ bind(object_has_monitor);
3745       // The object's monitor m is unlocked iff m->owner == NULL,
3746       // otherwise m->owner may contain a thread or a stack address.
3747       //
3748       // Try to CAS m->owner from NULL to current thread.
3749       __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
3750       __ mov(disp_hdr, zr);
3751 
3752       {
3753         Label retry_load, fail;
3754         __ bind(retry_load);
3755         __ ldxr(rscratch1, tmp);
3756         __ cmp(disp_hdr, rscratch1);
3757         __ br(Assembler::NE, fail);
3758         // use stlxr to ensure update is immediately visible
3759         __ stlxr(rscratch1, rthread, tmp);
3760         __ cbnzw(rscratch1, retry_load);
3761         __ bind(fail);
3762       }
3763 
3764       // Label next;
3765       // __ cmpxchgptr(/*oldv=*/disp_hdr,
3766       //               /*newv=*/rthread,
3767       //               /*addr=*/tmp,
3768       //               /*tmp=*/rscratch1,
3769       //               /*succeed*/next,
3770       //               /*fail*/NULL);
3771       // __ bind(next);
3772 
3773       // store a non-null value into the box.
3774       __ str(box, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3775 
3776       // PPC port checks the following invariants
3777       // #ifdef ASSERT
3778       // bne(flag, cont);
3779       // We have acquired the monitor, check some invariants.
3780       // addw(/*monitor=*/tmp, tmp, -ObjectMonitor::owner_offset_in_bytes());
3781       // Invariant 1: _recursions should be 0.
3782       // assert(ObjectMonitor::recursions_size_in_bytes() == 8, "unexpected size");
3783       // assert_mem8_is_zero(ObjectMonitor::recursions_offset_in_bytes(), tmp,
3784       //                        "monitor->_recursions should be 0", -1);
3785       // Invariant 2: OwnerIsThread shouldn't be 0.
3786       // assert(ObjectMonitor::OwnerIsThread_size_in_bytes() == 4, "unexpected size");
3787       //assert_mem4_isnot_zero(ObjectMonitor::OwnerIsThread_offset_in_bytes(), tmp,
3788       //                           "monitor->OwnerIsThread shouldn't be 0", -1);
3789       // #endif
3790     }
3791 
3792     __ bind(cont);
3793     // flag == EQ indicates success
3794     // flag == NE indicates failure
3795 
3796   %}
3797 
3798   // TODO
3799   // reimplement this with custom cmpxchgptr code
3800   // which avoids some of the unnecessary branching
3801   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3802     MacroAssembler _masm(&cbuf);
3803     Register oop = as_Register($object$$reg);
3804     Register box = as_Register($box$$reg);
3805     Register disp_hdr = as_Register($tmp$$reg);
3806     Register tmp = as_Register($tmp2$$reg);
3807     Label cont;
3808     Label object_has_monitor;
3809     Label cas_failed;
3810 
3811     assert_different_registers(oop, box, tmp, disp_hdr);
3812 
3813     // Always do locking in runtime.
3814     if (EmitSync & 0x01) {
3815       __ cmp(oop, zr); // Oop can't be 0 here => always false.
3816       return;
3817     }
3818 
3819     if (UseBiasedLocking) {
3820       __ biased_locking_exit(oop, tmp, cont);
3821     }
3822 
3823     // Find the lock address and load the displaced header from the stack.
3824     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3825 
3826     // If the displaced header is 0, we have a recursive unlock.
3827     __ cmp(disp_hdr, zr);
3828     __ br(Assembler::EQ, cont);
3829 
3830 
3831     // Handle existing monitor.
3832     if ((EmitSync & 0x02) == 0) {
3833       __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3834       __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3835     }
3836 
3837     // Check if it is still a light weight lock, this is is true if we
3838     // see the stack address of the basicLock in the markOop of the
3839     // object.
3840 
3841       {
3842         Label retry_load;
3843         __ bind(retry_load);
3844         __ ldxr(tmp, oop);
3845         __ cmp(box, tmp);
3846         __ br(Assembler::NE, cas_failed);
3847         // use stlxr to ensure update is immediately visible
3848         __ stlxr(tmp, disp_hdr, oop);
3849         __ cbzw(tmp, cont);
3850         __ b(retry_load);
3851       }
3852 
3853     // __ cmpxchgptr(/*compare_value=*/box,
3854     //               /*exchange_value=*/disp_hdr,
3855     //               /*where=*/oop,
3856     //               /*result=*/tmp,
3857     //               cont,
3858     //               /*cas_failed*/NULL);
3859     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3860 
3861     __ bind(cas_failed);
3862 
3863     // Handle existing monitor.
3864     if ((EmitSync & 0x02) == 0) {
3865       __ b(cont);
3866 
3867       __ bind(object_has_monitor);
3868       __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor
3869       __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3870       __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3871       __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3872       __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3873       __ cmp(rscratch1, zr);
3874       __ br(Assembler::NE, cont);
3875 
3876       __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3877       __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3878       __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3879       __ cmp(rscratch1, zr);
3880       __ cbnz(rscratch1, cont);
3881       // need a release store here
3882       __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3883       __ stlr(rscratch1, tmp); // rscratch1 is zero
3884     }
3885 
3886     __ bind(cont);
3887     // flag == EQ indicates success
3888     // flag == NE indicates failure
3889   %}
3890 
3891 %}
3892 
3893 //----------FRAME--------------------------------------------------------------
3894 // Definition of frame structure and management information.
3895 //
3896 //  S T A C K   L A Y O U T    Allocators stack-slot number
3897 //                             |   (to get allocators register number
3898 //  G  Owned by    |        |  v    add OptoReg::stack0())
3899 //  r   CALLER     |        |
3900 //  o     |        +--------+      pad to even-align allocators stack-slot
3901 //  w     V        |  pad0  |        numbers; owned by CALLER
3902 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
3903 //  h     ^        |   in   |  5
3904 //        |        |  args  |  4   Holes in incoming args owned by SELF
3905 //  |     |        |        |  3
3906 //  |     |        +--------+
3907 //  V     |        | old out|      Empty on Intel, window on Sparc
3908 //        |    old |preserve|      Must be even aligned.
3909 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
3910 //        |        |   in   |  3   area for Intel ret address
3911 //     Owned by    |preserve|      Empty on Sparc.
3912 //       SELF      +--------+
3913 //        |        |  pad2  |  2   pad to align old SP
3914 //        |        +--------+  1
3915 //        |        | locks  |  0
3916 //        |        +--------+----> OptoReg::stack0(), even aligned
3917 //        |        |  pad1  | 11   pad to align new SP
3918 //        |        +--------+
3919 //        |        |        | 10
3920 //        |        | spills |  9   spills
3921 //        V        |        |  8   (pad0 slot for callee)
3922 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
3923 //        ^        |  out   |  7
3924 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
3925 //     Owned by    +--------+
3926 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
3927 //        |    new |preserve|      Must be even-aligned.
3928 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
3929 //        |        |        |
3930 //
3931 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
3932 //         known from SELF's arguments and the Java calling convention.
3933 //         Region 6-7 is determined per call site.
3934 // Note 2: If the calling convention leaves holes in the incoming argument
3935 //         area, those holes are owned by SELF.  Holes in the outgoing area
3936 //         are owned by the CALLEE.  Holes should not be nessecary in the
3937 //         incoming area, as the Java calling convention is completely under
3938 //         the control of the AD file.  Doubles can be sorted and packed to
3939 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
3940 //         varargs C calling conventions.
3941 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
3942 //         even aligned with pad0 as needed.
3943 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
3944 //           (the latter is true on Intel but is it false on AArch64?)
3945 //         region 6-11 is even aligned; it may be padded out more so that
3946 //         the region from SP to FP meets the minimum stack alignment.
3947 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3948 //         alignment.  Region 11, pad1, may be dynamically extended so that
3949 //         SP meets the minimum alignment.
3950 
3951 frame %{
3952   // What direction does stack grow in (assumed to be same for C & Java)
3953   stack_direction(TOWARDS_LOW);
3954 
3955   // These three registers define part of the calling convention
3956   // between compiled code and the interpreter.
3957 
3958   // Inline Cache Register or methodOop for I2C.
3959   inline_cache_reg(R12);
3960 
3961   // Method Oop Register when calling interpreter.
3962   interpreter_method_oop_reg(R12);
3963 
3964   // Number of stack slots consumed by locking an object
3965   sync_stack_slots(2);
3966 
3967   // Compiled code's Frame Pointer
3968   frame_pointer(R31);
3969 
3970   // Interpreter stores its frame pointer in a register which is
3971   // stored to the stack by I2CAdaptors.
3972   // I2CAdaptors convert from interpreted java to compiled java.
3973   interpreter_frame_pointer(R29);
3974 
3975   // Stack alignment requirement
3976   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3977 
3978   // Number of stack slots between incoming argument block and the start of
3979   // a new frame.  The PROLOG must add this many slots to the stack.  The
3980   // EPILOG must remove this many slots. aarch64 needs two slots for
3981   // return address and fp.
3982   // TODO think this is correct but check
3983   in_preserve_stack_slots(4);
3984 
3985   // Number of outgoing stack slots killed above the out_preserve_stack_slots
3986   // for calls to C.  Supports the var-args backing area for register parms.
3987   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3988 
3989   // The after-PROLOG location of the return address.  Location of
3990   // return address specifies a type (REG or STACK) and a number
3991   // representing the register number (i.e. - use a register name) or
3992   // stack slot.
3993   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3994   // Otherwise, it is above the locks and verification slot and alignment word
3995   // TODO this may well be correct but need to check why that - 2 is there
3996   // ppc port uses 0 but we definitely need to allow for fixed_slots
3997   // which folds in the space used for monitors
3998   return_addr(STACK - 2 +
3999               round_to((Compile::current()->in_preserve_stack_slots() +
4000                         Compile::current()->fixed_slots()),
4001                        stack_alignment_in_slots()));
4002 
4003   // Body of function which returns an integer array locating
4004   // arguments either in registers or in stack slots.  Passed an array
4005   // of ideal registers called "sig" and a "length" count.  Stack-slot
4006   // offsets are based on outgoing arguments, i.e. a CALLER setting up
4007   // arguments for a CALLEE.  Incoming stack arguments are
4008   // automatically biased by the preserve_stack_slots field above.
4009 
4010   calling_convention
4011   %{
4012     // No difference between ingoing/outgoing just pass false
4013     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
4014   %}
4015 
4016   c_calling_convention
4017   %{
4018     // This is obviously always outgoing
4019     (void) SharedRuntime::c_calling_convention(sig_bt, regs, NULL, length);
4020   %}
4021 
4022   // Location of compiled Java return values.  Same as C for now.
4023   return_value
4024   %{
4025     // TODO do we allow ideal_reg == Op_RegN???
4026     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4027            "only return normal values");
4028 
4029     static const int lo[Op_RegL + 1] = { // enum name
4030       0,                                 // Op_Node
4031       0,                                 // Op_Set
4032       R0_num,                            // Op_RegN
4033       R0_num,                            // Op_RegI
4034       R0_num,                            // Op_RegP
4035       V0_num,                            // Op_RegF
4036       V0_num,                            // Op_RegD
4037       R0_num                             // Op_RegL
4038     };
4039 
4040     static const int hi[Op_RegL + 1] = { // enum name
4041       0,                                 // Op_Node
4042       0,                                 // Op_Set
4043       OptoReg::Bad,                       // Op_RegN
4044       OptoReg::Bad,                      // Op_RegI
4045       R0_H_num,                          // Op_RegP
4046       OptoReg::Bad,                      // Op_RegF
4047       V0_H_num,                          // Op_RegD
4048       R0_H_num                           // Op_RegL
4049     };
4050 
4051     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4052   %}
4053 %}
4054 
4055 //----------ATTRIBUTES---------------------------------------------------------
4056 //----------Operand Attributes-------------------------------------------------
4057 op_attrib op_cost(1);        // Required cost attribute
4058 
4059 //----------Instruction Attributes---------------------------------------------
4060 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4061 ins_attrib ins_size(32);        // Required size attribute (in bits)
4062 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4063                                 // a non-matching short branch variant
4064                                 // of some long branch?
4065 ins_attrib ins_alignment(4);    // Required alignment attribute (must
4066                                 // be a power of 2) specifies the
4067                                 // alignment that some part of the
4068                                 // instruction (not necessarily the
4069                                 // start) requires.  If > 1, a
4070                                 // compute_padding() function must be
4071                                 // provided for the instruction
4072 
4073 //----------OPERANDS-----------------------------------------------------------
4074 // Operand definitions must precede instruction definitions for correct parsing
4075 // in the ADLC because operands constitute user defined types which are used in
4076 // instruction definitions.
4077 
4078 //----------Simple Operands----------------------------------------------------
4079 
4080 // Integer operands 32 bit
4081 // 32 bit immediate
4082 operand immI()
4083 %{
4084   match(ConI);
4085 
4086   op_cost(0);
4087   format %{ %}
4088   interface(CONST_INTER);
4089 %}
4090 
4091 // 32 bit zero
4092 operand immI0()
4093 %{
4094   predicate(n->get_int() == 0);
4095   match(ConI);
4096 
4097   op_cost(0);
4098   format %{ %}
4099   interface(CONST_INTER);
4100 %}
4101 
4102 // 32 bit unit increment
4103 operand immI_1()
4104 %{
4105   predicate(n->get_int() == 1);
4106   match(ConI);
4107 
4108   op_cost(0);
4109   format %{ %}
4110   interface(CONST_INTER);
4111 %}
4112 
4113 // 32 bit unit decrement
4114 operand immI_M1()
4115 %{
4116   predicate(n->get_int() == -1);
4117   match(ConI);
4118 
4119   op_cost(0);
4120   format %{ %}
4121   interface(CONST_INTER);
4122 %}
4123 
4124 operand immI_le_4()
4125 %{
4126   predicate(n->get_int() <= 4);
4127   match(ConI);
4128 
4129   op_cost(0);
4130   format %{ %}
4131   interface(CONST_INTER);
4132 %}
4133 
4134 operand immI_31()
4135 %{
4136   predicate(n->get_int() == 31);
4137   match(ConI);
4138 
4139   op_cost(0);
4140   format %{ %}
4141   interface(CONST_INTER);
4142 %}
4143 
4144 operand immI_8()
4145 %{
4146   predicate(n->get_int() == 8);
4147   match(ConI);
4148 
4149   op_cost(0);
4150   format %{ %}
4151   interface(CONST_INTER);
4152 %}
4153 
4154 operand immI_16()
4155 %{
4156   predicate(n->get_int() == 16);
4157   match(ConI);
4158 
4159   op_cost(0);
4160   format %{ %}
4161   interface(CONST_INTER);
4162 %}
4163 
4164 operand immI_24()
4165 %{
4166   predicate(n->get_int() == 24);
4167   match(ConI);
4168 
4169   op_cost(0);
4170   format %{ %}
4171   interface(CONST_INTER);
4172 %}
4173 
4174 operand immI_32()
4175 %{
4176   predicate(n->get_int() == 32);
4177   match(ConI);
4178 
4179   op_cost(0);
4180   format %{ %}
4181   interface(CONST_INTER);
4182 %}
4183 
4184 operand immI_48()
4185 %{
4186   predicate(n->get_int() == 48);
4187   match(ConI);
4188 
4189   op_cost(0);
4190   format %{ %}
4191   interface(CONST_INTER);
4192 %}
4193 
4194 operand immI_56()
4195 %{
4196   predicate(n->get_int() == 56);
4197   match(ConI);
4198 
4199   op_cost(0);
4200   format %{ %}
4201   interface(CONST_INTER);
4202 %}
4203 
4204 operand immI_64()
4205 %{
4206   predicate(n->get_int() == 64);
4207   match(ConI);
4208 
4209   op_cost(0);
4210   format %{ %}
4211   interface(CONST_INTER);
4212 %}
4213 
4214 operand immI_255()
4215 %{
4216   predicate(n->get_int() == 255);
4217   match(ConI);
4218 
4219   op_cost(0);
4220   format %{ %}
4221   interface(CONST_INTER);
4222 %}
4223 
4224 operand immI_65535()
4225 %{
4226   predicate(n->get_int() == 65535);
4227   match(ConI);
4228 
4229   op_cost(0);
4230   format %{ %}
4231   interface(CONST_INTER);
4232 %}
4233 
4234 operand immL_63()
4235 %{
4236   predicate(n->get_int() == 63);
4237   match(ConI);
4238 
4239   op_cost(0);
4240   format %{ %}
4241   interface(CONST_INTER);
4242 %}
4243 
4244 operand immL_255()
4245 %{
4246   predicate(n->get_int() == 255);
4247   match(ConI);
4248 
4249   op_cost(0);
4250   format %{ %}
4251   interface(CONST_INTER);
4252 %}
4253 
4254 operand immL_65535()
4255 %{
4256   predicate(n->get_long() == 65535L);
4257   match(ConL);
4258 
4259   op_cost(0);
4260   format %{ %}
4261   interface(CONST_INTER);
4262 %}
4263 
4264 operand immL_4294967295()
4265 %{
4266   predicate(n->get_long() == 4294967295L);
4267   match(ConL);
4268 
4269   op_cost(0);
4270   format %{ %}
4271   interface(CONST_INTER);
4272 %}
4273 
4274 operand immL_bitmask()
4275 %{
4276   predicate(((n->get_long() & 0xc000000000000000l) == 0)
4277             && is_power_of_2(n->get_long() + 1));
4278   match(ConL);
4279 
4280   op_cost(0);
4281   format %{ %}
4282   interface(CONST_INTER);
4283 %}
4284 
4285 operand immI_bitmask()
4286 %{
4287   predicate(((n->get_int() & 0xc0000000) == 0)
4288             && is_power_of_2(n->get_int() + 1));
4289   match(ConI);
4290 
4291   op_cost(0);
4292   format %{ %}
4293   interface(CONST_INTER);
4294 %}
4295 
4296 // Scale values for scaled offset addressing modes (up to long but not quad)
4297 operand immIScale()
4298 %{
4299   predicate(0 <= n->get_int() && (n->get_int() <= 3));
4300   match(ConI);
4301 
4302   op_cost(0);
4303   format %{ %}
4304   interface(CONST_INTER);
4305 %}
4306 
4307 // 26 bit signed offset -- for pc-relative branches
4308 operand immI26()
4309 %{
4310   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4311   match(ConI);
4312 
4313   op_cost(0);
4314   format %{ %}
4315   interface(CONST_INTER);
4316 %}
4317 
4318 // 19 bit signed offset -- for pc-relative loads
4319 operand immI19()
4320 %{
4321   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4322   match(ConI);
4323 
4324   op_cost(0);
4325   format %{ %}
4326   interface(CONST_INTER);
4327 %}
4328 
4329 // 12 bit unsigned offset -- for base plus immediate loads
4330 operand immIU12()
4331 %{
4332   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4333   match(ConI);
4334 
4335   op_cost(0);
4336   format %{ %}
4337   interface(CONST_INTER);
4338 %}
4339 
4340 operand immLU12()
4341 %{
4342   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4343   match(ConL);
4344 
4345   op_cost(0);
4346   format %{ %}
4347   interface(CONST_INTER);
4348 %}
4349 
4350 // Offset for scaled or unscaled immediate loads and stores
4351 operand immIOffset()
4352 %{
4353   predicate(Address::offset_ok_for_immed(n->get_int()));
4354   match(ConI);
4355 
4356   op_cost(0);
4357   format %{ %}
4358   interface(CONST_INTER);
4359 %}
4360 
4361 operand immLoffset()
4362 %{
4363   predicate(Address::offset_ok_for_immed(n->get_long()));
4364   match(ConL);
4365 
4366   op_cost(0);
4367   format %{ %}
4368   interface(CONST_INTER);
4369 %}
4370 
4371 // 32 bit integer valid for add sub immediate
4372 operand immIAddSub()
4373 %{
4374   predicate(Assembler::operand_valid_for_add_sub_immediate((long)n->get_int()));
4375   match(ConI);
4376   op_cost(0);
4377   format %{ %}
4378   interface(CONST_INTER);
4379 %}
4380 
4381 // 32 bit unsigned integer valid for logical immediate
4382 // TODO -- check this is right when e.g the mask is 0x80000000
4383 operand immILog()
4384 %{
4385   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (unsigned long)n->get_int()));
4386   match(ConI);
4387 
4388   op_cost(0);
4389   format %{ %}
4390   interface(CONST_INTER);
4391 %}
4392 
4393 // Integer operands 64 bit
4394 // 64 bit immediate
4395 operand immL()
4396 %{
4397   match(ConL);
4398 
4399   op_cost(0);
4400   format %{ %}
4401   interface(CONST_INTER);
4402 %}
4403 
4404 // 64 bit zero
4405 operand immL0()
4406 %{
4407   predicate(n->get_long() == 0);
4408   match(ConL);
4409 
4410   op_cost(0);
4411   format %{ %}
4412   interface(CONST_INTER);
4413 %}
4414 
4415 // 64 bit unit increment
4416 operand immL_1()
4417 %{
4418   predicate(n->get_long() == 1);
4419   match(ConL);
4420 
4421   op_cost(0);
4422   format %{ %}
4423   interface(CONST_INTER);
4424 %}
4425 
4426 // 64 bit unit decrement
4427 operand immL_M1()
4428 %{
4429   predicate(n->get_long() == -1);
4430   match(ConL);
4431 
4432   op_cost(0);
4433   format %{ %}
4434   interface(CONST_INTER);
4435 %}
4436 
4437 // 32 bit offset of pc in thread anchor
4438 
4439 operand immL_pc_off()
4440 %{
4441   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4442                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4443   match(ConL);
4444 
4445   op_cost(0);
4446   format %{ %}
4447   interface(CONST_INTER);
4448 %}
4449 
4450 // 64 bit integer valid for add sub immediate
4451 operand immLAddSub()
4452 %{
4453   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4454   match(ConL);
4455   op_cost(0);
4456   format %{ %}
4457   interface(CONST_INTER);
4458 %}
4459 
4460 // 64 bit integer valid for logical immediate
4461 operand immLLog()
4462 %{
4463   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (unsigned long)n->get_long()));
4464   match(ConL);
4465   op_cost(0);
4466   format %{ %}
4467   interface(CONST_INTER);
4468 %}
4469 
4470 // Long Immediate: low 32-bit mask
4471 operand immL_32bits()
4472 %{
4473   predicate(n->get_long() == 0xFFFFFFFFL);
4474   match(ConL);
4475   op_cost(0);
4476   format %{ %}
4477   interface(CONST_INTER);
4478 %}
4479 
4480 // Pointer operands
4481 // Pointer Immediate
4482 operand immP()
4483 %{
4484   match(ConP);
4485 
4486   op_cost(0);
4487   format %{ %}
4488   interface(CONST_INTER);
4489 %}
4490 
4491 // NULL Pointer Immediate
4492 operand immP0()
4493 %{
4494   predicate(n->get_ptr() == 0);
4495   match(ConP);
4496 
4497   op_cost(0);
4498   format %{ %}
4499   interface(CONST_INTER);
4500 %}
4501 
4502 // Pointer Immediate One
4503 // this is used in object initialization (initial object header)
4504 operand immP_1()
4505 %{
4506   predicate(n->get_ptr() == 1);
4507   match(ConP);
4508 
4509   op_cost(0);
4510   format %{ %}
4511   interface(CONST_INTER);
4512 %}
4513 
4514 // Polling Page Pointer Immediate
4515 operand immPollPage()
4516 %{
4517   predicate((address)n->get_ptr() == os::get_polling_page());
4518   match(ConP);
4519 
4520   op_cost(0);
4521   format %{ %}
4522   interface(CONST_INTER);
4523 %}
4524 
4525 // Card Table Byte Map Base
4526 operand immByteMapBase()
4527 %{
4528   // Get base of card map
4529   predicate((jbyte*)n->get_ptr() ==
4530         ((CardTableModRefBS*)(Universe::heap()->barrier_set()))->byte_map_base);
4531   match(ConP);
4532 
4533   op_cost(0);
4534   format %{ %}
4535   interface(CONST_INTER);
4536 %}
4537 
4538 // Pointer Immediate Minus One
4539 // this is used when we want to write the current PC to the thread anchor
4540 operand immP_M1()
4541 %{
4542   predicate(n->get_ptr() == -1);
4543   match(ConP);
4544 
4545   op_cost(0);
4546   format %{ %}
4547   interface(CONST_INTER);
4548 %}
4549 
4550 // Pointer Immediate Minus Two
4551 // this is used when we want to write the current PC to the thread anchor
4552 operand immP_M2()
4553 %{
4554   predicate(n->get_ptr() == -2);
4555   match(ConP);
4556 
4557   op_cost(0);
4558   format %{ %}
4559   interface(CONST_INTER);
4560 %}
4561 
4562 // Float and Double operands
4563 // Double Immediate
4564 operand immD()
4565 %{
4566   match(ConD);
4567   op_cost(0);
4568   format %{ %}
4569   interface(CONST_INTER);
4570 %}
4571 
4572 // Double Immediate: +0.0d
4573 operand immD0()
4574 %{
4575   predicate(jlong_cast(n->getd()) == 0);
4576   match(ConD);
4577 
4578   op_cost(0);
4579   format %{ %}
4580   interface(CONST_INTER);
4581 %}
4582 
4583 // constant 'double +0.0'.
4584 operand immDPacked()
4585 %{
4586   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4587   match(ConD);
4588   op_cost(0);
4589   format %{ %}
4590   interface(CONST_INTER);
4591 %}
4592 
4593 // Float Immediate
4594 operand immF()
4595 %{
4596   match(ConF);
4597   op_cost(0);
4598   format %{ %}
4599   interface(CONST_INTER);
4600 %}
4601 
4602 // Float Immediate: +0.0f.
4603 operand immF0()
4604 %{
4605   predicate(jint_cast(n->getf()) == 0);
4606   match(ConF);
4607 
4608   op_cost(0);
4609   format %{ %}
4610   interface(CONST_INTER);
4611 %}
4612 
4613 //
4614 operand immFPacked()
4615 %{
4616   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4617   match(ConF);
4618   op_cost(0);
4619   format %{ %}
4620   interface(CONST_INTER);
4621 %}
4622 
4623 // Narrow pointer operands
4624 // Narrow Pointer Immediate
4625 operand immN()
4626 %{
4627   match(ConN);
4628 
4629   op_cost(0);
4630   format %{ %}
4631   interface(CONST_INTER);
4632 %}
4633 
4634 // Narrow NULL Pointer Immediate
4635 operand immN0()
4636 %{
4637   predicate(n->get_narrowcon() == 0);
4638   match(ConN);
4639 
4640   op_cost(0);
4641   format %{ %}
4642   interface(CONST_INTER);
4643 %}
4644 
4645 operand immNKlass()
4646 %{
4647   match(ConNKlass);
4648 
4649   op_cost(0);
4650   format %{ %}
4651   interface(CONST_INTER);
4652 %}
4653 
4654 // Integer 32 bit Register Operands
4655 // Integer 32 bitRegister (excludes SP)
4656 operand iRegI()
4657 %{
4658   constraint(ALLOC_IN_RC(any_reg32));
4659   match(RegI);
4660   match(iRegINoSp);
4661   op_cost(0);
4662   format %{ %}
4663   interface(REG_INTER);
4664 %}
4665 
4666 // Integer 32 bit Register not Special
4667 operand iRegINoSp()
4668 %{
4669   constraint(ALLOC_IN_RC(no_special_reg32));
4670   match(RegI);
4671   op_cost(0);
4672   format %{ %}
4673   interface(REG_INTER);
4674 %}
4675 
4676 // Integer 64 bit Register Operands
4677 // Integer 64 bit Register (includes SP)
4678 operand iRegL()
4679 %{
4680   constraint(ALLOC_IN_RC(any_reg));
4681   match(RegL);
4682   match(iRegLNoSp);
4683   op_cost(0);
4684   format %{ %}
4685   interface(REG_INTER);
4686 %}
4687 
4688 // Integer 64 bit Register not Special
4689 operand iRegLNoSp()
4690 %{
4691   constraint(ALLOC_IN_RC(no_special_reg));
4692   match(RegL);
4693   format %{ %}
4694   interface(REG_INTER);
4695 %}
4696 
4697 // Pointer Register Operands
4698 // Pointer Register
4699 operand iRegP()
4700 %{
4701   constraint(ALLOC_IN_RC(ptr_reg));
4702   match(RegP);
4703   match(iRegPNoSp);
4704   match(iRegP_R0);
4705   //match(iRegP_R2);
4706   //match(iRegP_R4);
4707   //match(iRegP_R5);
4708   match(thread_RegP);
4709   op_cost(0);
4710   format %{ %}
4711   interface(REG_INTER);
4712 %}
4713 
4714 // Pointer 64 bit Register not Special
4715 operand iRegPNoSp()
4716 %{
4717   constraint(ALLOC_IN_RC(no_special_ptr_reg));
4718   match(RegP);
4719   // match(iRegP);
4720   // match(iRegP_R0);
4721   // match(iRegP_R2);
4722   // match(iRegP_R4);
4723   // match(iRegP_R5);
4724   // match(thread_RegP);
4725   op_cost(0);
4726   format %{ %}
4727   interface(REG_INTER);
4728 %}
4729 
4730 // Pointer 64 bit Register R0 only
4731 operand iRegP_R0()
4732 %{
4733   constraint(ALLOC_IN_RC(r0_reg));
4734   match(RegP);
4735   // match(iRegP);
4736   match(iRegPNoSp);
4737   op_cost(0);
4738   format %{ %}
4739   interface(REG_INTER);
4740 %}
4741 
4742 // Pointer 64 bit Register R1 only
4743 operand iRegP_R1()
4744 %{
4745   constraint(ALLOC_IN_RC(r1_reg));
4746   match(RegP);
4747   // match(iRegP);
4748   match(iRegPNoSp);
4749   op_cost(0);
4750   format %{ %}
4751   interface(REG_INTER);
4752 %}
4753 
4754 // Pointer 64 bit Register R2 only
4755 operand iRegP_R2()
4756 %{
4757   constraint(ALLOC_IN_RC(r2_reg));
4758   match(RegP);
4759   // match(iRegP);
4760   match(iRegPNoSp);
4761   op_cost(0);
4762   format %{ %}
4763   interface(REG_INTER);
4764 %}
4765 
4766 // Pointer 64 bit Register R3 only
4767 operand iRegP_R3()
4768 %{
4769   constraint(ALLOC_IN_RC(r3_reg));
4770   match(RegP);
4771   // match(iRegP);
4772   match(iRegPNoSp);
4773   op_cost(0);
4774   format %{ %}
4775   interface(REG_INTER);
4776 %}
4777 
4778 // Pointer 64 bit Register R4 only
4779 operand iRegP_R4()
4780 %{
4781   constraint(ALLOC_IN_RC(r4_reg));
4782   match(RegP);
4783   // match(iRegP);
4784   match(iRegPNoSp);
4785   op_cost(0);
4786   format %{ %}
4787   interface(REG_INTER);
4788 %}
4789 
4790 // Pointer 64 bit Register R5 only
4791 operand iRegP_R5()
4792 %{
4793   constraint(ALLOC_IN_RC(r5_reg));
4794   match(RegP);
4795   // match(iRegP);
4796   match(iRegPNoSp);
4797   op_cost(0);
4798   format %{ %}
4799   interface(REG_INTER);
4800 %}
4801 
4802 // Pointer 64 bit Register R10 only
4803 operand iRegP_R10()
4804 %{
4805   constraint(ALLOC_IN_RC(r10_reg));
4806   match(RegP);
4807   // match(iRegP);
4808   match(iRegPNoSp);
4809   op_cost(0);
4810   format %{ %}
4811   interface(REG_INTER);
4812 %}
4813 
4814 // Long 64 bit Register R11 only
4815 operand iRegL_R11()
4816 %{
4817   constraint(ALLOC_IN_RC(r11_reg));
4818   match(RegL);
4819   match(iRegLNoSp);
4820   op_cost(0);
4821   format %{ %}
4822   interface(REG_INTER);
4823 %}
4824 
4825 // Pointer 64 bit Register FP only
4826 operand iRegP_FP()
4827 %{
4828   constraint(ALLOC_IN_RC(fp_reg));
4829   match(RegP);
4830   // match(iRegP);
4831   op_cost(0);
4832   format %{ %}
4833   interface(REG_INTER);
4834 %}
4835 
4836 // Register R0 only
4837 operand iRegI_R0()
4838 %{
4839   constraint(ALLOC_IN_RC(int_r0_reg));
4840   match(RegI);
4841   match(iRegINoSp);
4842   op_cost(0);
4843   format %{ %}
4844   interface(REG_INTER);
4845 %}
4846 
4847 // Register R2 only
4848 operand iRegI_R2()
4849 %{
4850   constraint(ALLOC_IN_RC(int_r2_reg));
4851   match(RegI);
4852   match(iRegINoSp);
4853   op_cost(0);
4854   format %{ %}
4855   interface(REG_INTER);
4856 %}
4857 
4858 // Register R3 only
4859 operand iRegI_R3()
4860 %{
4861   constraint(ALLOC_IN_RC(int_r3_reg));
4862   match(RegI);
4863   match(iRegINoSp);
4864   op_cost(0);
4865   format %{ %}
4866   interface(REG_INTER);
4867 %}
4868 
4869 
4870 // Register R2 only
4871 operand iRegI_R4()
4872 %{
4873   constraint(ALLOC_IN_RC(int_r4_reg));
4874   match(RegI);
4875   match(iRegINoSp);
4876   op_cost(0);
4877   format %{ %}
4878   interface(REG_INTER);
4879 %}
4880 
4881 
4882 // Pointer Register Operands
4883 // Narrow Pointer Register
4884 operand iRegN()
4885 %{
4886   constraint(ALLOC_IN_RC(any_reg32));
4887   match(RegN);
4888   match(iRegNNoSp);
4889   op_cost(0);
4890   format %{ %}
4891   interface(REG_INTER);
4892 %}
4893 
4894 // Integer 64 bit Register not Special
4895 operand iRegNNoSp()
4896 %{
4897   constraint(ALLOC_IN_RC(no_special_reg32));
4898   match(RegN);
4899   op_cost(0);
4900   format %{ %}
4901   interface(REG_INTER);
4902 %}
4903 
4904 // heap base register -- used for encoding immN0
4905 
4906 operand iRegIHeapbase()
4907 %{
4908   constraint(ALLOC_IN_RC(heapbase_reg));
4909   match(RegI);
4910   op_cost(0);
4911   format %{ %}
4912   interface(REG_INTER);
4913 %}
4914 
4915 // Float Register
4916 // Float register operands
4917 operand vRegF()
4918 %{
4919   constraint(ALLOC_IN_RC(float_reg));
4920   match(RegF);
4921 
4922   op_cost(0);
4923   format %{ %}
4924   interface(REG_INTER);
4925 %}
4926 
4927 // Double Register
4928 // Double register operands
4929 operand vRegD()
4930 %{
4931   constraint(ALLOC_IN_RC(double_reg));
4932   match(RegD);
4933 
4934   op_cost(0);
4935   format %{ %}
4936   interface(REG_INTER);
4937 %}
4938 
4939 operand vRegD_V0()
4940 %{
4941   constraint(ALLOC_IN_RC(v0_reg));
4942   match(RegD);
4943   op_cost(0);
4944   format %{ %}
4945   interface(REG_INTER);
4946 %}
4947 
4948 operand vRegD_V1()
4949 %{
4950   constraint(ALLOC_IN_RC(v1_reg));
4951   match(RegD);
4952   op_cost(0);
4953   format %{ %}
4954   interface(REG_INTER);
4955 %}
4956 
4957 operand vRegD_V2()
4958 %{
4959   constraint(ALLOC_IN_RC(v2_reg));
4960   match(RegD);
4961   op_cost(0);
4962   format %{ %}
4963   interface(REG_INTER);
4964 %}
4965 
4966 operand vRegD_V3()
4967 %{
4968   constraint(ALLOC_IN_RC(v3_reg));
4969   match(RegD);
4970   op_cost(0);
4971   format %{ %}
4972   interface(REG_INTER);
4973 %}
4974 
4975 // Flags register, used as output of signed compare instructions
4976 
4977 // note that on AArch64 we also use this register as the output for
4978 // for floating point compare instructions (CmpF CmpD). this ensures
4979 // that ordered inequality tests use GT, GE, LT or LE none of which
4980 // pass through cases where the result is unordered i.e. one or both
4981 // inputs to the compare is a NaN. this means that the ideal code can
4982 // replace e.g. a GT with an LE and not end up capturing the NaN case
4983 // (where the comparison should always fail). EQ and NE tests are
4984 // always generated in ideal code so that unordered folds into the NE
4985 // case, matching the behaviour of AArch64 NE.
4986 //
4987 // This differs from x86 where the outputs of FP compares use a
4988 // special FP flags registers and where compares based on this
4989 // register are distinguished into ordered inequalities (cmpOpUCF) and
4990 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
4991 // to explicitly handle the unordered case in branches. x86 also has
4992 // to include extra CMoveX rules to accept a cmpOpUCF input.
4993 
4994 operand rFlagsReg()
4995 %{
4996   constraint(ALLOC_IN_RC(int_flags));
4997   match(RegFlags);
4998 
4999   op_cost(0);
5000   format %{ "RFLAGS" %}
5001   interface(REG_INTER);
5002 %}
5003 
5004 // Flags register, used as output of unsigned compare instructions
5005 operand rFlagsRegU()
5006 %{
5007   constraint(ALLOC_IN_RC(int_flags));
5008   match(RegFlags);
5009 
5010   op_cost(0);
5011   format %{ "RFLAGSU" %}
5012   interface(REG_INTER);
5013 %}
5014 
5015 // Special Registers
5016 
5017 // Method Register
5018 operand inline_cache_RegP(iRegP reg)
5019 %{
5020   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5021   match(reg);
5022   match(iRegPNoSp);
5023   op_cost(0);
5024   format %{ %}
5025   interface(REG_INTER);
5026 %}
5027 
5028 operand interpreter_method_oop_RegP(iRegP reg)
5029 %{
5030   constraint(ALLOC_IN_RC(method_reg)); // interpreter_method_oop_reg
5031   match(reg);
5032   match(iRegPNoSp);
5033   op_cost(0);
5034   format %{ %}
5035   interface(REG_INTER);
5036 %}
5037 
5038 // Thread Register
5039 operand thread_RegP(iRegP reg)
5040 %{
5041   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5042   match(reg);
5043   op_cost(0);
5044   format %{ %}
5045   interface(REG_INTER);
5046 %}
5047 
5048 operand lr_RegP(iRegP reg)
5049 %{
5050   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5051   match(reg);
5052   op_cost(0);
5053   format %{ %}
5054   interface(REG_INTER);
5055 %}
5056 
5057 //----------Memory Operands----------------------------------------------------
5058 
5059 operand indirect(iRegP reg)
5060 %{
5061   constraint(ALLOC_IN_RC(ptr_reg));
5062   match(reg);
5063   op_cost(0);
5064   format %{ "[$reg]" %}
5065   interface(MEMORY_INTER) %{
5066     base($reg);
5067     index(0xffffffff);
5068     scale(0x0);
5069     disp(0x0);
5070   %}
5071 %}
5072 
5073 operand indIndexScaledOffsetI(iRegP reg, iRegL lreg, immIScale scale, immIU12 off)
5074 %{
5075   constraint(ALLOC_IN_RC(ptr_reg));
5076   match(AddP (AddP reg (LShiftL lreg scale)) off);
5077   op_cost(INSN_COST);
5078   format %{ "$reg, $lreg lsl($scale), $off" %}
5079   interface(MEMORY_INTER) %{
5080     base($reg);
5081     index($lreg);
5082     scale($scale);
5083     disp($off);
5084   %}
5085 %}
5086 
5087 operand indIndexScaledOffsetL(iRegP reg, iRegL lreg, immIScale scale, immLU12 off)
5088 %{
5089   constraint(ALLOC_IN_RC(ptr_reg));
5090   match(AddP (AddP reg (LShiftL lreg scale)) off);
5091   op_cost(INSN_COST);
5092   format %{ "$reg, $lreg lsl($scale), $off" %}
5093   interface(MEMORY_INTER) %{
5094     base($reg);
5095     index($lreg);
5096     scale($scale);
5097     disp($off);
5098   %}
5099 %}
5100 
5101 operand indIndexOffsetI2L(iRegP reg, iRegI ireg, immLU12 off)
5102 %{
5103   constraint(ALLOC_IN_RC(ptr_reg));
5104   match(AddP (AddP reg (ConvI2L ireg)) off);
5105   op_cost(INSN_COST);
5106   format %{ "$reg, $ireg, $off I2L" %}
5107   interface(MEMORY_INTER) %{
5108     base($reg);
5109     index($ireg);
5110     scale(0x0);
5111     disp($off);
5112   %}
5113 %}
5114 
5115 operand indIndexScaledOffsetI2L(iRegP reg, iRegI ireg, immIScale scale, immLU12 off)
5116 %{
5117   constraint(ALLOC_IN_RC(ptr_reg));
5118   match(AddP (AddP reg (LShiftL (ConvI2L ireg) scale)) off);
5119   op_cost(INSN_COST);
5120   format %{ "$reg, $ireg sxtw($scale), $off I2L" %}
5121   interface(MEMORY_INTER) %{
5122     base($reg);
5123     index($ireg);
5124     scale($scale);
5125     disp($off);
5126   %}
5127 %}
5128 
5129 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5130 %{
5131   constraint(ALLOC_IN_RC(ptr_reg));
5132   match(AddP reg (LShiftL (ConvI2L ireg) scale));
5133   op_cost(0);
5134   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5135   interface(MEMORY_INTER) %{
5136     base($reg);
5137     index($ireg);
5138     scale($scale);
5139     disp(0x0);
5140   %}
5141 %}
5142 
5143 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5144 %{
5145   constraint(ALLOC_IN_RC(ptr_reg));
5146   match(AddP reg (LShiftL lreg scale));
5147   op_cost(0);
5148   format %{ "$reg, $lreg lsl($scale)" %}
5149   interface(MEMORY_INTER) %{
5150     base($reg);
5151     index($lreg);
5152     scale($scale);
5153     disp(0x0);
5154   %}
5155 %}
5156 
5157 operand indIndex(iRegP reg, iRegL lreg)
5158 %{
5159   constraint(ALLOC_IN_RC(ptr_reg));
5160   match(AddP reg lreg);
5161   op_cost(0);
5162   format %{ "$reg, $lreg" %}
5163   interface(MEMORY_INTER) %{
5164     base($reg);
5165     index($lreg);
5166     scale(0x0);
5167     disp(0x0);
5168   %}
5169 %}
5170 
5171 operand indOffI(iRegP reg, immIOffset off)
5172 %{
5173   constraint(ALLOC_IN_RC(ptr_reg));
5174   match(AddP reg off);
5175   op_cost(0);
5176   format %{ "[$reg, $off]" %}
5177   interface(MEMORY_INTER) %{
5178     base($reg);
5179     index(0xffffffff);
5180     scale(0x0);
5181     disp($off);
5182   %}
5183 %}
5184 
5185 operand indOffL(iRegP reg, immLoffset off)
5186 %{
5187   constraint(ALLOC_IN_RC(ptr_reg));
5188   match(AddP reg off);
5189   op_cost(0);
5190   format %{ "[$reg, $off]" %}
5191   interface(MEMORY_INTER) %{
5192     base($reg);
5193     index(0xffffffff);
5194     scale(0x0);
5195     disp($off);
5196   %}
5197 %}
5198 
5199 
5200 operand indirectN(iRegN reg)
5201 %{
5202   predicate(Universe::narrow_oop_shift() == 0);
5203   constraint(ALLOC_IN_RC(ptr_reg));
5204   match(DecodeN reg);
5205   op_cost(0);
5206   format %{ "[$reg]\t# narrow" %}
5207   interface(MEMORY_INTER) %{
5208     base($reg);
5209     index(0xffffffff);
5210     scale(0x0);
5211     disp(0x0);
5212   %}
5213 %}
5214 
5215 operand indIndexScaledOffsetIN(iRegN reg, iRegL lreg, immIScale scale, immIU12 off)
5216 %{
5217   predicate(Universe::narrow_oop_shift() == 0);
5218   constraint(ALLOC_IN_RC(ptr_reg));
5219   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5220   op_cost(0);
5221   format %{ "$reg, $lreg lsl($scale), $off\t# narrow" %}
5222   interface(MEMORY_INTER) %{
5223     base($reg);
5224     index($lreg);
5225     scale($scale);
5226     disp($off);
5227   %}
5228 %}
5229 
5230 operand indIndexScaledOffsetLN(iRegN reg, iRegL lreg, immIScale scale, immLU12 off)
5231 %{
5232   predicate(Universe::narrow_oop_shift() == 0);
5233   constraint(ALLOC_IN_RC(ptr_reg));
5234   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5235   op_cost(INSN_COST);
5236   format %{ "$reg, $lreg lsl($scale), $off\t# narrow" %}
5237   interface(MEMORY_INTER) %{
5238     base($reg);
5239     index($lreg);
5240     scale($scale);
5241     disp($off);
5242   %}
5243 %}
5244 
5245 operand indIndexOffsetI2LN(iRegN reg, iRegI ireg, immLU12 off)
5246 %{
5247   predicate(Universe::narrow_oop_shift() == 0);
5248   constraint(ALLOC_IN_RC(ptr_reg));
5249   match(AddP (AddP (DecodeN reg) (ConvI2L ireg)) off);
5250   op_cost(INSN_COST);
5251   format %{ "$reg, $ireg, $off I2L\t# narrow" %}
5252   interface(MEMORY_INTER) %{
5253     base($reg);
5254     index($ireg);
5255     scale(0x0);
5256     disp($off);
5257   %}
5258 %}
5259 
5260 operand indIndexScaledOffsetI2LN(iRegN reg, iRegI ireg, immIScale scale, immLU12 off)
5261 %{
5262   predicate(Universe::narrow_oop_shift() == 0);
5263   constraint(ALLOC_IN_RC(ptr_reg));
5264   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale)) off);
5265   op_cost(INSN_COST);
5266   format %{ "$reg, $ireg sxtw($scale), $off I2L\t# narrow" %}
5267   interface(MEMORY_INTER) %{
5268     base($reg);
5269     index($ireg);
5270     scale($scale);
5271     disp($off);
5272   %}
5273 %}
5274 
5275 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5276 %{
5277   predicate(Universe::narrow_oop_shift() == 0);
5278   constraint(ALLOC_IN_RC(ptr_reg));
5279   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5280   op_cost(0);
5281   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5282   interface(MEMORY_INTER) %{
5283     base($reg);
5284     index($ireg);
5285     scale($scale);
5286     disp(0x0);
5287   %}
5288 %}
5289 
5290 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5291 %{
5292   predicate(Universe::narrow_oop_shift() == 0);
5293   constraint(ALLOC_IN_RC(ptr_reg));
5294   match(AddP (DecodeN reg) (LShiftL lreg scale));
5295   op_cost(0);
5296   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5297   interface(MEMORY_INTER) %{
5298     base($reg);
5299     index($lreg);
5300     scale($scale);
5301     disp(0x0);
5302   %}
5303 %}
5304 
5305 operand indIndexN(iRegN reg, iRegL lreg)
5306 %{
5307   predicate(Universe::narrow_oop_shift() == 0);
5308   constraint(ALLOC_IN_RC(ptr_reg));
5309   match(AddP (DecodeN reg) lreg);
5310   op_cost(0);
5311   format %{ "$reg, $lreg\t# narrow" %}
5312   interface(MEMORY_INTER) %{
5313     base($reg);
5314     index($lreg);
5315     scale(0x0);
5316     disp(0x0);
5317   %}
5318 %}
5319 
5320 operand indOffIN(iRegN reg, immIOffset off)
5321 %{
5322   predicate(Universe::narrow_oop_shift() == 0);
5323   constraint(ALLOC_IN_RC(ptr_reg));
5324   match(AddP (DecodeN reg) off);
5325   op_cost(0);
5326   format %{ "[$reg, $off]\t# narrow" %}
5327   interface(MEMORY_INTER) %{
5328     base($reg);
5329     index(0xffffffff);
5330     scale(0x0);
5331     disp($off);
5332   %}
5333 %}
5334 
5335 operand indOffLN(iRegN reg, immLoffset off)
5336 %{
5337   predicate(Universe::narrow_oop_shift() == 0);
5338   constraint(ALLOC_IN_RC(ptr_reg));
5339   match(AddP (DecodeN reg) off);
5340   op_cost(0);
5341   format %{ "[$reg, $off]\t# narrow" %}
5342   interface(MEMORY_INTER) %{
5343     base($reg);
5344     index(0xffffffff);
5345     scale(0x0);
5346     disp($off);
5347   %}
5348 %}
5349 
5350 
5351 
5352 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5353 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5354 %{
5355   constraint(ALLOC_IN_RC(ptr_reg));
5356   match(AddP reg off);
5357   op_cost(0);
5358   format %{ "[$reg, $off]" %}
5359   interface(MEMORY_INTER) %{
5360     base($reg);
5361     index(0xffffffff);
5362     scale(0x0);
5363     disp($off);
5364   %}
5365 %}
5366 
5367 //----------Special Memory Operands--------------------------------------------
5368 // Stack Slot Operand - This operand is used for loading and storing temporary
5369 //                      values on the stack where a match requires a value to
5370 //                      flow through memory.
5371 operand stackSlotP(sRegP reg)
5372 %{
5373   constraint(ALLOC_IN_RC(stack_slots));
5374   op_cost(100);
5375   // No match rule because this operand is only generated in matching
5376   // match(RegP);
5377   format %{ "[$reg]" %}
5378   interface(MEMORY_INTER) %{
5379     base(0x1e);  // RSP
5380     index(0x0);  // No Index
5381     scale(0x0);  // No Scale
5382     disp($reg);  // Stack Offset
5383   %}
5384 %}
5385 
5386 operand stackSlotI(sRegI reg)
5387 %{
5388   constraint(ALLOC_IN_RC(stack_slots));
5389   // No match rule because this operand is only generated in matching
5390   // match(RegI);
5391   format %{ "[$reg]" %}
5392   interface(MEMORY_INTER) %{
5393     base(0x1e);  // RSP
5394     index(0x0);  // No Index
5395     scale(0x0);  // No Scale
5396     disp($reg);  // Stack Offset
5397   %}
5398 %}
5399 
5400 operand stackSlotF(sRegF reg)
5401 %{
5402   constraint(ALLOC_IN_RC(stack_slots));
5403   // No match rule because this operand is only generated in matching
5404   // match(RegF);
5405   format %{ "[$reg]" %}
5406   interface(MEMORY_INTER) %{
5407     base(0x1e);  // RSP
5408     index(0x0);  // No Index
5409     scale(0x0);  // No Scale
5410     disp($reg);  // Stack Offset
5411   %}
5412 %}
5413 
5414 operand stackSlotD(sRegD reg)
5415 %{
5416   constraint(ALLOC_IN_RC(stack_slots));
5417   // No match rule because this operand is only generated in matching
5418   // match(RegD);
5419   format %{ "[$reg]" %}
5420   interface(MEMORY_INTER) %{
5421     base(0x1e);  // RSP
5422     index(0x0);  // No Index
5423     scale(0x0);  // No Scale
5424     disp($reg);  // Stack Offset
5425   %}
5426 %}
5427 
5428 operand stackSlotL(sRegL reg)
5429 %{
5430   constraint(ALLOC_IN_RC(stack_slots));
5431   // No match rule because this operand is only generated in matching
5432   // match(RegL);
5433   format %{ "[$reg]" %}
5434   interface(MEMORY_INTER) %{
5435     base(0x1e);  // RSP
5436     index(0x0);  // No Index
5437     scale(0x0);  // No Scale
5438     disp($reg);  // Stack Offset
5439   %}
5440 %}
5441 
5442 // Operands for expressing Control Flow
5443 // NOTE: Label is a predefined operand which should not be redefined in
5444 //       the AD file. It is generically handled within the ADLC.
5445 
5446 //----------Conditional Branch Operands----------------------------------------
5447 // Comparison Op  - This is the operation of the comparison, and is limited to
5448 //                  the following set of codes:
5449 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5450 //
5451 // Other attributes of the comparison, such as unsignedness, are specified
5452 // by the comparison instruction that sets a condition code flags register.
5453 // That result is represented by a flags operand whose subtype is appropriate
5454 // to the unsignedness (etc.) of the comparison.
5455 //
5456 // Later, the instruction which matches both the Comparison Op (a Bool) and
5457 // the flags (produced by the Cmp) specifies the coding of the comparison op
5458 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5459 
5460 // used for signed integral comparisons and fp comparisons
5461 
5462 operand cmpOp()
5463 %{
5464   match(Bool);
5465 
5466   format %{ "" %}
5467   interface(COND_INTER) %{
5468     equal(0x0, "eq");
5469     not_equal(0x1, "ne");
5470     less(0xb, "lt");
5471     greater_equal(0xa, "ge");
5472     less_equal(0xd, "le");
5473     greater(0xc, "gt");
5474     overflow(0x6, "vs");
5475     no_overflow(0x7, "vc");
5476   %}
5477 %}
5478 
5479 // used for unsigned integral comparisons
5480 
5481 operand cmpOpU()
5482 %{
5483   match(Bool);
5484 
5485   format %{ "" %}
5486   interface(COND_INTER) %{
5487     equal(0x0, "eq");
5488     not_equal(0x1, "ne");
5489     less(0x3, "lo");
5490     greater_equal(0x2, "hs");
5491     less_equal(0x9, "ls");
5492     greater(0x8, "hi");
5493     overflow(0x6, "vs");
5494     no_overflow(0x7, "vc");
5495   %}
5496 %}
5497 
5498 // Special operand allowing long args to int ops to be truncated for free
5499 
5500 operand iRegL2I(iRegL reg) %{
5501 
5502   op_cost(0);
5503 
5504   match(ConvL2I reg);
5505 
5506   format %{ "l2i($reg)" %}
5507 
5508   interface(REG_INTER)
5509 %}
5510 
5511 
5512 //----------OPERAND CLASSES----------------------------------------------------
5513 // Operand Classes are groups of operands that are used as to simplify
5514 // instruction definitions by not requiring the AD writer to specify
5515 // separate instructions for every form of operand when the
5516 // instruction accepts multiple operand types with the same basic
5517 // encoding and format. The classic case of this is memory operands.
5518 
5519 // memory is used to define read/write location for load/store
5520 // instruction defs. we can turn a memory op into an Address
5521 
5522 opclass memory(indirect, indIndexScaledOffsetI, indIndexScaledOffsetL, indIndexOffsetI2L, indIndexScaledOffsetI2L, indIndexScaled, indIndexScaledI2L, indIndex, indOffI, indOffL,
5523                indirectN, indIndexScaledOffsetIN, indIndexScaledOffsetLN, indIndexOffsetI2LN, indIndexScaledOffsetI2LN, indIndexScaledN, indIndexScaledI2LN, indIndexN, indOffIN, indOffLN);
5524 
5525 
5526 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5527 // operations. it allows the src to be either an iRegI or a (ConvL2I
5528 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5529 // can be elided because the 32-bit instruction will just employ the
5530 // lower 32 bits anyway.
5531 //
5532 // n.b. this does not elide all L2I conversions. if the truncated
5533 // value is consumed by more than one operation then the ConvL2I
5534 // cannot be bundled into the consuming nodes so an l2i gets planted
5535 // (actually a movw $dst $src) and the downstream instructions consume
5536 // the result of the l2i as an iRegI input. That's a shame since the
5537 // movw is actually redundant but its not too costly.
5538 
5539 opclass iRegIorL2I(iRegI, iRegL2I);
5540 
5541 //----------PIPELINE-----------------------------------------------------------
5542 // Rules which define the behavior of the target architectures pipeline.
5543 // Integer ALU reg operation
5544 pipeline %{
5545 
5546 attributes %{
5547   // ARM instructions are of fixed length
5548   fixed_size_instructions;        // Fixed size instructions TODO does
5549   max_instructions_per_bundle = 2;   // A53 = 2, A57 = 4
5550   // ARM instructions come in 32-bit word units
5551   instruction_unit_size = 4;         // An instruction is 4 bytes long
5552   instruction_fetch_unit_size = 64;  // The processor fetches one line
5553   instruction_fetch_units = 1;       // of 64 bytes
5554 
5555   // List of nop instructions
5556   nops( MachNop );
5557 %}
5558 
5559 // We don't use an actual pipeline model so don't care about resources
5560 // or description. we do use pipeline classes to introduce fixed
5561 // latencies
5562 
5563 //----------RESOURCES----------------------------------------------------------
5564 // Resources are the functional units available to the machine
5565 
5566 resources( INS0, INS1, INS01 = INS0 | INS1,
5567            ALU0, ALU1, ALU = ALU0 | ALU1,
5568            MAC,
5569            DIV,
5570            BRANCH,
5571            LDST,
5572            NEON_FP);
5573 
5574 //----------PIPELINE DESCRIPTION-----------------------------------------------
5575 // Pipeline Description specifies the stages in the machine's pipeline
5576 
5577 pipe_desc(ISS, EX1, EX2, WR);
5578 
5579 //----------PIPELINE CLASSES---------------------------------------------------
5580 // Pipeline Classes describe the stages in which input and output are
5581 // referenced by the hardware pipeline.
5582 
5583 //------- Integer ALU operations --------------------------
5584 
5585 // Integer ALU reg-reg operation
5586 // Operands needed in EX1, result generated in EX2
5587 // Eg.  ADD     x0, x1, x2
5588 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5589 %{
5590   single_instruction;
5591   dst    : EX2(write);
5592   src1   : EX1(read);
5593   src2   : EX1(read);
5594   INS01  : ISS; // Dual issue as instruction 0 or 1
5595   ALU    : EX2;
5596 %}
5597 
5598 // Integer ALU reg-reg operation with constant shift
5599 // Shifted register must be available in LATE_ISS instead of EX1
5600 // Eg.  ADD     x0, x1, x2, LSL #2
5601 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
5602 %{
5603   single_instruction;
5604   dst    : EX2(write);
5605   src1   : EX1(read);
5606   src2   : ISS(read);
5607   INS01  : ISS;
5608   ALU    : EX2;
5609 %}
5610 
5611 // Integer ALU reg operation with constant shift
5612 // Eg.  LSL     x0, x1, #shift
5613 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
5614 %{
5615   single_instruction;
5616   dst    : EX2(write);
5617   src1   : ISS(read);
5618   INS01  : ISS;
5619   ALU    : EX2;
5620 %}
5621 
5622 // Integer ALU reg-reg operation with variable shift
5623 // Both operands must be available in LATE_ISS instead of EX1
5624 // Result is available in EX1 instead of EX2
5625 // Eg.  LSLV    x0, x1, x2
5626 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
5627 %{
5628   single_instruction;
5629   dst    : EX1(write);
5630   src1   : ISS(read);
5631   src2   : ISS(read);
5632   INS01  : ISS;
5633   ALU    : EX1;
5634 %}
5635 
5636 // Integer ALU reg-reg operation with extract
5637 // As for _vshift above, but result generated in EX2
5638 // Eg.  EXTR    x0, x1, x2, #N
5639 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
5640 %{
5641   single_instruction;
5642   dst    : EX2(write);
5643   src1   : ISS(read);
5644   src2   : ISS(read);
5645   INS1   : ISS; // Can only dual issue as Instruction 1
5646   ALU    : EX1;
5647 %}
5648 
5649 // Integer ALU reg operation
5650 // Eg.  NEG     x0, x1
5651 pipe_class ialu_reg(iRegI dst, iRegI src)
5652 %{
5653   single_instruction;
5654   dst    : EX2(write);
5655   src    : EX1(read);
5656   INS01  : ISS;
5657   ALU    : EX2;
5658 %}
5659 
5660 // Integer ALU reg mmediate operation
5661 // Eg.  ADD     x0, x1, #N
5662 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
5663 %{
5664   single_instruction;
5665   dst    : EX2(write);
5666   src1   : EX1(read);
5667   INS01  : ISS;
5668   ALU    : EX2;
5669 %}
5670 
5671 // Integer ALU immediate operation (no source operands)
5672 // Eg.  MOV     x0, #N
5673 pipe_class ialu_imm(iRegI dst)
5674 %{
5675   single_instruction;
5676   dst    : EX1(write);
5677   INS01  : ISS;
5678   ALU    : EX1;
5679 %}
5680 
5681 //------- Compare operation -------------------------------
5682 
5683 // Compare reg-reg
5684 // Eg.  CMP     x0, x1
5685 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
5686 %{
5687   single_instruction;
5688 //  fixed_latency(16);
5689   cr     : EX2(write);
5690   op1    : EX1(read);
5691   op2    : EX1(read);
5692   INS01  : ISS;
5693   ALU    : EX2;
5694 %}
5695 
5696 // Compare reg-reg
5697 // Eg.  CMP     x0, #N
5698 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
5699 %{
5700   single_instruction;
5701 //  fixed_latency(16);
5702   cr     : EX2(write);
5703   op1    : EX1(read);
5704   INS01  : ISS;
5705   ALU    : EX2;
5706 %}
5707 
5708 //------- Conditional instructions ------------------------
5709 
5710 // Conditional no operands
5711 // Eg.  CSINC   x0, zr, zr, <cond>
5712 pipe_class icond_none(iRegI dst, rFlagsReg cr)
5713 %{
5714   single_instruction;
5715   cr     : EX1(read);
5716   dst    : EX2(write);
5717   INS01  : ISS;
5718   ALU    : EX2;
5719 %}
5720 
5721 // Conditional 2 operand
5722 // EG.  CSEL    X0, X1, X2, <cond>
5723 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
5724 %{
5725   single_instruction;
5726   cr     : EX1(read);
5727   src1   : EX1(read);
5728   src2   : EX1(read);
5729   dst    : EX2(write);
5730   INS01  : ISS;
5731   ALU    : EX2;
5732 %}
5733 
5734 // Conditional 2 operand
5735 // EG.  CSEL    X0, X1, X2, <cond>
5736 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
5737 %{
5738   single_instruction;
5739   cr     : EX1(read);
5740   src    : EX1(read);
5741   dst    : EX2(write);
5742   INS01  : ISS;
5743   ALU    : EX2;
5744 %}
5745 
5746 //------- Multiply pipeline operations --------------------
5747 
5748 // Multiply reg-reg
5749 // Eg.  MUL     w0, w1, w2
5750 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5751 %{
5752   single_instruction;
5753   dst    : WR(write);
5754   src1   : ISS(read);
5755   src2   : ISS(read);
5756   INS01  : ISS;
5757   MAC    : WR;
5758 %}
5759 
5760 // Multiply accumulate
5761 // Eg.  MADD    w0, w1, w2, w3
5762 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
5763 %{
5764   single_instruction;
5765   dst    : WR(write);
5766   src1   : ISS(read);
5767   src2   : ISS(read);
5768   src3   : ISS(read);
5769   INS01  : ISS;
5770   MAC    : WR;
5771 %}
5772 
5773 // Eg.  MUL     w0, w1, w2
5774 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5775 %{
5776   single_instruction;
5777   fixed_latency(3); // Maximum latency for 64 bit mul
5778   dst    : WR(write);
5779   src1   : ISS(read);
5780   src2   : ISS(read);
5781   INS01  : ISS;
5782   MAC    : WR;
5783 %}
5784 
5785 // Multiply accumulate
5786 // Eg.  MADD    w0, w1, w2, w3
5787 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
5788 %{
5789   single_instruction;
5790   fixed_latency(3); // Maximum latency for 64 bit mul
5791   dst    : WR(write);
5792   src1   : ISS(read);
5793   src2   : ISS(read);
5794   src3   : ISS(read);
5795   INS01  : ISS;
5796   MAC    : WR;
5797 %}
5798 
5799 //------- Divide pipeline operations --------------------
5800 
5801 // Eg.  SDIV    w0, w1, w2
5802 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5803 %{
5804   single_instruction;
5805   fixed_latency(8); // Maximum latency for 32 bit divide
5806   dst    : WR(write);
5807   src1   : ISS(read);
5808   src2   : ISS(read);
5809   INS0   : ISS; // Can only dual issue as instruction 0
5810   DIV    : WR;
5811 %}
5812 
5813 // Eg.  SDIV    x0, x1, x2
5814 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5815 %{
5816   single_instruction;
5817   fixed_latency(16); // Maximum latency for 64 bit divide
5818   dst    : WR(write);
5819   src1   : ISS(read);
5820   src2   : ISS(read);
5821   INS0   : ISS; // Can only dual issue as instruction 0
5822   DIV    : WR;
5823 %}
5824 
5825 //------- Load pipeline operations ------------------------
5826 
5827 // Load - prefetch
5828 // Eg.  PFRM    <mem>
5829 pipe_class iload_prefetch(memory mem)
5830 %{
5831   single_instruction;
5832   mem    : ISS(read);
5833   INS01  : ISS;
5834   LDST   : WR;
5835 %}
5836 
5837 // Load - reg, mem
5838 // Eg.  LDR     x0, <mem>
5839 pipe_class iload_reg_mem(iRegI dst, memory mem)
5840 %{
5841   single_instruction;
5842   dst    : WR(write);
5843   mem    : ISS(read);
5844   INS01  : ISS;
5845   LDST   : WR;
5846 %}
5847 
5848 // Load - reg, reg
5849 // Eg.  LDR     x0, [sp, x1]
5850 pipe_class iload_reg_reg(iRegI dst, iRegI src)
5851 %{
5852   single_instruction;
5853   dst    : WR(write);
5854   src    : ISS(read);
5855   INS01  : ISS;
5856   LDST   : WR;
5857 %}
5858 
5859 //------- Store pipeline operations -----------------------
5860 
5861 // Store - zr, mem
5862 // Eg.  STR     zr, <mem>
5863 pipe_class istore_mem(memory mem)
5864 %{
5865   single_instruction;
5866   mem    : ISS(read);
5867   INS01  : ISS;
5868   LDST   : WR;
5869 %}
5870 
5871 // Store - reg, mem
5872 // Eg.  STR     x0, <mem>
5873 pipe_class istore_reg_mem(iRegI src, memory mem)
5874 %{
5875   single_instruction;
5876   mem    : ISS(read);
5877   src    : EX2(read);
5878   INS01  : ISS;
5879   LDST   : WR;
5880 %}
5881 
5882 // Store - reg, reg
5883 // Eg. STR      x0, [sp, x1]
5884 pipe_class istore_reg_reg(iRegI dst, iRegI src)
5885 %{
5886   single_instruction;
5887   dst    : ISS(read);
5888   src    : EX2(read);
5889   INS01  : ISS;
5890   LDST   : WR;
5891 %}
5892 
5893 //------- Store pipeline operations -----------------------
5894 
5895 // Branch
5896 pipe_class pipe_branch()
5897 %{
5898   single_instruction;
5899   INS01  : ISS;
5900   BRANCH : EX1;
5901 %}
5902 
5903 // Conditional branch
5904 pipe_class pipe_branch_cond(rFlagsReg cr)
5905 %{
5906   single_instruction;
5907   cr     : EX1(read);
5908   INS01  : ISS;
5909   BRANCH : EX1;
5910 %}
5911 
5912 // Compare & Branch
5913 // EG.  CBZ/CBNZ
5914 pipe_class pipe_cmp_branch(iRegI op1)
5915 %{
5916   single_instruction;
5917   op1    : EX1(read);
5918   INS01  : ISS;
5919   BRANCH : EX1;
5920 %}
5921 
5922 //------- Synchronisation operations ----------------------
5923 
5924 // Any operation requiring serialization.
5925 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
5926 pipe_class pipe_serial()
5927 %{
5928   single_instruction;
5929   force_serialization;
5930   fixed_latency(16);
5931   INS01  : ISS(2); // Cannot dual issue with any other instruction
5932   LDST   : WR;
5933 %}
5934 
5935 // Generic big/slow expanded idiom - also serialized
5936 pipe_class pipe_slow()
5937 %{
5938   instruction_count(10);
5939   multiple_bundles;
5940   force_serialization;
5941   fixed_latency(16);
5942   INS01  : ISS(2); // Cannot dual issue with any other instruction
5943   LDST   : WR;
5944 %}
5945 
5946 // Empty pipeline class
5947 pipe_class pipe_class_empty()
5948 %{
5949   single_instruction;
5950   fixed_latency(0);
5951 %}
5952 
5953 // Default pipeline class.
5954 pipe_class pipe_class_default()
5955 %{
5956   single_instruction;
5957   fixed_latency(2);
5958 %}
5959 
5960 // Pipeline class for compares.
5961 pipe_class pipe_class_compare()
5962 %{
5963   single_instruction;
5964   fixed_latency(16);
5965 %}
5966 
5967 // Pipeline class for memory operations.
5968 pipe_class pipe_class_memory()
5969 %{
5970   single_instruction;
5971   fixed_latency(16);
5972 %}
5973 
5974 // Pipeline class for call.
5975 pipe_class pipe_class_call()
5976 %{
5977   single_instruction;
5978   fixed_latency(100);
5979 %}
5980 
5981 // Define the class for the Nop node.
5982 define %{
5983    MachNop = pipe_class_empty;
5984 %}
5985 
5986 %}
5987 //----------INSTRUCTIONS-------------------------------------------------------
5988 //
5989 // match      -- States which machine-independent subtree may be replaced
5990 //               by this instruction.
5991 // ins_cost   -- The estimated cost of this instruction is used by instruction
5992 //               selection to identify a minimum cost tree of machine
5993 //               instructions that matches a tree of machine-independent
5994 //               instructions.
5995 // format     -- A string providing the disassembly for this instruction.
5996 //               The value of an instruction's operand may be inserted
5997 //               by referring to it with a '$' prefix.
5998 // opcode     -- Three instruction opcodes may be provided.  These are referred
5999 //               to within an encode class as $primary, $secondary, and $tertiary
6000 //               rrspectively.  The primary opcode is commonly used to
6001 //               indicate the type of machine instruction, while secondary
6002 //               and tertiary are often used for prefix options or addressing
6003 //               modes.
6004 // ins_encode -- A list of encode classes with parameters. The encode class
6005 //               name must have been defined in an 'enc_class' specification
6006 //               in the encode section of the architecture description.
6007 
6008 // ============================================================================
6009 // Memory (Load/Store) Instructions
6010 
6011 // Load Instructions
6012 
6013 // Load Byte (8 bit signed)
6014 instruct loadB(iRegINoSp dst, memory mem)
6015 %{
6016   match(Set dst (LoadB mem));
6017   predicate(!needs_acquiring_load(n));
6018 
6019   ins_cost(4 * INSN_COST);
6020   format %{ "ldrsbw  $dst, $mem\t# byte" %}
6021 
6022   ins_encode(aarch64_enc_ldrsbw(dst, mem));
6023 
6024   ins_pipe(iload_reg_mem);
6025 %}
6026 
6027 // Load Byte (8 bit signed) into long
6028 instruct loadB2L(iRegLNoSp dst, memory mem)
6029 %{
6030   match(Set dst (ConvI2L (LoadB mem)));
6031   predicate(!needs_acquiring_load(n->in(1)));
6032 
6033   ins_cost(4 * INSN_COST);
6034   format %{ "ldrsb  $dst, $mem\t# byte" %}
6035 
6036   ins_encode(aarch64_enc_ldrsb(dst, mem));
6037 
6038   ins_pipe(iload_reg_mem);
6039 %}
6040 
6041 // Load Byte (8 bit unsigned)
6042 instruct loadUB(iRegINoSp dst, memory mem)
6043 %{
6044   match(Set dst (LoadUB mem));
6045   predicate(!needs_acquiring_load(n));
6046 
6047   ins_cost(4 * INSN_COST);
6048   format %{ "ldrbw  $dst, $mem\t# byte" %}
6049 
6050   ins_encode(aarch64_enc_ldrb(dst, mem));
6051 
6052   ins_pipe(iload_reg_mem);
6053 %}
6054 
6055 // Load Byte (8 bit unsigned) into long
6056 instruct loadUB2L(iRegLNoSp dst, memory mem)
6057 %{
6058   match(Set dst (ConvI2L (LoadUB mem)));
6059   predicate(!needs_acquiring_load(n->in(1)));
6060 
6061   ins_cost(4 * INSN_COST);
6062   format %{ "ldrb  $dst, $mem\t# byte" %}
6063 
6064   ins_encode(aarch64_enc_ldrb(dst, mem));
6065 
6066   ins_pipe(iload_reg_mem);
6067 %}
6068 
6069 // Load Short (16 bit signed)
6070 instruct loadS(iRegINoSp dst, memory mem)
6071 %{
6072   match(Set dst (LoadS mem));
6073   predicate(!needs_acquiring_load(n));
6074 
6075   ins_cost(4 * INSN_COST);
6076   format %{ "ldrshw  $dst, $mem\t# short" %}
6077 
6078   ins_encode(aarch64_enc_ldrshw(dst, mem));
6079 
6080   ins_pipe(iload_reg_mem);
6081 %}
6082 
6083 // Load Short (16 bit signed) into long
6084 instruct loadS2L(iRegLNoSp dst, memory mem)
6085 %{
6086   match(Set dst (ConvI2L (LoadS mem)));
6087   predicate(!needs_acquiring_load(n->in(1)));
6088 
6089   ins_cost(4 * INSN_COST);
6090   format %{ "ldrsh  $dst, $mem\t# short" %}
6091 
6092   ins_encode(aarch64_enc_ldrsh(dst, mem));
6093 
6094   ins_pipe(iload_reg_mem);
6095 %}
6096 
6097 // Load Char (16 bit unsigned)
6098 instruct loadUS(iRegINoSp dst, memory mem)
6099 %{
6100   match(Set dst (LoadUS mem));
6101   predicate(!needs_acquiring_load(n));
6102 
6103   ins_cost(4 * INSN_COST);
6104   format %{ "ldrh  $dst, $mem\t# short" %}
6105 
6106   ins_encode(aarch64_enc_ldrh(dst, mem));
6107 
6108   ins_pipe(iload_reg_mem);
6109 %}
6110 
6111 // Load Short/Char (16 bit unsigned) into long
6112 instruct loadUS2L(iRegLNoSp dst, memory mem)
6113 %{
6114   match(Set dst (ConvI2L (LoadUS mem)));
6115   predicate(!needs_acquiring_load(n->in(1)));
6116 
6117   ins_cost(4 * INSN_COST);
6118   format %{ "ldrh  $dst, $mem\t# short" %}
6119 
6120   ins_encode(aarch64_enc_ldrh(dst, mem));
6121 
6122   ins_pipe(iload_reg_mem);
6123 %}
6124 
6125 // Load Integer (32 bit signed)
6126 instruct loadI(iRegINoSp dst, memory mem)
6127 %{
6128   match(Set dst (LoadI mem));
6129   predicate(!needs_acquiring_load(n));
6130 
6131   ins_cost(4 * INSN_COST);
6132   format %{ "ldrw  $dst, $mem\t# int" %}
6133 
6134   ins_encode(aarch64_enc_ldrw(dst, mem));
6135 
6136   ins_pipe(iload_reg_mem);
6137 %}
6138 
6139 // Load Integer (32 bit signed) into long
6140 instruct loadI2L(iRegLNoSp dst, memory mem)
6141 %{
6142   match(Set dst (ConvI2L (LoadI mem)));
6143   predicate(!needs_acquiring_load(n->in(1)));
6144 
6145   ins_cost(4 * INSN_COST);
6146   format %{ "ldrsw  $dst, $mem\t# int" %}
6147 
6148   ins_encode(aarch64_enc_ldrsw(dst, mem));
6149 
6150   ins_pipe(iload_reg_mem);
6151 %}
6152 
6153 // Load Integer (32 bit unsigned) into long
6154 instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
6155 %{
6156   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6157   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6158 
6159   ins_cost(4 * INSN_COST);
6160   format %{ "ldrw  $dst, $mem\t# int" %}
6161 
6162   ins_encode(aarch64_enc_ldrw(dst, mem));
6163 
6164   ins_pipe(iload_reg_mem);
6165 %}
6166 
6167 // Load Long (64 bit signed)
6168 instruct loadL(iRegLNoSp dst, memory mem)
6169 %{
6170   match(Set dst (LoadL mem));
6171   predicate(!needs_acquiring_load(n));
6172 
6173   ins_cost(4 * INSN_COST);
6174   format %{ "ldr  $dst, $mem\t# int" %}
6175 
6176   ins_encode(aarch64_enc_ldr(dst, mem));
6177 
6178   ins_pipe(iload_reg_mem);
6179 %}
6180 
6181 // Load Range
6182 instruct loadRange(iRegINoSp dst, memory mem)
6183 %{
6184   match(Set dst (LoadRange mem));
6185 
6186   ins_cost(4 * INSN_COST);
6187   format %{ "ldrw  $dst, $mem\t# range" %}
6188 
6189   ins_encode(aarch64_enc_ldrw(dst, mem));
6190 
6191   ins_pipe(iload_reg_mem);
6192 %}
6193 
6194 // Load Pointer
6195 instruct loadP(iRegPNoSp dst, memory mem)
6196 %{
6197   match(Set dst (LoadP mem));
6198   predicate(!needs_acquiring_load(n));
6199 
6200   ins_cost(4 * INSN_COST);
6201   format %{ "ldr  $dst, $mem\t# ptr" %}
6202 
6203   ins_encode(aarch64_enc_ldr(dst, mem));
6204 
6205   ins_pipe(iload_reg_mem);
6206 %}
6207 
6208 // Load Compressed Pointer
6209 instruct loadN(iRegNNoSp dst, memory mem)
6210 %{
6211   match(Set dst (LoadN mem));
6212   predicate(!needs_acquiring_load(n));
6213 
6214   ins_cost(4 * INSN_COST);
6215   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
6216 
6217   ins_encode(aarch64_enc_ldrw(dst, mem));
6218 
6219   ins_pipe(iload_reg_mem);
6220 %}
6221 
6222 // Load Klass Pointer
6223 instruct loadKlass(iRegPNoSp dst, memory mem)
6224 %{
6225   match(Set dst (LoadKlass mem));
6226   predicate(!needs_acquiring_load(n));
6227 
6228   ins_cost(4 * INSN_COST);
6229   format %{ "ldr  $dst, $mem\t# class" %}
6230 
6231   ins_encode(aarch64_enc_ldr(dst, mem));
6232 
6233   ins_pipe(iload_reg_mem);
6234 %}
6235 
6236 // Load Narrow Klass Pointer
6237 instruct loadNKlass(iRegNNoSp dst, memory mem)
6238 %{
6239   match(Set dst (LoadNKlass mem));
6240   predicate(!needs_acquiring_load(n));
6241 
6242   ins_cost(4 * INSN_COST);
6243   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
6244 
6245   ins_encode(aarch64_enc_ldrw(dst, mem));
6246 
6247   ins_pipe(iload_reg_mem);
6248 %}
6249 
6250 // Load Float
6251 instruct loadF(vRegF dst, memory mem)
6252 %{
6253   match(Set dst (LoadF mem));
6254   predicate(!needs_acquiring_load(n));
6255 
6256   ins_cost(4 * INSN_COST);
6257   format %{ "ldrs  $dst, $mem\t# float" %}
6258 
6259   ins_encode( aarch64_enc_ldrs(dst, mem) );
6260 
6261   ins_pipe(pipe_class_memory);
6262 %}
6263 
6264 // Load Double
6265 instruct loadD(vRegD dst, memory mem)
6266 %{
6267   match(Set dst (LoadD mem));
6268   predicate(!needs_acquiring_load(n));
6269 
6270   ins_cost(4 * INSN_COST);
6271   format %{ "ldrd  $dst, $mem\t# double" %}
6272 
6273   ins_encode( aarch64_enc_ldrd(dst, mem) );
6274 
6275   ins_pipe(pipe_class_memory);
6276 %}
6277 
6278 
6279 // Load Int Constant
6280 instruct loadConI(iRegINoSp dst, immI src)
6281 %{
6282   match(Set dst src);
6283 
6284   ins_cost(INSN_COST);
6285   format %{ "mov $dst, $src\t# int" %}
6286 
6287   ins_encode( aarch64_enc_movw_imm(dst, src) );
6288 
6289   ins_pipe(ialu_imm);
6290 %}
6291 
6292 // Load Long Constant
6293 instruct loadConL(iRegLNoSp dst, immL src)
6294 %{
6295   match(Set dst src);
6296 
6297   ins_cost(INSN_COST);
6298   format %{ "mov $dst, $src\t# long" %}
6299 
6300   ins_encode( aarch64_enc_mov_imm(dst, src) );
6301 
6302   ins_pipe(ialu_imm);
6303 %}
6304 
6305 // Load Pointer Constant
6306 
6307 instruct loadConP(iRegPNoSp dst, immP con)
6308 %{
6309   match(Set dst con);
6310 
6311   ins_cost(INSN_COST * 4);
6312   format %{
6313     "mov  $dst, $con\t# ptr\n\t"
6314   %}
6315 
6316   ins_encode(aarch64_enc_mov_p(dst, con));
6317 
6318   ins_pipe(ialu_imm);
6319 %}
6320 
6321 // Load Null Pointer Constant
6322 
6323 instruct loadConP0(iRegPNoSp dst, immP0 con)
6324 %{
6325   match(Set dst con);
6326 
6327   ins_cost(INSN_COST);
6328   format %{ "mov  $dst, $con\t# NULL ptr" %}
6329 
6330   ins_encode(aarch64_enc_mov_p0(dst, con));
6331 
6332   ins_pipe(ialu_imm);
6333 %}
6334 
6335 // Load Pointer Constant One
6336 
6337 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6338 %{
6339   match(Set dst con);
6340 
6341   ins_cost(INSN_COST);
6342   format %{ "mov  $dst, $con\t# NULL ptr" %}
6343 
6344   ins_encode(aarch64_enc_mov_p1(dst, con));
6345 
6346   ins_pipe(ialu_imm);
6347 %}
6348 
6349 // Load Poll Page Constant
6350 
6351 instruct loadConPollPage(iRegPNoSp dst, immPollPage con)
6352 %{
6353   match(Set dst con);
6354 
6355   ins_cost(INSN_COST);
6356   format %{ "adr  $dst, $con\t# Poll Page Ptr" %}
6357 
6358   ins_encode(aarch64_enc_mov_poll_page(dst, con));
6359 
6360   ins_pipe(ialu_imm);
6361 %}
6362 
6363 // Load Byte Map Base Constant
6364 
6365 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6366 %{
6367   match(Set dst con);
6368 
6369   ins_cost(INSN_COST);
6370   format %{ "adr  $dst, $con\t# Byte Map Base" %}
6371 
6372   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6373 
6374   ins_pipe(ialu_imm);
6375 %}
6376 
6377 // Load Narrow Pointer Constant
6378 
6379 instruct loadConN(iRegNNoSp dst, immN con)
6380 %{
6381   match(Set dst con);
6382 
6383   ins_cost(INSN_COST * 4);
6384   format %{ "mov  $dst, $con\t# compressed ptr" %}
6385 
6386   ins_encode(aarch64_enc_mov_n(dst, con));
6387 
6388   ins_pipe(ialu_imm);
6389 %}
6390 
6391 // Load Narrow Null Pointer Constant
6392 
6393 instruct loadConN0(iRegNNoSp dst, immN0 con)
6394 %{
6395   match(Set dst con);
6396 
6397   ins_cost(INSN_COST);
6398   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
6399 
6400   ins_encode(aarch64_enc_mov_n0(dst, con));
6401 
6402   ins_pipe(ialu_imm);
6403 %}
6404 
6405 // Load Narrow Klass Constant
6406 
6407 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6408 %{
6409   match(Set dst con);
6410 
6411   ins_cost(INSN_COST);
6412   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
6413 
6414   ins_encode(aarch64_enc_mov_nk(dst, con));
6415 
6416   ins_pipe(ialu_imm);
6417 %}
6418 
6419 // Load Packed Float Constant
6420 
6421 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6422   match(Set dst con);
6423   ins_cost(INSN_COST * 4);
6424   format %{ "fmovs  $dst, $con"%}
6425   ins_encode %{
6426     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6427   %}
6428 
6429   ins_pipe(pipe_class_default);
6430 %}
6431 
6432 // Load Float Constant
6433 
6434 instruct loadConF(vRegF dst, immF con) %{
6435   match(Set dst con);
6436 
6437   ins_cost(INSN_COST * 4);
6438 
6439   format %{
6440     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6441   %}
6442 
6443   ins_encode %{
6444     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6445   %}
6446 
6447   ins_pipe(pipe_class_default);
6448 %}
6449 
6450 // Load Packed Double Constant
6451 
6452 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6453   match(Set dst con);
6454   ins_cost(INSN_COST);
6455   format %{ "fmovd  $dst, $con"%}
6456   ins_encode %{
6457     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6458   %}
6459 
6460   ins_pipe(pipe_class_default);
6461 %}
6462 
6463 // Load Double Constant
6464 
6465 instruct loadConD(vRegD dst, immD con) %{
6466   match(Set dst con);
6467 
6468   ins_cost(INSN_COST * 5);
6469   format %{
6470     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6471   %}
6472 
6473   ins_encode %{
6474     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6475   %}
6476 
6477   ins_pipe(pipe_class_default);
6478 %}
6479 
6480 // Store Instructions
6481 
6482 // Store CMS card-mark Immediate
6483 instruct storeimmCM0(immI0 zero, memory mem)
6484 %{
6485   match(Set mem (StoreCM mem zero));
6486 
6487   ins_cost(INSN_COST);
6488   format %{ "strb zr, $mem\t# byte" %}
6489 
6490   ins_encode(aarch64_enc_strb0(mem));
6491 
6492   ins_pipe(istore_mem);
6493 %}
6494 
6495 // Store Byte
6496 instruct storeB(iRegIorL2I src, memory mem)
6497 %{
6498   match(Set mem (StoreB mem src));
6499   predicate(!needs_releasing_store(n));
6500 
6501   ins_cost(INSN_COST);
6502   format %{ "strb  $src, $mem\t# byte" %}
6503 
6504   ins_encode(aarch64_enc_strb(src, mem));
6505 
6506   ins_pipe(istore_reg_mem);
6507 %}
6508 
6509 
6510 instruct storeimmB0(immI0 zero, memory mem)
6511 %{
6512   match(Set mem (StoreB mem zero));
6513   predicate(!needs_releasing_store(n));
6514 
6515   ins_cost(INSN_COST);
6516   format %{ "strb zr, $mem\t# byte" %}
6517 
6518   ins_encode(aarch64_enc_strb0(mem));
6519 
6520   ins_pipe(istore_mem);
6521 %}
6522 
6523 // Store Char/Short
6524 instruct storeC(iRegIorL2I src, memory mem)
6525 %{
6526   match(Set mem (StoreC mem src));
6527   predicate(!needs_releasing_store(n));
6528 
6529   ins_cost(INSN_COST);
6530   format %{ "strh  $src, $mem\t# short" %}
6531 
6532   ins_encode(aarch64_enc_strh(src, mem));
6533 
6534   ins_pipe(istore_reg_mem);
6535 %}
6536 
6537 instruct storeimmC0(immI0 zero, memory mem)
6538 %{
6539   match(Set mem (StoreC mem zero));
6540   predicate(!needs_releasing_store(n));
6541 
6542   ins_cost(INSN_COST);
6543   format %{ "strh  zr, $mem\t# short" %}
6544 
6545   ins_encode(aarch64_enc_strh0(mem));
6546 
6547   ins_pipe(istore_mem);
6548 %}
6549 
6550 // Store Integer
6551 
6552 instruct storeI(iRegIorL2I src, memory mem)
6553 %{
6554   match(Set mem(StoreI mem src));
6555   predicate(!needs_releasing_store(n));
6556 
6557   ins_cost(INSN_COST);
6558   format %{ "strw  $src, $mem\t# int" %}
6559 
6560   ins_encode(aarch64_enc_strw(src, mem));
6561 
6562   ins_pipe(istore_reg_mem);
6563 %}
6564 
6565 instruct storeimmI0(immI0 zero, memory mem)
6566 %{
6567   match(Set mem(StoreI mem zero));
6568   predicate(!needs_releasing_store(n));
6569 
6570   ins_cost(INSN_COST);
6571   format %{ "strw  zr, $mem\t# int" %}
6572 
6573   ins_encode(aarch64_enc_strw0(mem));
6574 
6575   ins_pipe(istore_mem);
6576 %}
6577 
6578 // Store Long (64 bit signed)
6579 instruct storeL(iRegL src, memory mem)
6580 %{
6581   match(Set mem (StoreL mem src));
6582   predicate(!needs_releasing_store(n));
6583 
6584   ins_cost(INSN_COST);
6585   format %{ "str  $src, $mem\t# int" %}
6586 
6587   ins_encode(aarch64_enc_str(src, mem));
6588 
6589   ins_pipe(istore_reg_mem);
6590 %}
6591 
6592 // Store Long (64 bit signed)
6593 instruct storeimmL0(immL0 zero, memory mem)
6594 %{
6595   match(Set mem (StoreL mem zero));
6596   predicate(!needs_releasing_store(n));
6597 
6598   ins_cost(INSN_COST);
6599   format %{ "str  zr, $mem\t# int" %}
6600 
6601   ins_encode(aarch64_enc_str0(mem));
6602 
6603   ins_pipe(istore_mem);
6604 %}
6605 
6606 // Store Pointer
6607 instruct storeP(iRegP src, memory mem)
6608 %{
6609   match(Set mem (StoreP mem src));
6610   predicate(!needs_releasing_store(n));
6611 
6612   ins_cost(INSN_COST);
6613   format %{ "str  $src, $mem\t# ptr" %}
6614 
6615   ins_encode(aarch64_enc_str(src, mem));
6616 
6617   ins_pipe(istore_reg_mem);
6618 %}
6619 
6620 // Store Pointer
6621 instruct storeimmP0(immP0 zero, memory mem)
6622 %{
6623   match(Set mem (StoreP mem zero));
6624   predicate(!needs_releasing_store(n));
6625 
6626   ins_cost(INSN_COST);
6627   format %{ "str zr, $mem\t# ptr" %}
6628 
6629   ins_encode(aarch64_enc_str0(mem));
6630 
6631   ins_pipe(istore_mem);
6632 %}
6633 
6634 // Store Compressed Pointer
6635 instruct storeN(iRegN src, memory mem)
6636 %{
6637   match(Set mem (StoreN mem src));
6638   predicate(!needs_releasing_store(n));
6639 
6640   ins_cost(INSN_COST);
6641   format %{ "strw  $src, $mem\t# compressed ptr" %}
6642 
6643   ins_encode(aarch64_enc_strw(src, mem));
6644 
6645   ins_pipe(istore_reg_mem);
6646 %}
6647 
6648 instruct storeImmN0(iRegIHeapbase heapbase, immN0 zero, memory mem)
6649 %{
6650   match(Set mem (StoreN mem zero));
6651   predicate(Universe::narrow_oop_base() == NULL &&
6652             Universe::narrow_klass_base() == NULL &&
6653             (!needs_releasing_store(n)));
6654 
6655   ins_cost(INSN_COST);
6656   format %{ "strw  rheapbase, $mem\t# compressed ptr (rheapbase==0)" %}
6657 
6658   ins_encode(aarch64_enc_strw(heapbase, mem));
6659 
6660   ins_pipe(istore_reg_mem);
6661 %}
6662 
6663 // Store Float
6664 instruct storeF(vRegF src, memory mem)
6665 %{
6666   match(Set mem (StoreF mem src));
6667   predicate(!needs_releasing_store(n));
6668 
6669   ins_cost(INSN_COST);
6670   format %{ "strs  $src, $mem\t# float" %}
6671 
6672   ins_encode( aarch64_enc_strs(src, mem) );
6673 
6674   ins_pipe(pipe_class_memory);
6675 %}
6676 
6677 // TODO
6678 // implement storeImmF0 and storeFImmPacked
6679 
6680 // Store Double
6681 instruct storeD(vRegD src, memory mem)
6682 %{
6683   match(Set mem (StoreD mem src));
6684   predicate(!needs_releasing_store(n));
6685 
6686   ins_cost(INSN_COST);
6687   format %{ "strd  $src, $mem\t# double" %}
6688 
6689   ins_encode( aarch64_enc_strd(src, mem) );
6690 
6691   ins_pipe(pipe_class_memory);
6692 %}
6693 
6694 // Store Compressed Klass Pointer
6695 instruct storeNKlass(iRegN src, memory mem)
6696 %{
6697   predicate(!needs_releasing_store(n));
6698   match(Set mem (StoreNKlass mem src));
6699 
6700   ins_cost(INSN_COST);
6701   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
6702 
6703   ins_encode(aarch64_enc_strw(src, mem));
6704 
6705   ins_pipe(istore_reg_mem);
6706 %}
6707 
6708 // TODO
6709 // implement storeImmD0 and storeDImmPacked
6710 
6711 // prefetch instructions
6712 // Must be safe to execute with invalid address (cannot fault).
6713 
6714 instruct prefetchalloc( memory mem ) %{
6715   match(PrefetchAllocation mem);
6716 
6717   ins_cost(INSN_COST);
6718   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
6719 
6720   ins_encode( aarch64_enc_prefetchw(mem) );
6721 
6722   ins_pipe(iload_prefetch);
6723 %}
6724 
6725 //  ---------------- volatile loads and stores ----------------
6726 
6727 // Load Byte (8 bit signed)
6728 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
6729 %{
6730   match(Set dst (LoadB mem));
6731 
6732   ins_cost(VOLATILE_REF_COST);
6733   format %{ "ldarsb  $dst, $mem\t# byte" %}
6734 
6735   ins_encode(aarch64_enc_ldarsb(dst, mem));
6736 
6737   ins_pipe(pipe_serial);
6738 %}
6739 
6740 // Load Byte (8 bit signed) into long
6741 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
6742 %{
6743   match(Set dst (ConvI2L (LoadB mem)));
6744 
6745   ins_cost(VOLATILE_REF_COST);
6746   format %{ "ldarsb  $dst, $mem\t# byte" %}
6747 
6748   ins_encode(aarch64_enc_ldarsb(dst, mem));
6749 
6750   ins_pipe(pipe_serial);
6751 %}
6752 
6753 // Load Byte (8 bit unsigned)
6754 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
6755 %{
6756   match(Set dst (LoadUB mem));
6757 
6758   ins_cost(VOLATILE_REF_COST);
6759   format %{ "ldarb  $dst, $mem\t# byte" %}
6760 
6761   ins_encode(aarch64_enc_ldarb(dst, mem));
6762 
6763   ins_pipe(pipe_serial);
6764 %}
6765 
6766 // Load Byte (8 bit unsigned) into long
6767 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
6768 %{
6769   match(Set dst (ConvI2L (LoadUB mem)));
6770 
6771   ins_cost(VOLATILE_REF_COST);
6772   format %{ "ldarb  $dst, $mem\t# byte" %}
6773 
6774   ins_encode(aarch64_enc_ldarb(dst, mem));
6775 
6776   ins_pipe(pipe_serial);
6777 %}
6778 
6779 // Load Short (16 bit signed)
6780 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
6781 %{
6782   match(Set dst (LoadS mem));
6783 
6784   ins_cost(VOLATILE_REF_COST);
6785   format %{ "ldarshw  $dst, $mem\t# short" %}
6786 
6787   ins_encode(aarch64_enc_ldarshw(dst, mem));
6788 
6789   ins_pipe(pipe_serial);
6790 %}
6791 
6792 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
6793 %{
6794   match(Set dst (LoadUS mem));
6795 
6796   ins_cost(VOLATILE_REF_COST);
6797   format %{ "ldarhw  $dst, $mem\t# short" %}
6798 
6799   ins_encode(aarch64_enc_ldarhw(dst, mem));
6800 
6801   ins_pipe(pipe_serial);
6802 %}
6803 
6804 // Load Short/Char (16 bit unsigned) into long
6805 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
6806 %{
6807   match(Set dst (ConvI2L (LoadUS mem)));
6808 
6809   ins_cost(VOLATILE_REF_COST);
6810   format %{ "ldarh  $dst, $mem\t# short" %}
6811 
6812   ins_encode(aarch64_enc_ldarh(dst, mem));
6813 
6814   ins_pipe(pipe_serial);
6815 %}
6816 
6817 // Load Short/Char (16 bit signed) into long
6818 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
6819 %{
6820   match(Set dst (ConvI2L (LoadS mem)));
6821 
6822   ins_cost(VOLATILE_REF_COST);
6823   format %{ "ldarh  $dst, $mem\t# short" %}
6824 
6825   ins_encode(aarch64_enc_ldarsh(dst, mem));
6826 
6827   ins_pipe(pipe_serial);
6828 %}
6829 
6830 // Load Integer (32 bit signed)
6831 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
6832 %{
6833   match(Set dst (LoadI mem));
6834 
6835   ins_cost(VOLATILE_REF_COST);
6836   format %{ "ldarw  $dst, $mem\t# int" %}
6837 
6838   ins_encode(aarch64_enc_ldarw(dst, mem));
6839 
6840   ins_pipe(pipe_serial);
6841 %}
6842 
6843 // Load Integer (32 bit unsigned) into long
6844 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
6845 %{
6846   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6847 
6848   ins_cost(VOLATILE_REF_COST);
6849   format %{ "ldarw  $dst, $mem\t# int" %}
6850 
6851   ins_encode(aarch64_enc_ldarw(dst, mem));
6852 
6853   ins_pipe(pipe_serial);
6854 %}
6855 
6856 // Load Long (64 bit signed)
6857 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
6858 %{
6859   match(Set dst (LoadL mem));
6860 
6861   ins_cost(VOLATILE_REF_COST);
6862   format %{ "ldar  $dst, $mem\t# int" %}
6863 
6864   ins_encode(aarch64_enc_ldar(dst, mem));
6865 
6866   ins_pipe(pipe_serial);
6867 %}
6868 
6869 // Load Pointer
6870 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
6871 %{
6872   match(Set dst (LoadP mem));
6873 
6874   ins_cost(VOLATILE_REF_COST);
6875   format %{ "ldar  $dst, $mem\t# ptr" %}
6876 
6877   ins_encode(aarch64_enc_ldar(dst, mem));
6878 
6879   ins_pipe(pipe_serial);
6880 %}
6881 
6882 // Load Compressed Pointer
6883 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
6884 %{
6885   match(Set dst (LoadN mem));
6886 
6887   ins_cost(VOLATILE_REF_COST);
6888   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
6889 
6890   ins_encode(aarch64_enc_ldarw(dst, mem));
6891 
6892   ins_pipe(pipe_serial);
6893 %}
6894 
6895 // Load Float
6896 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
6897 %{
6898   match(Set dst (LoadF mem));
6899 
6900   ins_cost(VOLATILE_REF_COST);
6901   format %{ "ldars  $dst, $mem\t# float" %}
6902 
6903   ins_encode( aarch64_enc_fldars(dst, mem) );
6904 
6905   ins_pipe(pipe_serial);
6906 %}
6907 
6908 // Load Double
6909 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
6910 %{
6911   match(Set dst (LoadD mem));
6912 
6913   ins_cost(VOLATILE_REF_COST);
6914   format %{ "ldard  $dst, $mem\t# double" %}
6915 
6916   ins_encode( aarch64_enc_fldard(dst, mem) );
6917 
6918   ins_pipe(pipe_serial);
6919 %}
6920 
6921 // Store Byte
6922 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
6923 %{
6924   match(Set mem (StoreB mem src));
6925 
6926   ins_cost(VOLATILE_REF_COST);
6927   format %{ "stlrb  $src, $mem\t# byte" %}
6928 
6929   ins_encode(aarch64_enc_stlrb(src, mem));
6930 
6931   ins_pipe(pipe_class_memory);
6932 %}
6933 
6934 // Store Char/Short
6935 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
6936 %{
6937   match(Set mem (StoreC mem src));
6938 
6939   ins_cost(VOLATILE_REF_COST);
6940   format %{ "stlrh  $src, $mem\t# short" %}
6941 
6942   ins_encode(aarch64_enc_stlrh(src, mem));
6943 
6944   ins_pipe(pipe_class_memory);
6945 %}
6946 
6947 // Store Integer
6948 
6949 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
6950 %{
6951   match(Set mem(StoreI mem src));
6952 
6953   ins_cost(VOLATILE_REF_COST);
6954   format %{ "stlrw  $src, $mem\t# int" %}
6955 
6956   ins_encode(aarch64_enc_stlrw(src, mem));
6957 
6958   ins_pipe(pipe_class_memory);
6959 %}
6960 
6961 // Store Long (64 bit signed)
6962 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
6963 %{
6964   match(Set mem (StoreL mem src));
6965 
6966   ins_cost(VOLATILE_REF_COST);
6967   format %{ "stlr  $src, $mem\t# int" %}
6968 
6969   ins_encode(aarch64_enc_stlr(src, mem));
6970 
6971   ins_pipe(pipe_class_memory);
6972 %}
6973 
6974 // Store Pointer
6975 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
6976 %{
6977   match(Set mem (StoreP mem src));
6978 
6979   ins_cost(VOLATILE_REF_COST);
6980   format %{ "stlr  $src, $mem\t# ptr" %}
6981 
6982   ins_encode(aarch64_enc_stlr(src, mem));
6983 
6984   ins_pipe(pipe_class_memory);
6985 %}
6986 
6987 // Store Compressed Pointer
6988 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
6989 %{
6990   match(Set mem (StoreN mem src));
6991 
6992   ins_cost(VOLATILE_REF_COST);
6993   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
6994 
6995   ins_encode(aarch64_enc_stlrw(src, mem));
6996 
6997   ins_pipe(pipe_class_memory);
6998 %}
6999 
7000 // Store Float
7001 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7002 %{
7003   match(Set mem (StoreF mem src));
7004 
7005   ins_cost(VOLATILE_REF_COST);
7006   format %{ "stlrs  $src, $mem\t# float" %}
7007 
7008   ins_encode( aarch64_enc_fstlrs(src, mem) );
7009 
7010   ins_pipe(pipe_class_memory);
7011 %}
7012 
7013 // TODO
7014 // implement storeImmF0 and storeFImmPacked
7015 
7016 // Store Double
7017 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7018 %{
7019   match(Set mem (StoreD mem src));
7020 
7021   ins_cost(VOLATILE_REF_COST);
7022   format %{ "stlrd  $src, $mem\t# double" %}
7023 
7024   ins_encode( aarch64_enc_fstlrd(src, mem) );
7025 
7026   ins_pipe(pipe_class_memory);
7027 %}
7028 
7029 //  ---------------- end of volatile loads and stores ----------------
7030 
7031 // ============================================================================
7032 // BSWAP Instructions
7033 
7034 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7035   match(Set dst (ReverseBytesI src));
7036 
7037   ins_cost(INSN_COST);
7038   format %{ "revw  $dst, $src" %}
7039 
7040   ins_encode %{
7041     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7042   %}
7043 
7044   ins_pipe(ialu_reg);
7045 %}
7046 
7047 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7048   match(Set dst (ReverseBytesL src));
7049 
7050   ins_cost(INSN_COST);
7051   format %{ "rev  $dst, $src" %}
7052 
7053   ins_encode %{
7054     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7055   %}
7056 
7057   ins_pipe(ialu_reg);
7058 %}
7059 
7060 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7061   match(Set dst (ReverseBytesUS src));
7062 
7063   ins_cost(INSN_COST);
7064   format %{ "rev16w  $dst, $src" %}
7065 
7066   ins_encode %{
7067     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7068   %}
7069 
7070   ins_pipe(ialu_reg);
7071 %}
7072 
7073 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7074   match(Set dst (ReverseBytesS src));
7075 
7076   ins_cost(INSN_COST);
7077   format %{ "rev16w  $dst, $src\n\t"
7078             "sbfmw $dst, $dst, #0, #15" %}
7079 
7080   ins_encode %{
7081     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7082     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7083   %}
7084 
7085   ins_pipe(ialu_reg);
7086 %}
7087 
7088 // ============================================================================
7089 // Zero Count Instructions
7090 
7091 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7092   match(Set dst (CountLeadingZerosI src));
7093 
7094   ins_cost(INSN_COST);
7095   format %{ "clzw  $dst, $src" %}
7096   ins_encode %{
7097     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7098   %}
7099 
7100   ins_pipe(ialu_reg);
7101 %}
7102 
7103 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7104   match(Set dst (CountLeadingZerosL src));
7105 
7106   ins_cost(INSN_COST);
7107   format %{ "clz   $dst, $src" %}
7108   ins_encode %{
7109     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7110   %}
7111 
7112   ins_pipe(ialu_reg);
7113 %}
7114 
7115 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7116   match(Set dst (CountTrailingZerosI src));
7117 
7118   ins_cost(INSN_COST * 2);
7119   format %{ "rbitw  $dst, $src\n\t"
7120             "clzw   $dst, $dst" %}
7121   ins_encode %{
7122     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7123     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7124   %}
7125 
7126   ins_pipe(ialu_reg);
7127 %}
7128 
7129 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7130   match(Set dst (CountTrailingZerosL src));
7131 
7132   ins_cost(INSN_COST * 2);
7133   format %{ "rbit   $dst, $src\n\t"
7134             "clz    $dst, $dst" %}
7135   ins_encode %{
7136     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7137     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7138   %}
7139 
7140   ins_pipe(ialu_reg);
7141 %}
7142 
7143 // ============================================================================
7144 // MemBar Instruction
7145 
7146 instruct load_fence() %{
7147   match(LoadFence);
7148   ins_cost(VOLATILE_REF_COST);
7149 
7150   format %{ "load_fence" %}
7151 
7152   ins_encode %{
7153     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7154   %}
7155   ins_pipe(pipe_serial);
7156 %}
7157 
7158 instruct unnecessary_membar_acquire() %{
7159   predicate(unnecessary_acquire(n));
7160   match(MemBarAcquire);
7161   ins_cost(0);
7162 
7163   format %{ "membar_acquire (elided)" %}
7164 
7165   ins_encode %{
7166     __ block_comment("membar_acquire (elided)");
7167   %}
7168 
7169   ins_pipe(pipe_class_empty);
7170 %}
7171 
7172 instruct membar_acquire() %{
7173   match(MemBarAcquire);
7174   ins_cost(VOLATILE_REF_COST);
7175 
7176   format %{ "membar_acquire" %}
7177 
7178   ins_encode %{
7179     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7180   %}
7181 
7182   ins_pipe(pipe_serial);
7183 %}
7184 
7185 
7186 instruct membar_acquire_lock() %{
7187   match(MemBarAcquireLock);
7188   ins_cost(VOLATILE_REF_COST);
7189 
7190   format %{ "membar_acquire_lock" %}
7191 
7192   ins_encode %{
7193     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7194   %}
7195 
7196   ins_pipe(pipe_serial);
7197 %}
7198 
7199 instruct store_fence() %{
7200   match(StoreFence);
7201   ins_cost(VOLATILE_REF_COST);
7202 
7203   format %{ "store_fence" %}
7204 
7205   ins_encode %{
7206     __ membar(Assembler::LoadStore|Assembler::StoreStore);
7207   %}
7208   ins_pipe(pipe_serial);
7209 %}
7210 
7211 instruct unnecessary_membar_release() %{
7212   predicate(unnecessary_release(n));
7213   match(MemBarRelease);
7214   ins_cost(0);
7215 
7216   format %{ "membar_release (elided)" %}
7217 
7218   ins_encode %{
7219     __ block_comment("membar_release (elided)");
7220   %}
7221   ins_pipe(pipe_serial);
7222 %}
7223 
7224 instruct membar_release() %{
7225   match(MemBarRelease);
7226   ins_cost(VOLATILE_REF_COST);
7227 
7228   format %{ "membar_release" %}
7229 
7230   ins_encode %{
7231     __ membar(Assembler::LoadStore|Assembler::StoreStore);
7232   %}
7233   ins_pipe(pipe_serial);
7234 %}
7235 
7236 instruct membar_storestore() %{
7237   match(MemBarStoreStore);
7238   ins_cost(VOLATILE_REF_COST);
7239 
7240   format %{ "MEMBAR-store-store" %}
7241 
7242   ins_encode %{
7243     __ membar(Assembler::StoreStore);
7244   %}
7245   ins_pipe(pipe_serial);
7246 %}
7247 
7248 instruct membar_release_lock() %{
7249   match(MemBarReleaseLock);
7250   ins_cost(VOLATILE_REF_COST);
7251 
7252   format %{ "membar_release_lock" %}
7253 
7254   ins_encode %{
7255     __ membar(Assembler::LoadStore|Assembler::StoreStore);
7256   %}
7257 
7258   ins_pipe(pipe_serial);
7259 %}
7260 
7261 instruct unnecessary_membar_volatile() %{
7262   predicate(unnecessary_volatile(n));
7263   match(MemBarVolatile);
7264   ins_cost(0);
7265 
7266   format %{ "membar_volatile (elided)" %}
7267 
7268   ins_encode %{
7269     __ block_comment("membar_volatile (elided)");
7270   %}
7271 
7272   ins_pipe(pipe_serial);
7273 %}
7274 
7275 instruct membar_volatile() %{
7276   match(MemBarVolatile);
7277   ins_cost(VOLATILE_REF_COST*100);
7278 
7279   format %{ "membar_volatile" %}
7280 
7281   ins_encode %{
7282     __ membar(Assembler::StoreLoad);
7283   %}
7284 
7285   ins_pipe(pipe_serial);
7286 %}
7287 
7288 // ============================================================================
7289 // Cast/Convert Instructions
7290 
7291 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7292   match(Set dst (CastX2P src));
7293 
7294   ins_cost(INSN_COST);
7295   format %{ "mov $dst, $src\t# long -> ptr" %}
7296 
7297   ins_encode %{
7298     if ($dst$$reg != $src$$reg) {
7299       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7300     }
7301   %}
7302 
7303   ins_pipe(ialu_reg);
7304 %}
7305 
7306 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7307   match(Set dst (CastP2X src));
7308 
7309   ins_cost(INSN_COST);
7310   format %{ "mov $dst, $src\t# ptr -> long" %}
7311 
7312   ins_encode %{
7313     if ($dst$$reg != $src$$reg) {
7314       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7315     }
7316   %}
7317 
7318   ins_pipe(ialu_reg);
7319 %}
7320 
7321 // Convert oop into int for vectors alignment masking
7322 instruct convP2I(iRegINoSp dst, iRegP src) %{
7323   match(Set dst (ConvL2I (CastP2X src)));
7324 
7325   ins_cost(INSN_COST);
7326   format %{ "movw $dst, $src\t# ptr -> int" %}
7327   ins_encode %{
7328     __ movw($dst$$Register, $src$$Register);
7329   %}
7330 
7331   ins_pipe(ialu_reg);
7332 %}
7333 
7334 // Convert compressed oop into int for vectors alignment masking
7335 // in case of 32bit oops (heap < 4Gb).
7336 instruct convN2I(iRegINoSp dst, iRegN src)
7337 %{
7338   predicate(Universe::narrow_oop_shift() == 0);
7339   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
7340 
7341   ins_cost(INSN_COST);
7342   format %{ "mov dst, $src\t# compressed ptr -> int" %}
7343   ins_encode %{
7344     __ movw($dst$$Register, $src$$Register);
7345   %}
7346 
7347   ins_pipe(ialu_reg);
7348 %}
7349 
7350 
7351 // Convert oop pointer into compressed form
7352 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7353   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
7354   match(Set dst (EncodeP src));
7355   effect(KILL cr);
7356   ins_cost(INSN_COST * 3);
7357   format %{ "encode_heap_oop $dst, $src" %}
7358   ins_encode %{
7359     Register s = $src$$Register;
7360     Register d = $dst$$Register;
7361     __ encode_heap_oop(d, s);
7362   %}
7363   ins_pipe(ialu_reg);
7364 %}
7365 
7366 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7367   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
7368   match(Set dst (EncodeP src));
7369   ins_cost(INSN_COST * 3);
7370   format %{ "encode_heap_oop_not_null $dst, $src" %}
7371   ins_encode %{
7372     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
7373   %}
7374   ins_pipe(ialu_reg);
7375 %}
7376 
7377 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7378   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
7379             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
7380   match(Set dst (DecodeN src));
7381   ins_cost(INSN_COST * 3);
7382   format %{ "decode_heap_oop $dst, $src" %}
7383   ins_encode %{
7384     Register s = $src$$Register;
7385     Register d = $dst$$Register;
7386     __ decode_heap_oop(d, s);
7387   %}
7388   ins_pipe(ialu_reg);
7389 %}
7390 
7391 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7392   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
7393             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
7394   match(Set dst (DecodeN src));
7395   ins_cost(INSN_COST * 3);
7396   format %{ "decode_heap_oop_not_null $dst, $src" %}
7397   ins_encode %{
7398     Register s = $src$$Register;
7399     Register d = $dst$$Register;
7400     __ decode_heap_oop_not_null(d, s);
7401   %}
7402   ins_pipe(ialu_reg);
7403 %}
7404 
7405 // n.b. AArch64 implementations of encode_klass_not_null and
7406 // decode_klass_not_null do not modify the flags register so, unlike
7407 // Intel, we don't kill CR as a side effect here
7408 
7409 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
7410   match(Set dst (EncodePKlass src));
7411 
7412   ins_cost(INSN_COST * 3);
7413   format %{ "encode_klass_not_null $dst,$src" %}
7414 
7415   ins_encode %{
7416     Register src_reg = as_Register($src$$reg);
7417     Register dst_reg = as_Register($dst$$reg);
7418     __ encode_klass_not_null(dst_reg, src_reg);
7419   %}
7420 
7421    ins_pipe(ialu_reg);
7422 %}
7423 
7424 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
7425   match(Set dst (DecodeNKlass src));
7426 
7427   ins_cost(INSN_COST * 3);
7428   format %{ "decode_klass_not_null $dst,$src" %}
7429 
7430   ins_encode %{
7431     Register src_reg = as_Register($src$$reg);
7432     Register dst_reg = as_Register($dst$$reg);
7433     if (dst_reg != src_reg) {
7434       __ decode_klass_not_null(dst_reg, src_reg);
7435     } else {
7436       __ decode_klass_not_null(dst_reg);
7437     }
7438   %}
7439 
7440    ins_pipe(ialu_reg);
7441 %}
7442 
7443 instruct checkCastPP(iRegPNoSp dst)
7444 %{
7445   match(Set dst (CheckCastPP dst));
7446 
7447   size(0);
7448   format %{ "# checkcastPP of $dst" %}
7449   ins_encode(/* empty encoding */);
7450   ins_pipe(pipe_class_empty);
7451 %}
7452 
7453 instruct castPP(iRegPNoSp dst)
7454 %{
7455   match(Set dst (CastPP dst));
7456 
7457   size(0);
7458   format %{ "# castPP of $dst" %}
7459   ins_encode(/* empty encoding */);
7460   ins_pipe(pipe_class_empty);
7461 %}
7462 
7463 instruct castII(iRegI dst)
7464 %{
7465   match(Set dst (CastII dst));
7466 
7467   size(0);
7468   format %{ "# castII of $dst" %}
7469   ins_encode(/* empty encoding */);
7470   ins_cost(0);
7471   ins_pipe(pipe_class_empty);
7472 %}
7473 
7474 // ============================================================================
7475 // Atomic operation instructions
7476 //
7477 // Intel and SPARC both implement Ideal Node LoadPLocked and
7478 // Store{PIL}Conditional instructions using a normal load for the
7479 // LoadPLocked and a CAS for the Store{PIL}Conditional.
7480 //
7481 // The ideal code appears only to use LoadPLocked/StorePLocked as a
7482 // pair to lock object allocations from Eden space when not using
7483 // TLABs.
7484 //
7485 // There does not appear to be a Load{IL}Locked Ideal Node and the
7486 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
7487 // and to use StoreIConditional only for 32-bit and StoreLConditional
7488 // only for 64-bit.
7489 //
7490 // We implement LoadPLocked and StorePLocked instructions using,
7491 // respectively the AArch64 hw load-exclusive and store-conditional
7492 // instructions. Whereas we must implement each of
7493 // Store{IL}Conditional using a CAS which employs a pair of
7494 // instructions comprising a load-exclusive followed by a
7495 // store-conditional.
7496 
7497 
7498 // Locked-load (linked load) of the current heap-top
7499 // used when updating the eden heap top
7500 // implemented using ldaxr on AArch64
7501 
7502 instruct loadPLocked(iRegPNoSp dst, indirect mem)
7503 %{
7504   match(Set dst (LoadPLocked mem));
7505 
7506   ins_cost(VOLATILE_REF_COST);
7507 
7508   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
7509 
7510   ins_encode(aarch64_enc_ldaxr(dst, mem));
7511 
7512   ins_pipe(pipe_serial);
7513 %}
7514 
7515 // Conditional-store of the updated heap-top.
7516 // Used during allocation of the shared heap.
7517 // Sets flag (EQ) on success.
7518 // implemented using stlxr on AArch64.
7519 
7520 instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
7521 %{
7522   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7523 
7524   ins_cost(VOLATILE_REF_COST);
7525 
7526  // TODO
7527  // do we need to do a store-conditional release or can we just use a
7528  // plain store-conditional?
7529 
7530   format %{
7531     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
7532     "cmpw rscratch1, zr\t# EQ on successful write"
7533   %}
7534 
7535   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
7536 
7537   ins_pipe(pipe_serial);
7538 %}
7539 
7540 // this has to be implemented as a CAS
7541 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
7542 %{
7543   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7544 
7545   ins_cost(VOLATILE_REF_COST);
7546 
7547   format %{
7548     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
7549     "cmpw rscratch1, zr\t# EQ on successful write"
7550   %}
7551 
7552   ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval));
7553 
7554   ins_pipe(pipe_slow);
7555 %}
7556 
7557 // this has to be implemented as a CAS
7558 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
7559 %{
7560   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7561 
7562   ins_cost(VOLATILE_REF_COST);
7563 
7564   format %{
7565     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
7566     "cmpw rscratch1, zr\t# EQ on successful write"
7567   %}
7568 
7569   ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval));
7570 
7571   ins_pipe(pipe_slow);
7572 %}
7573 
7574 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
7575 // can't match them
7576 
7577 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
7578 
7579   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
7580 
7581   effect(KILL cr);
7582 
7583  format %{
7584     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
7585     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
7586  %}
7587 
7588  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
7589             aarch64_enc_cset_eq(res));
7590 
7591   ins_pipe(pipe_slow);
7592 %}
7593 
7594 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
7595 
7596   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
7597 
7598   effect(KILL cr);
7599 
7600  format %{
7601     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
7602     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
7603  %}
7604 
7605  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
7606             aarch64_enc_cset_eq(res));
7607 
7608   ins_pipe(pipe_slow);
7609 %}
7610 
7611 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
7612 
7613   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
7614 
7615   effect(KILL cr);
7616 
7617  format %{
7618     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
7619     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
7620  %}
7621 
7622  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
7623             aarch64_enc_cset_eq(res));
7624 
7625   ins_pipe(pipe_slow);
7626 %}
7627 
7628 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
7629 
7630   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
7631 
7632   effect(KILL cr);
7633 
7634  format %{
7635     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
7636     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
7637  %}
7638 
7639  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
7640             aarch64_enc_cset_eq(res));
7641 
7642   ins_pipe(pipe_slow);
7643 %}
7644 
7645 
7646 instruct get_and_setI(indirect mem, iRegINoSp newv, iRegI prev) %{
7647   match(Set prev (GetAndSetI mem newv));
7648   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
7649   ins_encode %{
7650     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
7651   %}
7652   ins_pipe(pipe_serial);
7653 %}
7654 
7655 instruct get_and_setL(indirect mem, iRegLNoSp newv, iRegL prev) %{
7656   match(Set prev (GetAndSetL mem newv));
7657   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
7658   ins_encode %{
7659     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
7660   %}
7661   ins_pipe(pipe_serial);
7662 %}
7663 
7664 instruct get_and_setN(indirect mem, iRegNNoSp newv, iRegI prev) %{
7665   match(Set prev (GetAndSetN mem newv));
7666   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
7667   ins_encode %{
7668     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
7669   %}
7670   ins_pipe(pipe_serial);
7671 %}
7672 
7673 instruct get_and_setP(indirect mem, iRegPNoSp newv, iRegP prev) %{
7674   match(Set prev (GetAndSetP mem newv));
7675   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
7676   ins_encode %{
7677     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
7678   %}
7679   ins_pipe(pipe_serial);
7680 %}
7681 
7682 
7683 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
7684   match(Set newval (GetAndAddL mem incr));
7685   ins_cost(INSN_COST * 10);
7686   format %{ "get_and_addL $newval, [$mem], $incr" %}
7687   ins_encode %{
7688     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
7689   %}
7690   ins_pipe(pipe_serial);
7691 %}
7692 
7693 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
7694   predicate(n->as_LoadStore()->result_not_used());
7695   match(Set dummy (GetAndAddL mem incr));
7696   ins_cost(INSN_COST * 9);
7697   format %{ "get_and_addL [$mem], $incr" %}
7698   ins_encode %{
7699     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
7700   %}
7701   ins_pipe(pipe_serial);
7702 %}
7703 
7704 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
7705   match(Set newval (GetAndAddL mem incr));
7706   ins_cost(INSN_COST * 10);
7707   format %{ "get_and_addL $newval, [$mem], $incr" %}
7708   ins_encode %{
7709     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
7710   %}
7711   ins_pipe(pipe_serial);
7712 %}
7713 
7714 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
7715   predicate(n->as_LoadStore()->result_not_used());
7716   match(Set dummy (GetAndAddL mem incr));
7717   ins_cost(INSN_COST * 9);
7718   format %{ "get_and_addL [$mem], $incr" %}
7719   ins_encode %{
7720     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
7721   %}
7722   ins_pipe(pipe_serial);
7723 %}
7724 
7725 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
7726   match(Set newval (GetAndAddI mem incr));
7727   ins_cost(INSN_COST * 10);
7728   format %{ "get_and_addI $newval, [$mem], $incr" %}
7729   ins_encode %{
7730     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
7731   %}
7732   ins_pipe(pipe_serial);
7733 %}
7734 
7735 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
7736   predicate(n->as_LoadStore()->result_not_used());
7737   match(Set dummy (GetAndAddI mem incr));
7738   ins_cost(INSN_COST * 9);
7739   format %{ "get_and_addI [$mem], $incr" %}
7740   ins_encode %{
7741     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
7742   %}
7743   ins_pipe(pipe_serial);
7744 %}
7745 
7746 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
7747   match(Set newval (GetAndAddI mem incr));
7748   ins_cost(INSN_COST * 10);
7749   format %{ "get_and_addI $newval, [$mem], $incr" %}
7750   ins_encode %{
7751     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
7752   %}
7753   ins_pipe(pipe_serial);
7754 %}
7755 
7756 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
7757   predicate(n->as_LoadStore()->result_not_used());
7758   match(Set dummy (GetAndAddI mem incr));
7759   ins_cost(INSN_COST * 9);
7760   format %{ "get_and_addI [$mem], $incr" %}
7761   ins_encode %{
7762     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
7763   %}
7764   ins_pipe(pipe_serial);
7765 %}
7766 
7767 // Manifest a CmpL result in an integer register.
7768 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
7769 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
7770 %{
7771   match(Set dst (CmpL3 src1 src2));
7772   effect(KILL flags);
7773 
7774   ins_cost(INSN_COST * 6);
7775   format %{
7776       "cmp $src1, $src2"
7777       "csetw $dst, ne"
7778       "cnegw $dst, lt"
7779   %}
7780   // format %{ "CmpL3 $dst, $src1, $src2" %}
7781   ins_encode %{
7782     __ cmp($src1$$Register, $src2$$Register);
7783     __ csetw($dst$$Register, Assembler::NE);
7784     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
7785   %}
7786 
7787   ins_pipe(pipe_class_default);
7788 %}
7789 
7790 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
7791 %{
7792   match(Set dst (CmpL3 src1 src2));
7793   effect(KILL flags);
7794 
7795   ins_cost(INSN_COST * 6);
7796   format %{
7797       "cmp $src1, $src2"
7798       "csetw $dst, ne"
7799       "cnegw $dst, lt"
7800   %}
7801   ins_encode %{
7802     int32_t con = (int32_t)$src2$$constant;
7803      if (con < 0) {
7804       __ adds(zr, $src1$$Register, -con);
7805     } else {
7806       __ subs(zr, $src1$$Register, con);
7807     }
7808     __ csetw($dst$$Register, Assembler::NE);
7809     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
7810   %}
7811 
7812   ins_pipe(pipe_class_default);
7813 %}
7814 
7815 // ============================================================================
7816 // Conditional Move Instructions
7817 
7818 // n.b. we have identical rules for both a signed compare op (cmpOp)
7819 // and an unsigned compare op (cmpOpU). it would be nice if we could
7820 // define an op class which merged both inputs and use it to type the
7821 // argument to a single rule. unfortunatelyt his fails because the
7822 // opclass does not live up to the COND_INTER interface of its
7823 // component operands. When the generic code tries to negate the
7824 // operand it ends up running the generci Machoper::negate method
7825 // which throws a ShouldNotHappen. So, we have to provide two flavours
7826 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
7827 
7828 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7829   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
7830 
7831   ins_cost(INSN_COST * 2);
7832   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
7833 
7834   ins_encode %{
7835     __ cselw(as_Register($dst$$reg),
7836              as_Register($src2$$reg),
7837              as_Register($src1$$reg),
7838              (Assembler::Condition)$cmp$$cmpcode);
7839   %}
7840 
7841   ins_pipe(icond_reg_reg);
7842 %}
7843 
7844 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
7845   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
7846 
7847   ins_cost(INSN_COST * 2);
7848   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
7849 
7850   ins_encode %{
7851     __ cselw(as_Register($dst$$reg),
7852              as_Register($src2$$reg),
7853              as_Register($src1$$reg),
7854              (Assembler::Condition)$cmp$$cmpcode);
7855   %}
7856 
7857   ins_pipe(icond_reg_reg);
7858 %}
7859 
7860 // special cases where one arg is zero
7861 
7862 // n.b. this is selected in preference to the rule above because it
7863 // avoids loading constant 0 into a source register
7864 
7865 // TODO
7866 // we ought only to be able to cull one of these variants as the ideal
7867 // transforms ought always to order the zero consistently (to left/right?)
7868 
7869 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
7870   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
7871 
7872   ins_cost(INSN_COST * 2);
7873   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
7874 
7875   ins_encode %{
7876     __ cselw(as_Register($dst$$reg),
7877              as_Register($src$$reg),
7878              zr,
7879              (Assembler::Condition)$cmp$$cmpcode);
7880   %}
7881 
7882   ins_pipe(icond_reg);
7883 %}
7884 
7885 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
7886   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
7887 
7888   ins_cost(INSN_COST * 2);
7889   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
7890 
7891   ins_encode %{
7892     __ cselw(as_Register($dst$$reg),
7893              as_Register($src$$reg),
7894              zr,
7895              (Assembler::Condition)$cmp$$cmpcode);
7896   %}
7897 
7898   ins_pipe(icond_reg);
7899 %}
7900 
7901 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
7902   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
7903 
7904   ins_cost(INSN_COST * 2);
7905   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
7906 
7907   ins_encode %{
7908     __ cselw(as_Register($dst$$reg),
7909              zr,
7910              as_Register($src$$reg),
7911              (Assembler::Condition)$cmp$$cmpcode);
7912   %}
7913 
7914   ins_pipe(icond_reg);
7915 %}
7916 
7917 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
7918   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
7919 
7920   ins_cost(INSN_COST * 2);
7921   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
7922 
7923   ins_encode %{
7924     __ cselw(as_Register($dst$$reg),
7925              zr,
7926              as_Register($src$$reg),
7927              (Assembler::Condition)$cmp$$cmpcode);
7928   %}
7929 
7930   ins_pipe(icond_reg);
7931 %}
7932 
7933 // special case for creating a boolean 0 or 1
7934 
7935 // n.b. this is selected in preference to the rule above because it
7936 // avoids loading constants 0 and 1 into a source register
7937 
7938 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
7939   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
7940 
7941   ins_cost(INSN_COST * 2);
7942   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
7943 
7944   ins_encode %{
7945     // equivalently
7946     // cset(as_Register($dst$$reg),
7947     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
7948     __ csincw(as_Register($dst$$reg),
7949              zr,
7950              zr,
7951              (Assembler::Condition)$cmp$$cmpcode);
7952   %}
7953 
7954   ins_pipe(icond_none);
7955 %}
7956 
7957 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
7958   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
7959 
7960   ins_cost(INSN_COST * 2);
7961   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
7962 
7963   ins_encode %{
7964     // equivalently
7965     // cset(as_Register($dst$$reg),
7966     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
7967     __ csincw(as_Register($dst$$reg),
7968              zr,
7969              zr,
7970              (Assembler::Condition)$cmp$$cmpcode);
7971   %}
7972 
7973   ins_pipe(icond_none);
7974 %}
7975 
7976 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
7977   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
7978 
7979   ins_cost(INSN_COST * 2);
7980   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
7981 
7982   ins_encode %{
7983     __ csel(as_Register($dst$$reg),
7984             as_Register($src2$$reg),
7985             as_Register($src1$$reg),
7986             (Assembler::Condition)$cmp$$cmpcode);
7987   %}
7988 
7989   ins_pipe(icond_reg_reg);
7990 %}
7991 
7992 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
7993   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
7994 
7995   ins_cost(INSN_COST * 2);
7996   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
7997 
7998   ins_encode %{
7999     __ csel(as_Register($dst$$reg),
8000             as_Register($src2$$reg),
8001             as_Register($src1$$reg),
8002             (Assembler::Condition)$cmp$$cmpcode);
8003   %}
8004 
8005   ins_pipe(icond_reg_reg);
8006 %}
8007 
8008 // special cases where one arg is zero
8009 
8010 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8011   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8012 
8013   ins_cost(INSN_COST * 2);
8014   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
8015 
8016   ins_encode %{
8017     __ csel(as_Register($dst$$reg),
8018             zr,
8019             as_Register($src$$reg),
8020             (Assembler::Condition)$cmp$$cmpcode);
8021   %}
8022 
8023   ins_pipe(icond_reg);
8024 %}
8025 
8026 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8027   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8028 
8029   ins_cost(INSN_COST * 2);
8030   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
8031 
8032   ins_encode %{
8033     __ csel(as_Register($dst$$reg),
8034             zr,
8035             as_Register($src$$reg),
8036             (Assembler::Condition)$cmp$$cmpcode);
8037   %}
8038 
8039   ins_pipe(icond_reg);
8040 %}
8041 
8042 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8043   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8044 
8045   ins_cost(INSN_COST * 2);
8046   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
8047 
8048   ins_encode %{
8049     __ csel(as_Register($dst$$reg),
8050             as_Register($src$$reg),
8051             zr,
8052             (Assembler::Condition)$cmp$$cmpcode);
8053   %}
8054 
8055   ins_pipe(icond_reg);
8056 %}
8057 
8058 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8059   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8060 
8061   ins_cost(INSN_COST * 2);
8062   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
8063 
8064   ins_encode %{
8065     __ csel(as_Register($dst$$reg),
8066             as_Register($src$$reg),
8067             zr,
8068             (Assembler::Condition)$cmp$$cmpcode);
8069   %}
8070 
8071   ins_pipe(icond_reg);
8072 %}
8073 
8074 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8075   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8076 
8077   ins_cost(INSN_COST * 2);
8078   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
8079 
8080   ins_encode %{
8081     __ csel(as_Register($dst$$reg),
8082             as_Register($src2$$reg),
8083             as_Register($src1$$reg),
8084             (Assembler::Condition)$cmp$$cmpcode);
8085   %}
8086 
8087   ins_pipe(icond_reg_reg);
8088 %}
8089 
8090 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8091   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8092 
8093   ins_cost(INSN_COST * 2);
8094   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
8095 
8096   ins_encode %{
8097     __ csel(as_Register($dst$$reg),
8098             as_Register($src2$$reg),
8099             as_Register($src1$$reg),
8100             (Assembler::Condition)$cmp$$cmpcode);
8101   %}
8102 
8103   ins_pipe(icond_reg_reg);
8104 %}
8105 
8106 // special cases where one arg is zero
8107 
8108 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8109   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8110 
8111   ins_cost(INSN_COST * 2);
8112   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
8113 
8114   ins_encode %{
8115     __ csel(as_Register($dst$$reg),
8116             zr,
8117             as_Register($src$$reg),
8118             (Assembler::Condition)$cmp$$cmpcode);
8119   %}
8120 
8121   ins_pipe(icond_reg);
8122 %}
8123 
8124 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8125   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8126 
8127   ins_cost(INSN_COST * 2);
8128   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
8129 
8130   ins_encode %{
8131     __ csel(as_Register($dst$$reg),
8132             zr,
8133             as_Register($src$$reg),
8134             (Assembler::Condition)$cmp$$cmpcode);
8135   %}
8136 
8137   ins_pipe(icond_reg);
8138 %}
8139 
8140 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8141   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8142 
8143   ins_cost(INSN_COST * 2);
8144   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
8145 
8146   ins_encode %{
8147     __ csel(as_Register($dst$$reg),
8148             as_Register($src$$reg),
8149             zr,
8150             (Assembler::Condition)$cmp$$cmpcode);
8151   %}
8152 
8153   ins_pipe(icond_reg);
8154 %}
8155 
8156 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8157   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8158 
8159   ins_cost(INSN_COST * 2);
8160   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
8161 
8162   ins_encode %{
8163     __ csel(as_Register($dst$$reg),
8164             as_Register($src$$reg),
8165             zr,
8166             (Assembler::Condition)$cmp$$cmpcode);
8167   %}
8168 
8169   ins_pipe(icond_reg);
8170 %}
8171 
8172 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8173   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8174 
8175   ins_cost(INSN_COST * 2);
8176   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
8177 
8178   ins_encode %{
8179     __ cselw(as_Register($dst$$reg),
8180              as_Register($src2$$reg),
8181              as_Register($src1$$reg),
8182              (Assembler::Condition)$cmp$$cmpcode);
8183   %}
8184 
8185   ins_pipe(icond_reg_reg);
8186 %}
8187 
8188 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8189   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8190 
8191   ins_cost(INSN_COST * 2);
8192   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
8193 
8194   ins_encode %{
8195     __ cselw(as_Register($dst$$reg),
8196              as_Register($src2$$reg),
8197              as_Register($src1$$reg),
8198              (Assembler::Condition)$cmp$$cmpcode);
8199   %}
8200 
8201   ins_pipe(icond_reg_reg);
8202 %}
8203 
8204 // special cases where one arg is zero
8205 
8206 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8207   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8208 
8209   ins_cost(INSN_COST * 2);
8210   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
8211 
8212   ins_encode %{
8213     __ cselw(as_Register($dst$$reg),
8214              zr,
8215              as_Register($src$$reg),
8216              (Assembler::Condition)$cmp$$cmpcode);
8217   %}
8218 
8219   ins_pipe(icond_reg);
8220 %}
8221 
8222 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8223   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8224 
8225   ins_cost(INSN_COST * 2);
8226   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
8227 
8228   ins_encode %{
8229     __ cselw(as_Register($dst$$reg),
8230              zr,
8231              as_Register($src$$reg),
8232              (Assembler::Condition)$cmp$$cmpcode);
8233   %}
8234 
8235   ins_pipe(icond_reg);
8236 %}
8237 
8238 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8239   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8240 
8241   ins_cost(INSN_COST * 2);
8242   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
8243 
8244   ins_encode %{
8245     __ cselw(as_Register($dst$$reg),
8246              as_Register($src$$reg),
8247              zr,
8248              (Assembler::Condition)$cmp$$cmpcode);
8249   %}
8250 
8251   ins_pipe(icond_reg);
8252 %}
8253 
8254 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8255   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8256 
8257   ins_cost(INSN_COST * 2);
8258   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
8259 
8260   ins_encode %{
8261     __ cselw(as_Register($dst$$reg),
8262              as_Register($src$$reg),
8263              zr,
8264              (Assembler::Condition)$cmp$$cmpcode);
8265   %}
8266 
8267   ins_pipe(icond_reg);
8268 %}
8269 
8270 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
8271 %{
8272   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8273 
8274   ins_cost(INSN_COST * 3);
8275 
8276   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8277   ins_encode %{
8278     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8279     __ fcsels(as_FloatRegister($dst$$reg),
8280               as_FloatRegister($src2$$reg),
8281               as_FloatRegister($src1$$reg),
8282               cond);
8283   %}
8284 
8285   ins_pipe(pipe_class_default);
8286 %}
8287 
8288 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
8289 %{
8290   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8291 
8292   ins_cost(INSN_COST * 3);
8293 
8294   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8295   ins_encode %{
8296     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8297     __ fcsels(as_FloatRegister($dst$$reg),
8298               as_FloatRegister($src2$$reg),
8299               as_FloatRegister($src1$$reg),
8300               cond);
8301   %}
8302 
8303   ins_pipe(pipe_class_default);
8304 %}
8305 
8306 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
8307 %{
8308   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8309 
8310   ins_cost(INSN_COST * 3);
8311 
8312   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8313   ins_encode %{
8314     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8315     __ fcseld(as_FloatRegister($dst$$reg),
8316               as_FloatRegister($src2$$reg),
8317               as_FloatRegister($src1$$reg),
8318               cond);
8319   %}
8320 
8321   ins_pipe(pipe_class_default);
8322 %}
8323 
8324 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
8325 %{
8326   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8327 
8328   ins_cost(INSN_COST * 3);
8329 
8330   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8331   ins_encode %{
8332     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8333     __ fcseld(as_FloatRegister($dst$$reg),
8334               as_FloatRegister($src2$$reg),
8335               as_FloatRegister($src1$$reg),
8336               cond);
8337   %}
8338 
8339   ins_pipe(pipe_class_default);
8340 %}
8341 
8342 // ============================================================================
8343 // Arithmetic Instructions
8344 //
8345 
8346 // Integer Addition
8347 
8348 // TODO
8349 // these currently employ operations which do not set CR and hence are
8350 // not flagged as killing CR but we would like to isolate the cases
8351 // where we want to set flags from those where we don't. need to work
8352 // out how to do that.
8353 
8354 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8355   match(Set dst (AddI src1 src2));
8356 
8357   ins_cost(INSN_COST);
8358   format %{ "addw  $dst, $src1, $src2" %}
8359 
8360   ins_encode %{
8361     __ addw(as_Register($dst$$reg),
8362             as_Register($src1$$reg),
8363             as_Register($src2$$reg));
8364   %}
8365 
8366   ins_pipe(ialu_reg_reg);
8367 %}
8368 
8369 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
8370   match(Set dst (AddI src1 src2));
8371 
8372   ins_cost(INSN_COST);
8373   format %{ "addw $dst, $src1, $src2" %}
8374 
8375   // use opcode to indicate that this is an add not a sub
8376   opcode(0x0);
8377 
8378   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
8379 
8380   ins_pipe(ialu_reg_imm);
8381 %}
8382 
8383 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
8384   match(Set dst (AddI (ConvL2I src1) src2));
8385 
8386   ins_cost(INSN_COST);
8387   format %{ "addw $dst, $src1, $src2" %}
8388 
8389   // use opcode to indicate that this is an add not a sub
8390   opcode(0x0);
8391 
8392   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
8393 
8394   ins_pipe(ialu_reg_imm);
8395 %}
8396 
8397 // Pointer Addition
8398 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
8399   match(Set dst (AddP src1 src2));
8400 
8401   ins_cost(INSN_COST);
8402   format %{ "add $dst, $src1, $src2\t# ptr" %}
8403 
8404   ins_encode %{
8405     __ add(as_Register($dst$$reg),
8406            as_Register($src1$$reg),
8407            as_Register($src2$$reg));
8408   %}
8409 
8410   ins_pipe(ialu_reg_reg);
8411 %}
8412 
8413 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
8414   match(Set dst (AddP src1 (ConvI2L src2)));
8415 
8416   ins_cost(1.9 * INSN_COST);
8417   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
8418 
8419   ins_encode %{
8420     __ add(as_Register($dst$$reg),
8421            as_Register($src1$$reg),
8422            as_Register($src2$$reg), ext::sxtw);
8423   %}
8424 
8425   ins_pipe(ialu_reg_reg);
8426 %}
8427 
8428 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
8429   match(Set dst (AddP src1 (LShiftL src2 scale)));
8430 
8431   ins_cost(1.9 * INSN_COST);
8432   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
8433 
8434   ins_encode %{
8435     __ lea(as_Register($dst$$reg),
8436            Address(as_Register($src1$$reg), as_Register($src2$$reg),
8437                    Address::lsl($scale$$constant)));
8438   %}
8439 
8440   ins_pipe(ialu_reg_reg_shift);
8441 %}
8442 
8443 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
8444   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
8445 
8446   ins_cost(1.9 * INSN_COST);
8447   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
8448 
8449   ins_encode %{
8450     __ lea(as_Register($dst$$reg),
8451            Address(as_Register($src1$$reg), as_Register($src2$$reg),
8452                    Address::sxtw($scale$$constant)));
8453   %}
8454 
8455   ins_pipe(ialu_reg_reg_shift);
8456 %}
8457 
8458 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
8459   match(Set dst (LShiftL (ConvI2L src) scale));
8460 
8461   ins_cost(INSN_COST);
8462   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
8463 
8464   ins_encode %{
8465     __ sbfiz(as_Register($dst$$reg),
8466           as_Register($src$$reg),
8467           $scale$$constant & 63, MIN(32, (-$scale$$constant) & 63));
8468   %}
8469 
8470   ins_pipe(ialu_reg_shift);
8471 %}
8472 
8473 // Pointer Immediate Addition
8474 // n.b. this needs to be more expensive than using an indirect memory
8475 // operand
8476 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
8477   match(Set dst (AddP src1 src2));
8478 
8479   ins_cost(INSN_COST);
8480   format %{ "add $dst, $src1, $src2\t# ptr" %}
8481 
8482   // use opcode to indicate that this is an add not a sub
8483   opcode(0x0);
8484 
8485   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
8486 
8487   ins_pipe(ialu_reg_imm);
8488 %}
8489 
8490 // Long Addition
8491 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
8492 
8493   match(Set dst (AddL src1 src2));
8494 
8495   ins_cost(INSN_COST);
8496   format %{ "add  $dst, $src1, $src2" %}
8497 
8498   ins_encode %{
8499     __ add(as_Register($dst$$reg),
8500            as_Register($src1$$reg),
8501            as_Register($src2$$reg));
8502   %}
8503 
8504   ins_pipe(ialu_reg_reg);
8505 %}
8506 
8507 // No constant pool entries requiredLong Immediate Addition.
8508 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
8509   match(Set dst (AddL src1 src2));
8510 
8511   ins_cost(INSN_COST);
8512   format %{ "add $dst, $src1, $src2" %}
8513 
8514   // use opcode to indicate that this is an add not a sub
8515   opcode(0x0);
8516 
8517   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
8518 
8519   ins_pipe(ialu_reg_imm);
8520 %}
8521 
8522 // Integer Subtraction
8523 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8524   match(Set dst (SubI src1 src2));
8525 
8526   ins_cost(INSN_COST);
8527   format %{ "subw  $dst, $src1, $src2" %}
8528 
8529   ins_encode %{
8530     __ subw(as_Register($dst$$reg),
8531             as_Register($src1$$reg),
8532             as_Register($src2$$reg));
8533   %}
8534 
8535   ins_pipe(ialu_reg_reg);
8536 %}
8537 
8538 // Immediate Subtraction
8539 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
8540   match(Set dst (SubI src1 src2));
8541 
8542   ins_cost(INSN_COST);
8543   format %{ "subw $dst, $src1, $src2" %}
8544 
8545   // use opcode to indicate that this is a sub not an add
8546   opcode(0x1);
8547 
8548   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
8549 
8550   ins_pipe(ialu_reg_imm);
8551 %}
8552 
8553 // Long Subtraction
8554 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
8555 
8556   match(Set dst (SubL src1 src2));
8557 
8558   ins_cost(INSN_COST);
8559   format %{ "sub  $dst, $src1, $src2" %}
8560 
8561   ins_encode %{
8562     __ sub(as_Register($dst$$reg),
8563            as_Register($src1$$reg),
8564            as_Register($src2$$reg));
8565   %}
8566 
8567   ins_pipe(ialu_reg_reg);
8568 %}
8569 
8570 // No constant pool entries requiredLong Immediate Subtraction.
8571 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
8572   match(Set dst (SubL src1 src2));
8573 
8574   ins_cost(INSN_COST);
8575   format %{ "sub$dst, $src1, $src2" %}
8576 
8577   // use opcode to indicate that this is a sub not an add
8578   opcode(0x1);
8579 
8580   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
8581 
8582   ins_pipe(ialu_reg_imm);
8583 %}
8584 
8585 // Integer Negation (special case for sub)
8586 
8587 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
8588   match(Set dst (SubI zero src));
8589 
8590   ins_cost(INSN_COST);
8591   format %{ "negw $dst, $src\t# int" %}
8592 
8593   ins_encode %{
8594     __ negw(as_Register($dst$$reg),
8595             as_Register($src$$reg));
8596   %}
8597 
8598   ins_pipe(ialu_reg);
8599 %}
8600 
8601 // Long Negation
8602 
8603 instruct negL_reg(iRegLNoSp dst, iRegIorL2I src, immL0 zero, rFlagsReg cr) %{
8604   match(Set dst (SubL zero src));
8605 
8606   ins_cost(INSN_COST);
8607   format %{ "neg $dst, $src\t# long" %}
8608 
8609   ins_encode %{
8610     __ neg(as_Register($dst$$reg),
8611            as_Register($src$$reg));
8612   %}
8613 
8614   ins_pipe(ialu_reg);
8615 %}
8616 
8617 // Integer Multiply
8618 
8619 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8620   match(Set dst (MulI src1 src2));
8621 
8622   ins_cost(INSN_COST * 3);
8623   format %{ "mulw  $dst, $src1, $src2" %}
8624 
8625   ins_encode %{
8626     __ mulw(as_Register($dst$$reg),
8627             as_Register($src1$$reg),
8628             as_Register($src2$$reg));
8629   %}
8630 
8631   ins_pipe(imul_reg_reg);
8632 %}
8633 
8634 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8635   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
8636 
8637   ins_cost(INSN_COST * 3);
8638   format %{ "smull  $dst, $src1, $src2" %}
8639 
8640   ins_encode %{
8641     __ smull(as_Register($dst$$reg),
8642              as_Register($src1$$reg),
8643              as_Register($src2$$reg));
8644   %}
8645 
8646   ins_pipe(imul_reg_reg);
8647 %}
8648 
8649 // Long Multiply
8650 
8651 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
8652   match(Set dst (MulL src1 src2));
8653 
8654   ins_cost(INSN_COST * 5);
8655   format %{ "mul  $dst, $src1, $src2" %}
8656 
8657   ins_encode %{
8658     __ mul(as_Register($dst$$reg),
8659            as_Register($src1$$reg),
8660            as_Register($src2$$reg));
8661   %}
8662 
8663   ins_pipe(lmul_reg_reg);
8664 %}
8665 
8666 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
8667 %{
8668   match(Set dst (MulHiL src1 src2));
8669 
8670   ins_cost(INSN_COST * 7);
8671   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
8672 
8673   ins_encode %{
8674     __ smulh(as_Register($dst$$reg),
8675              as_Register($src1$$reg),
8676              as_Register($src2$$reg));
8677   %}
8678 
8679   ins_pipe(lmul_reg_reg);
8680 %}
8681 
8682 // Combined Integer Multiply & Add/Sub
8683 
8684 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
8685   match(Set dst (AddI src3 (MulI src1 src2)));
8686 
8687   ins_cost(INSN_COST * 3);
8688   format %{ "madd  $dst, $src1, $src2, $src3" %}
8689 
8690   ins_encode %{
8691     __ maddw(as_Register($dst$$reg),
8692              as_Register($src1$$reg),
8693              as_Register($src2$$reg),
8694              as_Register($src3$$reg));
8695   %}
8696 
8697   ins_pipe(imac_reg_reg);
8698 %}
8699 
8700 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
8701   match(Set dst (SubI src3 (MulI src1 src2)));
8702 
8703   ins_cost(INSN_COST * 3);
8704   format %{ "msub  $dst, $src1, $src2, $src3" %}
8705 
8706   ins_encode %{
8707     __ msubw(as_Register($dst$$reg),
8708              as_Register($src1$$reg),
8709              as_Register($src2$$reg),
8710              as_Register($src3$$reg));
8711   %}
8712 
8713   ins_pipe(imac_reg_reg);
8714 %}
8715 
8716 // Combined Long Multiply & Add/Sub
8717 
8718 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
8719   match(Set dst (AddL src3 (MulL src1 src2)));
8720 
8721   ins_cost(INSN_COST * 5);
8722   format %{ "madd  $dst, $src1, $src2, $src3" %}
8723 
8724   ins_encode %{
8725     __ madd(as_Register($dst$$reg),
8726             as_Register($src1$$reg),
8727             as_Register($src2$$reg),
8728             as_Register($src3$$reg));
8729   %}
8730 
8731   ins_pipe(lmac_reg_reg);
8732 %}
8733 
8734 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
8735   match(Set dst (SubL src3 (MulL src1 src2)));
8736 
8737   ins_cost(INSN_COST * 5);
8738   format %{ "msub  $dst, $src1, $src2, $src3" %}
8739 
8740   ins_encode %{
8741     __ msub(as_Register($dst$$reg),
8742             as_Register($src1$$reg),
8743             as_Register($src2$$reg),
8744             as_Register($src3$$reg));
8745   %}
8746 
8747   ins_pipe(lmac_reg_reg);
8748 %}
8749 
8750 // Integer Divide
8751 
8752 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8753   match(Set dst (DivI src1 src2));
8754 
8755   ins_cost(INSN_COST * 19);
8756   format %{ "sdivw  $dst, $src1, $src2" %}
8757 
8758   ins_encode(aarch64_enc_divw(dst, src1, src2));
8759   ins_pipe(idiv_reg_reg);
8760 %}
8761 
8762 instruct signExtract(iRegINoSp dst, iRegIorL2I src1, immI_31 div1, immI_31 div2) %{
8763   match(Set dst (URShiftI (RShiftI src1 div1) div2));
8764   ins_cost(INSN_COST);
8765   format %{ "lsrw $dst, $src1, $div1" %}
8766   ins_encode %{
8767     __ lsrw(as_Register($dst$$reg), as_Register($src1$$reg), 31);
8768   %}
8769   ins_pipe(ialu_reg_shift);
8770 %}
8771 
8772 instruct div2Round(iRegINoSp dst, iRegIorL2I src, immI_31 div1, immI_31 div2) %{
8773   match(Set dst (AddI src (URShiftI (RShiftI src div1) div2)));
8774   ins_cost(INSN_COST);
8775   format %{ "addw $dst, $src, LSR $div1" %}
8776 
8777   ins_encode %{
8778     __ addw(as_Register($dst$$reg),
8779               as_Register($src$$reg),
8780               as_Register($src$$reg),
8781               Assembler::LSR, 31);
8782   %}
8783   ins_pipe(ialu_reg);
8784 %}
8785 
8786 // Long Divide
8787 
8788 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
8789   match(Set dst (DivL src1 src2));
8790 
8791   ins_cost(INSN_COST * 35);
8792   format %{ "sdiv   $dst, $src1, $src2" %}
8793 
8794   ins_encode(aarch64_enc_div(dst, src1, src2));
8795   ins_pipe(ldiv_reg_reg);
8796 %}
8797 
8798 instruct signExtractL(iRegLNoSp dst, iRegL src1, immL_63 div1, immL_63 div2) %{
8799   match(Set dst (URShiftL (RShiftL src1 div1) div2));
8800   ins_cost(INSN_COST);
8801   format %{ "lsr $dst, $src1, $div1" %}
8802   ins_encode %{
8803     __ lsr(as_Register($dst$$reg), as_Register($src1$$reg), 63);
8804   %}
8805   ins_pipe(ialu_reg_shift);
8806 %}
8807 
8808 instruct div2RoundL(iRegLNoSp dst, iRegL src, immL_63 div1, immL_63 div2) %{
8809   match(Set dst (AddL src (URShiftL (RShiftL src div1) div2)));
8810   ins_cost(INSN_COST);
8811   format %{ "add $dst, $src, $div1" %}
8812 
8813   ins_encode %{
8814     __ add(as_Register($dst$$reg),
8815               as_Register($src$$reg),
8816               as_Register($src$$reg),
8817               Assembler::LSR, 63);
8818   %}
8819   ins_pipe(ialu_reg);
8820 %}
8821 
8822 // Integer Remainder
8823 
8824 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8825   match(Set dst (ModI src1 src2));
8826 
8827   ins_cost(INSN_COST * 22);
8828   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
8829             "msubw($dst, rscratch1, $src2, $src1" %}
8830 
8831   ins_encode(aarch64_enc_modw(dst, src1, src2));
8832   ins_pipe(idiv_reg_reg);
8833 %}
8834 
8835 // Long Remainder
8836 
8837 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
8838   match(Set dst (ModL src1 src2));
8839 
8840   ins_cost(INSN_COST * 38);
8841   format %{ "sdiv   rscratch1, $src1, $src2\n"
8842             "msub($dst, rscratch1, $src2, $src1" %}
8843 
8844   ins_encode(aarch64_enc_mod(dst, src1, src2));
8845   ins_pipe(ldiv_reg_reg);
8846 %}
8847 
8848 // Integer Shifts
8849 
8850 // Shift Left Register
8851 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8852   match(Set dst (LShiftI src1 src2));
8853 
8854   ins_cost(INSN_COST * 2);
8855   format %{ "lslvw  $dst, $src1, $src2" %}
8856 
8857   ins_encode %{
8858     __ lslvw(as_Register($dst$$reg),
8859              as_Register($src1$$reg),
8860              as_Register($src2$$reg));
8861   %}
8862 
8863   ins_pipe(ialu_reg_reg_vshift);
8864 %}
8865 
8866 // Shift Left Immediate
8867 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
8868   match(Set dst (LShiftI src1 src2));
8869 
8870   ins_cost(INSN_COST);
8871   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
8872 
8873   ins_encode %{
8874     __ lslw(as_Register($dst$$reg),
8875             as_Register($src1$$reg),
8876             $src2$$constant & 0x1f);
8877   %}
8878 
8879   ins_pipe(ialu_reg_shift);
8880 %}
8881 
8882 // Shift Right Logical Register
8883 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8884   match(Set dst (URShiftI src1 src2));
8885 
8886   ins_cost(INSN_COST * 2);
8887   format %{ "lsrvw  $dst, $src1, $src2" %}
8888 
8889   ins_encode %{
8890     __ lsrvw(as_Register($dst$$reg),
8891              as_Register($src1$$reg),
8892              as_Register($src2$$reg));
8893   %}
8894 
8895   ins_pipe(ialu_reg_reg_vshift);
8896 %}
8897 
8898 // Shift Right Logical Immediate
8899 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
8900   match(Set dst (URShiftI src1 src2));
8901 
8902   ins_cost(INSN_COST);
8903   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
8904 
8905   ins_encode %{
8906     __ lsrw(as_Register($dst$$reg),
8907             as_Register($src1$$reg),
8908             $src2$$constant & 0x1f);
8909   %}
8910 
8911   ins_pipe(ialu_reg_shift);
8912 %}
8913 
8914 // Shift Right Arithmetic Register
8915 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8916   match(Set dst (RShiftI src1 src2));
8917 
8918   ins_cost(INSN_COST * 2);
8919   format %{ "asrvw  $dst, $src1, $src2" %}
8920 
8921   ins_encode %{
8922     __ asrvw(as_Register($dst$$reg),
8923              as_Register($src1$$reg),
8924              as_Register($src2$$reg));
8925   %}
8926 
8927   ins_pipe(ialu_reg_reg_vshift);
8928 %}
8929 
8930 // Shift Right Arithmetic Immediate
8931 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
8932   match(Set dst (RShiftI src1 src2));
8933 
8934   ins_cost(INSN_COST);
8935   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
8936 
8937   ins_encode %{
8938     __ asrw(as_Register($dst$$reg),
8939             as_Register($src1$$reg),
8940             $src2$$constant & 0x1f);
8941   %}
8942 
8943   ins_pipe(ialu_reg_shift);
8944 %}
8945 
8946 // Combined Int Mask and Right Shift (using UBFM)
8947 // TODO
8948 
8949 // Long Shifts
8950 
8951 // Shift Left Register
8952 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
8953   match(Set dst (LShiftL src1 src2));
8954 
8955   ins_cost(INSN_COST * 2);
8956   format %{ "lslv  $dst, $src1, $src2" %}
8957 
8958   ins_encode %{
8959     __ lslv(as_Register($dst$$reg),
8960             as_Register($src1$$reg),
8961             as_Register($src2$$reg));
8962   %}
8963 
8964   ins_pipe(ialu_reg_reg_vshift);
8965 %}
8966 
8967 // Shift Left Immediate
8968 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
8969   match(Set dst (LShiftL src1 src2));
8970 
8971   ins_cost(INSN_COST);
8972   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
8973 
8974   ins_encode %{
8975     __ lsl(as_Register($dst$$reg),
8976             as_Register($src1$$reg),
8977             $src2$$constant & 0x3f);
8978   %}
8979 
8980   ins_pipe(ialu_reg_shift);
8981 %}
8982 
8983 // Shift Right Logical Register
8984 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
8985   match(Set dst (URShiftL src1 src2));
8986 
8987   ins_cost(INSN_COST * 2);
8988   format %{ "lsrv  $dst, $src1, $src2" %}
8989 
8990   ins_encode %{
8991     __ lsrv(as_Register($dst$$reg),
8992             as_Register($src1$$reg),
8993             as_Register($src2$$reg));
8994   %}
8995 
8996   ins_pipe(ialu_reg_reg_vshift);
8997 %}
8998 
8999 // Shift Right Logical Immediate
9000 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9001   match(Set dst (URShiftL src1 src2));
9002 
9003   ins_cost(INSN_COST);
9004   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9005 
9006   ins_encode %{
9007     __ lsr(as_Register($dst$$reg),
9008            as_Register($src1$$reg),
9009            $src2$$constant & 0x3f);
9010   %}
9011 
9012   ins_pipe(ialu_reg_shift);
9013 %}
9014 
9015 // A special-case pattern for card table stores.
9016 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9017   match(Set dst (URShiftL (CastP2X src1) src2));
9018 
9019   ins_cost(INSN_COST);
9020   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9021 
9022   ins_encode %{
9023     __ lsr(as_Register($dst$$reg),
9024            as_Register($src1$$reg),
9025            $src2$$constant & 0x3f);
9026   %}
9027 
9028   ins_pipe(ialu_reg_shift);
9029 %}
9030 
9031 // Shift Right Arithmetic Register
9032 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9033   match(Set dst (RShiftL src1 src2));
9034 
9035   ins_cost(INSN_COST * 2);
9036   format %{ "asrv  $dst, $src1, $src2" %}
9037 
9038   ins_encode %{
9039     __ asrv(as_Register($dst$$reg),
9040             as_Register($src1$$reg),
9041             as_Register($src2$$reg));
9042   %}
9043 
9044   ins_pipe(ialu_reg_reg_vshift);
9045 %}
9046 
9047 // Shift Right Arithmetic Immediate
9048 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9049   match(Set dst (RShiftL src1 src2));
9050 
9051   ins_cost(INSN_COST);
9052   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9053 
9054   ins_encode %{
9055     __ asr(as_Register($dst$$reg),
9056            as_Register($src1$$reg),
9057            $src2$$constant & 0x3f);
9058   %}
9059 
9060   ins_pipe(ialu_reg_shift);
9061 %}
9062 
9063 // BEGIN This section of the file is automatically generated. Do not edit --------------
9064 
9065 instruct regL_not_reg(iRegLNoSp dst,
9066                          iRegL src1, immL_M1 m1,
9067                          rFlagsReg cr) %{
9068   match(Set dst (XorL src1 m1));
9069   ins_cost(INSN_COST);
9070   format %{ "eon  $dst, $src1, zr" %}
9071 
9072   ins_encode %{
9073     __ eon(as_Register($dst$$reg),
9074               as_Register($src1$$reg),
9075               zr,
9076               Assembler::LSL, 0);
9077   %}
9078 
9079   ins_pipe(ialu_reg);
9080 %}
9081 instruct regI_not_reg(iRegINoSp dst,
9082                          iRegIorL2I src1, immI_M1 m1,
9083                          rFlagsReg cr) %{
9084   match(Set dst (XorI src1 m1));
9085   ins_cost(INSN_COST);
9086   format %{ "eonw  $dst, $src1, zr" %}
9087 
9088   ins_encode %{
9089     __ eonw(as_Register($dst$$reg),
9090               as_Register($src1$$reg),
9091               zr,
9092               Assembler::LSL, 0);
9093   %}
9094 
9095   ins_pipe(ialu_reg);
9096 %}
9097 
9098 instruct AndI_reg_not_reg(iRegINoSp dst,
9099                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
9100                          rFlagsReg cr) %{
9101   match(Set dst (AndI src1 (XorI src2 m1)));
9102   ins_cost(INSN_COST);
9103   format %{ "bicw  $dst, $src1, $src2" %}
9104 
9105   ins_encode %{
9106     __ bicw(as_Register($dst$$reg),
9107               as_Register($src1$$reg),
9108               as_Register($src2$$reg),
9109               Assembler::LSL, 0);
9110   %}
9111 
9112   ins_pipe(ialu_reg_reg);
9113 %}
9114 
9115 instruct AndL_reg_not_reg(iRegLNoSp dst,
9116                          iRegL src1, iRegL src2, immL_M1 m1,
9117                          rFlagsReg cr) %{
9118   match(Set dst (AndL src1 (XorL src2 m1)));
9119   ins_cost(INSN_COST);
9120   format %{ "bic  $dst, $src1, $src2" %}
9121 
9122   ins_encode %{
9123     __ bic(as_Register($dst$$reg),
9124               as_Register($src1$$reg),
9125               as_Register($src2$$reg),
9126               Assembler::LSL, 0);
9127   %}
9128 
9129   ins_pipe(ialu_reg_reg);
9130 %}
9131 
9132 instruct OrI_reg_not_reg(iRegINoSp dst,
9133                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
9134                          rFlagsReg cr) %{
9135   match(Set dst (OrI src1 (XorI src2 m1)));
9136   ins_cost(INSN_COST);
9137   format %{ "ornw  $dst, $src1, $src2" %}
9138 
9139   ins_encode %{
9140     __ ornw(as_Register($dst$$reg),
9141               as_Register($src1$$reg),
9142               as_Register($src2$$reg),
9143               Assembler::LSL, 0);
9144   %}
9145 
9146   ins_pipe(ialu_reg_reg);
9147 %}
9148 
9149 instruct OrL_reg_not_reg(iRegLNoSp dst,
9150                          iRegL src1, iRegL src2, immL_M1 m1,
9151                          rFlagsReg cr) %{
9152   match(Set dst (OrL src1 (XorL src2 m1)));
9153   ins_cost(INSN_COST);
9154   format %{ "orn  $dst, $src1, $src2" %}
9155 
9156   ins_encode %{
9157     __ orn(as_Register($dst$$reg),
9158               as_Register($src1$$reg),
9159               as_Register($src2$$reg),
9160               Assembler::LSL, 0);
9161   %}
9162 
9163   ins_pipe(ialu_reg_reg);
9164 %}
9165 
9166 instruct XorI_reg_not_reg(iRegINoSp dst,
9167                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
9168                          rFlagsReg cr) %{
9169   match(Set dst (XorI m1 (XorI src2 src1)));
9170   ins_cost(INSN_COST);
9171   format %{ "eonw  $dst, $src1, $src2" %}
9172 
9173   ins_encode %{
9174     __ eonw(as_Register($dst$$reg),
9175               as_Register($src1$$reg),
9176               as_Register($src2$$reg),
9177               Assembler::LSL, 0);
9178   %}
9179 
9180   ins_pipe(ialu_reg_reg);
9181 %}
9182 
9183 instruct XorL_reg_not_reg(iRegLNoSp dst,
9184                          iRegL src1, iRegL src2, immL_M1 m1,
9185                          rFlagsReg cr) %{
9186   match(Set dst (XorL m1 (XorL src2 src1)));
9187   ins_cost(INSN_COST);
9188   format %{ "eon  $dst, $src1, $src2" %}
9189 
9190   ins_encode %{
9191     __ eon(as_Register($dst$$reg),
9192               as_Register($src1$$reg),
9193               as_Register($src2$$reg),
9194               Assembler::LSL, 0);
9195   %}
9196 
9197   ins_pipe(ialu_reg_reg);
9198 %}
9199 
9200 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
9201                          iRegIorL2I src1, iRegIorL2I src2,
9202                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9203   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
9204   ins_cost(1.9 * INSN_COST);
9205   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
9206 
9207   ins_encode %{
9208     __ bicw(as_Register($dst$$reg),
9209               as_Register($src1$$reg),
9210               as_Register($src2$$reg),
9211               Assembler::LSR,
9212               $src3$$constant & 0x3f);
9213   %}
9214 
9215   ins_pipe(ialu_reg_reg_shift);
9216 %}
9217 
9218 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
9219                          iRegL src1, iRegL src2,
9220                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9221   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
9222   ins_cost(1.9 * INSN_COST);
9223   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
9224 
9225   ins_encode %{
9226     __ bic(as_Register($dst$$reg),
9227               as_Register($src1$$reg),
9228               as_Register($src2$$reg),
9229               Assembler::LSR,
9230               $src3$$constant & 0x3f);
9231   %}
9232 
9233   ins_pipe(ialu_reg_reg_shift);
9234 %}
9235 
9236 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
9237                          iRegIorL2I src1, iRegIorL2I src2,
9238                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9239   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
9240   ins_cost(1.9 * INSN_COST);
9241   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
9242 
9243   ins_encode %{
9244     __ bicw(as_Register($dst$$reg),
9245               as_Register($src1$$reg),
9246               as_Register($src2$$reg),
9247               Assembler::ASR,
9248               $src3$$constant & 0x3f);
9249   %}
9250 
9251   ins_pipe(ialu_reg_reg_shift);
9252 %}
9253 
9254 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
9255                          iRegL src1, iRegL src2,
9256                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9257   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
9258   ins_cost(1.9 * INSN_COST);
9259   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
9260 
9261   ins_encode %{
9262     __ bic(as_Register($dst$$reg),
9263               as_Register($src1$$reg),
9264               as_Register($src2$$reg),
9265               Assembler::ASR,
9266               $src3$$constant & 0x3f);
9267   %}
9268 
9269   ins_pipe(ialu_reg_reg_shift);
9270 %}
9271 
9272 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
9273                          iRegIorL2I src1, iRegIorL2I src2,
9274                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9275   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
9276   ins_cost(1.9 * INSN_COST);
9277   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
9278 
9279   ins_encode %{
9280     __ bicw(as_Register($dst$$reg),
9281               as_Register($src1$$reg),
9282               as_Register($src2$$reg),
9283               Assembler::LSL,
9284               $src3$$constant & 0x3f);
9285   %}
9286 
9287   ins_pipe(ialu_reg_reg_shift);
9288 %}
9289 
9290 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
9291                          iRegL src1, iRegL src2,
9292                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9293   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
9294   ins_cost(1.9 * INSN_COST);
9295   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
9296 
9297   ins_encode %{
9298     __ bic(as_Register($dst$$reg),
9299               as_Register($src1$$reg),
9300               as_Register($src2$$reg),
9301               Assembler::LSL,
9302               $src3$$constant & 0x3f);
9303   %}
9304 
9305   ins_pipe(ialu_reg_reg_shift);
9306 %}
9307 
9308 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
9309                          iRegIorL2I src1, iRegIorL2I src2,
9310                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9311   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
9312   ins_cost(1.9 * INSN_COST);
9313   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
9314 
9315   ins_encode %{
9316     __ eonw(as_Register($dst$$reg),
9317               as_Register($src1$$reg),
9318               as_Register($src2$$reg),
9319               Assembler::LSR,
9320               $src3$$constant & 0x3f);
9321   %}
9322 
9323   ins_pipe(ialu_reg_reg_shift);
9324 %}
9325 
9326 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
9327                          iRegL src1, iRegL src2,
9328                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9329   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
9330   ins_cost(1.9 * INSN_COST);
9331   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
9332 
9333   ins_encode %{
9334     __ eon(as_Register($dst$$reg),
9335               as_Register($src1$$reg),
9336               as_Register($src2$$reg),
9337               Assembler::LSR,
9338               $src3$$constant & 0x3f);
9339   %}
9340 
9341   ins_pipe(ialu_reg_reg_shift);
9342 %}
9343 
9344 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
9345                          iRegIorL2I src1, iRegIorL2I src2,
9346                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9347   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
9348   ins_cost(1.9 * INSN_COST);
9349   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
9350 
9351   ins_encode %{
9352     __ eonw(as_Register($dst$$reg),
9353               as_Register($src1$$reg),
9354               as_Register($src2$$reg),
9355               Assembler::ASR,
9356               $src3$$constant & 0x3f);
9357   %}
9358 
9359   ins_pipe(ialu_reg_reg_shift);
9360 %}
9361 
9362 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
9363                          iRegL src1, iRegL src2,
9364                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9365   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
9366   ins_cost(1.9 * INSN_COST);
9367   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
9368 
9369   ins_encode %{
9370     __ eon(as_Register($dst$$reg),
9371               as_Register($src1$$reg),
9372               as_Register($src2$$reg),
9373               Assembler::ASR,
9374               $src3$$constant & 0x3f);
9375   %}
9376 
9377   ins_pipe(ialu_reg_reg_shift);
9378 %}
9379 
9380 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
9381                          iRegIorL2I src1, iRegIorL2I src2,
9382                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9383   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
9384   ins_cost(1.9 * INSN_COST);
9385   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
9386 
9387   ins_encode %{
9388     __ eonw(as_Register($dst$$reg),
9389               as_Register($src1$$reg),
9390               as_Register($src2$$reg),
9391               Assembler::LSL,
9392               $src3$$constant & 0x3f);
9393   %}
9394 
9395   ins_pipe(ialu_reg_reg_shift);
9396 %}
9397 
9398 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
9399                          iRegL src1, iRegL src2,
9400                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9401   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
9402   ins_cost(1.9 * INSN_COST);
9403   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
9404 
9405   ins_encode %{
9406     __ eon(as_Register($dst$$reg),
9407               as_Register($src1$$reg),
9408               as_Register($src2$$reg),
9409               Assembler::LSL,
9410               $src3$$constant & 0x3f);
9411   %}
9412 
9413   ins_pipe(ialu_reg_reg_shift);
9414 %}
9415 
9416 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
9417                          iRegIorL2I src1, iRegIorL2I src2,
9418                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9419   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
9420   ins_cost(1.9 * INSN_COST);
9421   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
9422 
9423   ins_encode %{
9424     __ ornw(as_Register($dst$$reg),
9425               as_Register($src1$$reg),
9426               as_Register($src2$$reg),
9427               Assembler::LSR,
9428               $src3$$constant & 0x3f);
9429   %}
9430 
9431   ins_pipe(ialu_reg_reg_shift);
9432 %}
9433 
9434 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
9435                          iRegL src1, iRegL src2,
9436                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9437   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
9438   ins_cost(1.9 * INSN_COST);
9439   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
9440 
9441   ins_encode %{
9442     __ orn(as_Register($dst$$reg),
9443               as_Register($src1$$reg),
9444               as_Register($src2$$reg),
9445               Assembler::LSR,
9446               $src3$$constant & 0x3f);
9447   %}
9448 
9449   ins_pipe(ialu_reg_reg_shift);
9450 %}
9451 
9452 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
9453                          iRegIorL2I src1, iRegIorL2I src2,
9454                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9455   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
9456   ins_cost(1.9 * INSN_COST);
9457   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
9458 
9459   ins_encode %{
9460     __ ornw(as_Register($dst$$reg),
9461               as_Register($src1$$reg),
9462               as_Register($src2$$reg),
9463               Assembler::ASR,
9464               $src3$$constant & 0x3f);
9465   %}
9466 
9467   ins_pipe(ialu_reg_reg_shift);
9468 %}
9469 
9470 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
9471                          iRegL src1, iRegL src2,
9472                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9473   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
9474   ins_cost(1.9 * INSN_COST);
9475   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
9476 
9477   ins_encode %{
9478     __ orn(as_Register($dst$$reg),
9479               as_Register($src1$$reg),
9480               as_Register($src2$$reg),
9481               Assembler::ASR,
9482               $src3$$constant & 0x3f);
9483   %}
9484 
9485   ins_pipe(ialu_reg_reg_shift);
9486 %}
9487 
9488 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
9489                          iRegIorL2I src1, iRegIorL2I src2,
9490                          immI src3, immI_M1 src4, rFlagsReg cr) %{
9491   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
9492   ins_cost(1.9 * INSN_COST);
9493   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
9494 
9495   ins_encode %{
9496     __ ornw(as_Register($dst$$reg),
9497               as_Register($src1$$reg),
9498               as_Register($src2$$reg),
9499               Assembler::LSL,
9500               $src3$$constant & 0x3f);
9501   %}
9502 
9503   ins_pipe(ialu_reg_reg_shift);
9504 %}
9505 
9506 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
9507                          iRegL src1, iRegL src2,
9508                          immI src3, immL_M1 src4, rFlagsReg cr) %{
9509   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
9510   ins_cost(1.9 * INSN_COST);
9511   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
9512 
9513   ins_encode %{
9514     __ orn(as_Register($dst$$reg),
9515               as_Register($src1$$reg),
9516               as_Register($src2$$reg),
9517               Assembler::LSL,
9518               $src3$$constant & 0x3f);
9519   %}
9520 
9521   ins_pipe(ialu_reg_reg_shift);
9522 %}
9523 
9524 instruct AndI_reg_URShift_reg(iRegINoSp dst,
9525                          iRegIorL2I src1, iRegIorL2I src2,
9526                          immI src3, rFlagsReg cr) %{
9527   match(Set dst (AndI src1 (URShiftI src2 src3)));
9528 
9529   ins_cost(1.9 * INSN_COST);
9530   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
9531 
9532   ins_encode %{
9533     __ andw(as_Register($dst$$reg),
9534               as_Register($src1$$reg),
9535               as_Register($src2$$reg),
9536               Assembler::LSR,
9537               $src3$$constant & 0x3f);
9538   %}
9539 
9540   ins_pipe(ialu_reg_reg_shift);
9541 %}
9542 
9543 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
9544                          iRegL src1, iRegL src2,
9545                          immI src3, rFlagsReg cr) %{
9546   match(Set dst (AndL src1 (URShiftL src2 src3)));
9547 
9548   ins_cost(1.9 * INSN_COST);
9549   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
9550 
9551   ins_encode %{
9552     __ andr(as_Register($dst$$reg),
9553               as_Register($src1$$reg),
9554               as_Register($src2$$reg),
9555               Assembler::LSR,
9556               $src3$$constant & 0x3f);
9557   %}
9558 
9559   ins_pipe(ialu_reg_reg_shift);
9560 %}
9561 
9562 instruct AndI_reg_RShift_reg(iRegINoSp dst,
9563                          iRegIorL2I src1, iRegIorL2I src2,
9564                          immI src3, rFlagsReg cr) %{
9565   match(Set dst (AndI src1 (RShiftI src2 src3)));
9566 
9567   ins_cost(1.9 * INSN_COST);
9568   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
9569 
9570   ins_encode %{
9571     __ andw(as_Register($dst$$reg),
9572               as_Register($src1$$reg),
9573               as_Register($src2$$reg),
9574               Assembler::ASR,
9575               $src3$$constant & 0x3f);
9576   %}
9577 
9578   ins_pipe(ialu_reg_reg_shift);
9579 %}
9580 
9581 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
9582                          iRegL src1, iRegL src2,
9583                          immI src3, rFlagsReg cr) %{
9584   match(Set dst (AndL src1 (RShiftL src2 src3)));
9585 
9586   ins_cost(1.9 * INSN_COST);
9587   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
9588 
9589   ins_encode %{
9590     __ andr(as_Register($dst$$reg),
9591               as_Register($src1$$reg),
9592               as_Register($src2$$reg),
9593               Assembler::ASR,
9594               $src3$$constant & 0x3f);
9595   %}
9596 
9597   ins_pipe(ialu_reg_reg_shift);
9598 %}
9599 
9600 instruct AndI_reg_LShift_reg(iRegINoSp dst,
9601                          iRegIorL2I src1, iRegIorL2I src2,
9602                          immI src3, rFlagsReg cr) %{
9603   match(Set dst (AndI src1 (LShiftI src2 src3)));
9604 
9605   ins_cost(1.9 * INSN_COST);
9606   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
9607 
9608   ins_encode %{
9609     __ andw(as_Register($dst$$reg),
9610               as_Register($src1$$reg),
9611               as_Register($src2$$reg),
9612               Assembler::LSL,
9613               $src3$$constant & 0x3f);
9614   %}
9615 
9616   ins_pipe(ialu_reg_reg_shift);
9617 %}
9618 
9619 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
9620                          iRegL src1, iRegL src2,
9621                          immI src3, rFlagsReg cr) %{
9622   match(Set dst (AndL src1 (LShiftL src2 src3)));
9623 
9624   ins_cost(1.9 * INSN_COST);
9625   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
9626 
9627   ins_encode %{
9628     __ andr(as_Register($dst$$reg),
9629               as_Register($src1$$reg),
9630               as_Register($src2$$reg),
9631               Assembler::LSL,
9632               $src3$$constant & 0x3f);
9633   %}
9634 
9635   ins_pipe(ialu_reg_reg_shift);
9636 %}
9637 
9638 instruct XorI_reg_URShift_reg(iRegINoSp dst,
9639                          iRegIorL2I src1, iRegIorL2I src2,
9640                          immI src3, rFlagsReg cr) %{
9641   match(Set dst (XorI src1 (URShiftI src2 src3)));
9642 
9643   ins_cost(1.9 * INSN_COST);
9644   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
9645 
9646   ins_encode %{
9647     __ eorw(as_Register($dst$$reg),
9648               as_Register($src1$$reg),
9649               as_Register($src2$$reg),
9650               Assembler::LSR,
9651               $src3$$constant & 0x3f);
9652   %}
9653 
9654   ins_pipe(ialu_reg_reg_shift);
9655 %}
9656 
9657 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
9658                          iRegL src1, iRegL src2,
9659                          immI src3, rFlagsReg cr) %{
9660   match(Set dst (XorL src1 (URShiftL src2 src3)));
9661 
9662   ins_cost(1.9 * INSN_COST);
9663   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
9664 
9665   ins_encode %{
9666     __ eor(as_Register($dst$$reg),
9667               as_Register($src1$$reg),
9668               as_Register($src2$$reg),
9669               Assembler::LSR,
9670               $src3$$constant & 0x3f);
9671   %}
9672 
9673   ins_pipe(ialu_reg_reg_shift);
9674 %}
9675 
9676 instruct XorI_reg_RShift_reg(iRegINoSp dst,
9677                          iRegIorL2I src1, iRegIorL2I src2,
9678                          immI src3, rFlagsReg cr) %{
9679   match(Set dst (XorI src1 (RShiftI src2 src3)));
9680 
9681   ins_cost(1.9 * INSN_COST);
9682   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
9683 
9684   ins_encode %{
9685     __ eorw(as_Register($dst$$reg),
9686               as_Register($src1$$reg),
9687               as_Register($src2$$reg),
9688               Assembler::ASR,
9689               $src3$$constant & 0x3f);
9690   %}
9691 
9692   ins_pipe(ialu_reg_reg_shift);
9693 %}
9694 
9695 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
9696                          iRegL src1, iRegL src2,
9697                          immI src3, rFlagsReg cr) %{
9698   match(Set dst (XorL src1 (RShiftL src2 src3)));
9699 
9700   ins_cost(1.9 * INSN_COST);
9701   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
9702 
9703   ins_encode %{
9704     __ eor(as_Register($dst$$reg),
9705               as_Register($src1$$reg),
9706               as_Register($src2$$reg),
9707               Assembler::ASR,
9708               $src3$$constant & 0x3f);
9709   %}
9710 
9711   ins_pipe(ialu_reg_reg_shift);
9712 %}
9713 
9714 instruct XorI_reg_LShift_reg(iRegINoSp dst,
9715                          iRegIorL2I src1, iRegIorL2I src2,
9716                          immI src3, rFlagsReg cr) %{
9717   match(Set dst (XorI src1 (LShiftI src2 src3)));
9718 
9719   ins_cost(1.9 * INSN_COST);
9720   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
9721 
9722   ins_encode %{
9723     __ eorw(as_Register($dst$$reg),
9724               as_Register($src1$$reg),
9725               as_Register($src2$$reg),
9726               Assembler::LSL,
9727               $src3$$constant & 0x3f);
9728   %}
9729 
9730   ins_pipe(ialu_reg_reg_shift);
9731 %}
9732 
9733 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
9734                          iRegL src1, iRegL src2,
9735                          immI src3, rFlagsReg cr) %{
9736   match(Set dst (XorL src1 (LShiftL src2 src3)));
9737 
9738   ins_cost(1.9 * INSN_COST);
9739   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
9740 
9741   ins_encode %{
9742     __ eor(as_Register($dst$$reg),
9743               as_Register($src1$$reg),
9744               as_Register($src2$$reg),
9745               Assembler::LSL,
9746               $src3$$constant & 0x3f);
9747   %}
9748 
9749   ins_pipe(ialu_reg_reg_shift);
9750 %}
9751 
9752 instruct OrI_reg_URShift_reg(iRegINoSp dst,
9753                          iRegIorL2I src1, iRegIorL2I src2,
9754                          immI src3, rFlagsReg cr) %{
9755   match(Set dst (OrI src1 (URShiftI src2 src3)));
9756 
9757   ins_cost(1.9 * INSN_COST);
9758   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
9759 
9760   ins_encode %{
9761     __ orrw(as_Register($dst$$reg),
9762               as_Register($src1$$reg),
9763               as_Register($src2$$reg),
9764               Assembler::LSR,
9765               $src3$$constant & 0x3f);
9766   %}
9767 
9768   ins_pipe(ialu_reg_reg_shift);
9769 %}
9770 
9771 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
9772                          iRegL src1, iRegL src2,
9773                          immI src3, rFlagsReg cr) %{
9774   match(Set dst (OrL src1 (URShiftL src2 src3)));
9775 
9776   ins_cost(1.9 * INSN_COST);
9777   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
9778 
9779   ins_encode %{
9780     __ orr(as_Register($dst$$reg),
9781               as_Register($src1$$reg),
9782               as_Register($src2$$reg),
9783               Assembler::LSR,
9784               $src3$$constant & 0x3f);
9785   %}
9786 
9787   ins_pipe(ialu_reg_reg_shift);
9788 %}
9789 
9790 instruct OrI_reg_RShift_reg(iRegINoSp dst,
9791                          iRegIorL2I src1, iRegIorL2I src2,
9792                          immI src3, rFlagsReg cr) %{
9793   match(Set dst (OrI src1 (RShiftI src2 src3)));
9794 
9795   ins_cost(1.9 * INSN_COST);
9796   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
9797 
9798   ins_encode %{
9799     __ orrw(as_Register($dst$$reg),
9800               as_Register($src1$$reg),
9801               as_Register($src2$$reg),
9802               Assembler::ASR,
9803               $src3$$constant & 0x3f);
9804   %}
9805 
9806   ins_pipe(ialu_reg_reg_shift);
9807 %}
9808 
9809 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
9810                          iRegL src1, iRegL src2,
9811                          immI src3, rFlagsReg cr) %{
9812   match(Set dst (OrL src1 (RShiftL src2 src3)));
9813 
9814   ins_cost(1.9 * INSN_COST);
9815   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
9816 
9817   ins_encode %{
9818     __ orr(as_Register($dst$$reg),
9819               as_Register($src1$$reg),
9820               as_Register($src2$$reg),
9821               Assembler::ASR,
9822               $src3$$constant & 0x3f);
9823   %}
9824 
9825   ins_pipe(ialu_reg_reg_shift);
9826 %}
9827 
9828 instruct OrI_reg_LShift_reg(iRegINoSp dst,
9829                          iRegIorL2I src1, iRegIorL2I src2,
9830                          immI src3, rFlagsReg cr) %{
9831   match(Set dst (OrI src1 (LShiftI src2 src3)));
9832 
9833   ins_cost(1.9 * INSN_COST);
9834   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
9835 
9836   ins_encode %{
9837     __ orrw(as_Register($dst$$reg),
9838               as_Register($src1$$reg),
9839               as_Register($src2$$reg),
9840               Assembler::LSL,
9841               $src3$$constant & 0x3f);
9842   %}
9843 
9844   ins_pipe(ialu_reg_reg_shift);
9845 %}
9846 
9847 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
9848                          iRegL src1, iRegL src2,
9849                          immI src3, rFlagsReg cr) %{
9850   match(Set dst (OrL src1 (LShiftL src2 src3)));
9851 
9852   ins_cost(1.9 * INSN_COST);
9853   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
9854 
9855   ins_encode %{
9856     __ orr(as_Register($dst$$reg),
9857               as_Register($src1$$reg),
9858               as_Register($src2$$reg),
9859               Assembler::LSL,
9860               $src3$$constant & 0x3f);
9861   %}
9862 
9863   ins_pipe(ialu_reg_reg_shift);
9864 %}
9865 
9866 instruct AddI_reg_URShift_reg(iRegINoSp dst,
9867                          iRegIorL2I src1, iRegIorL2I src2,
9868                          immI src3, rFlagsReg cr) %{
9869   match(Set dst (AddI src1 (URShiftI src2 src3)));
9870 
9871   ins_cost(1.9 * INSN_COST);
9872   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
9873 
9874   ins_encode %{
9875     __ addw(as_Register($dst$$reg),
9876               as_Register($src1$$reg),
9877               as_Register($src2$$reg),
9878               Assembler::LSR,
9879               $src3$$constant & 0x3f);
9880   %}
9881 
9882   ins_pipe(ialu_reg_reg_shift);
9883 %}
9884 
9885 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
9886                          iRegL src1, iRegL src2,
9887                          immI src3, rFlagsReg cr) %{
9888   match(Set dst (AddL src1 (URShiftL src2 src3)));
9889 
9890   ins_cost(1.9 * INSN_COST);
9891   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
9892 
9893   ins_encode %{
9894     __ add(as_Register($dst$$reg),
9895               as_Register($src1$$reg),
9896               as_Register($src2$$reg),
9897               Assembler::LSR,
9898               $src3$$constant & 0x3f);
9899   %}
9900 
9901   ins_pipe(ialu_reg_reg_shift);
9902 %}
9903 
9904 instruct AddI_reg_RShift_reg(iRegINoSp dst,
9905                          iRegIorL2I src1, iRegIorL2I src2,
9906                          immI src3, rFlagsReg cr) %{
9907   match(Set dst (AddI src1 (RShiftI src2 src3)));
9908 
9909   ins_cost(1.9 * INSN_COST);
9910   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
9911 
9912   ins_encode %{
9913     __ addw(as_Register($dst$$reg),
9914               as_Register($src1$$reg),
9915               as_Register($src2$$reg),
9916               Assembler::ASR,
9917               $src3$$constant & 0x3f);
9918   %}
9919 
9920   ins_pipe(ialu_reg_reg_shift);
9921 %}
9922 
9923 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
9924                          iRegL src1, iRegL src2,
9925                          immI src3, rFlagsReg cr) %{
9926   match(Set dst (AddL src1 (RShiftL src2 src3)));
9927 
9928   ins_cost(1.9 * INSN_COST);
9929   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
9930 
9931   ins_encode %{
9932     __ add(as_Register($dst$$reg),
9933               as_Register($src1$$reg),
9934               as_Register($src2$$reg),
9935               Assembler::ASR,
9936               $src3$$constant & 0x3f);
9937   %}
9938 
9939   ins_pipe(ialu_reg_reg_shift);
9940 %}
9941 
9942 instruct AddI_reg_LShift_reg(iRegINoSp dst,
9943                          iRegIorL2I src1, iRegIorL2I src2,
9944                          immI src3, rFlagsReg cr) %{
9945   match(Set dst (AddI src1 (LShiftI src2 src3)));
9946 
9947   ins_cost(1.9 * INSN_COST);
9948   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
9949 
9950   ins_encode %{
9951     __ addw(as_Register($dst$$reg),
9952               as_Register($src1$$reg),
9953               as_Register($src2$$reg),
9954               Assembler::LSL,
9955               $src3$$constant & 0x3f);
9956   %}
9957 
9958   ins_pipe(ialu_reg_reg_shift);
9959 %}
9960 
9961 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
9962                          iRegL src1, iRegL src2,
9963                          immI src3, rFlagsReg cr) %{
9964   match(Set dst (AddL src1 (LShiftL src2 src3)));
9965 
9966   ins_cost(1.9 * INSN_COST);
9967   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
9968 
9969   ins_encode %{
9970     __ add(as_Register($dst$$reg),
9971               as_Register($src1$$reg),
9972               as_Register($src2$$reg),
9973               Assembler::LSL,
9974               $src3$$constant & 0x3f);
9975   %}
9976 
9977   ins_pipe(ialu_reg_reg_shift);
9978 %}
9979 
9980 instruct SubI_reg_URShift_reg(iRegINoSp dst,
9981                          iRegIorL2I src1, iRegIorL2I src2,
9982                          immI src3, rFlagsReg cr) %{
9983   match(Set dst (SubI src1 (URShiftI src2 src3)));
9984 
9985   ins_cost(1.9 * INSN_COST);
9986   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
9987 
9988   ins_encode %{
9989     __ subw(as_Register($dst$$reg),
9990               as_Register($src1$$reg),
9991               as_Register($src2$$reg),
9992               Assembler::LSR,
9993               $src3$$constant & 0x3f);
9994   %}
9995 
9996   ins_pipe(ialu_reg_reg_shift);
9997 %}
9998 
9999 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
10000                          iRegL src1, iRegL src2,
10001                          immI src3, rFlagsReg cr) %{
10002   match(Set dst (SubL src1 (URShiftL src2 src3)));
10003 
10004   ins_cost(1.9 * INSN_COST);
10005   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
10006 
10007   ins_encode %{
10008     __ sub(as_Register($dst$$reg),
10009               as_Register($src1$$reg),
10010               as_Register($src2$$reg),
10011               Assembler::LSR,
10012               $src3$$constant & 0x3f);
10013   %}
10014 
10015   ins_pipe(ialu_reg_reg_shift);
10016 %}
10017 
10018 instruct SubI_reg_RShift_reg(iRegINoSp dst,
10019                          iRegIorL2I src1, iRegIorL2I src2,
10020                          immI src3, rFlagsReg cr) %{
10021   match(Set dst (SubI src1 (RShiftI src2 src3)));
10022 
10023   ins_cost(1.9 * INSN_COST);
10024   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
10025 
10026   ins_encode %{
10027     __ subw(as_Register($dst$$reg),
10028               as_Register($src1$$reg),
10029               as_Register($src2$$reg),
10030               Assembler::ASR,
10031               $src3$$constant & 0x3f);
10032   %}
10033 
10034   ins_pipe(ialu_reg_reg_shift);
10035 %}
10036 
10037 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
10038                          iRegL src1, iRegL src2,
10039                          immI src3, rFlagsReg cr) %{
10040   match(Set dst (SubL src1 (RShiftL src2 src3)));
10041 
10042   ins_cost(1.9 * INSN_COST);
10043   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
10044 
10045   ins_encode %{
10046     __ sub(as_Register($dst$$reg),
10047               as_Register($src1$$reg),
10048               as_Register($src2$$reg),
10049               Assembler::ASR,
10050               $src3$$constant & 0x3f);
10051   %}
10052 
10053   ins_pipe(ialu_reg_reg_shift);
10054 %}
10055 
10056 instruct SubI_reg_LShift_reg(iRegINoSp dst,
10057                          iRegIorL2I src1, iRegIorL2I src2,
10058                          immI src3, rFlagsReg cr) %{
10059   match(Set dst (SubI src1 (LShiftI src2 src3)));
10060 
10061   ins_cost(1.9 * INSN_COST);
10062   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
10063 
10064   ins_encode %{
10065     __ subw(as_Register($dst$$reg),
10066               as_Register($src1$$reg),
10067               as_Register($src2$$reg),
10068               Assembler::LSL,
10069               $src3$$constant & 0x3f);
10070   %}
10071 
10072   ins_pipe(ialu_reg_reg_shift);
10073 %}
10074 
10075 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
10076                          iRegL src1, iRegL src2,
10077                          immI src3, rFlagsReg cr) %{
10078   match(Set dst (SubL src1 (LShiftL src2 src3)));
10079 
10080   ins_cost(1.9 * INSN_COST);
10081   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
10082 
10083   ins_encode %{
10084     __ sub(as_Register($dst$$reg),
10085               as_Register($src1$$reg),
10086               as_Register($src2$$reg),
10087               Assembler::LSL,
10088               $src3$$constant & 0x3f);
10089   %}
10090 
10091   ins_pipe(ialu_reg_reg_shift);
10092 %}
10093 
10094 
10095 
10096 // Shift Left followed by Shift Right.
10097 // This idiom is used by the compiler for the i2b bytecode etc.
10098 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
10099 %{
10100   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
10101   // Make sure we are not going to exceed what sbfm can do.
10102   predicate((unsigned int)n->in(2)->get_int() <= 63
10103             && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
10104 
10105   ins_cost(INSN_COST * 2);
10106   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
10107   ins_encode %{
10108     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10109     int s = 63 - lshift;
10110     int r = (rshift - lshift) & 63;
10111     __ sbfm(as_Register($dst$$reg),
10112             as_Register($src$$reg),
10113             r, s);
10114   %}
10115 
10116   ins_pipe(ialu_reg_shift);
10117 %}
10118 
10119 // Shift Left followed by Shift Right.
10120 // This idiom is used by the compiler for the i2b bytecode etc.
10121 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
10122 %{
10123   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
10124   // Make sure we are not going to exceed what sbfmw can do.
10125   predicate((unsigned int)n->in(2)->get_int() <= 31
10126             && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
10127 
10128   ins_cost(INSN_COST * 2);
10129   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
10130   ins_encode %{
10131     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10132     int s = 31 - lshift;
10133     int r = (rshift - lshift) & 31;
10134     __ sbfmw(as_Register($dst$$reg),
10135             as_Register($src$$reg),
10136             r, s);
10137   %}
10138 
10139   ins_pipe(ialu_reg_shift);
10140 %}
10141 
10142 // Shift Left followed by Shift Right.
10143 // This idiom is used by the compiler for the i2b bytecode etc.
10144 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
10145 %{
10146   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
10147   // Make sure we are not going to exceed what ubfm can do.
10148   predicate((unsigned int)n->in(2)->get_int() <= 63
10149             && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
10150 
10151   ins_cost(INSN_COST * 2);
10152   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
10153   ins_encode %{
10154     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10155     int s = 63 - lshift;
10156     int r = (rshift - lshift) & 63;
10157     __ ubfm(as_Register($dst$$reg),
10158             as_Register($src$$reg),
10159             r, s);
10160   %}
10161 
10162   ins_pipe(ialu_reg_shift);
10163 %}
10164 
10165 // Shift Left followed by Shift Right.
10166 // This idiom is used by the compiler for the i2b bytecode etc.
10167 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
10168 %{
10169   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
10170   // Make sure we are not going to exceed what ubfmw can do.
10171   predicate((unsigned int)n->in(2)->get_int() <= 31
10172             && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
10173 
10174   ins_cost(INSN_COST * 2);
10175   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
10176   ins_encode %{
10177     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10178     int s = 31 - lshift;
10179     int r = (rshift - lshift) & 31;
10180     __ ubfmw(as_Register($dst$$reg),
10181             as_Register($src$$reg),
10182             r, s);
10183   %}
10184 
10185   ins_pipe(ialu_reg_shift);
10186 %}
10187 // Bitfield extract with shift & mask
10188 
10189 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
10190 %{
10191   match(Set dst (AndI (URShiftI src rshift) mask));
10192 
10193   ins_cost(INSN_COST);
10194   format %{ "ubfxw $dst, $src, $mask" %}
10195   ins_encode %{
10196     int rshift = $rshift$$constant;
10197     long mask = $mask$$constant;
10198     int width = exact_log2(mask+1);
10199     __ ubfxw(as_Register($dst$$reg),
10200             as_Register($src$$reg), rshift, width);
10201   %}
10202   ins_pipe(ialu_reg_shift);
10203 %}
10204 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
10205 %{
10206   match(Set dst (AndL (URShiftL src rshift) mask));
10207 
10208   ins_cost(INSN_COST);
10209   format %{ "ubfx $dst, $src, $mask" %}
10210   ins_encode %{
10211     int rshift = $rshift$$constant;
10212     long mask = $mask$$constant;
10213     int width = exact_log2(mask+1);
10214     __ ubfx(as_Register($dst$$reg),
10215             as_Register($src$$reg), rshift, width);
10216   %}
10217   ins_pipe(ialu_reg_shift);
10218 %}
10219 
10220 // We can use ubfx when extending an And with a mask when we know mask
10221 // is positive.  We know that because immI_bitmask guarantees it.
10222 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
10223 %{
10224   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
10225 
10226   ins_cost(INSN_COST * 2);
10227   format %{ "ubfx $dst, $src, $mask" %}
10228   ins_encode %{
10229     int rshift = $rshift$$constant;
10230     long mask = $mask$$constant;
10231     int width = exact_log2(mask+1);
10232     __ ubfx(as_Register($dst$$reg),
10233             as_Register($src$$reg), rshift, width);
10234   %}
10235   ins_pipe(ialu_reg_shift);
10236 %}
10237 
10238 // Rotations
10239 
10240 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
10241 %{
10242   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
10243   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
10244 
10245   ins_cost(INSN_COST);
10246   format %{ "extr $dst, $src1, $src2, #$rshift" %}
10247 
10248   ins_encode %{
10249     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
10250             $rshift$$constant & 63);
10251   %}
10252   ins_pipe(ialu_reg_reg_extr);
10253 %}
10254 
10255 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
10256 %{
10257   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
10258   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
10259 
10260   ins_cost(INSN_COST);
10261   format %{ "extr $dst, $src1, $src2, #$rshift" %}
10262 
10263   ins_encode %{
10264     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
10265             $rshift$$constant & 31);
10266   %}
10267   ins_pipe(ialu_reg_reg_extr);
10268 %}
10269 
10270 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
10271 %{
10272   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
10273   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
10274 
10275   ins_cost(INSN_COST);
10276   format %{ "extr $dst, $src1, $src2, #$rshift" %}
10277 
10278   ins_encode %{
10279     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
10280             $rshift$$constant & 63);
10281   %}
10282   ins_pipe(ialu_reg_reg_extr);
10283 %}
10284 
10285 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
10286 %{
10287   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
10288   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
10289 
10290   ins_cost(INSN_COST);
10291   format %{ "extr $dst, $src1, $src2, #$rshift" %}
10292 
10293   ins_encode %{
10294     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
10295             $rshift$$constant & 31);
10296   %}
10297   ins_pipe(ialu_reg_reg_extr);
10298 %}
10299 
10300 
10301 // rol expander
10302 
10303 instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
10304 %{
10305   effect(DEF dst, USE src, USE shift);
10306 
10307   format %{ "rol    $dst, $src, $shift" %}
10308   ins_cost(INSN_COST * 3);
10309   ins_encode %{
10310     __ subw(rscratch1, zr, as_Register($shift$$reg));
10311     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
10312             rscratch1);
10313     %}
10314   ins_pipe(ialu_reg_reg_vshift);
10315 %}
10316 
10317 // rol expander
10318 
10319 instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
10320 %{
10321   effect(DEF dst, USE src, USE shift);
10322 
10323   format %{ "rol    $dst, $src, $shift" %}
10324   ins_cost(INSN_COST * 3);
10325   ins_encode %{
10326     __ subw(rscratch1, zr, as_Register($shift$$reg));
10327     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
10328             rscratch1);
10329     %}
10330   ins_pipe(ialu_reg_reg_vshift);
10331 %}
10332 
10333 instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
10334 %{
10335   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
10336 
10337   expand %{
10338     rolL_rReg(dst, src, shift, cr);
10339   %}
10340 %}
10341 
10342 instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
10343 %{
10344   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
10345 
10346   expand %{
10347     rolL_rReg(dst, src, shift, cr);
10348   %}
10349 %}
10350 
10351 instruct rolI_rReg_Var_C_32(iRegLNoSp dst, iRegL src, iRegI shift, immI_32 c_32, rFlagsReg cr)
10352 %{
10353   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
10354 
10355   expand %{
10356     rolL_rReg(dst, src, shift, cr);
10357   %}
10358 %}
10359 
10360 instruct rolI_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
10361 %{
10362   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
10363 
10364   expand %{
10365     rolL_rReg(dst, src, shift, cr);
10366   %}
10367 %}
10368 
10369 // ror expander
10370 
10371 instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
10372 %{
10373   effect(DEF dst, USE src, USE shift);
10374 
10375   format %{ "ror    $dst, $src, $shift" %}
10376   ins_cost(INSN_COST);
10377   ins_encode %{
10378     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
10379             as_Register($shift$$reg));
10380     %}
10381   ins_pipe(ialu_reg_reg_vshift);
10382 %}
10383 
10384 // ror expander
10385 
10386 instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
10387 %{
10388   effect(DEF dst, USE src, USE shift);
10389 
10390   format %{ "ror    $dst, $src, $shift" %}
10391   ins_cost(INSN_COST);
10392   ins_encode %{
10393     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
10394             as_Register($shift$$reg));
10395     %}
10396   ins_pipe(ialu_reg_reg_vshift);
10397 %}
10398 
10399 instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
10400 %{
10401   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
10402 
10403   expand %{
10404     rorL_rReg(dst, src, shift, cr);
10405   %}
10406 %}
10407 
10408 instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
10409 %{
10410   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
10411 
10412   expand %{
10413     rorL_rReg(dst, src, shift, cr);
10414   %}
10415 %}
10416 
10417 instruct rorI_rReg_Var_C_32(iRegLNoSp dst, iRegL src, iRegI shift, immI_32 c_32, rFlagsReg cr)
10418 %{
10419   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
10420 
10421   expand %{
10422     rorL_rReg(dst, src, shift, cr);
10423   %}
10424 %}
10425 
10426 instruct rorI_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
10427 %{
10428   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
10429 
10430   expand %{
10431     rorL_rReg(dst, src, shift, cr);
10432   %}
10433 %}
10434 
10435 // Add/subtract (extended)
10436 
10437 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
10438 %{
10439   match(Set dst (AddL src1 (ConvI2L src2)));
10440   ins_cost(INSN_COST);
10441   format %{ "add  $dst, $src1, sxtw $src2" %}
10442 
10443    ins_encode %{
10444      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10445             as_Register($src2$$reg), ext::sxtw);
10446    %}
10447   ins_pipe(ialu_reg_reg);
10448 %};
10449 
10450 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
10451 %{
10452   match(Set dst (SubL src1 (ConvI2L src2)));
10453   ins_cost(INSN_COST);
10454   format %{ "sub  $dst, $src1, sxtw $src2" %}
10455 
10456    ins_encode %{
10457      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
10458             as_Register($src2$$reg), ext::sxtw);
10459    %}
10460   ins_pipe(ialu_reg_reg);
10461 %};
10462 
10463 
10464 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
10465 %{
10466   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
10467   ins_cost(INSN_COST);
10468   format %{ "add  $dst, $src1, sxth $src2" %}
10469 
10470    ins_encode %{
10471      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10472             as_Register($src2$$reg), ext::sxth);
10473    %}
10474   ins_pipe(ialu_reg_reg);
10475 %}
10476 
10477 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
10478 %{
10479   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
10480   ins_cost(INSN_COST);
10481   format %{ "add  $dst, $src1, sxtb $src2" %}
10482 
10483    ins_encode %{
10484      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10485             as_Register($src2$$reg), ext::sxtb);
10486    %}
10487   ins_pipe(ialu_reg_reg);
10488 %}
10489 
10490 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
10491 %{
10492   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
10493   ins_cost(INSN_COST);
10494   format %{ "add  $dst, $src1, uxtb $src2" %}
10495 
10496    ins_encode %{
10497      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10498             as_Register($src2$$reg), ext::uxtb);
10499    %}
10500   ins_pipe(ialu_reg_reg);
10501 %}
10502 
10503 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
10504 %{
10505   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
10506   ins_cost(INSN_COST);
10507   format %{ "add  $dst, $src1, sxth $src2" %}
10508 
10509    ins_encode %{
10510      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10511             as_Register($src2$$reg), ext::sxth);
10512    %}
10513   ins_pipe(ialu_reg_reg);
10514 %}
10515 
10516 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
10517 %{
10518   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
10519   ins_cost(INSN_COST);
10520   format %{ "add  $dst, $src1, sxtw $src2" %}
10521 
10522    ins_encode %{
10523      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10524             as_Register($src2$$reg), ext::sxtw);
10525    %}
10526   ins_pipe(ialu_reg_reg);
10527 %}
10528 
10529 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
10530 %{
10531   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
10532   ins_cost(INSN_COST);
10533   format %{ "add  $dst, $src1, sxtb $src2" %}
10534 
10535    ins_encode %{
10536      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10537             as_Register($src2$$reg), ext::sxtb);
10538    %}
10539   ins_pipe(ialu_reg_reg);
10540 %}
10541 
10542 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
10543 %{
10544   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
10545   ins_cost(INSN_COST);
10546   format %{ "add  $dst, $src1, uxtb $src2" %}
10547 
10548    ins_encode %{
10549      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10550             as_Register($src2$$reg), ext::uxtb);
10551    %}
10552   ins_pipe(ialu_reg_reg);
10553 %}
10554 
10555 
10556 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
10557 %{
10558   match(Set dst (AddI src1 (AndI src2 mask)));
10559   ins_cost(INSN_COST);
10560   format %{ "addw  $dst, $src1, $src2, uxtb" %}
10561 
10562    ins_encode %{
10563      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
10564             as_Register($src2$$reg), ext::uxtb);
10565    %}
10566   ins_pipe(ialu_reg_reg);
10567 %}
10568 
10569 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
10570 %{
10571   match(Set dst (AddI src1 (AndI src2 mask)));
10572   ins_cost(INSN_COST);
10573   format %{ "addw  $dst, $src1, $src2, uxth" %}
10574 
10575    ins_encode %{
10576      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
10577             as_Register($src2$$reg), ext::uxth);
10578    %}
10579   ins_pipe(ialu_reg_reg);
10580 %}
10581 
10582 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
10583 %{
10584   match(Set dst (AddL src1 (AndL src2 mask)));
10585   ins_cost(INSN_COST);
10586   format %{ "add  $dst, $src1, $src2, uxtb" %}
10587 
10588    ins_encode %{
10589      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10590             as_Register($src2$$reg), ext::uxtb);
10591    %}
10592   ins_pipe(ialu_reg_reg);
10593 %}
10594 
10595 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
10596 %{
10597   match(Set dst (AddL src1 (AndL src2 mask)));
10598   ins_cost(INSN_COST);
10599   format %{ "add  $dst, $src1, $src2, uxth" %}
10600 
10601    ins_encode %{
10602      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10603             as_Register($src2$$reg), ext::uxth);
10604    %}
10605   ins_pipe(ialu_reg_reg);
10606 %}
10607 
10608 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
10609 %{
10610   match(Set dst (AddL src1 (AndL src2 mask)));
10611   ins_cost(INSN_COST);
10612   format %{ "add  $dst, $src1, $src2, uxtw" %}
10613 
10614    ins_encode %{
10615      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
10616             as_Register($src2$$reg), ext::uxtw);
10617    %}
10618   ins_pipe(ialu_reg_reg);
10619 %}
10620 
10621 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
10622 %{
10623   match(Set dst (SubI src1 (AndI src2 mask)));
10624   ins_cost(INSN_COST);
10625   format %{ "subw  $dst, $src1, $src2, uxtb" %}
10626 
10627    ins_encode %{
10628      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
10629             as_Register($src2$$reg), ext::uxtb);
10630    %}
10631   ins_pipe(ialu_reg_reg);
10632 %}
10633 
10634 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
10635 %{
10636   match(Set dst (SubI src1 (AndI src2 mask)));
10637   ins_cost(INSN_COST);
10638   format %{ "subw  $dst, $src1, $src2, uxth" %}
10639 
10640    ins_encode %{
10641      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
10642             as_Register($src2$$reg), ext::uxth);
10643    %}
10644   ins_pipe(ialu_reg_reg);
10645 %}
10646 
10647 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
10648 %{
10649   match(Set dst (SubL src1 (AndL src2 mask)));
10650   ins_cost(INSN_COST);
10651   format %{ "sub  $dst, $src1, $src2, uxtb" %}
10652 
10653    ins_encode %{
10654      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
10655             as_Register($src2$$reg), ext::uxtb);
10656    %}
10657   ins_pipe(ialu_reg_reg);
10658 %}
10659 
10660 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
10661 %{
10662   match(Set dst (SubL src1 (AndL src2 mask)));
10663   ins_cost(INSN_COST);
10664   format %{ "sub  $dst, $src1, $src2, uxth" %}
10665 
10666    ins_encode %{
10667      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
10668             as_Register($src2$$reg), ext::uxth);
10669    %}
10670   ins_pipe(ialu_reg_reg);
10671 %}
10672 
10673 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
10674 %{
10675   match(Set dst (SubL src1 (AndL src2 mask)));
10676   ins_cost(INSN_COST);
10677   format %{ "sub  $dst, $src1, $src2, uxtw" %}
10678 
10679    ins_encode %{
10680      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
10681             as_Register($src2$$reg), ext::uxtw);
10682    %}
10683   ins_pipe(ialu_reg_reg);
10684 %}
10685 
10686 // END This section of the file is automatically generated. Do not edit --------------
10687 
10688 // ============================================================================
10689 // Floating Point Arithmetic Instructions
10690 
10691 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
10692   match(Set dst (AddF src1 src2));
10693 
10694   ins_cost(INSN_COST * 5);
10695   format %{ "fadds   $dst, $src1, $src2" %}
10696 
10697   ins_encode %{
10698     __ fadds(as_FloatRegister($dst$$reg),
10699              as_FloatRegister($src1$$reg),
10700              as_FloatRegister($src2$$reg));
10701   %}
10702 
10703   ins_pipe(pipe_class_default);
10704 %}
10705 
10706 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
10707   match(Set dst (AddD src1 src2));
10708 
10709   ins_cost(INSN_COST * 5);
10710   format %{ "faddd   $dst, $src1, $src2" %}
10711 
10712   ins_encode %{
10713     __ faddd(as_FloatRegister($dst$$reg),
10714              as_FloatRegister($src1$$reg),
10715              as_FloatRegister($src2$$reg));
10716   %}
10717 
10718   ins_pipe(pipe_class_default);
10719 %}
10720 
10721 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
10722   match(Set dst (SubF src1 src2));
10723 
10724   ins_cost(INSN_COST * 5);
10725   format %{ "fsubs   $dst, $src1, $src2" %}
10726 
10727   ins_encode %{
10728     __ fsubs(as_FloatRegister($dst$$reg),
10729              as_FloatRegister($src1$$reg),
10730              as_FloatRegister($src2$$reg));
10731   %}
10732 
10733   ins_pipe(pipe_class_default);
10734 %}
10735 
10736 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
10737   match(Set dst (SubD src1 src2));
10738 
10739   ins_cost(INSN_COST * 5);
10740   format %{ "fsubd   $dst, $src1, $src2" %}
10741 
10742   ins_encode %{
10743     __ fsubd(as_FloatRegister($dst$$reg),
10744              as_FloatRegister($src1$$reg),
10745              as_FloatRegister($src2$$reg));
10746   %}
10747 
10748   ins_pipe(pipe_class_default);
10749 %}
10750 
10751 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
10752   match(Set dst (MulF src1 src2));
10753 
10754   ins_cost(INSN_COST * 6);
10755   format %{ "fmuls   $dst, $src1, $src2" %}
10756 
10757   ins_encode %{
10758     __ fmuls(as_FloatRegister($dst$$reg),
10759              as_FloatRegister($src1$$reg),
10760              as_FloatRegister($src2$$reg));
10761   %}
10762 
10763   ins_pipe(pipe_class_default);
10764 %}
10765 
10766 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
10767   match(Set dst (MulD src1 src2));
10768 
10769   ins_cost(INSN_COST * 6);
10770   format %{ "fmuld   $dst, $src1, $src2" %}
10771 
10772   ins_encode %{
10773     __ fmuld(as_FloatRegister($dst$$reg),
10774              as_FloatRegister($src1$$reg),
10775              as_FloatRegister($src2$$reg));
10776   %}
10777 
10778   ins_pipe(pipe_class_default);
10779 %}
10780 
10781 // We cannot use these fused mul w add/sub ops because they don't
10782 // produce the same result as the equivalent separated ops
10783 // (essentially they don't round the intermediate result). that's a
10784 // shame. leaving them here in case we can idenitfy cases where it is
10785 // legitimate to use them
10786 
10787 
10788 // instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
10789 //   match(Set dst (AddF (MulF src1 src2) src3));
10790 
10791 //   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
10792 
10793 //   ins_encode %{
10794 //     __ fmadds(as_FloatRegister($dst$$reg),
10795 //              as_FloatRegister($src1$$reg),
10796 //              as_FloatRegister($src2$$reg),
10797 //              as_FloatRegister($src3$$reg));
10798 //   %}
10799 
10800 //   ins_pipe(pipe_class_default);
10801 // %}
10802 
10803 // instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
10804 //   match(Set dst (AddD (MulD src1 src2) src3));
10805 
10806 //   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
10807 
10808 //   ins_encode %{
10809 //     __ fmaddd(as_FloatRegister($dst$$reg),
10810 //              as_FloatRegister($src1$$reg),
10811 //              as_FloatRegister($src2$$reg),
10812 //              as_FloatRegister($src3$$reg));
10813 //   %}
10814 
10815 //   ins_pipe(pipe_class_default);
10816 // %}
10817 
10818 // instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
10819 //   match(Set dst (AddF (MulF (NegF src1) src2) src3));
10820 //   match(Set dst (AddF (NegF (MulF src1 src2)) src3));
10821 
10822 //   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
10823 
10824 //   ins_encode %{
10825 //     __ fmsubs(as_FloatRegister($dst$$reg),
10826 //               as_FloatRegister($src1$$reg),
10827 //               as_FloatRegister($src2$$reg),
10828 //              as_FloatRegister($src3$$reg));
10829 //   %}
10830 
10831 //   ins_pipe(pipe_class_default);
10832 // %}
10833 
10834 // instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
10835 //   match(Set dst (AddD (MulD (NegD src1) src2) src3));
10836 //   match(Set dst (AddD (NegD (MulD src1 src2)) src3));
10837 
10838 //   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
10839 
10840 //   ins_encode %{
10841 //     __ fmsubd(as_FloatRegister($dst$$reg),
10842 //               as_FloatRegister($src1$$reg),
10843 //               as_FloatRegister($src2$$reg),
10844 //               as_FloatRegister($src3$$reg));
10845 //   %}
10846 
10847 //   ins_pipe(pipe_class_default);
10848 // %}
10849 
10850 // instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
10851 //   match(Set dst (SubF (MulF (NegF src1) src2) src3));
10852 //   match(Set dst (SubF (NegF (MulF src1 src2)) src3));
10853 
10854 //   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
10855 
10856 //   ins_encode %{
10857 //     __ fnmadds(as_FloatRegister($dst$$reg),
10858 //                as_FloatRegister($src1$$reg),
10859 //                as_FloatRegister($src2$$reg),
10860 //                as_FloatRegister($src3$$reg));
10861 //   %}
10862 
10863 //   ins_pipe(pipe_class_default);
10864 // %}
10865 
10866 // instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
10867 //   match(Set dst (SubD (MulD (NegD src1) src2) src3));
10868 //   match(Set dst (SubD (NegD (MulD src1 src2)) src3));
10869 
10870 //   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
10871 
10872 //   ins_encode %{
10873 //     __ fnmaddd(as_FloatRegister($dst$$reg),
10874 //                as_FloatRegister($src1$$reg),
10875 //                as_FloatRegister($src2$$reg),
10876 //                as_FloatRegister($src3$$reg));
10877 //   %}
10878 
10879 //   ins_pipe(pipe_class_default);
10880 // %}
10881 
10882 // instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
10883 //   match(Set dst (SubF (MulF src1 src2) src3));
10884 
10885 //   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
10886 
10887 //   ins_encode %{
10888 //     __ fnmsubs(as_FloatRegister($dst$$reg),
10889 //                as_FloatRegister($src1$$reg),
10890 //                as_FloatRegister($src2$$reg),
10891 //                as_FloatRegister($src3$$reg));
10892 //   %}
10893 
10894 //   ins_pipe(pipe_class_default);
10895 // %}
10896 
10897 // instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
10898 //   match(Set dst (SubD (MulD src1 src2) src3));
10899 
10900 //   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
10901 
10902 //   ins_encode %{
10903 //   // n.b. insn name should be fnmsubd
10904 //     __ fnmsub(as_FloatRegister($dst$$reg),
10905 //                as_FloatRegister($src1$$reg),
10906 //                as_FloatRegister($src2$$reg),
10907 //                as_FloatRegister($src3$$reg));
10908 //   %}
10909 
10910 //   ins_pipe(pipe_class_default);
10911 // %}
10912 
10913 
10914 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
10915   match(Set dst (DivF src1  src2));
10916 
10917   ins_cost(INSN_COST * 18);
10918   format %{ "fdivs   $dst, $src1, $src2" %}
10919 
10920   ins_encode %{
10921     __ fdivs(as_FloatRegister($dst$$reg),
10922              as_FloatRegister($src1$$reg),
10923              as_FloatRegister($src2$$reg));
10924   %}
10925 
10926   ins_pipe(pipe_class_default);
10927 %}
10928 
10929 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
10930   match(Set dst (DivD src1  src2));
10931 
10932   ins_cost(INSN_COST * 32);
10933   format %{ "fdivd   $dst, $src1, $src2" %}
10934 
10935   ins_encode %{
10936     __ fdivd(as_FloatRegister($dst$$reg),
10937              as_FloatRegister($src1$$reg),
10938              as_FloatRegister($src2$$reg));
10939   %}
10940 
10941   ins_pipe(pipe_class_default);
10942 %}
10943 
10944 instruct negF_reg_reg(vRegF dst, vRegF src) %{
10945   match(Set dst (NegF src));
10946 
10947   ins_cost(INSN_COST * 3);
10948   format %{ "fneg   $dst, $src" %}
10949 
10950   ins_encode %{
10951     __ fnegs(as_FloatRegister($dst$$reg),
10952              as_FloatRegister($src$$reg));
10953   %}
10954 
10955   ins_pipe(pipe_class_default);
10956 %}
10957 
10958 instruct negD_reg_reg(vRegD dst, vRegD src) %{
10959   match(Set dst (NegD src));
10960 
10961   ins_cost(INSN_COST * 3);
10962   format %{ "fnegd   $dst, $src" %}
10963 
10964   ins_encode %{
10965     __ fnegd(as_FloatRegister($dst$$reg),
10966              as_FloatRegister($src$$reg));
10967   %}
10968 
10969   ins_pipe(pipe_class_default);
10970 %}
10971 
10972 instruct absF_reg(vRegF dst, vRegF src) %{
10973   match(Set dst (AbsF src));
10974 
10975   ins_cost(INSN_COST * 3);
10976   format %{ "fabss   $dst, $src" %}
10977   ins_encode %{
10978     __ fabss(as_FloatRegister($dst$$reg),
10979              as_FloatRegister($src$$reg));
10980   %}
10981 
10982   ins_pipe(pipe_class_default);
10983 %}
10984 
10985 instruct absD_reg(vRegD dst, vRegD src) %{
10986   match(Set dst (AbsD src));
10987 
10988   ins_cost(INSN_COST * 3);
10989   format %{ "fabsd   $dst, $src" %}
10990   ins_encode %{
10991     __ fabsd(as_FloatRegister($dst$$reg),
10992              as_FloatRegister($src$$reg));
10993   %}
10994 
10995   ins_pipe(pipe_class_default);
10996 %}
10997 
10998 instruct sqrtD_reg(vRegD dst, vRegD src) %{
10999   match(Set dst (SqrtD src));
11000 
11001   ins_cost(INSN_COST * 50);
11002   format %{ "fsqrtd  $dst, $src" %}
11003   ins_encode %{
11004     __ fsqrtd(as_FloatRegister($dst$$reg),
11005              as_FloatRegister($src$$reg));
11006   %}
11007 
11008   ins_pipe(pipe_class_default);
11009 %}
11010 
11011 instruct sqrtF_reg(vRegF dst, vRegF src) %{
11012   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
11013 
11014   ins_cost(INSN_COST * 50);
11015   format %{ "fsqrts  $dst, $src" %}
11016   ins_encode %{
11017     __ fsqrts(as_FloatRegister($dst$$reg),
11018              as_FloatRegister($src$$reg));
11019   %}
11020 
11021   ins_pipe(pipe_class_default);
11022 %}
11023 
11024 // ============================================================================
11025 // Logical Instructions
11026 
11027 // Integer Logical Instructions
11028 
11029 // And Instructions
11030 
11031 
11032 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
11033   match(Set dst (AndI src1 src2));
11034 
11035   format %{ "andw  $dst, $src1, $src2\t# int" %}
11036 
11037   ins_cost(INSN_COST);
11038   ins_encode %{
11039     __ andw(as_Register($dst$$reg),
11040             as_Register($src1$$reg),
11041             as_Register($src2$$reg));
11042   %}
11043 
11044   ins_pipe(ialu_reg_reg);
11045 %}
11046 
11047 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
11048   match(Set dst (AndI src1 src2));
11049 
11050   format %{ "andsw  $dst, $src1, $src2\t# int" %}
11051 
11052   ins_cost(INSN_COST);
11053   ins_encode %{
11054     __ andw(as_Register($dst$$reg),
11055             as_Register($src1$$reg),
11056             (unsigned long)($src2$$constant));
11057   %}
11058 
11059   ins_pipe(ialu_reg_imm);
11060 %}
11061 
11062 // Or Instructions
11063 
11064 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11065   match(Set dst (OrI src1 src2));
11066 
11067   format %{ "orrw  $dst, $src1, $src2\t# int" %}
11068 
11069   ins_cost(INSN_COST);
11070   ins_encode %{
11071     __ orrw(as_Register($dst$$reg),
11072             as_Register($src1$$reg),
11073             as_Register($src2$$reg));
11074   %}
11075 
11076   ins_pipe(ialu_reg_reg);
11077 %}
11078 
11079 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
11080   match(Set dst (OrI src1 src2));
11081 
11082   format %{ "orrw  $dst, $src1, $src2\t# int" %}
11083 
11084   ins_cost(INSN_COST);
11085   ins_encode %{
11086     __ orrw(as_Register($dst$$reg),
11087             as_Register($src1$$reg),
11088             (unsigned long)($src2$$constant));
11089   %}
11090 
11091   ins_pipe(ialu_reg_imm);
11092 %}
11093 
11094 // Xor Instructions
11095 
11096 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11097   match(Set dst (XorI src1 src2));
11098 
11099   format %{ "eorw  $dst, $src1, $src2\t# int" %}
11100 
11101   ins_cost(INSN_COST);
11102   ins_encode %{
11103     __ eorw(as_Register($dst$$reg),
11104             as_Register($src1$$reg),
11105             as_Register($src2$$reg));
11106   %}
11107 
11108   ins_pipe(ialu_reg_reg);
11109 %}
11110 
11111 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
11112   match(Set dst (XorI src1 src2));
11113 
11114   format %{ "eorw  $dst, $src1, $src2\t# int" %}
11115 
11116   ins_cost(INSN_COST);
11117   ins_encode %{
11118     __ eorw(as_Register($dst$$reg),
11119             as_Register($src1$$reg),
11120             (unsigned long)($src2$$constant));
11121   %}
11122 
11123   ins_pipe(ialu_reg_imm);
11124 %}
11125 
11126 // Long Logical Instructions
11127 // TODO
11128 
11129 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
11130   match(Set dst (AndL src1 src2));
11131 
11132   format %{ "and  $dst, $src1, $src2\t# int" %}
11133 
11134   ins_cost(INSN_COST);
11135   ins_encode %{
11136     __ andr(as_Register($dst$$reg),
11137             as_Register($src1$$reg),
11138             as_Register($src2$$reg));
11139   %}
11140 
11141   ins_pipe(ialu_reg_reg);
11142 %}
11143 
11144 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
11145   match(Set dst (AndL src1 src2));
11146 
11147   format %{ "and  $dst, $src1, $src2\t# int" %}
11148 
11149   ins_cost(INSN_COST);
11150   ins_encode %{
11151     __ andr(as_Register($dst$$reg),
11152             as_Register($src1$$reg),
11153             (unsigned long)($src2$$constant));
11154   %}
11155 
11156   ins_pipe(ialu_reg_imm);
11157 %}
11158 
11159 // Or Instructions
11160 
11161 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11162   match(Set dst (OrL src1 src2));
11163 
11164   format %{ "orr  $dst, $src1, $src2\t# int" %}
11165 
11166   ins_cost(INSN_COST);
11167   ins_encode %{
11168     __ orr(as_Register($dst$$reg),
11169            as_Register($src1$$reg),
11170            as_Register($src2$$reg));
11171   %}
11172 
11173   ins_pipe(ialu_reg_reg);
11174 %}
11175 
11176 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
11177   match(Set dst (OrL src1 src2));
11178 
11179   format %{ "orr  $dst, $src1, $src2\t# int" %}
11180 
11181   ins_cost(INSN_COST);
11182   ins_encode %{
11183     __ orr(as_Register($dst$$reg),
11184            as_Register($src1$$reg),
11185            (unsigned long)($src2$$constant));
11186   %}
11187 
11188   ins_pipe(ialu_reg_imm);
11189 %}
11190 
11191 // Xor Instructions
11192 
11193 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11194   match(Set dst (XorL src1 src2));
11195 
11196   format %{ "eor  $dst, $src1, $src2\t# int" %}
11197 
11198   ins_cost(INSN_COST);
11199   ins_encode %{
11200     __ eor(as_Register($dst$$reg),
11201            as_Register($src1$$reg),
11202            as_Register($src2$$reg));
11203   %}
11204 
11205   ins_pipe(ialu_reg_reg);
11206 %}
11207 
11208 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
11209   match(Set dst (XorL src1 src2));
11210 
11211   ins_cost(INSN_COST);
11212   format %{ "eor  $dst, $src1, $src2\t# int" %}
11213 
11214   ins_encode %{
11215     __ eor(as_Register($dst$$reg),
11216            as_Register($src1$$reg),
11217            (unsigned long)($src2$$constant));
11218   %}
11219 
11220   ins_pipe(ialu_reg_imm);
11221 %}
11222 
11223 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
11224 %{
11225   match(Set dst (ConvI2L src));
11226 
11227   ins_cost(INSN_COST);
11228   format %{ "sxtw  $dst, $src\t# i2l" %}
11229   ins_encode %{
11230     __ sbfm($dst$$Register, $src$$Register, 0, 31);
11231   %}
11232   ins_pipe(ialu_reg_shift);
11233 %}
11234 
11235 // this pattern occurs in bigmath arithmetic
11236 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
11237 %{
11238   match(Set dst (AndL (ConvI2L src) mask));
11239 
11240   ins_cost(INSN_COST);
11241   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
11242   ins_encode %{
11243     __ ubfm($dst$$Register, $src$$Register, 0, 31);
11244   %}
11245 
11246   ins_pipe(ialu_reg_shift);
11247 %}
11248 
11249 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
11250   match(Set dst (ConvL2I src));
11251 
11252   ins_cost(INSN_COST);
11253   format %{ "movw  $dst, $src \t// l2i" %}
11254 
11255   ins_encode %{
11256     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
11257   %}
11258 
11259   ins_pipe(ialu_reg);
11260 %}
11261 
11262 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
11263 %{
11264   match(Set dst (Conv2B src));
11265   effect(KILL cr);
11266 
11267   format %{
11268     "cmpw $src, zr\n\t"
11269     "cset $dst, ne"
11270   %}
11271 
11272   ins_encode %{
11273     __ cmpw(as_Register($src$$reg), zr);
11274     __ cset(as_Register($dst$$reg), Assembler::NE);
11275   %}
11276 
11277   ins_pipe(ialu_reg);
11278 %}
11279 
11280 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
11281 %{
11282   match(Set dst (Conv2B src));
11283   effect(KILL cr);
11284 
11285   format %{
11286     "cmp  $src, zr\n\t"
11287     "cset $dst, ne"
11288   %}
11289 
11290   ins_encode %{
11291     __ cmp(as_Register($src$$reg), zr);
11292     __ cset(as_Register($dst$$reg), Assembler::NE);
11293   %}
11294 
11295   ins_pipe(ialu_reg);
11296 %}
11297 
11298 instruct convD2F_reg(vRegF dst, vRegD src) %{
11299   match(Set dst (ConvD2F src));
11300 
11301   ins_cost(INSN_COST * 5);
11302   format %{ "fcvtd  $dst, $src \t// d2f" %}
11303 
11304   ins_encode %{
11305     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
11306   %}
11307 
11308   ins_pipe(pipe_class_default);
11309 %}
11310 
11311 instruct convF2D_reg(vRegD dst, vRegF src) %{
11312   match(Set dst (ConvF2D src));
11313 
11314   ins_cost(INSN_COST * 5);
11315   format %{ "fcvts  $dst, $src \t// f2d" %}
11316 
11317   ins_encode %{
11318     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
11319   %}
11320 
11321   ins_pipe(pipe_class_default);
11322 %}
11323 
11324 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
11325   match(Set dst (ConvF2I src));
11326 
11327   ins_cost(INSN_COST * 5);
11328   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
11329 
11330   ins_encode %{
11331     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
11332   %}
11333 
11334   ins_pipe(pipe_class_default);
11335 %}
11336 
11337 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
11338   match(Set dst (ConvF2L src));
11339 
11340   ins_cost(INSN_COST * 5);
11341   format %{ "fcvtzs  $dst, $src \t// f2l" %}
11342 
11343   ins_encode %{
11344     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
11345   %}
11346 
11347   ins_pipe(pipe_class_default);
11348 %}
11349 
11350 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
11351   match(Set dst (ConvI2F src));
11352 
11353   ins_cost(INSN_COST * 5);
11354   format %{ "scvtfws  $dst, $src \t// i2f" %}
11355 
11356   ins_encode %{
11357     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
11358   %}
11359 
11360   ins_pipe(pipe_class_default);
11361 %}
11362 
11363 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
11364   match(Set dst (ConvL2F src));
11365 
11366   ins_cost(INSN_COST * 5);
11367   format %{ "scvtfs  $dst, $src \t// l2f" %}
11368 
11369   ins_encode %{
11370     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
11371   %}
11372 
11373   ins_pipe(pipe_class_default);
11374 %}
11375 
11376 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
11377   match(Set dst (ConvD2I src));
11378 
11379   ins_cost(INSN_COST * 5);
11380   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
11381 
11382   ins_encode %{
11383     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
11384   %}
11385 
11386   ins_pipe(pipe_class_default);
11387 %}
11388 
11389 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
11390   match(Set dst (ConvD2L src));
11391 
11392   ins_cost(INSN_COST * 5);
11393   format %{ "fcvtzd  $dst, $src \t// d2l" %}
11394 
11395   ins_encode %{
11396     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
11397   %}
11398 
11399   ins_pipe(pipe_class_default);
11400 %}
11401 
11402 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
11403   match(Set dst (ConvI2D src));
11404 
11405   ins_cost(INSN_COST * 5);
11406   format %{ "scvtfwd  $dst, $src \t// i2d" %}
11407 
11408   ins_encode %{
11409     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
11410   %}
11411 
11412   ins_pipe(pipe_class_default);
11413 %}
11414 
11415 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
11416   match(Set dst (ConvL2D src));
11417 
11418   ins_cost(INSN_COST * 5);
11419   format %{ "scvtfd  $dst, $src \t// l2d" %}
11420 
11421   ins_encode %{
11422     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
11423   %}
11424 
11425   ins_pipe(pipe_class_default);
11426 %}
11427 
11428 // stack <-> reg and reg <-> reg shuffles with no conversion
11429 
11430 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
11431 
11432   match(Set dst (MoveF2I src));
11433 
11434   effect(DEF dst, USE src);
11435 
11436   ins_cost(4 * INSN_COST);
11437 
11438   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
11439 
11440   ins_encode %{
11441     __ ldrw($dst$$Register, Address(sp, $src$$disp));
11442   %}
11443 
11444   ins_pipe(iload_reg_reg);
11445 
11446 %}
11447 
11448 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
11449 
11450   match(Set dst (MoveI2F src));
11451 
11452   effect(DEF dst, USE src);
11453 
11454   ins_cost(4 * INSN_COST);
11455 
11456   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
11457 
11458   ins_encode %{
11459     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
11460   %}
11461 
11462   ins_pipe(pipe_class_memory);
11463 
11464 %}
11465 
11466 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
11467 
11468   match(Set dst (MoveD2L src));
11469 
11470   effect(DEF dst, USE src);
11471 
11472   ins_cost(4 * INSN_COST);
11473 
11474   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
11475 
11476   ins_encode %{
11477     __ ldr($dst$$Register, Address(sp, $src$$disp));
11478   %}
11479 
11480   ins_pipe(iload_reg_reg);
11481 
11482 %}
11483 
11484 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
11485 
11486   match(Set dst (MoveL2D src));
11487 
11488   effect(DEF dst, USE src);
11489 
11490   ins_cost(4 * INSN_COST);
11491 
11492   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
11493 
11494   ins_encode %{
11495     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
11496   %}
11497 
11498   ins_pipe(pipe_class_memory);
11499 
11500 %}
11501 
11502 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
11503 
11504   match(Set dst (MoveF2I src));
11505 
11506   effect(DEF dst, USE src);
11507 
11508   ins_cost(INSN_COST);
11509 
11510   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
11511 
11512   ins_encode %{
11513     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
11514   %}
11515 
11516   ins_pipe(pipe_class_memory);
11517 
11518 %}
11519 
11520 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
11521 
11522   match(Set dst (MoveI2F src));
11523 
11524   effect(DEF dst, USE src);
11525 
11526   ins_cost(INSN_COST);
11527 
11528   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
11529 
11530   ins_encode %{
11531     __ strw($src$$Register, Address(sp, $dst$$disp));
11532   %}
11533 
11534   ins_pipe(istore_reg_reg);
11535 
11536 %}
11537 
11538 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
11539 
11540   match(Set dst (MoveD2L src));
11541 
11542   effect(DEF dst, USE src);
11543 
11544   ins_cost(INSN_COST);
11545 
11546   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
11547 
11548   ins_encode %{
11549     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
11550   %}
11551 
11552   ins_pipe(pipe_class_memory);
11553 
11554 %}
11555 
11556 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
11557 
11558   match(Set dst (MoveL2D src));
11559 
11560   effect(DEF dst, USE src);
11561 
11562   ins_cost(INSN_COST);
11563 
11564   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
11565 
11566   ins_encode %{
11567     __ str($src$$Register, Address(sp, $dst$$disp));
11568   %}
11569 
11570   ins_pipe(istore_reg_reg);
11571 
11572 %}
11573 
11574 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
11575 
11576   match(Set dst (MoveF2I src));
11577 
11578   effect(DEF dst, USE src);
11579 
11580   ins_cost(INSN_COST);
11581 
11582   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
11583 
11584   ins_encode %{
11585     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
11586   %}
11587 
11588   ins_pipe(pipe_class_memory);
11589 
11590 %}
11591 
11592 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
11593 
11594   match(Set dst (MoveI2F src));
11595 
11596   effect(DEF dst, USE src);
11597 
11598   ins_cost(INSN_COST);
11599 
11600   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
11601 
11602   ins_encode %{
11603     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
11604   %}
11605 
11606   ins_pipe(pipe_class_memory);
11607 
11608 %}
11609 
11610 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
11611 
11612   match(Set dst (MoveD2L src));
11613 
11614   effect(DEF dst, USE src);
11615 
11616   ins_cost(INSN_COST);
11617 
11618   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
11619 
11620   ins_encode %{
11621     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
11622   %}
11623 
11624   ins_pipe(pipe_class_memory);
11625 
11626 %}
11627 
11628 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
11629 
11630   match(Set dst (MoveL2D src));
11631 
11632   effect(DEF dst, USE src);
11633 
11634   ins_cost(INSN_COST);
11635 
11636   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
11637 
11638   ins_encode %{
11639     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
11640   %}
11641 
11642   ins_pipe(pipe_class_memory);
11643 
11644 %}
11645 
11646 // ============================================================================
11647 // clearing of an array
11648 
11649 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
11650 %{
11651   match(Set dummy (ClearArray cnt base));
11652   effect(USE_KILL cnt, USE_KILL base);
11653 
11654   ins_cost(4 * INSN_COST);
11655   format %{ "ClearArray $cnt, $base" %}
11656 
11657   ins_encode(aarch64_enc_clear_array_reg_reg(cnt, base));
11658 
11659   ins_pipe(pipe_class_memory);
11660 %}
11661 
11662 // ============================================================================
11663 // Overflow Math Instructions
11664 
11665 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
11666 %{
11667   match(Set cr (OverflowAddI op1 op2));
11668 
11669   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
11670   ins_cost(INSN_COST);
11671   ins_encode %{
11672     __ cmnw($op1$$Register, $op2$$Register);
11673   %}
11674 
11675   ins_pipe(icmp_reg_reg);
11676 %}
11677 
11678 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
11679 %{
11680   match(Set cr (OverflowAddI op1 op2));
11681 
11682   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
11683   ins_cost(INSN_COST);
11684   ins_encode %{
11685     __ cmnw($op1$$Register, $op2$$constant);
11686   %}
11687 
11688   ins_pipe(icmp_reg_imm);
11689 %}
11690 
11691 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
11692 %{
11693   match(Set cr (OverflowAddL op1 op2));
11694 
11695   format %{ "cmn   $op1, $op2\t# overflow check long" %}
11696   ins_cost(INSN_COST);
11697   ins_encode %{
11698     __ cmn($op1$$Register, $op2$$Register);
11699   %}
11700 
11701   ins_pipe(icmp_reg_reg);
11702 %}
11703 
11704 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
11705 %{
11706   match(Set cr (OverflowAddL op1 op2));
11707 
11708   format %{ "cmn   $op1, $op2\t# overflow check long" %}
11709   ins_cost(INSN_COST);
11710   ins_encode %{
11711     __ cmn($op1$$Register, $op2$$constant);
11712   %}
11713 
11714   ins_pipe(icmp_reg_imm);
11715 %}
11716 
11717 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
11718 %{
11719   match(Set cr (OverflowSubI op1 op2));
11720 
11721   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
11722   ins_cost(INSN_COST);
11723   ins_encode %{
11724     __ cmpw($op1$$Register, $op2$$Register);
11725   %}
11726 
11727   ins_pipe(icmp_reg_reg);
11728 %}
11729 
11730 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
11731 %{
11732   match(Set cr (OverflowSubI op1 op2));
11733 
11734   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
11735   ins_cost(INSN_COST);
11736   ins_encode %{
11737     __ cmpw($op1$$Register, $op2$$constant);
11738   %}
11739 
11740   ins_pipe(icmp_reg_imm);
11741 %}
11742 
11743 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
11744 %{
11745   match(Set cr (OverflowSubL op1 op2));
11746 
11747   format %{ "cmp   $op1, $op2\t# overflow check long" %}
11748   ins_cost(INSN_COST);
11749   ins_encode %{
11750     __ cmp($op1$$Register, $op2$$Register);
11751   %}
11752 
11753   ins_pipe(icmp_reg_reg);
11754 %}
11755 
11756 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
11757 %{
11758   match(Set cr (OverflowSubL op1 op2));
11759 
11760   format %{ "cmp   $op1, $op2\t# overflow check long" %}
11761   ins_cost(INSN_COST);
11762   ins_encode %{
11763     __ cmp($op1$$Register, $op2$$constant);
11764   %}
11765 
11766   ins_pipe(icmp_reg_imm);
11767 %}
11768 
11769 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
11770 %{
11771   match(Set cr (OverflowSubI zero op1));
11772 
11773   format %{ "cmpw  zr, $op1\t# overflow check int" %}
11774   ins_cost(INSN_COST);
11775   ins_encode %{
11776     __ cmpw(zr, $op1$$Register);
11777   %}
11778 
11779   ins_pipe(icmp_reg_imm);
11780 %}
11781 
11782 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
11783 %{
11784   match(Set cr (OverflowSubL zero op1));
11785 
11786   format %{ "cmp   zr, $op1\t# overflow check long" %}
11787   ins_cost(INSN_COST);
11788   ins_encode %{
11789     __ cmp(zr, $op1$$Register);
11790   %}
11791 
11792   ins_pipe(icmp_reg_imm);
11793 %}
11794 
11795 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
11796 %{
11797   match(Set cr (OverflowMulI op1 op2));
11798 
11799   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
11800             "cmp   rscratch1, rscratch1, sxtw\n\t"
11801             "movw  rscratch1, #0x80000000\n\t"
11802             "cselw rscratch1, rscratch1, zr, NE\n\t"
11803             "cmpw  rscratch1, #1" %}
11804   ins_cost(5 * INSN_COST);
11805   ins_encode %{
11806     __ smull(rscratch1, $op1$$Register, $op2$$Register);
11807     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
11808     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
11809     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
11810     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
11811   %}
11812 
11813   ins_pipe(pipe_slow);
11814 %}
11815 
11816 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
11817 %{
11818   match(If cmp (OverflowMulI op1 op2));
11819   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
11820             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
11821   effect(USE labl, KILL cr);
11822 
11823   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
11824             "cmp   rscratch1, rscratch1, sxtw\n\t"
11825             "b$cmp   $labl" %}
11826   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
11827   ins_encode %{
11828     Label* L = $labl$$label;
11829     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
11830     __ smull(rscratch1, $op1$$Register, $op2$$Register);
11831     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
11832     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
11833   %}
11834 
11835   ins_pipe(pipe_serial);
11836 %}
11837 
11838 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
11839 %{
11840   match(Set cr (OverflowMulL op1 op2));
11841 
11842   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
11843             "smulh rscratch2, $op1, $op2\n\t"
11844             "cmp   rscratch2, rscratch1, ASR #31\n\t"
11845             "movw  rscratch1, #0x80000000\n\t"
11846             "cselw rscratch1, rscratch1, zr, NE\n\t"
11847             "cmpw  rscratch1, #1" %}
11848   ins_cost(6 * INSN_COST);
11849   ins_encode %{
11850     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
11851     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
11852     __ cmp(rscratch2, rscratch1, Assembler::ASR, 31);    // Top is pure sign ext
11853     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
11854     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
11855     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
11856   %}
11857 
11858   ins_pipe(pipe_slow);
11859 %}
11860 
11861 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
11862 %{
11863   match(If cmp (OverflowMulL op1 op2));
11864   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
11865             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
11866   effect(USE labl, KILL cr);
11867 
11868   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
11869             "smulh rscratch2, $op1, $op2\n\t"
11870             "cmp   rscratch2, rscratch1, ASR #31\n\t"
11871             "b$cmp $labl" %}
11872   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
11873   ins_encode %{
11874     Label* L = $labl$$label;
11875     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
11876     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
11877     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
11878     __ cmp(rscratch2, rscratch1, Assembler::ASR, 31);    // Top is pure sign ext
11879     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
11880   %}
11881 
11882   ins_pipe(pipe_serial);
11883 %}
11884 
11885 // ============================================================================
11886 // Compare Instructions
11887 
11888 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
11889 %{
11890   match(Set cr (CmpI op1 op2));
11891 
11892   effect(DEF cr, USE op1, USE op2);
11893 
11894   ins_cost(INSN_COST);
11895   format %{ "cmpw  $op1, $op2" %}
11896 
11897   ins_encode(aarch64_enc_cmpw(op1, op2));
11898 
11899   ins_pipe(icmp_reg_reg);
11900 %}
11901 
11902 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
11903 %{
11904   match(Set cr (CmpI op1 zero));
11905 
11906   effect(DEF cr, USE op1);
11907 
11908   ins_cost(INSN_COST);
11909   format %{ "cmpw $op1, 0" %}
11910 
11911   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
11912 
11913   ins_pipe(icmp_reg_imm);
11914 %}
11915 
11916 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
11917 %{
11918   match(Set cr (CmpI op1 op2));
11919 
11920   effect(DEF cr, USE op1);
11921 
11922   ins_cost(INSN_COST);
11923   format %{ "cmpw  $op1, $op2" %}
11924 
11925   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
11926 
11927   ins_pipe(icmp_reg_imm);
11928 %}
11929 
11930 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
11931 %{
11932   match(Set cr (CmpI op1 op2));
11933 
11934   effect(DEF cr, USE op1);
11935 
11936   ins_cost(INSN_COST * 2);
11937   format %{ "cmpw  $op1, $op2" %}
11938 
11939   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
11940 
11941   ins_pipe(icmp_reg_imm);
11942 %}
11943 
11944 // Unsigned compare Instructions; really, same as signed compare
11945 // except it should only be used to feed an If or a CMovI which takes a
11946 // cmpOpU.
11947 
11948 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
11949 %{
11950   match(Set cr (CmpU op1 op2));
11951 
11952   effect(DEF cr, USE op1, USE op2);
11953 
11954   ins_cost(INSN_COST);
11955   format %{ "cmpw  $op1, $op2\t# unsigned" %}
11956 
11957   ins_encode(aarch64_enc_cmpw(op1, op2));
11958 
11959   ins_pipe(icmp_reg_reg);
11960 %}
11961 
11962 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
11963 %{
11964   match(Set cr (CmpU op1 zero));
11965 
11966   effect(DEF cr, USE op1);
11967 
11968   ins_cost(INSN_COST);
11969   format %{ "cmpw $op1, #0\t# unsigned" %}
11970 
11971   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
11972 
11973   ins_pipe(icmp_reg_imm);
11974 %}
11975 
11976 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
11977 %{
11978   match(Set cr (CmpU op1 op2));
11979 
11980   effect(DEF cr, USE op1);
11981 
11982   ins_cost(INSN_COST);
11983   format %{ "cmpw  $op1, $op2\t# unsigned" %}
11984 
11985   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
11986 
11987   ins_pipe(icmp_reg_imm);
11988 %}
11989 
11990 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
11991 %{
11992   match(Set cr (CmpU op1 op2));
11993 
11994   effect(DEF cr, USE op1);
11995 
11996   ins_cost(INSN_COST * 2);
11997   format %{ "cmpw  $op1, $op2\t# unsigned" %}
11998 
11999   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
12000 
12001   ins_pipe(icmp_reg_imm);
12002 %}
12003 
12004 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
12005 %{
12006   match(Set cr (CmpL op1 op2));
12007 
12008   effect(DEF cr, USE op1, USE op2);
12009 
12010   ins_cost(INSN_COST);
12011   format %{ "cmp  $op1, $op2" %}
12012 
12013   ins_encode(aarch64_enc_cmp(op1, op2));
12014 
12015   ins_pipe(icmp_reg_reg);
12016 %}
12017 
12018 instruct compL_reg_immI0(rFlagsReg cr, iRegL op1, immI0 zero)
12019 %{
12020   match(Set cr (CmpL op1 zero));
12021 
12022   effect(DEF cr, USE op1);
12023 
12024   ins_cost(INSN_COST);
12025   format %{ "tst  $op1" %}
12026 
12027   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
12028 
12029   ins_pipe(icmp_reg_imm);
12030 %}
12031 
12032 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
12033 %{
12034   match(Set cr (CmpL op1 op2));
12035 
12036   effect(DEF cr, USE op1);
12037 
12038   ins_cost(INSN_COST);
12039   format %{ "cmp  $op1, $op2" %}
12040 
12041   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
12042 
12043   ins_pipe(icmp_reg_imm);
12044 %}
12045 
12046 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
12047 %{
12048   match(Set cr (CmpL op1 op2));
12049 
12050   effect(DEF cr, USE op1);
12051 
12052   ins_cost(INSN_COST * 2);
12053   format %{ "cmp  $op1, $op2" %}
12054 
12055   ins_encode(aarch64_enc_cmp_imm(op1, op2));
12056 
12057   ins_pipe(icmp_reg_imm);
12058 %}
12059 
12060 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
12061 %{
12062   match(Set cr (CmpP op1 op2));
12063 
12064   effect(DEF cr, USE op1, USE op2);
12065 
12066   ins_cost(INSN_COST);
12067   format %{ "cmp  $op1, $op2\t // ptr" %}
12068 
12069   ins_encode(aarch64_enc_cmpp(op1, op2));
12070 
12071   ins_pipe(icmp_reg_reg);
12072 %}
12073 
12074 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
12075 %{
12076   match(Set cr (CmpN op1 op2));
12077 
12078   effect(DEF cr, USE op1, USE op2);
12079 
12080   ins_cost(INSN_COST);
12081   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
12082 
12083   ins_encode(aarch64_enc_cmpn(op1, op2));
12084 
12085   ins_pipe(icmp_reg_reg);
12086 %}
12087 
12088 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
12089 %{
12090   match(Set cr (CmpP op1 zero));
12091 
12092   effect(DEF cr, USE op1, USE zero);
12093 
12094   ins_cost(INSN_COST);
12095   format %{ "cmp  $op1, 0\t // ptr" %}
12096 
12097   ins_encode(aarch64_enc_testp(op1));
12098 
12099   ins_pipe(icmp_reg_imm);
12100 %}
12101 
12102 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
12103 %{
12104   match(Set cr (CmpN op1 zero));
12105 
12106   effect(DEF cr, USE op1, USE zero);
12107 
12108   ins_cost(INSN_COST);
12109   format %{ "cmp  $op1, 0\t // compressed ptr" %}
12110 
12111   ins_encode(aarch64_enc_testn(op1));
12112 
12113   ins_pipe(icmp_reg_imm);
12114 %}
12115 
12116 // FP comparisons
12117 //
12118 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
12119 // using normal cmpOp. See declaration of rFlagsReg for details.
12120 
12121 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
12122 %{
12123   match(Set cr (CmpF src1 src2));
12124 
12125   ins_cost(3 * INSN_COST);
12126   format %{ "fcmps $src1, $src2" %}
12127 
12128   ins_encode %{
12129     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
12130   %}
12131 
12132   ins_pipe(pipe_class_compare);
12133 %}
12134 
12135 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
12136 %{
12137   match(Set cr (CmpF src1 src2));
12138 
12139   ins_cost(3 * INSN_COST);
12140   format %{ "fcmps $src1, 0.0" %}
12141 
12142   ins_encode %{
12143     __ fcmps(as_FloatRegister($src1$$reg), 0.0D);
12144   %}
12145 
12146   ins_pipe(pipe_class_compare);
12147 %}
12148 // FROM HERE
12149 
12150 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
12151 %{
12152   match(Set cr (CmpD src1 src2));
12153 
12154   ins_cost(3 * INSN_COST);
12155   format %{ "fcmpd $src1, $src2" %}
12156 
12157   ins_encode %{
12158     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
12159   %}
12160 
12161   ins_pipe(pipe_class_compare);
12162 %}
12163 
12164 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
12165 %{
12166   match(Set cr (CmpD src1 src2));
12167 
12168   ins_cost(3 * INSN_COST);
12169   format %{ "fcmpd $src1, 0.0" %}
12170 
12171   ins_encode %{
12172     __ fcmpd(as_FloatRegister($src1$$reg), 0.0D);
12173   %}
12174 
12175   ins_pipe(pipe_class_compare);
12176 %}
12177 
12178 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
12179 %{
12180   match(Set dst (CmpF3 src1 src2));
12181   effect(KILL cr);
12182 
12183   ins_cost(5 * INSN_COST);
12184   format %{ "fcmps $src1, $src2\n\t"
12185             "csinvw($dst, zr, zr, eq\n\t"
12186             "csnegw($dst, $dst, $dst, lt)"
12187   %}
12188 
12189   ins_encode %{
12190     Label done;
12191     FloatRegister s1 = as_FloatRegister($src1$$reg);
12192     FloatRegister s2 = as_FloatRegister($src2$$reg);
12193     Register d = as_Register($dst$$reg);
12194     __ fcmps(s1, s2);
12195     // installs 0 if EQ else -1
12196     __ csinvw(d, zr, zr, Assembler::EQ);
12197     // keeps -1 if less or unordered else installs 1
12198     __ csnegw(d, d, d, Assembler::LT);
12199     __ bind(done);
12200   %}
12201 
12202   ins_pipe(pipe_class_default);
12203 
12204 %}
12205 
12206 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
12207 %{
12208   match(Set dst (CmpD3 src1 src2));
12209   effect(KILL cr);
12210 
12211   ins_cost(5 * INSN_COST);
12212   format %{ "fcmpd $src1, $src2\n\t"
12213             "csinvw($dst, zr, zr, eq\n\t"
12214             "csnegw($dst, $dst, $dst, lt)"
12215   %}
12216 
12217   ins_encode %{
12218     Label done;
12219     FloatRegister s1 = as_FloatRegister($src1$$reg);
12220     FloatRegister s2 = as_FloatRegister($src2$$reg);
12221     Register d = as_Register($dst$$reg);
12222     __ fcmpd(s1, s2);
12223     // installs 0 if EQ else -1
12224     __ csinvw(d, zr, zr, Assembler::EQ);
12225     // keeps -1 if less or unordered else installs 1
12226     __ csnegw(d, d, d, Assembler::LT);
12227     __ bind(done);
12228   %}
12229   ins_pipe(pipe_class_default);
12230 
12231 %}
12232 
12233 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
12234 %{
12235   match(Set dst (CmpF3 src1 zero));
12236   effect(KILL cr);
12237 
12238   ins_cost(5 * INSN_COST);
12239   format %{ "fcmps $src1, 0.0\n\t"
12240             "csinvw($dst, zr, zr, eq\n\t"
12241             "csnegw($dst, $dst, $dst, lt)"
12242   %}
12243 
12244   ins_encode %{
12245     Label done;
12246     FloatRegister s1 = as_FloatRegister($src1$$reg);
12247     Register d = as_Register($dst$$reg);
12248     __ fcmps(s1, 0.0D);
12249     // installs 0 if EQ else -1
12250     __ csinvw(d, zr, zr, Assembler::EQ);
12251     // keeps -1 if less or unordered else installs 1
12252     __ csnegw(d, d, d, Assembler::LT);
12253     __ bind(done);
12254   %}
12255 
12256   ins_pipe(pipe_class_default);
12257 
12258 %}
12259 
12260 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
12261 %{
12262   match(Set dst (CmpD3 src1 zero));
12263   effect(KILL cr);
12264 
12265   ins_cost(5 * INSN_COST);
12266   format %{ "fcmpd $src1, 0.0\n\t"
12267             "csinvw($dst, zr, zr, eq\n\t"
12268             "csnegw($dst, $dst, $dst, lt)"
12269   %}
12270 
12271   ins_encode %{
12272     Label done;
12273     FloatRegister s1 = as_FloatRegister($src1$$reg);
12274     Register d = as_Register($dst$$reg);
12275     __ fcmpd(s1, 0.0D);
12276     // installs 0 if EQ else -1
12277     __ csinvw(d, zr, zr, Assembler::EQ);
12278     // keeps -1 if less or unordered else installs 1
12279     __ csnegw(d, d, d, Assembler::LT);
12280     __ bind(done);
12281   %}
12282   ins_pipe(pipe_class_default);
12283 
12284 %}
12285 
12286 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
12287 %{
12288   match(Set dst (CmpLTMask p q));
12289   effect(KILL cr);
12290 
12291   ins_cost(3 * INSN_COST);
12292 
12293   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
12294             "csetw $dst, lt\n\t"
12295             "subw $dst, zr, $dst"
12296   %}
12297 
12298   ins_encode %{
12299     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
12300     __ csetw(as_Register($dst$$reg), Assembler::LT);
12301     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
12302   %}
12303 
12304   ins_pipe(ialu_reg_reg);
12305 %}
12306 
12307 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
12308 %{
12309   match(Set dst (CmpLTMask src zero));
12310   effect(KILL cr);
12311 
12312   ins_cost(INSN_COST);
12313 
12314   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
12315 
12316   ins_encode %{
12317     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
12318   %}
12319 
12320   ins_pipe(ialu_reg_shift);
12321 %}
12322 
12323 // ============================================================================
12324 // Max and Min
12325 
12326 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12327 %{
12328   match(Set dst (MinI src1 src2));
12329 
12330   effect(DEF dst, USE src1, USE src2, KILL cr);
12331   size(8);
12332 
12333   ins_cost(INSN_COST * 3);
12334   format %{
12335     "cmpw $src1 $src2\t signed int\n\t"
12336     "cselw $dst, $src1, $src2 lt\t"
12337   %}
12338 
12339   ins_encode %{
12340     __ cmpw(as_Register($src1$$reg),
12341             as_Register($src2$$reg));
12342     __ cselw(as_Register($dst$$reg),
12343              as_Register($src1$$reg),
12344              as_Register($src2$$reg),
12345              Assembler::LT);
12346   %}
12347 
12348   ins_pipe(ialu_reg_reg);
12349 %}
12350 // FROM HERE
12351 
12352 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12353 %{
12354   match(Set dst (MaxI src1 src2));
12355 
12356   effect(DEF dst, USE src1, USE src2, KILL cr);
12357   size(8);
12358 
12359   ins_cost(INSN_COST * 3);
12360   format %{
12361     "cmpw $src1 $src2\t signed int\n\t"
12362     "cselw $dst, $src1, $src2 gt\t"
12363   %}
12364 
12365   ins_encode %{
12366     __ cmpw(as_Register($src1$$reg),
12367             as_Register($src2$$reg));
12368     __ cselw(as_Register($dst$$reg),
12369              as_Register($src1$$reg),
12370              as_Register($src2$$reg),
12371              Assembler::GT);
12372   %}
12373 
12374   ins_pipe(ialu_reg_reg);
12375 %}
12376 
12377 // ============================================================================
12378 // Branch Instructions
12379 
12380 // Direct Branch.
12381 instruct branch(label lbl)
12382 %{
12383   match(Goto);
12384 
12385   effect(USE lbl);
12386 
12387   ins_cost(BRANCH_COST);
12388   format %{ "b  $lbl" %}
12389 
12390   ins_encode(aarch64_enc_b(lbl));
12391 
12392   ins_pipe(pipe_branch);
12393 %}
12394 
12395 // Conditional Near Branch
12396 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
12397 %{
12398   // Same match rule as `branchConFar'.
12399   match(If cmp cr);
12400 
12401   effect(USE lbl);
12402 
12403   ins_cost(BRANCH_COST);
12404   // If set to 1 this indicates that the current instruction is a
12405   // short variant of a long branch. This avoids using this
12406   // instruction in first-pass matching. It will then only be used in
12407   // the `Shorten_branches' pass.
12408   // ins_short_branch(1);
12409   format %{ "b$cmp  $lbl" %}
12410 
12411   ins_encode(aarch64_enc_br_con(cmp, lbl));
12412 
12413   ins_pipe(pipe_branch_cond);
12414 %}
12415 
12416 // Conditional Near Branch Unsigned
12417 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
12418 %{
12419   // Same match rule as `branchConFar'.
12420   match(If cmp cr);
12421 
12422   effect(USE lbl);
12423 
12424   ins_cost(BRANCH_COST);
12425   // If set to 1 this indicates that the current instruction is a
12426   // short variant of a long branch. This avoids using this
12427   // instruction in first-pass matching. It will then only be used in
12428   // the `Shorten_branches' pass.
12429   // ins_short_branch(1);
12430   format %{ "b$cmp  $lbl\t# unsigned" %}
12431 
12432   ins_encode(aarch64_enc_br_conU(cmp, lbl));
12433 
12434   ins_pipe(pipe_branch_cond);
12435 %}
12436 
12437 // Make use of CBZ and CBNZ.  These instructions, as well as being
12438 // shorter than (cmp; branch), have the additional benefit of not
12439 // killing the flags.
12440 
12441 instruct cmpI_imm0_branch(cmpOp cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
12442   match(If cmp (CmpI op1 op2));
12443   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
12444             || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
12445   effect(USE labl);
12446 
12447   ins_cost(BRANCH_COST);
12448   format %{ "cbw$cmp   $op1, $labl" %}
12449   ins_encode %{
12450     Label* L = $labl$$label;
12451     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
12452     if (cond == Assembler::EQ)
12453       __ cbzw($op1$$Register, *L);
12454     else
12455       __ cbnzw($op1$$Register, *L);
12456   %}
12457   ins_pipe(pipe_cmp_branch);
12458 %}
12459 
12460 instruct cmpL_imm0_branch(cmpOp cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
12461   match(If cmp (CmpL op1 op2));
12462   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
12463             || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
12464   effect(USE labl);
12465 
12466   ins_cost(BRANCH_COST);
12467   format %{ "cb$cmp   $op1, $labl" %}
12468   ins_encode %{
12469     Label* L = $labl$$label;
12470     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
12471     if (cond == Assembler::EQ)
12472       __ cbz($op1$$Register, *L);
12473     else
12474       __ cbnz($op1$$Register, *L);
12475   %}
12476   ins_pipe(pipe_cmp_branch);
12477 %}
12478 
12479 instruct cmpP_imm0_branch(cmpOp cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
12480   match(If cmp (CmpP op1 op2));
12481   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
12482             || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
12483   effect(USE labl);
12484 
12485   ins_cost(BRANCH_COST);
12486   format %{ "cb$cmp   $op1, $labl" %}
12487   ins_encode %{
12488     Label* L = $labl$$label;
12489     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
12490     if (cond == Assembler::EQ)
12491       __ cbz($op1$$Register, *L);
12492     else
12493       __ cbnz($op1$$Register, *L);
12494   %}
12495   ins_pipe(pipe_cmp_branch);
12496 %}
12497 
12498 // Conditional Far Branch
12499 // Conditional Far Branch Unsigned
12500 // TODO: fixme
12501 
12502 // counted loop end branch near
12503 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
12504 %{
12505   match(CountedLoopEnd cmp cr);
12506 
12507   effect(USE lbl);
12508 
12509   ins_cost(BRANCH_COST);
12510   // short variant.
12511   // ins_short_branch(1);
12512   format %{ "b$cmp $lbl \t// counted loop end" %}
12513 
12514   ins_encode(aarch64_enc_br_con(cmp, lbl));
12515 
12516   ins_pipe(pipe_branch);
12517 %}
12518 
12519 // counted loop end branch near Unsigned
12520 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
12521 %{
12522   match(CountedLoopEnd cmp cr);
12523 
12524   effect(USE lbl);
12525 
12526   ins_cost(BRANCH_COST);
12527   // short variant.
12528   // ins_short_branch(1);
12529   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
12530 
12531   ins_encode(aarch64_enc_br_conU(cmp, lbl));
12532 
12533   ins_pipe(pipe_branch);
12534 %}
12535 
12536 // counted loop end branch far
12537 // counted loop end branch far unsigned
12538 // TODO: fixme
12539 
12540 // ============================================================================
12541 // inlined locking and unlocking
12542 
12543 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
12544 %{
12545   match(Set cr (FastLock object box));
12546   effect(TEMP tmp, TEMP tmp2);
12547 
12548   // TODO
12549   // identify correct cost
12550   ins_cost(5 * INSN_COST);
12551   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
12552 
12553   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
12554 
12555   ins_pipe(pipe_serial);
12556 %}
12557 
12558 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
12559 %{
12560   match(Set cr (FastUnlock object box));
12561   effect(TEMP tmp, TEMP tmp2);
12562 
12563   ins_cost(5 * INSN_COST);
12564   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
12565 
12566   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
12567 
12568   ins_pipe(pipe_serial);
12569 %}
12570 
12571 
12572 // ============================================================================
12573 // Safepoint Instructions
12574 
12575 // TODO
12576 // provide a near and far version of this code
12577 
12578 instruct safePoint(iRegP poll)
12579 %{
12580   match(SafePoint poll);
12581 
12582   format %{
12583     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
12584   %}
12585   ins_encode %{
12586     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
12587   %}
12588   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
12589 %}
12590 
12591 
12592 // ============================================================================
12593 // Procedure Call/Return Instructions
12594 
12595 // Call Java Static Instruction
12596 
12597 instruct CallStaticJavaDirect(method meth)
12598 %{
12599   match(CallStaticJava);
12600 
12601   effect(USE meth);
12602 
12603   ins_cost(CALL_COST);
12604 
12605   format %{ "call,static $meth \t// ==> " %}
12606 
12607   ins_encode( aarch64_enc_java_static_call(meth),
12608               aarch64_enc_call_epilog );
12609 
12610   ins_pipe(pipe_class_call);
12611 %}
12612 
12613 // TO HERE
12614 
12615 // Call Java Dynamic Instruction
12616 instruct CallDynamicJavaDirect(method meth)
12617 %{
12618   match(CallDynamicJava);
12619 
12620   effect(USE meth);
12621 
12622   ins_cost(CALL_COST);
12623 
12624   format %{ "CALL,dynamic $meth \t// ==> " %}
12625 
12626   ins_encode( aarch64_enc_java_dynamic_call(meth),
12627                aarch64_enc_call_epilog );
12628 
12629   ins_pipe(pipe_class_call);
12630 %}
12631 
12632 // Call Runtime Instruction
12633 
12634 instruct CallRuntimeDirect(method meth)
12635 %{
12636   match(CallRuntime);
12637 
12638   effect(USE meth);
12639 
12640   ins_cost(CALL_COST);
12641 
12642   format %{ "CALL, runtime $meth" %}
12643 
12644   ins_encode( aarch64_enc_java_to_runtime(meth) );
12645 
12646   ins_pipe(pipe_class_call);
12647 %}
12648 
12649 // Call Runtime Instruction
12650 
12651 instruct CallLeafDirect(method meth)
12652 %{
12653   match(CallLeaf);
12654 
12655   effect(USE meth);
12656 
12657   ins_cost(CALL_COST);
12658 
12659   format %{ "CALL, runtime leaf $meth" %}
12660 
12661   ins_encode( aarch64_enc_java_to_runtime(meth) );
12662 
12663   ins_pipe(pipe_class_call);
12664 %}
12665 
12666 // Call Runtime Instruction
12667 
12668 instruct CallLeafNoFPDirect(method meth)
12669 %{
12670   match(CallLeafNoFP);
12671 
12672   effect(USE meth);
12673 
12674   ins_cost(CALL_COST);
12675 
12676   format %{ "CALL, runtime leaf nofp $meth" %}
12677 
12678   ins_encode( aarch64_enc_java_to_runtime(meth) );
12679 
12680   ins_pipe(pipe_class_call);
12681 %}
12682 
12683 // Tail Call; Jump from runtime stub to Java code.
12684 // Also known as an 'interprocedural jump'.
12685 // Target of jump will eventually return to caller.
12686 // TailJump below removes the return address.
12687 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop)
12688 %{
12689   match(TailCall jump_target method_oop);
12690 
12691   ins_cost(CALL_COST);
12692 
12693   format %{ "br $jump_target\t# $method_oop holds method oop" %}
12694 
12695   ins_encode(aarch64_enc_tail_call(jump_target));
12696 
12697   ins_pipe(pipe_class_call);
12698 %}
12699 
12700 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
12701 %{
12702   match(TailJump jump_target ex_oop);
12703 
12704   ins_cost(CALL_COST);
12705 
12706   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
12707 
12708   ins_encode(aarch64_enc_tail_jmp(jump_target));
12709 
12710   ins_pipe(pipe_class_call);
12711 %}
12712 
12713 // Create exception oop: created by stack-crawling runtime code.
12714 // Created exception is now available to this handler, and is setup
12715 // just prior to jumping to this handler. No code emitted.
12716 // TODO check
12717 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
12718 instruct CreateException(iRegP_R0 ex_oop)
12719 %{
12720   match(Set ex_oop (CreateEx));
12721 
12722   format %{ " -- \t// exception oop; no code emitted" %}
12723 
12724   size(0);
12725 
12726   ins_encode( /*empty*/ );
12727 
12728   ins_pipe(pipe_class_empty);
12729 %}
12730 
12731 // Rethrow exception: The exception oop will come in the first
12732 // argument position. Then JUMP (not call) to the rethrow stub code.
12733 instruct RethrowException() %{
12734   match(Rethrow);
12735   ins_cost(CALL_COST);
12736 
12737   format %{ "b rethrow_stub" %}
12738 
12739   ins_encode( aarch64_enc_rethrow() );
12740 
12741   ins_pipe(pipe_class_call);
12742 %}
12743 
12744 
12745 // Return Instruction
12746 // epilog node loads ret address into lr as part of frame pop
12747 instruct Ret()
12748 %{
12749   match(Return);
12750 
12751   format %{ "ret\t// return register" %}
12752 
12753   ins_encode( aarch64_enc_ret() );
12754 
12755   ins_pipe(pipe_branch);
12756 %}
12757 
12758 // Die now.
12759 instruct ShouldNotReachHere() %{
12760   match(Halt);
12761 
12762   ins_cost(CALL_COST);
12763   format %{ "ShouldNotReachHere" %}
12764 
12765   ins_encode %{
12766     // TODO
12767     // implement proper trap call here
12768     __ brk(999);
12769   %}
12770 
12771   ins_pipe(pipe_class_default);
12772 %}
12773 
12774 // ============================================================================
12775 // Partial Subtype Check
12776 //
12777 // superklass array for an instance of the superklass.  Set a hidden
12778 // internal cache on a hit (cache is checked with exposed code in
12779 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12780 // encoding ALSO sets flags.
12781 
12782 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
12783 %{
12784   match(Set result (PartialSubtypeCheck sub super));
12785   effect(KILL cr, KILL temp);
12786 
12787   ins_cost(1100);  // slightly larger than the next version
12788   format %{ "partialSubtypeCheck $result, $sub, $super" %}
12789 
12790   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
12791 
12792   opcode(0x1); // Force zero of result reg on hit
12793 
12794   ins_pipe(pipe_class_memory);
12795 %}
12796 
12797 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
12798 %{
12799   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12800   effect(KILL temp, KILL result);
12801 
12802   ins_cost(1100);  // slightly larger than the next version
12803   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
12804 
12805   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
12806 
12807   opcode(0x0); // Don't zero result reg on hit
12808 
12809   ins_pipe(pipe_class_memory);
12810 %}
12811 
12812 instruct string_compare(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
12813                         iRegI_R0 result, iRegP_R10 tmp1, rFlagsReg cr)
12814 %{
12815   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12816   effect(KILL tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12817 
12818   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
12819   ins_encode %{
12820     __ string_compare($str1$$Register, $str2$$Register,
12821                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12822                       $tmp1$$Register);
12823   %}
12824   ins_pipe(pipe_class_memory);
12825 %}
12826 
12827 instruct string_indexof(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
12828        iRegI_R0 result, iRegI tmp1, iRegI tmp2, iRegI tmp3, iRegI tmp4, rFlagsReg cr)
12829 %{
12830   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12831   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
12832          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
12833   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result" %}
12834 
12835   ins_encode %{
12836     __ string_indexof($str1$$Register, $str2$$Register,
12837                       $cnt1$$Register, $cnt2$$Register,
12838                       $tmp1$$Register, $tmp2$$Register,
12839                       $tmp3$$Register, $tmp4$$Register,
12840                       -1, $result$$Register);
12841   %}
12842   ins_pipe(pipe_class_memory);
12843 %}
12844 
12845 instruct string_indexof_con(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
12846                  immI_le_4 int_cnt2, iRegI_R0 result, iRegI tmp1, iRegI tmp2,
12847                  iRegI tmp3, iRegI tmp4, rFlagsReg cr)
12848 %{
12849   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12850   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
12851          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
12852   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result" %}
12853 
12854   ins_encode %{
12855     int icnt2 = (int)$int_cnt2$$constant;
12856     __ string_indexof($str1$$Register, $str2$$Register,
12857                       $cnt1$$Register, zr,
12858                       $tmp1$$Register, $tmp2$$Register,
12859                       $tmp3$$Register, $tmp4$$Register,
12860                       icnt2, $result$$Register);
12861   %}
12862   ins_pipe(pipe_class_memory);
12863 %}
12864 
12865 instruct string_equals(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
12866                         iRegI_R0 result, iRegP_R10 tmp, rFlagsReg cr)
12867 %{
12868   match(Set result (StrEquals (Binary str1 str2) cnt));
12869   effect(KILL tmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
12870 
12871   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp" %}
12872   ins_encode %{
12873     __ string_equals($str1$$Register, $str2$$Register,
12874                       $cnt$$Register, $result$$Register,
12875                       $tmp$$Register);
12876   %}
12877   ins_pipe(pipe_class_memory);
12878 %}
12879 
12880 instruct array_equals(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
12881                       iRegP_R10 tmp, rFlagsReg cr)
12882 %{
12883   match(Set result (AryEq ary1 ary2));
12884   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, KILL cr);
12885 
12886   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
12887   ins_encode %{
12888     __ char_arrays_equals($ary1$$Register, $ary2$$Register,
12889                           $result$$Register, $tmp$$Register);
12890   %}
12891   ins_pipe(pipe_class_memory);
12892 %}
12893 
12894 // encode char[] to byte[] in ISO_8859_1
12895 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
12896                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
12897                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
12898                           iRegI_R0 result, rFlagsReg cr)
12899 %{
12900   match(Set result (EncodeISOArray src (Binary dst len)));
12901   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
12902          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
12903 
12904   format %{ "Encode array $src,$dst,$len -> $result" %}
12905   ins_encode %{
12906     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12907          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
12908          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
12909   %}
12910   ins_pipe( pipe_class_memory );
12911 %}
12912 
12913 // ============================================================================
12914 // This name is KNOWN by the ADLC and cannot be changed.
12915 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12916 // for this guy.
12917 instruct tlsLoadP(thread_RegP dst)
12918 %{
12919   match(Set dst (ThreadLocal));
12920 
12921   ins_cost(0);
12922 
12923   format %{ " -- \t// $dst=Thread::current(), empty" %}
12924 
12925   size(0);
12926 
12927   ins_encode( /*empty*/ );
12928 
12929   ins_pipe(pipe_class_empty);
12930 %}
12931 
12932 
12933 
12934 //----------PEEPHOLE RULES-----------------------------------------------------
12935 // These must follow all instruction definitions as they use the names
12936 // defined in the instructions definitions.
12937 //
12938 // peepmatch ( root_instr_name [preceding_instruction]* );
12939 //
12940 // peepconstraint %{
12941 // (instruction_number.operand_name relational_op instruction_number.operand_name
12942 //  [, ...] );
12943 // // instruction numbers are zero-based using left to right order in peepmatch
12944 //
12945 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12946 // // provide an instruction_number.operand_name for each operand that appears
12947 // // in the replacement instruction's match rule
12948 //
12949 // ---------VM FLAGS---------------------------------------------------------
12950 //
12951 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12952 //
12953 // Each peephole rule is given an identifying number starting with zero and
12954 // increasing by one in the order seen by the parser.  An individual peephole
12955 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12956 // on the command-line.
12957 //
12958 // ---------CURRENT LIMITATIONS----------------------------------------------
12959 //
12960 // Only match adjacent instructions in same basic block
12961 // Only equality constraints
12962 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12963 // Only one replacement instruction
12964 //
12965 // ---------EXAMPLE----------------------------------------------------------
12966 //
12967 // // pertinent parts of existing instructions in architecture description
12968 // instruct movI(iRegINoSp dst, iRegI src)
12969 // %{
12970 //   match(Set dst (CopyI src));
12971 // %}
12972 //
12973 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
12974 // %{
12975 //   match(Set dst (AddI dst src));
12976 //   effect(KILL cr);
12977 // %}
12978 //
12979 // // Change (inc mov) to lea
12980 // peephole %{
12981 //   // increment preceeded by register-register move
12982 //   peepmatch ( incI_iReg movI );
12983 //   // require that the destination register of the increment
12984 //   // match the destination register of the move
12985 //   peepconstraint ( 0.dst == 1.dst );
12986 //   // construct a replacement instruction that sets
12987 //   // the destination to ( move's source register + one )
12988 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
12989 // %}
12990 //
12991 
12992 // Implementation no longer uses movX instructions since
12993 // machine-independent system no longer uses CopyX nodes.
12994 //
12995 // peephole
12996 // %{
12997 //   peepmatch (incI_iReg movI);
12998 //   peepconstraint (0.dst == 1.dst);
12999 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
13000 // %}
13001 
13002 // peephole
13003 // %{
13004 //   peepmatch (decI_iReg movI);
13005 //   peepconstraint (0.dst == 1.dst);
13006 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
13007 // %}
13008 
13009 // peephole
13010 // %{
13011 //   peepmatch (addI_iReg_imm movI);
13012 //   peepconstraint (0.dst == 1.dst);
13013 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
13014 // %}
13015 
13016 // peephole
13017 // %{
13018 //   peepmatch (incL_iReg movL);
13019 //   peepconstraint (0.dst == 1.dst);
13020 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
13021 // %}
13022 
13023 // peephole
13024 // %{
13025 //   peepmatch (decL_iReg movL);
13026 //   peepconstraint (0.dst == 1.dst);
13027 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
13028 // %}
13029 
13030 // peephole
13031 // %{
13032 //   peepmatch (addL_iReg_imm movL);
13033 //   peepconstraint (0.dst == 1.dst);
13034 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
13035 // %}
13036 
13037 // peephole
13038 // %{
13039 //   peepmatch (addP_iReg_imm movP);
13040 //   peepconstraint (0.dst == 1.dst);
13041 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
13042 // %}
13043 
13044 // // Change load of spilled value to only a spill
13045 // instruct storeI(memory mem, iRegI src)
13046 // %{
13047 //   match(Set mem (StoreI mem src));
13048 // %}
13049 //
13050 // instruct loadI(iRegINoSp dst, memory mem)
13051 // %{
13052 //   match(Set dst (LoadI mem));
13053 // %}
13054 //
13055 
13056 //----------SMARTSPILL RULES---------------------------------------------------
13057 // These must follow all instruction definitions as they use the names
13058 // defined in the instructions definitions.
13059 
13060 // Local Variables:
13061 // mode: c++
13062 // End: