1 //
   2 // Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2014, 2020, Red Hat, Inc. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 // AArch64 Architecture Description File
  27 
  28 //----------REGISTER DEFINITION BLOCK------------------------------------------
  29 // This information is used by the matcher and the register allocator to
  30 // describe individual registers and classes of registers within the target
  31 // archtecture.
  32 
  33 register %{
  34 //----------Architecture Description Register Definitions----------------------
  35 // General Registers
  36 // "reg_def"  name ( register save type, C convention save type,
  37 //                   ideal register type, encoding );
  38 // Register Save Types:
  39 //
  40 // NS  = No-Save:       The register allocator assumes that these registers
  41 //                      can be used without saving upon entry to the method, &
  42 //                      that they do not need to be saved at call sites.
  43 //
  44 // SOC = Save-On-Call:  The register allocator assumes that these registers
  45 //                      can be used without saving upon entry to the method,
  46 //                      but that they must be saved at call sites.
  47 //
  48 // SOE = Save-On-Entry: The register allocator assumes that these registers
  49 //                      must be saved before using them upon entry to the
  50 //                      method, but they do not need to be saved at call
  51 //                      sites.
  52 //
  53 // AS  = Always-Save:   The register allocator assumes that these registers
  54 //                      must be saved before using them upon entry to the
  55 //                      method, & that they must be saved at call sites.
  56 //
  57 // Ideal Register Type is used to determine how to save & restore a
  58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  60 //
  61 // The encoding number is the actual bit-pattern placed into the opcodes.
  62 
  63 // We must define the 64 bit int registers in two 32 bit halves, the
  64 // real lower register and a virtual upper half register. upper halves
  65 // are used by the register allocator but are not actually supplied as
  66 // operands to memory ops.
  67 //
  68 // follow the C1 compiler in making registers
  69 //
  70 //   r0-r7,r10-r26 volatile (caller save)
  71 //   r27-r32 system (no save, no allocate)
  72 //   r8-r9 invisible to the allocator (so we can use them as scratch regs)
  73 //
  74 // as regards Java usage. we don't use any callee save registers
  75 // because this makes it difficult to de-optimise a frame (see comment
  76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
  77 //
  78 
  79 // General Registers
  80 
  81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
  82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
  83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
  84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
  85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
  86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
  87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
  88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
  89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
  90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
  91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
  92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
  93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
  94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
  95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
  96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
  97 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  98 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  99 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
 100 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 101 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
 102 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
 103 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
 104 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
 105 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
 106 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
 107 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
 108 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
 109 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
 110 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
 111 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
 112 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
 113 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()        );
 114 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
 115 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
 116 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
 117 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
 118 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
 119 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
 120 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
 121 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
 122 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
 123 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
 124 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
 125 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
 126 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
 127 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
 128 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
 129 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
 130 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
 131 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
 132 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
 133 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
 134 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
 135 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
 136 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
 137 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
 138 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
 139 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
 140 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
 141 
 142 // ----------------------------
 143 // Float/Double Registers
 144 // ----------------------------
 145 
 146 // Double Registers
 147 
 148 // The rules of ADL require that double registers be defined in pairs.
 149 // Each pair must be two 32-bit values, but not necessarily a pair of
 150 // single float registers. In each pair, ADLC-assigned register numbers
 151 // must be adjacent, with the lower number even. Finally, when the
 152 // CPU stores such a register pair to memory, the word associated with
 153 // the lower ADLC-assigned number must be stored to the lower address.
 154 
 155 // AArch64 has 32 floating-point registers. Each can store a vector of
 156 // single or double precision floating-point values up to 8 * 32
 157 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
 158 // use the first float or double element of the vector.
 159 
 160 // for Java use float registers v0-v15 are always save on call whereas
 161 // the platform ABI treats v8-v15 as callee save). float registers
 162 // v16-v31 are SOC as per the platform spec
 163 
 164   reg_def V0   ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()          );
 165   reg_def V0_H ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next()  );
 166   reg_def V0_J ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next(2) );
 167   reg_def V0_K ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next(3) );
 168 
 169   reg_def V1   ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()          );
 170   reg_def V1_H ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next()  );
 171   reg_def V1_J ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next(2) );
 172   reg_def V1_K ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next(3) );
 173 
 174   reg_def V2   ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()          );
 175   reg_def V2_H ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next()  );
 176   reg_def V2_J ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next(2) );
 177   reg_def V2_K ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next(3) );
 178 
 179   reg_def V3   ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()          );
 180   reg_def V3_H ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next()  );
 181   reg_def V3_J ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next(2) );
 182   reg_def V3_K ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next(3) );
 183 
 184   reg_def V4   ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()          );
 185   reg_def V4_H ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next()  );
 186   reg_def V4_J ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next(2) );
 187   reg_def V4_K ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next(3) );
 188 
 189   reg_def V5   ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()          );
 190   reg_def V5_H ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next()  );
 191   reg_def V5_J ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next(2) );
 192   reg_def V5_K ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next(3) );
 193 
 194   reg_def V6   ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()          );
 195   reg_def V6_H ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next()  );
 196   reg_def V6_J ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next(2) );
 197   reg_def V6_K ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next(3) );
 198 
 199   reg_def V7   ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()          );
 200   reg_def V7_H ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next()  );
 201   reg_def V7_J ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next(2) );
 202   reg_def V7_K ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next(3) );
 203 
 204   reg_def V8   ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()          );
 205   reg_def V8_H ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next()  );
 206   reg_def V8_J ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next(2) );
 207   reg_def V8_K ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next(3) );
 208 
 209   reg_def V9   ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()          );
 210   reg_def V9_H ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next()  );
 211   reg_def V9_J ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next(2) );
 212   reg_def V9_K ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next(3) );
 213 
 214   reg_def V10  ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()         );
 215   reg_def V10_H( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next() );
 216   reg_def V10_J( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2));
 217   reg_def V10_K( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3));
 218 
 219   reg_def V11  ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()         );
 220   reg_def V11_H( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next() );
 221   reg_def V11_J( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2));
 222   reg_def V11_K( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3));
 223 
 224   reg_def V12  ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()         );
 225   reg_def V12_H( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next() );
 226   reg_def V12_J( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2));
 227   reg_def V12_K( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3));
 228 
 229   reg_def V13  ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()         );
 230   reg_def V13_H( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next() );
 231   reg_def V13_J( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2));
 232   reg_def V13_K( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3));
 233 
 234   reg_def V14  ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()         );
 235   reg_def V14_H( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next() );
 236   reg_def V14_J( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2));
 237   reg_def V14_K( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3));
 238 
 239   reg_def V15  ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()         );
 240   reg_def V15_H( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next() );
 241   reg_def V15_J( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2));
 242   reg_def V15_K( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3));
 243 
 244   reg_def V16  ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()         );
 245   reg_def V16_H( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
 246   reg_def V16_J( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2));
 247   reg_def V16_K( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3));
 248 
 249   reg_def V17  ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()         );
 250   reg_def V17_H( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
 251   reg_def V17_J( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2));
 252   reg_def V17_K( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3));
 253 
 254   reg_def V18  ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()         );
 255   reg_def V18_H( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
 256   reg_def V18_J( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2));
 257   reg_def V18_K( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3));
 258 
 259   reg_def V19  ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()         );
 260   reg_def V19_H( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
 261   reg_def V19_J( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2));
 262   reg_def V19_K( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3));
 263 
 264   reg_def V20  ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()         );
 265   reg_def V20_H( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
 266   reg_def V20_J( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2));
 267   reg_def V20_K( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3));
 268 
 269   reg_def V21  ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()         );
 270   reg_def V21_H( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
 271   reg_def V21_J( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2));
 272   reg_def V21_K( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3));
 273 
 274   reg_def V22  ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()         );
 275   reg_def V22_H( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
 276   reg_def V22_J( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2));
 277   reg_def V22_K( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3));
 278 
 279   reg_def V23  ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()         );
 280   reg_def V23_H( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
 281   reg_def V23_J( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2));
 282   reg_def V23_K( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3));
 283 
 284   reg_def V24  ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()         );
 285   reg_def V24_H( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
 286   reg_def V24_J( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2));
 287   reg_def V24_K( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3));
 288 
 289   reg_def V25  ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()         );
 290   reg_def V25_H( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
 291   reg_def V25_J( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2));
 292   reg_def V25_K( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3));
 293 
 294   reg_def V26  ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()         );
 295   reg_def V26_H( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
 296   reg_def V26_J( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2));
 297   reg_def V26_K( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3));
 298 
 299   reg_def V27  ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()         );
 300   reg_def V27_H( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
 301   reg_def V27_J( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2));
 302   reg_def V27_K( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3));
 303 
 304   reg_def V28  ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()         );
 305   reg_def V28_H( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
 306   reg_def V28_J( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2));
 307   reg_def V28_K( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3));
 308 
 309   reg_def V29  ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()         );
 310   reg_def V29_H( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
 311   reg_def V29_J( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2));
 312   reg_def V29_K( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3));
 313 
 314   reg_def V30  ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()         );
 315   reg_def V30_H( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
 316   reg_def V30_J( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2));
 317   reg_def V30_K( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3));
 318 
 319   reg_def V31  ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()         );
 320   reg_def V31_H( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
 321   reg_def V31_J( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2));
 322   reg_def V31_K( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3));
 323 
 324 // ----------------------------
 325 // Special Registers
 326 // ----------------------------
 327 
 328 // the AArch64 CSPR status flag register is not directly acessible as
 329 // instruction operand. the FPSR status flag register is a system
 330 // register which can be written/read using MSR/MRS but again does not
 331 // appear as an operand (a code identifying the FSPR occurs as an
 332 // immediate value in the instruction).
 333 
 334 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
 335 
 336 
 337 // Specify priority of register selection within phases of register
 338 // allocation.  Highest priority is first.  A useful heuristic is to
 339 // give registers a low priority when they are required by machine
 340 // instructions, like EAX and EDX on I486, and choose no-save registers
 341 // before save-on-call, & save-on-call before save-on-entry.  Registers
 342 // which participate in fixed calling sequences should come last.
 343 // Registers which are used as pairs must fall on an even boundary.
 344 
 345 alloc_class chunk0(
 346     // volatiles
 347     R10, R10_H,
 348     R11, R11_H,
 349     R12, R12_H,
 350     R13, R13_H,
 351     R14, R14_H,
 352     R15, R15_H,
 353     R16, R16_H,
 354     R17, R17_H,
 355     R18, R18_H,
 356 
 357     // arg registers
 358     R0, R0_H,
 359     R1, R1_H,
 360     R2, R2_H,
 361     R3, R3_H,
 362     R4, R4_H,
 363     R5, R5_H,
 364     R6, R6_H,
 365     R7, R7_H,
 366 
 367     // non-volatiles
 368     R19, R19_H,
 369     R20, R20_H,
 370     R21, R21_H,
 371     R22, R22_H,
 372     R23, R23_H,
 373     R24, R24_H,
 374     R25, R25_H,
 375     R26, R26_H,
 376 
 377     // non-allocatable registers
 378 
 379     R27, R27_H, // heapbase
 380     R28, R28_H, // thread
 381     R29, R29_H, // fp
 382     R30, R30_H, // lr
 383     R31, R31_H, // sp
 384 );
 385 
 386 alloc_class chunk1(
 387 
 388     // no save
 389     V16, V16_H, V16_J, V16_K,
 390     V17, V17_H, V17_J, V17_K,
 391     V18, V18_H, V18_J, V18_K,
 392     V19, V19_H, V19_J, V19_K,
 393     V20, V20_H, V20_J, V20_K,
 394     V21, V21_H, V21_J, V21_K,
 395     V22, V22_H, V22_J, V22_K,
 396     V23, V23_H, V23_J, V23_K,
 397     V24, V24_H, V24_J, V24_K,
 398     V25, V25_H, V25_J, V25_K,
 399     V26, V26_H, V26_J, V26_K,
 400     V27, V27_H, V27_J, V27_K,
 401     V28, V28_H, V28_J, V28_K,
 402     V29, V29_H, V29_J, V29_K,
 403     V30, V30_H, V30_J, V30_K,
 404     V31, V31_H, V31_J, V31_K,
 405 
 406     // arg registers
 407     V0, V0_H, V0_J, V0_K,
 408     V1, V1_H, V1_J, V1_K,
 409     V2, V2_H, V2_J, V2_K,
 410     V3, V3_H, V3_J, V3_K,
 411     V4, V4_H, V4_J, V4_K,
 412     V5, V5_H, V5_J, V5_K,
 413     V6, V6_H, V6_J, V6_K,
 414     V7, V7_H, V7_J, V7_K,
 415 
 416     // non-volatiles
 417     V8, V8_H, V8_J, V8_K,
 418     V9, V9_H, V9_J, V9_K,
 419     V10, V10_H, V10_J, V10_K,
 420     V11, V11_H, V11_J, V11_K,
 421     V12, V12_H, V12_J, V12_K,
 422     V13, V13_H, V13_J, V13_K,
 423     V14, V14_H, V14_J, V14_K,
 424     V15, V15_H, V15_J, V15_K,
 425 );
 426 
 427 alloc_class chunk2(RFLAGS);
 428 
 429 //----------Architecture Description Register Classes--------------------------
 430 // Several register classes are automatically defined based upon information in
 431 // this architecture description.
 432 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 433 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 434 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 435 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 436 //
 437 
 438 // Class for all 32 bit general purpose registers
 439 reg_class all_reg32(
 440     R0,
 441     R1,
 442     R2,
 443     R3,
 444     R4,
 445     R5,
 446     R6,
 447     R7,
 448     R10,
 449     R11,
 450     R12,
 451     R13,
 452     R14,
 453     R15,
 454     R16,
 455     R17,
 456     R18,
 457     R19,
 458     R20,
 459     R21,
 460     R22,
 461     R23,
 462     R24,
 463     R25,
 464     R26,
 465     R27,
 466     R28,
 467     R29,
 468     R30,
 469     R31
 470 );
 471 
 472 
 473 // Class for all 32 bit integer registers (excluding SP which
 474 // will never be used as an integer register)
 475 reg_class any_reg32 %{
 476   return _ANY_REG32_mask;
 477 %}
 478 
 479 // Singleton class for R0 int register
 480 reg_class int_r0_reg(R0);
 481 
 482 // Singleton class for R2 int register
 483 reg_class int_r2_reg(R2);
 484 
 485 // Singleton class for R3 int register
 486 reg_class int_r3_reg(R3);
 487 
 488 // Singleton class for R4 int register
 489 reg_class int_r4_reg(R4);
 490 
 491 // Singleton class for R31 int register
 492 reg_class int_r31_reg(R31);
 493 
 494 // Class for all 64 bit general purpose registers
 495 reg_class all_reg(
 496     R0, R0_H,
 497     R1, R1_H,
 498     R2, R2_H,
 499     R3, R3_H,
 500     R4, R4_H,
 501     R5, R5_H,
 502     R6, R6_H,
 503     R7, R7_H,
 504     R10, R10_H,
 505     R11, R11_H,
 506     R12, R12_H,
 507     R13, R13_H,
 508     R14, R14_H,
 509     R15, R15_H,
 510     R16, R16_H,
 511     R17, R17_H,
 512     R18, R18_H,
 513     R19, R19_H,
 514     R20, R20_H,
 515     R21, R21_H,
 516     R22, R22_H,
 517     R23, R23_H,
 518     R24, R24_H,
 519     R25, R25_H,
 520     R26, R26_H,
 521     R27, R27_H,
 522     R28, R28_H,
 523     R29, R29_H,
 524     R30, R30_H,
 525     R31, R31_H
 526 );
 527 
 528 // Class for all long integer registers (including SP)
 529 reg_class any_reg %{
 530   return _ANY_REG_mask;
 531 %}
 532 
 533 // Class for non-allocatable 32 bit registers
 534 reg_class non_allocatable_reg32(
 535     R28,                        // thread
 536     R30,                        // lr
 537     R31                         // sp
 538 );
 539 
 540 // Class for non-allocatable 64 bit registers
 541 reg_class non_allocatable_reg(
 542     R28, R28_H,                 // thread
 543     R30, R30_H,                 // lr
 544     R31, R31_H                  // sp
 545 );
 546 
 547 // Class for all non-special integer registers
 548 reg_class no_special_reg32 %{
 549   return _NO_SPECIAL_REG32_mask;
 550 %}
 551 
 552 // Class for all non-special long integer registers
 553 reg_class no_special_reg %{
 554   return _NO_SPECIAL_REG_mask;
 555 %}
 556 
 557 // Class for 64 bit register r0
 558 reg_class r0_reg(
 559     R0, R0_H
 560 );
 561 
 562 // Class for 64 bit register r1
 563 reg_class r1_reg(
 564     R1, R1_H
 565 );
 566 
 567 // Class for 64 bit register r2
 568 reg_class r2_reg(
 569     R2, R2_H
 570 );
 571 
 572 // Class for 64 bit register r3
 573 reg_class r3_reg(
 574     R3, R3_H
 575 );
 576 
 577 // Class for 64 bit register r4
 578 reg_class r4_reg(
 579     R4, R4_H
 580 );
 581 
 582 // Class for 64 bit register r5
 583 reg_class r5_reg(
 584     R5, R5_H
 585 );
 586 
 587 // Class for 64 bit register r10
 588 reg_class r10_reg(
 589     R10, R10_H
 590 );
 591 
 592 // Class for 64 bit register r11
 593 reg_class r11_reg(
 594     R11, R11_H
 595 );
 596 
 597 // Class for method register
 598 reg_class method_reg(
 599     R12, R12_H
 600 );
 601 
 602 // Class for heapbase register
 603 reg_class heapbase_reg(
 604     R27, R27_H
 605 );
 606 
 607 // Class for thread register
 608 reg_class thread_reg(
 609     R28, R28_H
 610 );
 611 
 612 // Class for frame pointer register
 613 reg_class fp_reg(
 614     R29, R29_H
 615 );
 616 
 617 // Class for link register
 618 reg_class lr_reg(
 619     R30, R30_H
 620 );
 621 
 622 // Class for long sp register
 623 reg_class sp_reg(
 624   R31, R31_H
 625 );
 626 
 627 // Class for all pointer registers
 628 reg_class ptr_reg %{
 629   return _PTR_REG_mask;
 630 %}
 631 
 632 // Class for all non_special pointer registers
 633 reg_class no_special_ptr_reg %{
 634   return _NO_SPECIAL_PTR_REG_mask;
 635 %}
 636 
 637 // Class for all float registers
 638 reg_class float_reg(
 639     V0,
 640     V1,
 641     V2,
 642     V3,
 643     V4,
 644     V5,
 645     V6,
 646     V7,
 647     V8,
 648     V9,
 649     V10,
 650     V11,
 651     V12,
 652     V13,
 653     V14,
 654     V15,
 655     V16,
 656     V17,
 657     V18,
 658     V19,
 659     V20,
 660     V21,
 661     V22,
 662     V23,
 663     V24,
 664     V25,
 665     V26,
 666     V27,
 667     V28,
 668     V29,
 669     V30,
 670     V31
 671 );
 672 
 673 // Double precision float registers have virtual `high halves' that
 674 // are needed by the allocator.
 675 // Class for all double registers
 676 reg_class double_reg(
 677     V0, V0_H,
 678     V1, V1_H,
 679     V2, V2_H,
 680     V3, V3_H,
 681     V4, V4_H,
 682     V5, V5_H,
 683     V6, V6_H,
 684     V7, V7_H,
 685     V8, V8_H,
 686     V9, V9_H,
 687     V10, V10_H,
 688     V11, V11_H,
 689     V12, V12_H,
 690     V13, V13_H,
 691     V14, V14_H,
 692     V15, V15_H,
 693     V16, V16_H,
 694     V17, V17_H,
 695     V18, V18_H,
 696     V19, V19_H,
 697     V20, V20_H,
 698     V21, V21_H,
 699     V22, V22_H,
 700     V23, V23_H,
 701     V24, V24_H,
 702     V25, V25_H,
 703     V26, V26_H,
 704     V27, V27_H,
 705     V28, V28_H,
 706     V29, V29_H,
 707     V30, V30_H,
 708     V31, V31_H
 709 );
 710 
 711 // Class for all 64bit vector registers
 712 reg_class vectord_reg(
 713     V0, V0_H,
 714     V1, V1_H,
 715     V2, V2_H,
 716     V3, V3_H,
 717     V4, V4_H,
 718     V5, V5_H,
 719     V6, V6_H,
 720     V7, V7_H,
 721     V8, V8_H,
 722     V9, V9_H,
 723     V10, V10_H,
 724     V11, V11_H,
 725     V12, V12_H,
 726     V13, V13_H,
 727     V14, V14_H,
 728     V15, V15_H,
 729     V16, V16_H,
 730     V17, V17_H,
 731     V18, V18_H,
 732     V19, V19_H,
 733     V20, V20_H,
 734     V21, V21_H,
 735     V22, V22_H,
 736     V23, V23_H,
 737     V24, V24_H,
 738     V25, V25_H,
 739     V26, V26_H,
 740     V27, V27_H,
 741     V28, V28_H,
 742     V29, V29_H,
 743     V30, V30_H,
 744     V31, V31_H
 745 );
 746 
 747 // Class for all 128bit vector registers
 748 reg_class vectorx_reg(
 749     V0, V0_H, V0_J, V0_K,
 750     V1, V1_H, V1_J, V1_K,
 751     V2, V2_H, V2_J, V2_K,
 752     V3, V3_H, V3_J, V3_K,
 753     V4, V4_H, V4_J, V4_K,
 754     V5, V5_H, V5_J, V5_K,
 755     V6, V6_H, V6_J, V6_K,
 756     V7, V7_H, V7_J, V7_K,
 757     V8, V8_H, V8_J, V8_K,
 758     V9, V9_H, V9_J, V9_K,
 759     V10, V10_H, V10_J, V10_K,
 760     V11, V11_H, V11_J, V11_K,
 761     V12, V12_H, V12_J, V12_K,
 762     V13, V13_H, V13_J, V13_K,
 763     V14, V14_H, V14_J, V14_K,
 764     V15, V15_H, V15_J, V15_K,
 765     V16, V16_H, V16_J, V16_K,
 766     V17, V17_H, V17_J, V17_K,
 767     V18, V18_H, V18_J, V18_K,
 768     V19, V19_H, V19_J, V19_K,
 769     V20, V20_H, V20_J, V20_K,
 770     V21, V21_H, V21_J, V21_K,
 771     V22, V22_H, V22_J, V22_K,
 772     V23, V23_H, V23_J, V23_K,
 773     V24, V24_H, V24_J, V24_K,
 774     V25, V25_H, V25_J, V25_K,
 775     V26, V26_H, V26_J, V26_K,
 776     V27, V27_H, V27_J, V27_K,
 777     V28, V28_H, V28_J, V28_K,
 778     V29, V29_H, V29_J, V29_K,
 779     V30, V30_H, V30_J, V30_K,
 780     V31, V31_H, V31_J, V31_K
 781 );
 782 
 783 // Class for 128 bit register v0
 784 reg_class v0_reg(
 785     V0, V0_H
 786 );
 787 
 788 // Class for 128 bit register v1
 789 reg_class v1_reg(
 790     V1, V1_H
 791 );
 792 
 793 // Class for 128 bit register v2
 794 reg_class v2_reg(
 795     V2, V2_H
 796 );
 797 
 798 // Class for 128 bit register v3
 799 reg_class v3_reg(
 800     V3, V3_H
 801 );
 802 
 803 // Class for 128 bit register v4
 804 reg_class v4_reg(
 805     V4, V4_H
 806 );
 807 
 808 // Class for 128 bit register v5
 809 reg_class v5_reg(
 810     V5, V5_H
 811 );
 812 
 813 // Class for 128 bit register v6
 814 reg_class v6_reg(
 815     V6, V6_H
 816 );
 817 
 818 // Class for 128 bit register v7
 819 reg_class v7_reg(
 820     V7, V7_H
 821 );
 822 
 823 // Class for 128 bit register v8
 824 reg_class v8_reg(
 825     V8, V8_H
 826 );
 827 
 828 // Class for 128 bit register v9
 829 reg_class v9_reg(
 830     V9, V9_H
 831 );
 832 
 833 // Class for 128 bit register v10
 834 reg_class v10_reg(
 835     V10, V10_H
 836 );
 837 
 838 // Class for 128 bit register v11
 839 reg_class v11_reg(
 840     V11, V11_H
 841 );
 842 
 843 // Class for 128 bit register v12
 844 reg_class v12_reg(
 845     V12, V12_H
 846 );
 847 
 848 // Class for 128 bit register v13
 849 reg_class v13_reg(
 850     V13, V13_H
 851 );
 852 
 853 // Class for 128 bit register v14
 854 reg_class v14_reg(
 855     V14, V14_H
 856 );
 857 
 858 // Class for 128 bit register v15
 859 reg_class v15_reg(
 860     V15, V15_H
 861 );
 862 
 863 // Class for 128 bit register v16
 864 reg_class v16_reg(
 865     V16, V16_H
 866 );
 867 
 868 // Class for 128 bit register v17
 869 reg_class v17_reg(
 870     V17, V17_H
 871 );
 872 
 873 // Class for 128 bit register v18
 874 reg_class v18_reg(
 875     V18, V18_H
 876 );
 877 
 878 // Class for 128 bit register v19
 879 reg_class v19_reg(
 880     V19, V19_H
 881 );
 882 
 883 // Class for 128 bit register v20
 884 reg_class v20_reg(
 885     V20, V20_H
 886 );
 887 
 888 // Class for 128 bit register v21
 889 reg_class v21_reg(
 890     V21, V21_H
 891 );
 892 
 893 // Class for 128 bit register v22
 894 reg_class v22_reg(
 895     V22, V22_H
 896 );
 897 
 898 // Class for 128 bit register v23
 899 reg_class v23_reg(
 900     V23, V23_H
 901 );
 902 
 903 // Class for 128 bit register v24
 904 reg_class v24_reg(
 905     V24, V24_H
 906 );
 907 
 908 // Class for 128 bit register v25
 909 reg_class v25_reg(
 910     V25, V25_H
 911 );
 912 
 913 // Class for 128 bit register v26
 914 reg_class v26_reg(
 915     V26, V26_H
 916 );
 917 
 918 // Class for 128 bit register v27
 919 reg_class v27_reg(
 920     V27, V27_H
 921 );
 922 
 923 // Class for 128 bit register v28
 924 reg_class v28_reg(
 925     V28, V28_H
 926 );
 927 
 928 // Class for 128 bit register v29
 929 reg_class v29_reg(
 930     V29, V29_H
 931 );
 932 
 933 // Class for 128 bit register v30
 934 reg_class v30_reg(
 935     V30, V30_H
 936 );
 937 
 938 // Class for 128 bit register v31
 939 reg_class v31_reg(
 940     V31, V31_H
 941 );
 942 
 943 // Singleton class for condition codes
 944 reg_class int_flags(RFLAGS);
 945 
 946 %}
 947 
 948 //----------DEFINITION BLOCK---------------------------------------------------
 949 // Define name --> value mappings to inform the ADLC of an integer valued name
 950 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 951 // Format:
 952 //        int_def  <name>         ( <int_value>, <expression>);
 953 // Generated Code in ad_<arch>.hpp
 954 //        #define  <name>   (<expression>)
 955 //        // value == <int_value>
 956 // Generated code in ad_<arch>.cpp adlc_verification()
 957 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 958 //
 959 
 960 // we follow the ppc-aix port in using a simple cost model which ranks
 961 // register operations as cheap, memory ops as more expensive and
 962 // branches as most expensive. the first two have a low as well as a
 963 // normal cost. huge cost appears to be a way of saying don't do
 964 // something
 965 
 966 definitions %{
 967   // The default cost (of a register move instruction).
 968   int_def INSN_COST            (    100,     100);
 969   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 970   int_def CALL_COST            (    200,     2 * INSN_COST);
 971   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 972 %}
 973 
 974 
 975 //----------SOURCE BLOCK-------------------------------------------------------
 976 // This is a block of C++ code which provides values, functions, and
 977 // definitions necessary in the rest of the architecture description
 978 
 979 source_hpp %{
 980 
 981 #include "asm/macroAssembler.hpp"
 982 #include "gc/shared/cardTable.hpp"
 983 #include "gc/shared/cardTableBarrierSet.hpp"
 984 #include "gc/shared/collectedHeap.hpp"
 985 #include "opto/addnode.hpp"
 986 #include "opto/convertnode.hpp"
 987 
 988 extern RegMask _ANY_REG32_mask;
 989 extern RegMask _ANY_REG_mask;
 990 extern RegMask _PTR_REG_mask;
 991 extern RegMask _NO_SPECIAL_REG32_mask;
 992 extern RegMask _NO_SPECIAL_REG_mask;
 993 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 994 
 995 class CallStubImpl {
 996 
 997   //--------------------------------------------------------------
 998   //---<  Used for optimization in Compile::shorten_branches  >---
 999   //--------------------------------------------------------------
1000 
1001  public:
1002   // Size of call trampoline stub.
1003   static uint size_call_trampoline() {
1004     return 0; // no call trampolines on this platform
1005   }
1006 
1007   // number of relocations needed by a call trampoline stub
1008   static uint reloc_call_trampoline() {
1009     return 0; // no call trampolines on this platform
1010   }
1011 };
1012 
1013 class HandlerImpl {
1014 
1015  public:
1016 
1017   static int emit_exception_handler(CodeBuffer &cbuf);
1018   static int emit_deopt_handler(CodeBuffer& cbuf);
1019 
1020   static uint size_exception_handler() {
1021     return MacroAssembler::far_branch_size();
1022   }
1023 
1024   static uint size_deopt_handler() {
1025     // count one adr and one far branch instruction
1026     return 4 * NativeInstruction::instruction_size;
1027   }
1028 };
1029 
1030 class Node::PD {
1031 public:
1032   enum NodeFlags {
1033     _last_flag = Node::_last_flag
1034   };
1035 };
1036 
1037  bool is_CAS(int opcode, bool maybe_volatile);
1038 
1039   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1040 
1041   bool unnecessary_acquire(const Node *barrier);
1042   bool needs_acquiring_load(const Node *load);
1043 
1044   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1045 
1046   bool unnecessary_release(const Node *barrier);
1047   bool unnecessary_volatile(const Node *barrier);
1048   bool needs_releasing_store(const Node *store);
1049 
1050   // predicate controlling translation of CompareAndSwapX
1051   bool needs_acquiring_load_exclusive(const Node *load);
1052 
1053   // predicate controlling addressing modes
1054   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1055 %}
1056 
1057 source %{
1058 
1059   // Derived RegMask with conditionally allocatable registers
1060 
1061   void PhaseOutput::pd_perform_mach_node_analysis() {
1062   }
1063 
1064   int MachNode::pd_alignment_required() const {
1065     return 1;
1066   }
1067 
1068   int MachNode::compute_padding(int current_offset) const {
1069     return 0;
1070   }
1071 
1072   RegMask _ANY_REG32_mask;
1073   RegMask _ANY_REG_mask;
1074   RegMask _PTR_REG_mask;
1075   RegMask _NO_SPECIAL_REG32_mask;
1076   RegMask _NO_SPECIAL_REG_mask;
1077   RegMask _NO_SPECIAL_PTR_REG_mask;
1078 
1079   void reg_mask_init() {
1080     // We derive below RegMask(s) from the ones which are auto-generated from
1081     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1082     // registers conditionally reserved.
1083 
1084     _ANY_REG32_mask = _ALL_REG32_mask;
1085     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1086 
1087     _ANY_REG_mask = _ALL_REG_mask;
1088 
1089     _PTR_REG_mask = _ALL_REG_mask;
1090 
1091     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1092     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1093 
1094     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1095     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1096 
1097     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1098     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1099 
1100     // r27 is not allocatable when compressed oops is on and heapbase is not
1101     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1102     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL || UseAOT)) {
1103       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1104       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1105       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1106     }
1107 
1108     // r29 is not allocatable when PreserveFramePointer is on
1109     if (PreserveFramePointer) {
1110       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1111       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
1112       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
1113     }
1114   }
1115 
1116   // Optimizaton of volatile gets and puts
1117   // -------------------------------------
1118   //
1119   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1120   // use to implement volatile reads and writes. For a volatile read
1121   // we simply need
1122   //
1123   //   ldar<x>
1124   //
1125   // and for a volatile write we need
1126   //
1127   //   stlr<x>
1128   //
1129   // Alternatively, we can implement them by pairing a normal
1130   // load/store with a memory barrier. For a volatile read we need
1131   //
1132   //   ldr<x>
1133   //   dmb ishld
1134   //
1135   // for a volatile write
1136   //
1137   //   dmb ish
1138   //   str<x>
1139   //   dmb ish
1140   //
1141   // We can also use ldaxr and stlxr to implement compare and swap CAS
1142   // sequences. These are normally translated to an instruction
1143   // sequence like the following
1144   //
1145   //   dmb      ish
1146   // retry:
1147   //   ldxr<x>   rval raddr
1148   //   cmp       rval rold
1149   //   b.ne done
1150   //   stlxr<x>  rval, rnew, rold
1151   //   cbnz      rval retry
1152   // done:
1153   //   cset      r0, eq
1154   //   dmb ishld
1155   //
1156   // Note that the exclusive store is already using an stlxr
1157   // instruction. That is required to ensure visibility to other
1158   // threads of the exclusive write (assuming it succeeds) before that
1159   // of any subsequent writes.
1160   //
1161   // The following instruction sequence is an improvement on the above
1162   //
1163   // retry:
1164   //   ldaxr<x>  rval raddr
1165   //   cmp       rval rold
1166   //   b.ne done
1167   //   stlxr<x>  rval, rnew, rold
1168   //   cbnz      rval retry
1169   // done:
1170   //   cset      r0, eq
1171   //
1172   // We don't need the leading dmb ish since the stlxr guarantees
1173   // visibility of prior writes in the case that the swap is
1174   // successful. Crucially we don't have to worry about the case where
1175   // the swap is not successful since no valid program should be
1176   // relying on visibility of prior changes by the attempting thread
1177   // in the case where the CAS fails.
1178   //
1179   // Similarly, we don't need the trailing dmb ishld if we substitute
1180   // an ldaxr instruction since that will provide all the guarantees we
1181   // require regarding observation of changes made by other threads
1182   // before any change to the CAS address observed by the load.
1183   //
1184   // In order to generate the desired instruction sequence we need to
1185   // be able to identify specific 'signature' ideal graph node
1186   // sequences which i) occur as a translation of a volatile reads or
1187   // writes or CAS operations and ii) do not occur through any other
1188   // translation or graph transformation. We can then provide
1189   // alternative aldc matching rules which translate these node
1190   // sequences to the desired machine code sequences. Selection of the
1191   // alternative rules can be implemented by predicates which identify
1192   // the relevant node sequences.
1193   //
1194   // The ideal graph generator translates a volatile read to the node
1195   // sequence
1196   //
1197   //   LoadX[mo_acquire]
1198   //   MemBarAcquire
1199   //
1200   // As a special case when using the compressed oops optimization we
1201   // may also see this variant
1202   //
1203   //   LoadN[mo_acquire]
1204   //   DecodeN
1205   //   MemBarAcquire
1206   //
1207   // A volatile write is translated to the node sequence
1208   //
1209   //   MemBarRelease
1210   //   StoreX[mo_release] {CardMark}-optional
1211   //   MemBarVolatile
1212   //
1213   // n.b. the above node patterns are generated with a strict
1214   // 'signature' configuration of input and output dependencies (see
1215   // the predicates below for exact details). The card mark may be as
1216   // simple as a few extra nodes or, in a few GC configurations, may
1217   // include more complex control flow between the leading and
1218   // trailing memory barriers. However, whatever the card mark
1219   // configuration these signatures are unique to translated volatile
1220   // reads/stores -- they will not appear as a result of any other
1221   // bytecode translation or inlining nor as a consequence of
1222   // optimizing transforms.
1223   //
1224   // We also want to catch inlined unsafe volatile gets and puts and
1225   // be able to implement them using either ldar<x>/stlr<x> or some
1226   // combination of ldr<x>/stlr<x> and dmb instructions.
1227   //
1228   // Inlined unsafe volatiles puts manifest as a minor variant of the
1229   // normal volatile put node sequence containing an extra cpuorder
1230   // membar
1231   //
1232   //   MemBarRelease
1233   //   MemBarCPUOrder
1234   //   StoreX[mo_release] {CardMark}-optional
1235   //   MemBarCPUOrder
1236   //   MemBarVolatile
1237   //
1238   // n.b. as an aside, a cpuorder membar is not itself subject to
1239   // matching and translation by adlc rules.  However, the rule
1240   // predicates need to detect its presence in order to correctly
1241   // select the desired adlc rules.
1242   //
1243   // Inlined unsafe volatile gets manifest as a slightly different
1244   // node sequence to a normal volatile get because of the
1245   // introduction of some CPUOrder memory barriers to bracket the
1246   // Load. However, but the same basic skeleton of a LoadX feeding a
1247   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
1248   // present
1249   //
1250   //   MemBarCPUOrder
1251   //        ||       \\
1252   //   MemBarCPUOrder LoadX[mo_acquire]
1253   //        ||            |
1254   //        ||       {DecodeN} optional
1255   //        ||       /
1256   //     MemBarAcquire
1257   //
1258   // In this case the acquire membar does not directly depend on the
1259   // load. However, we can be sure that the load is generated from an
1260   // inlined unsafe volatile get if we see it dependent on this unique
1261   // sequence of membar nodes. Similarly, given an acquire membar we
1262   // can know that it was added because of an inlined unsafe volatile
1263   // get if it is fed and feeds a cpuorder membar and if its feed
1264   // membar also feeds an acquiring load.
1265   //
1266   // Finally an inlined (Unsafe) CAS operation is translated to the
1267   // following ideal graph
1268   //
1269   //   MemBarRelease
1270   //   MemBarCPUOrder
1271   //   CompareAndSwapX {CardMark}-optional
1272   //   MemBarCPUOrder
1273   //   MemBarAcquire
1274   //
1275   // So, where we can identify these volatile read and write
1276   // signatures we can choose to plant either of the above two code
1277   // sequences. For a volatile read we can simply plant a normal
1278   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1279   // also choose to inhibit translation of the MemBarAcquire and
1280   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1281   //
1282   // When we recognise a volatile store signature we can choose to
1283   // plant at a dmb ish as a translation for the MemBarRelease, a
1284   // normal str<x> and then a dmb ish for the MemBarVolatile.
1285   // Alternatively, we can inhibit translation of the MemBarRelease
1286   // and MemBarVolatile and instead plant a simple stlr<x>
1287   // instruction.
1288   //
1289   // when we recognise a CAS signature we can choose to plant a dmb
1290   // ish as a translation for the MemBarRelease, the conventional
1291   // macro-instruction sequence for the CompareAndSwap node (which
1292   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1293   // Alternatively, we can elide generation of the dmb instructions
1294   // and plant the alternative CompareAndSwap macro-instruction
1295   // sequence (which uses ldaxr<x>).
1296   //
1297   // Of course, the above only applies when we see these signature
1298   // configurations. We still want to plant dmb instructions in any
1299   // other cases where we may see a MemBarAcquire, MemBarRelease or
1300   // MemBarVolatile. For example, at the end of a constructor which
1301   // writes final/volatile fields we will see a MemBarRelease
1302   // instruction and this needs a 'dmb ish' lest we risk the
1303   // constructed object being visible without making the
1304   // final/volatile field writes visible.
1305   //
1306   // n.b. the translation rules below which rely on detection of the
1307   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1308   // If we see anything other than the signature configurations we
1309   // always just translate the loads and stores to ldr<x> and str<x>
1310   // and translate acquire, release and volatile membars to the
1311   // relevant dmb instructions.
1312   //
1313 
1314   // is_CAS(int opcode, bool maybe_volatile)
1315   //
1316   // return true if opcode is one of the possible CompareAndSwapX
1317   // values otherwise false.
1318 
1319   bool is_CAS(int opcode, bool maybe_volatile)
1320   {
1321     switch(opcode) {
1322       // We handle these
1323     case Op_CompareAndSwapI:
1324     case Op_CompareAndSwapL:
1325     case Op_CompareAndSwapP:
1326     case Op_CompareAndSwapN:
1327     case Op_ShenandoahCompareAndSwapP:
1328     case Op_ShenandoahCompareAndSwapN:
1329     case Op_CompareAndSwapB:
1330     case Op_CompareAndSwapS:
1331     case Op_GetAndSetI:
1332     case Op_GetAndSetL:
1333     case Op_GetAndSetP:
1334     case Op_GetAndSetN:
1335     case Op_GetAndAddI:
1336     case Op_GetAndAddL:
1337       return true;
1338     case Op_CompareAndExchangeI:
1339     case Op_CompareAndExchangeN:
1340     case Op_CompareAndExchangeB:
1341     case Op_CompareAndExchangeS:
1342     case Op_CompareAndExchangeL:
1343     case Op_CompareAndExchangeP:
1344     case Op_WeakCompareAndSwapB:
1345     case Op_WeakCompareAndSwapS:
1346     case Op_WeakCompareAndSwapI:
1347     case Op_WeakCompareAndSwapL:
1348     case Op_WeakCompareAndSwapP:
1349     case Op_WeakCompareAndSwapN:
1350     case Op_ShenandoahWeakCompareAndSwapP:
1351     case Op_ShenandoahWeakCompareAndSwapN:
1352     case Op_ShenandoahCompareAndExchangeP:
1353     case Op_ShenandoahCompareAndExchangeN:
1354       return maybe_volatile;
1355     default:
1356       return false;
1357     }
1358   }
1359 
1360   // helper to determine the maximum number of Phi nodes we may need to
1361   // traverse when searching from a card mark membar for the merge mem
1362   // feeding a trailing membar or vice versa
1363 
1364 // predicates controlling emit of ldr<x>/ldar<x>
1365 
1366 bool unnecessary_acquire(const Node *barrier)
1367 {
1368   assert(barrier->is_MemBar(), "expecting a membar");
1369 
1370   MemBarNode* mb = barrier->as_MemBar();
1371 
1372   if (mb->trailing_load()) {
1373     return true;
1374   }
1375 
1376   if (mb->trailing_load_store()) {
1377     Node* load_store = mb->in(MemBarNode::Precedent);
1378     assert(load_store->is_LoadStore(), "unexpected graph shape");
1379     return is_CAS(load_store->Opcode(), true);
1380   }
1381 
1382   return false;
1383 }
1384 
1385 bool needs_acquiring_load(const Node *n)
1386 {
1387   assert(n->is_Load(), "expecting a load");
1388   LoadNode *ld = n->as_Load();
1389   return ld->is_acquire();
1390 }
1391 
1392 bool unnecessary_release(const Node *n)
1393 {
1394   assert((n->is_MemBar() &&
1395           n->Opcode() == Op_MemBarRelease),
1396          "expecting a release membar");
1397 
1398   MemBarNode *barrier = n->as_MemBar();
1399   if (!barrier->leading()) {
1400     return false;
1401   } else {
1402     Node* trailing = barrier->trailing_membar();
1403     MemBarNode* trailing_mb = trailing->as_MemBar();
1404     assert(trailing_mb->trailing(), "Not a trailing membar?");
1405     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1406 
1407     Node* mem = trailing_mb->in(MemBarNode::Precedent);
1408     if (mem->is_Store()) {
1409       assert(mem->as_Store()->is_release(), "");
1410       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1411       return true;
1412     } else {
1413       assert(mem->is_LoadStore(), "");
1414       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1415       return is_CAS(mem->Opcode(), true);
1416     }
1417   }
1418   return false;
1419 }
1420 
1421 bool unnecessary_volatile(const Node *n)
1422 {
1423   // assert n->is_MemBar();
1424   MemBarNode *mbvol = n->as_MemBar();
1425 
1426   bool release = mbvol->trailing_store();
1427   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1428 #ifdef ASSERT
1429   if (release) {
1430     Node* leading = mbvol->leading_membar();
1431     assert(leading->Opcode() == Op_MemBarRelease, "");
1432     assert(leading->as_MemBar()->leading_store(), "");
1433     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1434   }
1435 #endif
1436 
1437   return release;
1438 }
1439 
1440 // predicates controlling emit of str<x>/stlr<x>
1441 
1442 bool needs_releasing_store(const Node *n)
1443 {
1444   // assert n->is_Store();
1445   StoreNode *st = n->as_Store();
1446   return st->trailing_membar() != NULL;
1447 }
1448 
1449 // predicate controlling translation of CAS
1450 //
1451 // returns true if CAS needs to use an acquiring load otherwise false
1452 
1453 bool needs_acquiring_load_exclusive(const Node *n)
1454 {
1455   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1456   LoadStoreNode* ldst = n->as_LoadStore();
1457   if (is_CAS(n->Opcode(), false)) {
1458     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1459   } else {
1460     return ldst->trailing_membar() != NULL;
1461   }
1462 
1463   // so we can just return true here
1464   return true;
1465 }
1466 
1467 #define __ _masm.
1468 
1469 // advance declarations for helper functions to convert register
1470 // indices to register objects
1471 
1472 // the ad file has to provide implementations of certain methods
1473 // expected by the generic code
1474 //
1475 // REQUIRED FUNCTIONALITY
1476 
1477 //=============================================================================
1478 
1479 // !!!!! Special hack to get all types of calls to specify the byte offset
1480 //       from the start of the call to the point where the return address
1481 //       will point.
1482 
1483 int MachCallStaticJavaNode::ret_addr_offset()
1484 {
1485   // call should be a simple bl
1486   int off = 4;
1487   return off;
1488 }
1489 
1490 int MachCallDynamicJavaNode::ret_addr_offset()
1491 {
1492   return 16; // movz, movk, movk, bl
1493 }
1494 
1495 int MachCallRuntimeNode::ret_addr_offset() {
1496   // for generated stubs the call will be
1497   //   far_call(addr)
1498   // for real runtime callouts it will be six instructions
1499   // see aarch64_enc_java_to_runtime
1500   //   adr(rscratch2, retaddr)
1501   //   lea(rscratch1, RuntimeAddress(addr)
1502   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1503   //   blr(rscratch1)
1504   CodeBlob *cb = CodeCache::find_blob(_entry_point);
1505   if (cb) {
1506     return MacroAssembler::far_branch_size();
1507   } else {
1508     return 6 * NativeInstruction::instruction_size;
1509   }
1510 }
1511 
1512 // Indicate if the safepoint node needs the polling page as an input
1513 
1514 // the shared code plants the oop data at the start of the generated
1515 // code for the safepoint node and that needs ot be at the load
1516 // instruction itself. so we cannot plant a mov of the safepoint poll
1517 // address followed by a load. setting this to true means the mov is
1518 // scheduled as a prior instruction. that's better for scheduling
1519 // anyway.
1520 
1521 bool SafePointNode::needs_polling_address_input()
1522 {
1523   return true;
1524 }
1525 
1526 //=============================================================================
1527 
1528 #ifndef PRODUCT
1529 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1530   st->print("BREAKPOINT");
1531 }
1532 #endif
1533 
1534 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1535   C2_MacroAssembler _masm(&cbuf);
1536   __ brk(0);
1537 }
1538 
1539 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1540   return MachNode::size(ra_);
1541 }
1542 
1543 //=============================================================================
1544 
1545 #ifndef PRODUCT
1546   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1547     st->print("nop \t# %d bytes pad for loops and calls", _count);
1548   }
1549 #endif
1550 
1551   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1552     C2_MacroAssembler _masm(&cbuf);
1553     for (int i = 0; i < _count; i++) {
1554       __ nop();
1555     }
1556   }
1557 
1558   uint MachNopNode::size(PhaseRegAlloc*) const {
1559     return _count * NativeInstruction::instruction_size;
1560   }
1561 
1562 //=============================================================================
1563 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1564 
1565 int ConstantTable::calculate_table_base_offset() const {
1566   return 0;  // absolute addressing, no offset
1567 }
1568 
1569 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1570 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1571   ShouldNotReachHere();
1572 }
1573 
1574 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1575   // Empty encoding
1576 }
1577 
1578 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1579   return 0;
1580 }
1581 
1582 #ifndef PRODUCT
1583 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1584   st->print("-- \t// MachConstantBaseNode (empty encoding)");
1585 }
1586 #endif
1587 
1588 #ifndef PRODUCT
1589 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1590   Compile* C = ra_->C;
1591 
1592   int framesize = C->output()->frame_slots() << LogBytesPerInt;
1593 
1594   if (C->output()->need_stack_bang(framesize))
1595     st->print("# stack bang size=%d\n\t", framesize);
1596 
1597   if (framesize < ((1 << 9) + 2 * wordSize)) {
1598     st->print("sub  sp, sp, #%d\n\t", framesize);
1599     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1600     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
1601   } else {
1602     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1603     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
1604     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1605     st->print("sub  sp, sp, rscratch1");
1606   }
1607   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1608     st->print("\n\t");
1609     st->print("ldr  rscratch1, [guard]\n\t");
1610     st->print("dmb ishld\n\t");
1611     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
1612     st->print("cmp  rscratch1, rscratch2\n\t");
1613     st->print("b.eq skip");
1614     st->print("\n\t");
1615     st->print("blr #nmethod_entry_barrier_stub\n\t");
1616     st->print("b skip\n\t");
1617     st->print("guard: int\n\t");
1618     st->print("\n\t");
1619     st->print("skip:\n\t");
1620   }
1621 }
1622 #endif
1623 
1624 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1625   Compile* C = ra_->C;
1626   C2_MacroAssembler _masm(&cbuf);
1627 
1628   // n.b. frame size includes space for return pc and rfp
1629   const int framesize = C->output()->frame_size_in_bytes();
1630   assert(framesize%(2*wordSize) == 0, "must preserve 2*wordSize alignment");
1631 
1632   // insert a nop at the start of the prolog so we can patch in a
1633   // branch if we need to invalidate the method later
1634   __ nop();
1635 
1636   if (C->clinit_barrier_on_entry()) {
1637     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1638 
1639     Label L_skip_barrier;
1640 
1641     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1642     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1643     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1644     __ bind(L_skip_barrier);
1645   }
1646 
1647   int bangsize = C->output()->bang_size_in_bytes();
1648   if (C->output()->need_stack_bang(bangsize) && UseStackBanging)
1649     __ generate_stack_overflow_check(bangsize);
1650 
1651   __ build_frame(framesize);
1652 
1653   if (C->stub_function() == NULL) {
1654     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1655     bs->nmethod_entry_barrier(&_masm);
1656   }
1657 
1658   if (VerifyStackAtCalls) {
1659     Unimplemented();
1660   }
1661 
1662   C->output()->set_frame_complete(cbuf.insts_size());
1663 
1664   if (C->has_mach_constant_base_node()) {
1665     // NOTE: We set the table base offset here because users might be
1666     // emitted before MachConstantBaseNode.
1667     ConstantTable& constant_table = C->output()->constant_table();
1668     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1669   }
1670 }
1671 
1672 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1673 {
1674   return MachNode::size(ra_); // too many variables; just compute it
1675                               // the hard way
1676 }
1677 
1678 int MachPrologNode::reloc() const
1679 {
1680   return 0;
1681 }
1682 
1683 //=============================================================================
1684 
1685 #ifndef PRODUCT
1686 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1687   Compile* C = ra_->C;
1688   int framesize = C->output()->frame_slots() << LogBytesPerInt;
1689 
1690   st->print("# pop frame %d\n\t",framesize);
1691 
1692   if (framesize == 0) {
1693     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1694   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1695     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1696     st->print("add  sp, sp, #%d\n\t", framesize);
1697   } else {
1698     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1699     st->print("add  sp, sp, rscratch1\n\t");
1700     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1701   }
1702 
1703   if (do_polling() && C->is_method_compilation()) {
1704     st->print("# touch polling page\n\t");
1705     st->print("ldr rscratch1, [rthread],#polling_page_offset\n\t");
1706     st->print("ldr zr, [rscratch1]");
1707   }
1708 }
1709 #endif
1710 
1711 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1712   Compile* C = ra_->C;
1713   C2_MacroAssembler _masm(&cbuf);
1714   int framesize = C->output()->frame_slots() << LogBytesPerInt;
1715 
1716   __ remove_frame(framesize);
1717 
1718   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1719     __ reserved_stack_check();
1720   }
1721 
1722   if (do_polling() && C->is_method_compilation()) {
1723     __ fetch_and_read_polling_page(rscratch1, relocInfo::poll_return_type);
1724   }
1725 }
1726 
1727 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1728   // Variable size. Determine dynamically.
1729   return MachNode::size(ra_);
1730 }
1731 
1732 int MachEpilogNode::reloc() const {
1733   // Return number of relocatable values contained in this instruction.
1734   return 1; // 1 for polling page.
1735 }
1736 
1737 const Pipeline * MachEpilogNode::pipeline() const {
1738   return MachNode::pipeline_class();
1739 }
1740 
1741 //=============================================================================
1742 
1743 // Figure out which register class each belongs in: rc_int, rc_float or
1744 // rc_stack.
1745 enum RC { rc_bad, rc_int, rc_float, rc_stack };
1746 
1747 static enum RC rc_class(OptoReg::Name reg) {
1748 
1749   if (reg == OptoReg::Bad) {
1750     return rc_bad;
1751   }
1752 
1753   // we have 30 int registers * 2 halves
1754   // (rscratch1 and rscratch2 are omitted)
1755   int slots_of_int_registers = RegisterImpl::max_slots_per_register * (RegisterImpl::number_of_registers - 2);
1756 
1757   if (reg < slots_of_int_registers) {
1758     return rc_int;
1759   }
1760 
1761   // we have 32 float register * 4 halves
1762   if (reg < slots_of_int_registers + FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers) {
1763     return rc_float;
1764   }
1765 
1766   // Between float regs & stack is the flags regs.
1767   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1768 
1769   return rc_stack;
1770 }
1771 
1772 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1773   Compile* C = ra_->C;
1774 
1775   // Get registers to move.
1776   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1777   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1778   OptoReg::Name dst_hi = ra_->get_reg_second(this);
1779   OptoReg::Name dst_lo = ra_->get_reg_first(this);
1780 
1781   enum RC src_hi_rc = rc_class(src_hi);
1782   enum RC src_lo_rc = rc_class(src_lo);
1783   enum RC dst_hi_rc = rc_class(dst_hi);
1784   enum RC dst_lo_rc = rc_class(dst_lo);
1785 
1786   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1787 
1788   if (src_hi != OptoReg::Bad) {
1789     assert((src_lo&1)==0 && src_lo+1==src_hi &&
1790            (dst_lo&1)==0 && dst_lo+1==dst_hi,
1791            "expected aligned-adjacent pairs");
1792   }
1793 
1794   if (src_lo == dst_lo && src_hi == dst_hi) {
1795     return 0;            // Self copy, no move.
1796   }
1797 
1798   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1799               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1800   int src_offset = ra_->reg2offset(src_lo);
1801   int dst_offset = ra_->reg2offset(dst_lo);
1802 
1803   if (bottom_type()->isa_vect() != NULL) {
1804     uint ireg = ideal_reg();
1805     assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1806     if (cbuf) {
1807       C2_MacroAssembler _masm(cbuf);
1808       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1809       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1810         // stack->stack
1811         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1812         if (ireg == Op_VecD) {
1813           __ unspill(rscratch1, true, src_offset);
1814           __ spill(rscratch1, true, dst_offset);
1815         } else {
1816           __ spill_copy128(src_offset, dst_offset);
1817         }
1818       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1819         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1820                ireg == Op_VecD ? __ T8B : __ T16B,
1821                as_FloatRegister(Matcher::_regEncode[src_lo]));
1822       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1823         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1824                        ireg == Op_VecD ? __ D : __ Q,
1825                        ra_->reg2offset(dst_lo));
1826       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1827         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1828                        ireg == Op_VecD ? __ D : __ Q,
1829                        ra_->reg2offset(src_lo));
1830       } else {
1831         ShouldNotReachHere();
1832       }
1833     }
1834   } else if (cbuf) {
1835     C2_MacroAssembler _masm(cbuf);
1836     switch (src_lo_rc) {
1837     case rc_int:
1838       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
1839         if (is64) {
1840             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1841                    as_Register(Matcher::_regEncode[src_lo]));
1842         } else {
1843             C2_MacroAssembler _masm(cbuf);
1844             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1845                     as_Register(Matcher::_regEncode[src_lo]));
1846         }
1847       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1848         if (is64) {
1849             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1850                      as_Register(Matcher::_regEncode[src_lo]));
1851         } else {
1852             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1853                      as_Register(Matcher::_regEncode[src_lo]));
1854         }
1855       } else {                    // gpr --> stack spill
1856         assert(dst_lo_rc == rc_stack, "spill to bad register class");
1857         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
1858       }
1859       break;
1860     case rc_float:
1861       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
1862         if (is64) {
1863             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
1864                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1865         } else {
1866             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
1867                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1868         }
1869       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
1870           if (cbuf) {
1871             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1872                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1873         } else {
1874             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1875                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1876         }
1877       } else {                    // fpr --> stack spill
1878         assert(dst_lo_rc == rc_stack, "spill to bad register class");
1879         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1880                  is64 ? __ D : __ S, dst_offset);
1881       }
1882       break;
1883     case rc_stack:
1884       if (dst_lo_rc == rc_int) {  // stack --> gpr load
1885         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
1886       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
1887         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1888                    is64 ? __ D : __ S, src_offset);
1889       } else {                    // stack --> stack copy
1890         assert(dst_lo_rc == rc_stack, "spill to bad register class");
1891         __ unspill(rscratch1, is64, src_offset);
1892         __ spill(rscratch1, is64, dst_offset);
1893       }
1894       break;
1895     default:
1896       assert(false, "bad rc_class for spill");
1897       ShouldNotReachHere();
1898     }
1899   }
1900 
1901   if (st) {
1902     st->print("spill ");
1903     if (src_lo_rc == rc_stack) {
1904       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
1905     } else {
1906       st->print("%s -> ", Matcher::regName[src_lo]);
1907     }
1908     if (dst_lo_rc == rc_stack) {
1909       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
1910     } else {
1911       st->print("%s", Matcher::regName[dst_lo]);
1912     }
1913     if (bottom_type()->isa_vect() != NULL) {
1914       st->print("\t# vector spill size = %d", ideal_reg()==Op_VecD ? 64:128);
1915     } else {
1916       st->print("\t# spill size = %d", is64 ? 64:32);
1917     }
1918   }
1919 
1920   return 0;
1921 
1922 }
1923 
1924 #ifndef PRODUCT
1925 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1926   if (!ra_)
1927     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
1928   else
1929     implementation(NULL, ra_, false, st);
1930 }
1931 #endif
1932 
1933 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1934   implementation(&cbuf, ra_, false, NULL);
1935 }
1936 
1937 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1938   return MachNode::size(ra_);
1939 }
1940 
1941 //=============================================================================
1942 
1943 #ifndef PRODUCT
1944 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1945   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1946   int reg = ra_->get_reg_first(this);
1947   st->print("add %s, rsp, #%d]\t# box lock",
1948             Matcher::regName[reg], offset);
1949 }
1950 #endif
1951 
1952 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1953   C2_MacroAssembler _masm(&cbuf);
1954 
1955   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1956   int reg    = ra_->get_encode(this);
1957 
1958   // This add will handle any 24-bit signed offset. 24 bits allows an
1959   // 8 megabyte stack frame.
1960   __ add(as_Register(reg), sp, offset);
1961 }
1962 
1963 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
1964   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
1965   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1966 
1967   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
1968     return NativeInstruction::instruction_size;
1969   } else {
1970     return 2 * NativeInstruction::instruction_size;
1971   }
1972 }
1973 
1974 //=============================================================================
1975 
1976 #ifndef PRODUCT
1977 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1978 {
1979   st->print_cr("# MachUEPNode");
1980   if (UseCompressedClassPointers) {
1981     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1982     if (CompressedKlassPointers::shift() != 0) {
1983       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1984     }
1985   } else {
1986    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1987   }
1988   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
1989   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
1990 }
1991 #endif
1992 
1993 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1994 {
1995   // This is the unverified entry point.
1996   C2_MacroAssembler _masm(&cbuf);
1997 
1998   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
1999   Label skip;
2000   // TODO
2001   // can we avoid this skip and still use a reloc?
2002   __ br(Assembler::EQ, skip);
2003   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2004   __ bind(skip);
2005 }
2006 
2007 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2008 {
2009   return MachNode::size(ra_);
2010 }
2011 
2012 // REQUIRED EMIT CODE
2013 
2014 //=============================================================================
2015 
2016 // Emit exception handler code.
2017 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2018 {
2019   // mov rscratch1 #exception_blob_entry_point
2020   // br rscratch1
2021   // Note that the code buffer's insts_mark is always relative to insts.
2022   // That's why we must use the macroassembler to generate a handler.
2023   C2_MacroAssembler _masm(&cbuf);
2024   address base = __ start_a_stub(size_exception_handler());
2025   if (base == NULL) {
2026     ciEnv::current()->record_failure("CodeCache is full");
2027     return 0;  // CodeBuffer::expand failed
2028   }
2029   int offset = __ offset();
2030   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2031   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2032   __ end_a_stub();
2033   return offset;
2034 }
2035 
2036 // Emit deopt handler code.
2037 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2038 {
2039   // Note that the code buffer's insts_mark is always relative to insts.
2040   // That's why we must use the macroassembler to generate a handler.
2041   C2_MacroAssembler _masm(&cbuf);
2042   address base = __ start_a_stub(size_deopt_handler());
2043   if (base == NULL) {
2044     ciEnv::current()->record_failure("CodeCache is full");
2045     return 0;  // CodeBuffer::expand failed
2046   }
2047   int offset = __ offset();
2048 
2049   __ adr(lr, __ pc());
2050   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2051 
2052   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
2053   __ end_a_stub();
2054   return offset;
2055 }
2056 
2057 // REQUIRED MATCHER CODE
2058 
2059 //=============================================================================
2060 
2061 const bool Matcher::match_rule_supported(int opcode) {
2062   if (!has_match_rule(opcode))
2063     return false;
2064 
2065   bool ret_value = true;
2066   switch (opcode) {
2067     case Op_CacheWB:
2068     case Op_CacheWBPreSync:
2069     case Op_CacheWBPostSync:
2070       if (!VM_Version::supports_data_cache_line_flush()) {
2071         ret_value = false;
2072       }
2073       break;
2074   }
2075 
2076   return ret_value; // Per default match rules are supported.
2077 }
2078 
2079 // Identify extra cases that we might want to provide match rules for vector nodes and
2080 // other intrinsics guarded with vector length (vlen) and element type (bt).
2081 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
2082   if (!match_rule_supported(opcode)) {
2083     return false;
2084   }
2085 
2086   // Special cases which require vector length
2087   switch (opcode) {
2088     case Op_MulAddVS2VI: {
2089       if (vlen != 4) {
2090         return false;
2091       }
2092       break;
2093     }
2094   }
2095 
2096   return true; // Per default match rules are supported.
2097 }
2098 
2099 const bool Matcher::has_predicated_vectors(void) {
2100   return false;
2101 }
2102 
2103 const int Matcher::float_pressure(int default_pressure_threshold) {
2104   return default_pressure_threshold;
2105 }
2106 
2107 int Matcher::regnum_to_fpu_offset(int regnum)
2108 {
2109   Unimplemented();
2110   return 0;
2111 }
2112 
2113 // Is this branch offset short enough that a short branch can be used?
2114 //
2115 // NOTE: If the platform does not provide any short branch variants, then
2116 //       this method should return false for offset 0.
2117 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2118   // The passed offset is relative to address of the branch.
2119 
2120   return (-32768 <= offset && offset < 32768);
2121 }
2122 
2123 const bool Matcher::isSimpleConstant64(jlong value) {
2124   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
2125   // Probably always true, even if a temp register is required.
2126   return true;
2127 }
2128 
2129 // true just means we have fast l2f conversion
2130 const bool Matcher::convL2FSupported(void) {
2131   return true;
2132 }
2133 
2134 // Vector width in bytes.
2135 const int Matcher::vector_width_in_bytes(BasicType bt) {
2136   int size = MIN2(16,(int)MaxVectorSize);
2137   // Minimum 2 values in vector
2138   if (size < 2*type2aelembytes(bt)) size = 0;
2139   // But never < 4
2140   if (size < 4) size = 0;
2141   return size;
2142 }
2143 
2144 // Limits on vector size (number of elements) loaded into vector.
2145 const int Matcher::max_vector_size(const BasicType bt) {
2146   return vector_width_in_bytes(bt)/type2aelembytes(bt);
2147 }
2148 const int Matcher::min_vector_size(const BasicType bt) {
2149 //  For the moment limit the vector size to 8 bytes
2150     int size = 8 / type2aelembytes(bt);
2151     if (size < 2) size = 2;
2152     return size;
2153 }
2154 
2155 // Vector ideal reg.
2156 const uint Matcher::vector_ideal_reg(int len) {
2157   switch(len) {
2158     case  8: return Op_VecD;
2159     case 16: return Op_VecX;
2160   }
2161   ShouldNotReachHere();
2162   return 0;
2163 }
2164 
2165 // AES support not yet implemented
2166 const bool Matcher::pass_original_key_for_aes() {
2167   return false;
2168 }
2169 
2170 // aarch64 supports misaligned vectors store/load.
2171 const bool Matcher::misaligned_vectors_ok() {
2172   return true;
2173 }
2174 
2175 // false => size gets scaled to BytesPerLong, ok.
2176 const bool Matcher::init_array_count_is_in_bytes = false;
2177 
2178 // Use conditional move (CMOVL)
2179 const int Matcher::long_cmove_cost() {
2180   // long cmoves are no more expensive than int cmoves
2181   return 0;
2182 }
2183 
2184 const int Matcher::float_cmove_cost() {
2185   // float cmoves are no more expensive than int cmoves
2186   return 0;
2187 }
2188 
2189 // Does the CPU require late expand (see block.cpp for description of late expand)?
2190 const bool Matcher::require_postalloc_expand = false;
2191 
2192 // Do we need to mask the count passed to shift instructions or does
2193 // the cpu only look at the lower 5/6 bits anyway?
2194 const bool Matcher::need_masked_shift_count = false;
2195 
2196 // No support for generic vector operands.
2197 const bool Matcher::supports_generic_vector_operands  = false;
2198 
2199 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
2200   ShouldNotReachHere(); // generic vector operands not supported
2201   return NULL;
2202 }
2203 
2204 bool Matcher::is_generic_reg2reg_move(MachNode* m) {
2205   ShouldNotReachHere();  // generic vector operands not supported
2206   return false;
2207 }
2208 
2209 bool Matcher::is_generic_vector(MachOper* opnd)  {
2210   ShouldNotReachHere();  // generic vector operands not supported
2211   return false;
2212 }
2213 
2214 // This affects two different things:
2215 //  - how Decode nodes are matched
2216 //  - how ImplicitNullCheck opportunities are recognized
2217 // If true, the matcher will try to remove all Decodes and match them
2218 // (as operands) into nodes. NullChecks are not prepared to deal with
2219 // Decodes by final_graph_reshaping().
2220 // If false, final_graph_reshaping() forces the decode behind the Cmp
2221 // for a NullCheck. The matcher matches the Decode node into a register.
2222 // Implicit_null_check optimization moves the Decode along with the
2223 // memory operation back up before the NullCheck.
2224 bool Matcher::narrow_oop_use_complex_address() {
2225   return CompressedOops::shift() == 0;
2226 }
2227 
2228 bool Matcher::narrow_klass_use_complex_address() {
2229 // TODO
2230 // decide whether we need to set this to true
2231   return false;
2232 }
2233 
2234 bool Matcher::const_oop_prefer_decode() {
2235   // Prefer ConN+DecodeN over ConP in simple compressed oops mode.
2236   return CompressedOops::base() == NULL;
2237 }
2238 
2239 bool Matcher::const_klass_prefer_decode() {
2240   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
2241   return CompressedKlassPointers::base() == NULL;
2242 }
2243 
2244 // Is it better to copy float constants, or load them directly from
2245 // memory?  Intel can load a float constant from a direct address,
2246 // requiring no extra registers.  Most RISCs will have to materialize
2247 // an address into a register first, so they would do better to copy
2248 // the constant from stack.
2249 const bool Matcher::rematerialize_float_constants = false;
2250 
2251 // If CPU can load and store mis-aligned doubles directly then no
2252 // fixup is needed.  Else we split the double into 2 integer pieces
2253 // and move it piece-by-piece.  Only happens when passing doubles into
2254 // C code as the Java calling convention forces doubles to be aligned.
2255 const bool Matcher::misaligned_doubles_ok = true;
2256 
2257 // No-op on amd64
2258 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
2259   Unimplemented();
2260 }
2261 
2262 // Advertise here if the CPU requires explicit rounding operations to implement strictfp mode.
2263 const bool Matcher::strict_fp_requires_explicit_rounding = false;
2264 
2265 // Are floats converted to double when stored to stack during
2266 // deoptimization?
2267 bool Matcher::float_in_double() { return false; }
2268 
2269 // Do ints take an entire long register or just half?
2270 // The relevant question is how the int is callee-saved:
2271 // the whole long is written but de-opt'ing will have to extract
2272 // the relevant 32 bits.
2273 const bool Matcher::int_in_long = true;
2274 
2275 // Return whether or not this register is ever used as an argument.
2276 // This function is used on startup to build the trampoline stubs in
2277 // generateOptoStub.  Registers not mentioned will be killed by the VM
2278 // call in the trampoline, and arguments in those registers not be
2279 // available to the callee.
2280 bool Matcher::can_be_java_arg(int reg)
2281 {
2282   return
2283     reg ==  R0_num || reg == R0_H_num ||
2284     reg ==  R1_num || reg == R1_H_num ||
2285     reg ==  R2_num || reg == R2_H_num ||
2286     reg ==  R3_num || reg == R3_H_num ||
2287     reg ==  R4_num || reg == R4_H_num ||
2288     reg ==  R5_num || reg == R5_H_num ||
2289     reg ==  R6_num || reg == R6_H_num ||
2290     reg ==  R7_num || reg == R7_H_num ||
2291     reg ==  V0_num || reg == V0_H_num ||
2292     reg ==  V1_num || reg == V1_H_num ||
2293     reg ==  V2_num || reg == V2_H_num ||
2294     reg ==  V3_num || reg == V3_H_num ||
2295     reg ==  V4_num || reg == V4_H_num ||
2296     reg ==  V5_num || reg == V5_H_num ||
2297     reg ==  V6_num || reg == V6_H_num ||
2298     reg ==  V7_num || reg == V7_H_num;
2299 }
2300 
2301 bool Matcher::is_spillable_arg(int reg)
2302 {
2303   return can_be_java_arg(reg);
2304 }
2305 
2306 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2307   return false;
2308 }
2309 
2310 RegMask Matcher::divI_proj_mask() {
2311   ShouldNotReachHere();
2312   return RegMask();
2313 }
2314 
2315 // Register for MODI projection of divmodI.
2316 RegMask Matcher::modI_proj_mask() {
2317   ShouldNotReachHere();
2318   return RegMask();
2319 }
2320 
2321 // Register for DIVL projection of divmodL.
2322 RegMask Matcher::divL_proj_mask() {
2323   ShouldNotReachHere();
2324   return RegMask();
2325 }
2326 
2327 // Register for MODL projection of divmodL.
2328 RegMask Matcher::modL_proj_mask() {
2329   ShouldNotReachHere();
2330   return RegMask();
2331 }
2332 
2333 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2334   return FP_REG_mask();
2335 }
2336 
2337 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2338   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2339     Node* u = addp->fast_out(i);
2340     if (u->is_Mem()) {
2341       int opsize = u->as_Mem()->memory_size();
2342       assert(opsize > 0, "unexpected memory operand size");
2343       if (u->as_Mem()->memory_size() != (1<<shift)) {
2344         return false;
2345       }
2346     }
2347   }
2348   return true;
2349 }
2350 
2351 const bool Matcher::convi2l_type_required = false;
2352 
2353 // Should the matcher clone input 'm' of node 'n'?
2354 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2355   if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
2356     mstack.push(m, Visit);           // m = ShiftCntV
2357     return true;
2358   }
2359   return false;
2360 }
2361 
2362 // Should the Matcher clone shifts on addressing modes, expecting them
2363 // to be subsumed into complex addressing expressions or compute them
2364 // into registers?
2365 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2366   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2367     return true;
2368   }
2369 
2370   Node *off = m->in(AddPNode::Offset);
2371   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2372       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2373       // Are there other uses besides address expressions?
2374       !is_visited(off)) {
2375     address_visited.set(off->_idx); // Flag as address_visited
2376     mstack.push(off->in(2), Visit);
2377     Node *conv = off->in(1);
2378     if (conv->Opcode() == Op_ConvI2L &&
2379         // Are there other uses besides address expressions?
2380         !is_visited(conv)) {
2381       address_visited.set(conv->_idx); // Flag as address_visited
2382       mstack.push(conv->in(1), Pre_Visit);
2383     } else {
2384       mstack.push(conv, Pre_Visit);
2385     }
2386     address_visited.test_set(m->_idx); // Flag as address_visited
2387     mstack.push(m->in(AddPNode::Address), Pre_Visit);
2388     mstack.push(m->in(AddPNode::Base), Pre_Visit);
2389     return true;
2390   } else if (off->Opcode() == Op_ConvI2L &&
2391              // Are there other uses besides address expressions?
2392              !is_visited(off)) {
2393     address_visited.test_set(m->_idx); // Flag as address_visited
2394     address_visited.set(off->_idx); // Flag as address_visited
2395     mstack.push(off->in(1), Pre_Visit);
2396     mstack.push(m->in(AddPNode::Address), Pre_Visit);
2397     mstack.push(m->in(AddPNode::Base), Pre_Visit);
2398     return true;
2399   }
2400   return false;
2401 }
2402 
2403 void Compile::reshape_address(AddPNode* addp) {
2404 }
2405 
2406 
2407 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
2408   C2_MacroAssembler _masm(&cbuf);                                       \
2409   {                                                                     \
2410     guarantee(INDEX == -1, "mode not permitted for volatile");          \
2411     guarantee(DISP == 0, "mode not permitted for volatile");            \
2412     guarantee(SCALE == 0, "mode not permitted for volatile");           \
2413     __ INSN(REG, as_Register(BASE));                                    \
2414   }
2415 
2416 
2417 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2418   {
2419     Address::extend scale;
2420 
2421     // Hooboy, this is fugly.  We need a way to communicate to the
2422     // encoder that the index needs to be sign extended, so we have to
2423     // enumerate all the cases.
2424     switch (opcode) {
2425     case INDINDEXSCALEDI2L:
2426     case INDINDEXSCALEDI2LN:
2427     case INDINDEXI2L:
2428     case INDINDEXI2LN:
2429       scale = Address::sxtw(size);
2430       break;
2431     default:
2432       scale = Address::lsl(size);
2433     }
2434 
2435     if (index == -1) {
2436       return Address(base, disp);
2437     } else {
2438       assert(disp == 0, "unsupported address mode: disp = %d", disp);
2439       return Address(base, as_Register(index), scale);
2440     }
2441   }
2442 
2443 
2444 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2445 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2446 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2447 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2448                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
2449 
2450   // Used for all non-volatile memory accesses.  The use of
2451   // $mem->opcode() to discover whether this pattern uses sign-extended
2452   // offsets is something of a kludge.
2453   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2454                         Register reg, int opcode,
2455                         Register base, int index, int scale, int disp,
2456                         int size_in_memory)
2457   {
2458     Address addr = mem2address(opcode, base, index, scale, disp);
2459     if (addr.getMode() == Address::base_plus_offset) {
2460       /* If we get an out-of-range offset it is a bug in the compiler,
2461          so we assert here. */
2462       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2463              "c2 compiler bug");
2464       /* Fix up any out-of-range offsets. */
2465       assert_different_registers(rscratch1, base);
2466       assert_different_registers(rscratch1, reg);
2467       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2468     }
2469     (masm.*insn)(reg, addr);
2470   }
2471 
2472   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2473                         FloatRegister reg, int opcode,
2474                         Register base, int index, int size, int disp,
2475                         int size_in_memory)
2476   {
2477     Address::extend scale;
2478 
2479     switch (opcode) {
2480     case INDINDEXSCALEDI2L:
2481     case INDINDEXSCALEDI2LN:
2482       scale = Address::sxtw(size);
2483       break;
2484     default:
2485       scale = Address::lsl(size);
2486     }
2487 
2488     if (index == -1) {
2489       /* If we get an out-of-range offset it is a bug in the compiler,
2490          so we assert here. */
2491       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2492       /* Fix up any out-of-range offsets. */
2493       assert_different_registers(rscratch1, base);
2494       Address addr = Address(base, disp);
2495       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2496       (masm.*insn)(reg, addr);
2497     } else {
2498       assert(disp == 0, "unsupported address mode: disp = %d", disp);
2499       (masm.*insn)(reg, Address(base, as_Register(index), scale));
2500     }
2501   }
2502 
2503   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2504                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2505                         int opcode, Register base, int index, int size, int disp)
2506   {
2507     if (index == -1) {
2508       (masm.*insn)(reg, T, Address(base, disp));
2509     } else {
2510       assert(disp == 0, "unsupported address mode");
2511       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2512     }
2513   }
2514 
2515 %}
2516 
2517 
2518 
2519 //----------ENCODING BLOCK-----------------------------------------------------
2520 // This block specifies the encoding classes used by the compiler to
2521 // output byte streams.  Encoding classes are parameterized macros
2522 // used by Machine Instruction Nodes in order to generate the bit
2523 // encoding of the instruction.  Operands specify their base encoding
2524 // interface with the interface keyword.  There are currently
2525 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2526 // COND_INTER.  REG_INTER causes an operand to generate a function
2527 // which returns its register number when queried.  CONST_INTER causes
2528 // an operand to generate a function which returns the value of the
2529 // constant when queried.  MEMORY_INTER causes an operand to generate
2530 // four functions which return the Base Register, the Index Register,
2531 // the Scale Value, and the Offset Value of the operand when queried.
2532 // COND_INTER causes an operand to generate six functions which return
2533 // the encoding code (ie - encoding bits for the instruction)
2534 // associated with each basic boolean condition for a conditional
2535 // instruction.
2536 //
2537 // Instructions specify two basic values for encoding.  Again, a
2538 // function is available to check if the constant displacement is an
2539 // oop. They use the ins_encode keyword to specify their encoding
2540 // classes (which must be a sequence of enc_class names, and their
2541 // parameters, specified in the encoding block), and they use the
2542 // opcode keyword to specify, in order, their primary, secondary, and
2543 // tertiary opcode.  Only the opcode sections which a particular
2544 // instruction needs for encoding need to be specified.
2545 encode %{
2546   // Build emit functions for each basic byte or larger field in the
2547   // intel encoding scheme (opcode, rm, sib, immediate), and call them
2548   // from C++ code in the enc_class source block.  Emit functions will
2549   // live in the main source block for now.  In future, we can
2550   // generalize this by adding a syntax that specifies the sizes of
2551   // fields in an order, so that the adlc can build the emit functions
2552   // automagically
2553 
2554   // catch all for unimplemented encodings
2555   enc_class enc_unimplemented %{
2556     C2_MacroAssembler _masm(&cbuf);
2557     __ unimplemented("C2 catch all");
2558   %}
2559 
2560   // BEGIN Non-volatile memory access
2561 
2562   // This encoding class is generated automatically from ad_encode.m4.
2563   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2564   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2565     Register dst_reg = as_Register($dst$$reg);
2566     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2567                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2568   %}
2569 
2570   // This encoding class is generated automatically from ad_encode.m4.
2571   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2572   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2573     Register dst_reg = as_Register($dst$$reg);
2574     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2575                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2576   %}
2577 
2578   // This encoding class is generated automatically from ad_encode.m4.
2579   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2580   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2581     Register dst_reg = as_Register($dst$$reg);
2582     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2583                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2584   %}
2585 
2586   // This encoding class is generated automatically from ad_encode.m4.
2587   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2588   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2589     Register dst_reg = as_Register($dst$$reg);
2590     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2591                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2592   %}
2593 
2594   // This encoding class is generated automatically from ad_encode.m4.
2595   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2596   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2597     Register dst_reg = as_Register($dst$$reg);
2598     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2599                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2600   %}
2601 
2602   // This encoding class is generated automatically from ad_encode.m4.
2603   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2604   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2605     Register dst_reg = as_Register($dst$$reg);
2606     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2607                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2608   %}
2609 
2610   // This encoding class is generated automatically from ad_encode.m4.
2611   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2612   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2613     Register dst_reg = as_Register($dst$$reg);
2614     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2615                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2616   %}
2617 
2618   // This encoding class is generated automatically from ad_encode.m4.
2619   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2620   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2621     Register dst_reg = as_Register($dst$$reg);
2622     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2623                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2624   %}
2625 
2626   // This encoding class is generated automatically from ad_encode.m4.
2627   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2628   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2629     Register dst_reg = as_Register($dst$$reg);
2630     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2631                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2632   %}
2633 
2634   // This encoding class is generated automatically from ad_encode.m4.
2635   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2636   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2637     Register dst_reg = as_Register($dst$$reg);
2638     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2639                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2640   %}
2641 
2642   // This encoding class is generated automatically from ad_encode.m4.
2643   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2644   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2645     Register dst_reg = as_Register($dst$$reg);
2646     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2647                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2648   %}
2649 
2650   // This encoding class is generated automatically from ad_encode.m4.
2651   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2652   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2653     Register dst_reg = as_Register($dst$$reg);
2654     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2655                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2656   %}
2657 
2658   // This encoding class is generated automatically from ad_encode.m4.
2659   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2660   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2661     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2662     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2663                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2664   %}
2665 
2666   // This encoding class is generated automatically from ad_encode.m4.
2667   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2668   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2669     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2670     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2671                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2672   %}
2673 
2674   // This encoding class is generated automatically from ad_encode.m4.
2675   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2676   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2677     Register src_reg = as_Register($src$$reg);
2678     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2679                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2680   %}
2681 
2682   // This encoding class is generated automatically from ad_encode.m4.
2683   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2684   enc_class aarch64_enc_strb0(memory1 mem) %{
2685     C2_MacroAssembler _masm(&cbuf);
2686     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2687                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2688   %}
2689 
2690   // This encoding class is generated automatically from ad_encode.m4.
2691   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2692   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2693     Register src_reg = as_Register($src$$reg);
2694     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2695                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2696   %}
2697 
2698   // This encoding class is generated automatically from ad_encode.m4.
2699   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2700   enc_class aarch64_enc_strh0(memory2 mem) %{
2701     C2_MacroAssembler _masm(&cbuf);
2702     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2703                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2704   %}
2705 
2706   // This encoding class is generated automatically from ad_encode.m4.
2707   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2708   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2709     Register src_reg = as_Register($src$$reg);
2710     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2711                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2712   %}
2713 
2714   // This encoding class is generated automatically from ad_encode.m4.
2715   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2716   enc_class aarch64_enc_strw0(memory4 mem) %{
2717     C2_MacroAssembler _masm(&cbuf);
2718     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2719                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2720   %}
2721 
2722   // This encoding class is generated automatically from ad_encode.m4.
2723   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2724   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2725     Register src_reg = as_Register($src$$reg);
2726     // we sometimes get asked to store the stack pointer into the
2727     // current thread -- we cannot do that directly on AArch64
2728     if (src_reg == r31_sp) {
2729       C2_MacroAssembler _masm(&cbuf);
2730       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2731       __ mov(rscratch2, sp);
2732       src_reg = rscratch2;
2733     }
2734     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2735                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2736   %}
2737 
2738   // This encoding class is generated automatically from ad_encode.m4.
2739   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2740   enc_class aarch64_enc_str0(memory8 mem) %{
2741     C2_MacroAssembler _masm(&cbuf);
2742     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2743                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2744   %}
2745 
2746   // This encoding class is generated automatically from ad_encode.m4.
2747   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2748   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
2749     FloatRegister src_reg = as_FloatRegister($src$$reg);
2750     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2751                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2752   %}
2753 
2754   // This encoding class is generated automatically from ad_encode.m4.
2755   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2756   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
2757     FloatRegister src_reg = as_FloatRegister($src$$reg);
2758     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2759                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2760   %}
2761 
2762   // This encoding class is generated automatically from ad_encode.m4.
2763   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2764   enc_class aarch64_enc_strw_immn(immN src, memory1 mem) %{
2765     C2_MacroAssembler _masm(&cbuf);
2766     address con = (address)$src$$constant;
2767     // need to do this the hard way until we can manage relocs
2768     // for 32 bit constants
2769     __ movoop(rscratch2, (jobject)con);
2770     if (con) __ encode_heap_oop_not_null(rscratch2);
2771     loadStore(_masm, &MacroAssembler::strw, rscratch2, $mem->opcode(),
2772                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2773   %}
2774 
2775   // This encoding class is generated automatically from ad_encode.m4.
2776   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2777   enc_class aarch64_enc_strw_immnk(immN src, memory4 mem) %{
2778     C2_MacroAssembler _masm(&cbuf);
2779     address con = (address)$src$$constant;
2780     // need to do this the hard way until we can manage relocs
2781     // for 32 bit constants
2782     __ movoop(rscratch2, (jobject)con);
2783     __ encode_klass_not_null(rscratch2);
2784     loadStore(_masm, &MacroAssembler::strw, rscratch2, $mem->opcode(),
2785                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2786   %}
2787 
2788   // This encoding class is generated automatically from ad_encode.m4.
2789   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2790   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
2791       C2_MacroAssembler _masm(&cbuf);
2792       __ membar(Assembler::StoreStore);
2793       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2794                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2795   %}
2796 
2797   // END Non-volatile memory access
2798 
2799   // Vector loads and stores
2800   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
2801     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2802     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
2803        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2804   %}
2805 
2806   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
2807     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2808     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
2809        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2810   %}
2811 
2812   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
2813     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2814     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
2815        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2816   %}
2817 
2818   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
2819     FloatRegister src_reg = as_FloatRegister($src$$reg);
2820     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
2821        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2822   %}
2823 
2824   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
2825     FloatRegister src_reg = as_FloatRegister($src$$reg);
2826     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
2827        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2828   %}
2829 
2830   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
2831     FloatRegister src_reg = as_FloatRegister($src$$reg);
2832     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
2833        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2834   %}
2835 
2836   // volatile loads and stores
2837 
2838   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
2839     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2840                  rscratch1, stlrb);
2841   %}
2842 
2843   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
2844     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2845                  rscratch1, stlrh);
2846   %}
2847 
2848   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
2849     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2850                  rscratch1, stlrw);
2851   %}
2852 
2853 
2854   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
2855     Register dst_reg = as_Register($dst$$reg);
2856     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2857              rscratch1, ldarb);
2858     __ sxtbw(dst_reg, dst_reg);
2859   %}
2860 
2861   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
2862     Register dst_reg = as_Register($dst$$reg);
2863     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2864              rscratch1, ldarb);
2865     __ sxtb(dst_reg, dst_reg);
2866   %}
2867 
2868   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
2869     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2870              rscratch1, ldarb);
2871   %}
2872 
2873   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
2874     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2875              rscratch1, ldarb);
2876   %}
2877 
2878   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
2879     Register dst_reg = as_Register($dst$$reg);
2880     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2881              rscratch1, ldarh);
2882     __ sxthw(dst_reg, dst_reg);
2883   %}
2884 
2885   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
2886     Register dst_reg = as_Register($dst$$reg);
2887     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2888              rscratch1, ldarh);
2889     __ sxth(dst_reg, dst_reg);
2890   %}
2891 
2892   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
2893     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2894              rscratch1, ldarh);
2895   %}
2896 
2897   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
2898     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2899              rscratch1, ldarh);
2900   %}
2901 
2902   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
2903     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2904              rscratch1, ldarw);
2905   %}
2906 
2907   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
2908     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2909              rscratch1, ldarw);
2910   %}
2911 
2912   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
2913     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2914              rscratch1, ldar);
2915   %}
2916 
2917   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
2918     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2919              rscratch1, ldarw);
2920     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
2921   %}
2922 
2923   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
2924     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2925              rscratch1, ldar);
2926     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
2927   %}
2928 
2929   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
2930     Register src_reg = as_Register($src$$reg);
2931     // we sometimes get asked to store the stack pointer into the
2932     // current thread -- we cannot do that directly on AArch64
2933     if (src_reg == r31_sp) {
2934       C2_MacroAssembler _masm(&cbuf);
2935       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2936       __ mov(rscratch2, sp);
2937       src_reg = rscratch2;
2938     }
2939     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2940                  rscratch1, stlr);
2941   %}
2942 
2943   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
2944     {
2945       C2_MacroAssembler _masm(&cbuf);
2946       FloatRegister src_reg = as_FloatRegister($src$$reg);
2947       __ fmovs(rscratch2, src_reg);
2948     }
2949     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2950                  rscratch1, stlrw);
2951   %}
2952 
2953   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
2954     {
2955       C2_MacroAssembler _masm(&cbuf);
2956       FloatRegister src_reg = as_FloatRegister($src$$reg);
2957       __ fmovd(rscratch2, src_reg);
2958     }
2959     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2960                  rscratch1, stlr);
2961   %}
2962 
2963   // synchronized read/update encodings
2964 
2965   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
2966     C2_MacroAssembler _masm(&cbuf);
2967     Register dst_reg = as_Register($dst$$reg);
2968     Register base = as_Register($mem$$base);
2969     int index = $mem$$index;
2970     int scale = $mem$$scale;
2971     int disp = $mem$$disp;
2972     if (index == -1) {
2973        if (disp != 0) {
2974         __ lea(rscratch1, Address(base, disp));
2975         __ ldaxr(dst_reg, rscratch1);
2976       } else {
2977         // TODO
2978         // should we ever get anything other than this case?
2979         __ ldaxr(dst_reg, base);
2980       }
2981     } else {
2982       Register index_reg = as_Register(index);
2983       if (disp == 0) {
2984         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
2985         __ ldaxr(dst_reg, rscratch1);
2986       } else {
2987         __ lea(rscratch1, Address(base, disp));
2988         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
2989         __ ldaxr(dst_reg, rscratch1);
2990       }
2991     }
2992   %}
2993 
2994   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
2995     C2_MacroAssembler _masm(&cbuf);
2996     Register src_reg = as_Register($src$$reg);
2997     Register base = as_Register($mem$$base);
2998     int index = $mem$$index;
2999     int scale = $mem$$scale;
3000     int disp = $mem$$disp;
3001     if (index == -1) {
3002        if (disp != 0) {
3003         __ lea(rscratch2, Address(base, disp));
3004         __ stlxr(rscratch1, src_reg, rscratch2);
3005       } else {
3006         // TODO
3007         // should we ever get anything other than this case?
3008         __ stlxr(rscratch1, src_reg, base);
3009       }
3010     } else {
3011       Register index_reg = as_Register(index);
3012       if (disp == 0) {
3013         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3014         __ stlxr(rscratch1, src_reg, rscratch2);
3015       } else {
3016         __ lea(rscratch2, Address(base, disp));
3017         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3018         __ stlxr(rscratch1, src_reg, rscratch2);
3019       }
3020     }
3021     __ cmpw(rscratch1, zr);
3022   %}
3023 
3024   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3025     C2_MacroAssembler _masm(&cbuf);
3026     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3027     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3028                Assembler::xword, /*acquire*/ false, /*release*/ true,
3029                /*weak*/ false, noreg);
3030   %}
3031 
3032   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3033     C2_MacroAssembler _masm(&cbuf);
3034     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3035     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3036                Assembler::word, /*acquire*/ false, /*release*/ true,
3037                /*weak*/ false, noreg);
3038   %}
3039 
3040   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3041     C2_MacroAssembler _masm(&cbuf);
3042     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3043     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3044                Assembler::halfword, /*acquire*/ false, /*release*/ true,
3045                /*weak*/ false, noreg);
3046   %}
3047 
3048   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3049     C2_MacroAssembler _masm(&cbuf);
3050     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3051     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3052                Assembler::byte, /*acquire*/ false, /*release*/ true,
3053                /*weak*/ false, noreg);
3054   %}
3055 
3056 
3057   // The only difference between aarch64_enc_cmpxchg and
3058   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3059   // CompareAndSwap sequence to serve as a barrier on acquiring a
3060   // lock.
3061   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3062     C2_MacroAssembler _masm(&cbuf);
3063     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3064     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3065                Assembler::xword, /*acquire*/ true, /*release*/ true,
3066                /*weak*/ false, noreg);
3067   %}
3068 
3069   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3070     C2_MacroAssembler _masm(&cbuf);
3071     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3072     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3073                Assembler::word, /*acquire*/ true, /*release*/ true,
3074                /*weak*/ false, noreg);
3075   %}
3076 
3077   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3078     C2_MacroAssembler _masm(&cbuf);
3079     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3080     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3081                Assembler::halfword, /*acquire*/ true, /*release*/ true,
3082                /*weak*/ false, noreg);
3083   %}
3084 
3085   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3086     C2_MacroAssembler _masm(&cbuf);
3087     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3088     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3089                Assembler::byte, /*acquire*/ true, /*release*/ true,
3090                /*weak*/ false, noreg);
3091   %}
3092 
3093   // auxiliary used for CompareAndSwapX to set result register
3094   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3095     C2_MacroAssembler _masm(&cbuf);
3096     Register res_reg = as_Register($res$$reg);
3097     __ cset(res_reg, Assembler::EQ);
3098   %}
3099 
3100   // prefetch encodings
3101 
3102   enc_class aarch64_enc_prefetchw(memory mem) %{
3103     C2_MacroAssembler _masm(&cbuf);
3104     Register base = as_Register($mem$$base);
3105     int index = $mem$$index;
3106     int scale = $mem$$scale;
3107     int disp = $mem$$disp;
3108     if (index == -1) {
3109       __ prfm(Address(base, disp), PSTL1KEEP);
3110     } else {
3111       Register index_reg = as_Register(index);
3112       if (disp == 0) {
3113         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3114       } else {
3115         __ lea(rscratch1, Address(base, disp));
3116         __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3117       }
3118     }
3119   %}
3120 
3121   /// mov envcodings
3122 
3123   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3124     C2_MacroAssembler _masm(&cbuf);
3125     uint32_t con = (uint32_t)$src$$constant;
3126     Register dst_reg = as_Register($dst$$reg);
3127     if (con == 0) {
3128       __ movw(dst_reg, zr);
3129     } else {
3130       __ movw(dst_reg, con);
3131     }
3132   %}
3133 
3134   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3135     C2_MacroAssembler _masm(&cbuf);
3136     Register dst_reg = as_Register($dst$$reg);
3137     uint64_t con = (uint64_t)$src$$constant;
3138     if (con == 0) {
3139       __ mov(dst_reg, zr);
3140     } else {
3141       __ mov(dst_reg, con);
3142     }
3143   %}
3144 
3145   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3146     C2_MacroAssembler _masm(&cbuf);
3147     Register dst_reg = as_Register($dst$$reg);
3148     address con = (address)$src$$constant;
3149     if (con == NULL || con == (address)1) {
3150       ShouldNotReachHere();
3151     } else {
3152       relocInfo::relocType rtype = $src->constant_reloc();
3153       if (rtype == relocInfo::oop_type) {
3154         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
3155       } else if (rtype == relocInfo::metadata_type) {
3156         __ mov_metadata(dst_reg, (Metadata*)con);
3157       } else {
3158         assert(rtype == relocInfo::none, "unexpected reloc type");
3159         if (con < (address)(uintptr_t)os::vm_page_size()) {
3160           __ mov(dst_reg, con);
3161         } else {
3162           uintptr_t offset;
3163           __ adrp(dst_reg, con, offset);
3164           __ add(dst_reg, dst_reg, offset);
3165         }
3166       }
3167     }
3168   %}
3169 
3170   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3171     C2_MacroAssembler _masm(&cbuf);
3172     Register dst_reg = as_Register($dst$$reg);
3173     __ mov(dst_reg, zr);
3174   %}
3175 
3176   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3177     C2_MacroAssembler _masm(&cbuf);
3178     Register dst_reg = as_Register($dst$$reg);
3179     __ mov(dst_reg, (uint64_t)1);
3180   %}
3181 
3182   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3183     C2_MacroAssembler _masm(&cbuf);
3184     __ load_byte_map_base($dst$$Register);
3185   %}
3186 
3187   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3188     C2_MacroAssembler _masm(&cbuf);
3189     Register dst_reg = as_Register($dst$$reg);
3190     address con = (address)$src$$constant;
3191     if (con == NULL) {
3192       ShouldNotReachHere();
3193     } else {
3194       relocInfo::relocType rtype = $src->constant_reloc();
3195       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3196       __ set_narrow_oop(dst_reg, (jobject)con);
3197     }
3198   %}
3199 
3200   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3201     C2_MacroAssembler _masm(&cbuf);
3202     Register dst_reg = as_Register($dst$$reg);
3203     __ mov(dst_reg, zr);
3204   %}
3205 
3206   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3207     C2_MacroAssembler _masm(&cbuf);
3208     Register dst_reg = as_Register($dst$$reg);
3209     address con = (address)$src$$constant;
3210     if (con == NULL) {
3211       ShouldNotReachHere();
3212     } else {
3213       relocInfo::relocType rtype = $src->constant_reloc();
3214       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3215       __ set_narrow_klass(dst_reg, (Klass *)con);
3216     }
3217   %}
3218 
3219   // arithmetic encodings
3220 
3221   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3222     C2_MacroAssembler _masm(&cbuf);
3223     Register dst_reg = as_Register($dst$$reg);
3224     Register src_reg = as_Register($src1$$reg);
3225     int32_t con = (int32_t)$src2$$constant;
3226     // add has primary == 0, subtract has primary == 1
3227     if ($primary) { con = -con; }
3228     if (con < 0) {
3229       __ subw(dst_reg, src_reg, -con);
3230     } else {
3231       __ addw(dst_reg, src_reg, con);
3232     }
3233   %}
3234 
3235   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3236     C2_MacroAssembler _masm(&cbuf);
3237     Register dst_reg = as_Register($dst$$reg);
3238     Register src_reg = as_Register($src1$$reg);
3239     int32_t con = (int32_t)$src2$$constant;
3240     // add has primary == 0, subtract has primary == 1
3241     if ($primary) { con = -con; }
3242     if (con < 0) {
3243       __ sub(dst_reg, src_reg, -con);
3244     } else {
3245       __ add(dst_reg, src_reg, con);
3246     }
3247   %}
3248 
3249   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3250     C2_MacroAssembler _masm(&cbuf);
3251    Register dst_reg = as_Register($dst$$reg);
3252    Register src1_reg = as_Register($src1$$reg);
3253    Register src2_reg = as_Register($src2$$reg);
3254     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3255   %}
3256 
3257   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3258     C2_MacroAssembler _masm(&cbuf);
3259    Register dst_reg = as_Register($dst$$reg);
3260    Register src1_reg = as_Register($src1$$reg);
3261    Register src2_reg = as_Register($src2$$reg);
3262     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3263   %}
3264 
3265   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3266     C2_MacroAssembler _masm(&cbuf);
3267    Register dst_reg = as_Register($dst$$reg);
3268    Register src1_reg = as_Register($src1$$reg);
3269    Register src2_reg = as_Register($src2$$reg);
3270     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3271   %}
3272 
3273   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3274     C2_MacroAssembler _masm(&cbuf);
3275    Register dst_reg = as_Register($dst$$reg);
3276    Register src1_reg = as_Register($src1$$reg);
3277    Register src2_reg = as_Register($src2$$reg);
3278     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3279   %}
3280 
3281   // compare instruction encodings
3282 
3283   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3284     C2_MacroAssembler _masm(&cbuf);
3285     Register reg1 = as_Register($src1$$reg);
3286     Register reg2 = as_Register($src2$$reg);
3287     __ cmpw(reg1, reg2);
3288   %}
3289 
3290   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3291     C2_MacroAssembler _masm(&cbuf);
3292     Register reg = as_Register($src1$$reg);
3293     int32_t val = $src2$$constant;
3294     if (val >= 0) {
3295       __ subsw(zr, reg, val);
3296     } else {
3297       __ addsw(zr, reg, -val);
3298     }
3299   %}
3300 
3301   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3302     C2_MacroAssembler _masm(&cbuf);
3303     Register reg1 = as_Register($src1$$reg);
3304     uint32_t val = (uint32_t)$src2$$constant;
3305     __ movw(rscratch1, val);
3306     __ cmpw(reg1, rscratch1);
3307   %}
3308 
3309   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3310     C2_MacroAssembler _masm(&cbuf);
3311     Register reg1 = as_Register($src1$$reg);
3312     Register reg2 = as_Register($src2$$reg);
3313     __ cmp(reg1, reg2);
3314   %}
3315 
3316   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3317     C2_MacroAssembler _masm(&cbuf);
3318     Register reg = as_Register($src1$$reg);
3319     int64_t val = $src2$$constant;
3320     if (val >= 0) {
3321       __ subs(zr, reg, val);
3322     } else if (val != -val) {
3323       __ adds(zr, reg, -val);
3324     } else {
3325     // aargh, Long.MIN_VALUE is a special case
3326       __ orr(rscratch1, zr, (uint64_t)val);
3327       __ subs(zr, reg, rscratch1);
3328     }
3329   %}
3330 
3331   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3332     C2_MacroAssembler _masm(&cbuf);
3333     Register reg1 = as_Register($src1$$reg);
3334     uint64_t val = (uint64_t)$src2$$constant;
3335     __ mov(rscratch1, val);
3336     __ cmp(reg1, rscratch1);
3337   %}
3338 
3339   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3340     C2_MacroAssembler _masm(&cbuf);
3341     Register reg1 = as_Register($src1$$reg);
3342     Register reg2 = as_Register($src2$$reg);
3343     __ cmp(reg1, reg2);
3344   %}
3345 
3346   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3347     C2_MacroAssembler _masm(&cbuf);
3348     Register reg1 = as_Register($src1$$reg);
3349     Register reg2 = as_Register($src2$$reg);
3350     __ cmpw(reg1, reg2);
3351   %}
3352 
3353   enc_class aarch64_enc_testp(iRegP src) %{
3354     C2_MacroAssembler _masm(&cbuf);
3355     Register reg = as_Register($src$$reg);
3356     __ cmp(reg, zr);
3357   %}
3358 
3359   enc_class aarch64_enc_testn(iRegN src) %{
3360     C2_MacroAssembler _masm(&cbuf);
3361     Register reg = as_Register($src$$reg);
3362     __ cmpw(reg, zr);
3363   %}
3364 
3365   enc_class aarch64_enc_b(label lbl) %{
3366     C2_MacroAssembler _masm(&cbuf);
3367     Label *L = $lbl$$label;
3368     __ b(*L);
3369   %}
3370 
3371   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3372     C2_MacroAssembler _masm(&cbuf);
3373     Label *L = $lbl$$label;
3374     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3375   %}
3376 
3377   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3378     C2_MacroAssembler _masm(&cbuf);
3379     Label *L = $lbl$$label;
3380     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3381   %}
3382 
3383   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3384   %{
3385      Register sub_reg = as_Register($sub$$reg);
3386      Register super_reg = as_Register($super$$reg);
3387      Register temp_reg = as_Register($temp$$reg);
3388      Register result_reg = as_Register($result$$reg);
3389 
3390      Label miss;
3391      C2_MacroAssembler _masm(&cbuf);
3392      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3393                                      NULL, &miss,
3394                                      /*set_cond_codes:*/ true);
3395      if ($primary) {
3396        __ mov(result_reg, zr);
3397      }
3398      __ bind(miss);
3399   %}
3400 
3401   enc_class aarch64_enc_java_static_call(method meth) %{
3402     C2_MacroAssembler _masm(&cbuf);
3403 
3404     address addr = (address)$meth$$method;
3405     address call;
3406     if (!_method) {
3407       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3408       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3409     } else {
3410       int method_index = resolved_method_index(cbuf);
3411       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3412                                                   : static_call_Relocation::spec(method_index);
3413       call = __ trampoline_call(Address(addr, rspec), &cbuf);
3414 
3415       // Emit stub for static call
3416       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
3417       if (stub == NULL) {
3418         ciEnv::current()->record_failure("CodeCache is full");
3419         return;
3420       }
3421     }
3422     if (call == NULL) {
3423       ciEnv::current()->record_failure("CodeCache is full");
3424       return;
3425     }
3426   %}
3427 
3428   enc_class aarch64_enc_java_dynamic_call(method meth) %{
3429     C2_MacroAssembler _masm(&cbuf);
3430     int method_index = resolved_method_index(cbuf);
3431     address call = __ ic_call((address)$meth$$method, method_index);
3432     if (call == NULL) {
3433       ciEnv::current()->record_failure("CodeCache is full");
3434       return;
3435     }
3436   %}
3437 
3438   enc_class aarch64_enc_call_epilog() %{
3439     C2_MacroAssembler _masm(&cbuf);
3440     if (VerifyStackAtCalls) {
3441       // Check that stack depth is unchanged: find majik cookie on stack
3442       __ call_Unimplemented();
3443     }
3444   %}
3445 
3446   enc_class aarch64_enc_java_to_runtime(method meth) %{
3447     C2_MacroAssembler _masm(&cbuf);
3448 
3449     // some calls to generated routines (arraycopy code) are scheduled
3450     // by C2 as runtime calls. if so we can call them using a br (they
3451     // will be in a reachable segment) otherwise we have to use a blr
3452     // which loads the absolute address into a register.
3453     address entry = (address)$meth$$method;
3454     CodeBlob *cb = CodeCache::find_blob(entry);
3455     if (cb) {
3456       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3457       if (call == NULL) {
3458         ciEnv::current()->record_failure("CodeCache is full");
3459         return;
3460       }
3461     } else {
3462       Label retaddr;
3463       __ adr(rscratch2, retaddr);
3464       __ lea(rscratch1, RuntimeAddress(entry));
3465       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3466       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3467       __ blr(rscratch1);
3468       __ bind(retaddr);
3469       __ add(sp, sp, 2 * wordSize);
3470     }
3471   %}
3472 
3473   enc_class aarch64_enc_rethrow() %{
3474     C2_MacroAssembler _masm(&cbuf);
3475     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3476   %}
3477 
3478   enc_class aarch64_enc_ret() %{
3479     C2_MacroAssembler _masm(&cbuf);
3480     __ ret(lr);
3481   %}
3482 
3483   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3484     C2_MacroAssembler _masm(&cbuf);
3485     Register target_reg = as_Register($jump_target$$reg);
3486     __ br(target_reg);
3487   %}
3488 
3489   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3490     C2_MacroAssembler _masm(&cbuf);
3491     Register target_reg = as_Register($jump_target$$reg);
3492     // exception oop should be in r0
3493     // ret addr has been popped into lr
3494     // callee expects it in r3
3495     __ mov(r3, lr);
3496     __ br(target_reg);
3497   %}
3498 
3499   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3500     C2_MacroAssembler _masm(&cbuf);
3501     Register oop = as_Register($object$$reg);
3502     Register box = as_Register($box$$reg);
3503     Register disp_hdr = as_Register($tmp$$reg);
3504     Register tmp = as_Register($tmp2$$reg);
3505     Label cont;
3506     Label object_has_monitor;
3507     Label cas_failed;
3508 
3509     assert_different_registers(oop, box, tmp, disp_hdr);
3510 
3511     // Load markWord from object into displaced_header.
3512     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3513 
3514     if (UseBiasedLocking && !UseOptoBiasInlining) {
3515       __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
3516     }
3517 
3518     // Check for existing monitor
3519     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3520 
3521     // Set tmp to be (markWord of object | UNLOCK_VALUE).
3522     __ orr(tmp, disp_hdr, markWord::unlocked_value);
3523 
3524     // Initialize the box. (Must happen before we update the object mark!)
3525     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3526 
3527     // Compare object markWord with an unlocked value (tmp) and if
3528     // equal exchange the stack address of our box with object markWord.
3529     // On failure disp_hdr contains the possibly locked markWord.
3530     __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3531                /*release*/ true, /*weak*/ false, disp_hdr);
3532     __ br(Assembler::EQ, cont);
3533 
3534     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3535 
3536     // If the compare-and-exchange succeeded, then we found an unlocked
3537     // object, will have now locked it will continue at label cont
3538 
3539     __ bind(cas_failed);
3540     // We did not see an unlocked object so try the fast recursive case.
3541 
3542     // Check if the owner is self by comparing the value in the
3543     // markWord of object (disp_hdr) with the stack pointer.
3544     __ mov(rscratch1, sp);
3545     __ sub(disp_hdr, disp_hdr, rscratch1);
3546     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3547     // If condition is true we are cont and hence we can store 0 as the
3548     // displaced header in the box, which indicates that it is a recursive lock.
3549     __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
3550     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3551 
3552     __ b(cont);
3553 
3554     // Handle existing monitor.
3555     __ bind(object_has_monitor);
3556 
3557     // The object's monitor m is unlocked iff m->owner == NULL,
3558     // otherwise m->owner may contain a thread or a stack address.
3559     //
3560     // Try to CAS m->owner from NULL to current thread.
3561     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
3562     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3563                /*release*/ true, /*weak*/ false, noreg); // Sets flags for result
3564 
3565     // Store a non-null value into the box to avoid looking like a re-entrant
3566     // lock. The fast-path monitor unlock code checks for
3567     // markWord::monitor_value so use markWord::unused_mark which has the
3568     // relevant bit set, and also matches ObjectSynchronizer::enter.
3569     __ mov(tmp, (address)markWord::unused_mark().value());
3570     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3571 
3572     __ bind(cont);
3573     // flag == EQ indicates success
3574     // flag == NE indicates failure
3575   %}
3576 
3577   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3578     C2_MacroAssembler _masm(&cbuf);
3579     Register oop = as_Register($object$$reg);
3580     Register box = as_Register($box$$reg);
3581     Register disp_hdr = as_Register($tmp$$reg);
3582     Register tmp = as_Register($tmp2$$reg);
3583     Label cont;
3584     Label object_has_monitor;
3585 
3586     assert_different_registers(oop, box, tmp, disp_hdr);
3587 
3588     if (UseBiasedLocking && !UseOptoBiasInlining) {
3589       __ biased_locking_exit(oop, tmp, cont);
3590     }
3591 
3592     // Find the lock address and load the displaced header from the stack.
3593     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3594 
3595     // If the displaced header is 0, we have a recursive unlock.
3596     __ cmp(disp_hdr, zr);
3597     __ br(Assembler::EQ, cont);
3598 
3599     // Handle existing monitor.
3600     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3601     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3602 
3603     // Check if it is still a light weight lock, this is is true if we
3604     // see the stack address of the basicLock in the markWord of the
3605     // object.
3606 
3607     __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3608                /*release*/ true, /*weak*/ false, tmp);
3609     __ b(cont);
3610 
3611     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3612 
3613     // Handle existing monitor.
3614     __ bind(object_has_monitor);
3615     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
3616     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
3617     __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3618     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3619     __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3620     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3621     __ cmp(rscratch1, zr); // Sets flags for result
3622     __ br(Assembler::NE, cont);
3623 
3624     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3625     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3626     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3627     __ cmp(rscratch1, zr); // Sets flags for result
3628     __ cbnz(rscratch1, cont);
3629     // need a release store here
3630     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3631     __ stlr(zr, tmp); // set unowned
3632 
3633     __ bind(cont);
3634     // flag == EQ indicates success
3635     // flag == NE indicates failure
3636   %}
3637 
3638 %}
3639 
3640 //----------FRAME--------------------------------------------------------------
3641 // Definition of frame structure and management information.
3642 //
3643 //  S T A C K   L A Y O U T    Allocators stack-slot number
3644 //                             |   (to get allocators register number
3645 //  G  Owned by    |        |  v    add OptoReg::stack0())
3646 //  r   CALLER     |        |
3647 //  o     |        +--------+      pad to even-align allocators stack-slot
3648 //  w     V        |  pad0  |        numbers; owned by CALLER
3649 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
3650 //  h     ^        |   in   |  5
3651 //        |        |  args  |  4   Holes in incoming args owned by SELF
3652 //  |     |        |        |  3
3653 //  |     |        +--------+
3654 //  V     |        | old out|      Empty on Intel, window on Sparc
3655 //        |    old |preserve|      Must be even aligned.
3656 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
3657 //        |        |   in   |  3   area for Intel ret address
3658 //     Owned by    |preserve|      Empty on Sparc.
3659 //       SELF      +--------+
3660 //        |        |  pad2  |  2   pad to align old SP
3661 //        |        +--------+  1
3662 //        |        | locks  |  0
3663 //        |        +--------+----> OptoReg::stack0(), even aligned
3664 //        |        |  pad1  | 11   pad to align new SP
3665 //        |        +--------+
3666 //        |        |        | 10
3667 //        |        | spills |  9   spills
3668 //        V        |        |  8   (pad0 slot for callee)
3669 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
3670 //        ^        |  out   |  7
3671 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
3672 //     Owned by    +--------+
3673 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
3674 //        |    new |preserve|      Must be even-aligned.
3675 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
3676 //        |        |        |
3677 //
3678 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
3679 //         known from SELF's arguments and the Java calling convention.
3680 //         Region 6-7 is determined per call site.
3681 // Note 2: If the calling convention leaves holes in the incoming argument
3682 //         area, those holes are owned by SELF.  Holes in the outgoing area
3683 //         are owned by the CALLEE.  Holes should not be nessecary in the
3684 //         incoming area, as the Java calling convention is completely under
3685 //         the control of the AD file.  Doubles can be sorted and packed to
3686 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
3687 //         varargs C calling conventions.
3688 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
3689 //         even aligned with pad0 as needed.
3690 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
3691 //           (the latter is true on Intel but is it false on AArch64?)
3692 //         region 6-11 is even aligned; it may be padded out more so that
3693 //         the region from SP to FP meets the minimum stack alignment.
3694 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3695 //         alignment.  Region 11, pad1, may be dynamically extended so that
3696 //         SP meets the minimum alignment.
3697 
3698 frame %{
3699   // What direction does stack grow in (assumed to be same for C & Java)
3700   stack_direction(TOWARDS_LOW);
3701 
3702   // These three registers define part of the calling convention
3703   // between compiled code and the interpreter.
3704 
3705   // Inline Cache Register or Method for I2C.
3706   inline_cache_reg(R12);
3707 
3708   // Method Oop Register when calling interpreter.
3709   interpreter_method_oop_reg(R12);
3710 
3711   // Number of stack slots consumed by locking an object
3712   sync_stack_slots(2);
3713 
3714   // Compiled code's Frame Pointer
3715   frame_pointer(R31);
3716 
3717   // Interpreter stores its frame pointer in a register which is
3718   // stored to the stack by I2CAdaptors.
3719   // I2CAdaptors convert from interpreted java to compiled java.
3720   interpreter_frame_pointer(R29);
3721 
3722   // Stack alignment requirement
3723   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3724 
3725   // Number of stack slots between incoming argument block and the start of
3726   // a new frame.  The PROLOG must add this many slots to the stack.  The
3727   // EPILOG must remove this many slots. aarch64 needs two slots for
3728   // return address and fp.
3729   // TODO think this is correct but check
3730   in_preserve_stack_slots(4);
3731 
3732   // Number of outgoing stack slots killed above the out_preserve_stack_slots
3733   // for calls to C.  Supports the var-args backing area for register parms.
3734   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3735 
3736   // The after-PROLOG location of the return address.  Location of
3737   // return address specifies a type (REG or STACK) and a number
3738   // representing the register number (i.e. - use a register name) or
3739   // stack slot.
3740   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3741   // Otherwise, it is above the locks and verification slot and alignment word
3742   // TODO this may well be correct but need to check why that - 2 is there
3743   // ppc port uses 0 but we definitely need to allow for fixed_slots
3744   // which folds in the space used for monitors
3745   return_addr(STACK - 2 +
3746               align_up((Compile::current()->in_preserve_stack_slots() +
3747                         Compile::current()->fixed_slots()),
3748                        stack_alignment_in_slots()));
3749 
3750   // Body of function which returns an integer array locating
3751   // arguments either in registers or in stack slots.  Passed an array
3752   // of ideal registers called "sig" and a "length" count.  Stack-slot
3753   // offsets are based on outgoing arguments, i.e. a CALLER setting up
3754   // arguments for a CALLEE.  Incoming stack arguments are
3755   // automatically biased by the preserve_stack_slots field above.
3756 
3757   calling_convention
3758   %{
3759     // No difference between ingoing/outgoing just pass false
3760     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3761   %}
3762 
3763   c_calling_convention
3764   %{
3765     // This is obviously always outgoing
3766     (void) SharedRuntime::c_calling_convention(sig_bt, regs, NULL, length);
3767   %}
3768 
3769   // Location of compiled Java return values.  Same as C for now.
3770   return_value
3771   %{
3772     // TODO do we allow ideal_reg == Op_RegN???
3773     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3774            "only return normal values");
3775 
3776     static const int lo[Op_RegL + 1] = { // enum name
3777       0,                                 // Op_Node
3778       0,                                 // Op_Set
3779       R0_num,                            // Op_RegN
3780       R0_num,                            // Op_RegI
3781       R0_num,                            // Op_RegP
3782       V0_num,                            // Op_RegF
3783       V0_num,                            // Op_RegD
3784       R0_num                             // Op_RegL
3785     };
3786 
3787     static const int hi[Op_RegL + 1] = { // enum name
3788       0,                                 // Op_Node
3789       0,                                 // Op_Set
3790       OptoReg::Bad,                      // Op_RegN
3791       OptoReg::Bad,                      // Op_RegI
3792       R0_H_num,                          // Op_RegP
3793       OptoReg::Bad,                      // Op_RegF
3794       V0_H_num,                          // Op_RegD
3795       R0_H_num                           // Op_RegL
3796     };
3797 
3798     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3799   %}
3800 %}
3801 
3802 //----------ATTRIBUTES---------------------------------------------------------
3803 //----------Operand Attributes-------------------------------------------------
3804 op_attrib op_cost(1);        // Required cost attribute
3805 
3806 //----------Instruction Attributes---------------------------------------------
3807 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3808 ins_attrib ins_size(32);        // Required size attribute (in bits)
3809 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3810                                 // a non-matching short branch variant
3811                                 // of some long branch?
3812 ins_attrib ins_alignment(4);    // Required alignment attribute (must
3813                                 // be a power of 2) specifies the
3814                                 // alignment that some part of the
3815                                 // instruction (not necessarily the
3816                                 // start) requires.  If > 1, a
3817                                 // compute_padding() function must be
3818                                 // provided for the instruction
3819 
3820 //----------OPERANDS-----------------------------------------------------------
3821 // Operand definitions must precede instruction definitions for correct parsing
3822 // in the ADLC because operands constitute user defined types which are used in
3823 // instruction definitions.
3824 
3825 //----------Simple Operands----------------------------------------------------
3826 
3827 // Integer operands 32 bit
3828 // 32 bit immediate
3829 operand immI()
3830 %{
3831   match(ConI);
3832 
3833   op_cost(0);
3834   format %{ %}
3835   interface(CONST_INTER);
3836 %}
3837 
3838 // 32 bit zero
3839 operand immI0()
3840 %{
3841   predicate(n->get_int() == 0);
3842   match(ConI);
3843 
3844   op_cost(0);
3845   format %{ %}
3846   interface(CONST_INTER);
3847 %}
3848 
3849 // 32 bit unit increment
3850 operand immI_1()
3851 %{
3852   predicate(n->get_int() == 1);
3853   match(ConI);
3854 
3855   op_cost(0);
3856   format %{ %}
3857   interface(CONST_INTER);
3858 %}
3859 
3860 // 32 bit unit decrement
3861 operand immI_M1()
3862 %{
3863   predicate(n->get_int() == -1);
3864   match(ConI);
3865 
3866   op_cost(0);
3867   format %{ %}
3868   interface(CONST_INTER);
3869 %}
3870 
3871 // Shift values for add/sub extension shift
3872 operand immIExt()
3873 %{
3874   predicate(0 <= n->get_int() && (n->get_int() <= 4));
3875   match(ConI);
3876 
3877   op_cost(0);
3878   format %{ %}
3879   interface(CONST_INTER);
3880 %}
3881 
3882 operand immI_le_4()
3883 %{
3884   predicate(n->get_int() <= 4);
3885   match(ConI);
3886 
3887   op_cost(0);
3888   format %{ %}
3889   interface(CONST_INTER);
3890 %}
3891 
3892 operand immI_31()
3893 %{
3894   predicate(n->get_int() == 31);
3895   match(ConI);
3896 
3897   op_cost(0);
3898   format %{ %}
3899   interface(CONST_INTER);
3900 %}
3901 
3902 operand immI_8()
3903 %{
3904   predicate(n->get_int() == 8);
3905   match(ConI);
3906 
3907   op_cost(0);
3908   format %{ %}
3909   interface(CONST_INTER);
3910 %}
3911 
3912 operand immI_16()
3913 %{
3914   predicate(n->get_int() == 16);
3915   match(ConI);
3916 
3917   op_cost(0);
3918   format %{ %}
3919   interface(CONST_INTER);
3920 %}
3921 
3922 operand immI_24()
3923 %{
3924   predicate(n->get_int() == 24);
3925   match(ConI);
3926 
3927   op_cost(0);
3928   format %{ %}
3929   interface(CONST_INTER);
3930 %}
3931 
3932 operand immI_32()
3933 %{
3934   predicate(n->get_int() == 32);
3935   match(ConI);
3936 
3937   op_cost(0);
3938   format %{ %}
3939   interface(CONST_INTER);
3940 %}
3941 
3942 operand immI_48()
3943 %{
3944   predicate(n->get_int() == 48);
3945   match(ConI);
3946 
3947   op_cost(0);
3948   format %{ %}
3949   interface(CONST_INTER);
3950 %}
3951 
3952 operand immI_56()
3953 %{
3954   predicate(n->get_int() == 56);
3955   match(ConI);
3956 
3957   op_cost(0);
3958   format %{ %}
3959   interface(CONST_INTER);
3960 %}
3961 
3962 operand immI_63()
3963 %{
3964   predicate(n->get_int() == 63);
3965   match(ConI);
3966 
3967   op_cost(0);
3968   format %{ %}
3969   interface(CONST_INTER);
3970 %}
3971 
3972 operand immI_64()
3973 %{
3974   predicate(n->get_int() == 64);
3975   match(ConI);
3976 
3977   op_cost(0);
3978   format %{ %}
3979   interface(CONST_INTER);
3980 %}
3981 
3982 operand immI_255()
3983 %{
3984   predicate(n->get_int() == 255);
3985   match(ConI);
3986 
3987   op_cost(0);
3988   format %{ %}
3989   interface(CONST_INTER);
3990 %}
3991 
3992 operand immI_65535()
3993 %{
3994   predicate(n->get_int() == 65535);
3995   match(ConI);
3996 
3997   op_cost(0);
3998   format %{ %}
3999   interface(CONST_INTER);
4000 %}
4001 
4002 operand immL_255()
4003 %{
4004   predicate(n->get_long() == 255L);
4005   match(ConL);
4006 
4007   op_cost(0);
4008   format %{ %}
4009   interface(CONST_INTER);
4010 %}
4011 
4012 operand immL_65535()
4013 %{
4014   predicate(n->get_long() == 65535L);
4015   match(ConL);
4016 
4017   op_cost(0);
4018   format %{ %}
4019   interface(CONST_INTER);
4020 %}
4021 
4022 operand immL_4294967295()
4023 %{
4024   predicate(n->get_long() == 4294967295L);
4025   match(ConL);
4026 
4027   op_cost(0);
4028   format %{ %}
4029   interface(CONST_INTER);
4030 %}
4031 
4032 operand immL_bitmask()
4033 %{
4034   predicate((n->get_long() != 0)
4035             && ((n->get_long() & 0xc000000000000000l) == 0)
4036             && is_power_of_2(n->get_long() + 1));
4037   match(ConL);
4038 
4039   op_cost(0);
4040   format %{ %}
4041   interface(CONST_INTER);
4042 %}
4043 
4044 operand immI_bitmask()
4045 %{
4046   predicate((n->get_int() != 0)
4047             && ((n->get_int() & 0xc0000000) == 0)
4048             && is_power_of_2(n->get_int() + 1));
4049   match(ConI);
4050 
4051   op_cost(0);
4052   format %{ %}
4053   interface(CONST_INTER);
4054 %}
4055 
4056 operand immL_positive_bitmaskI()
4057 %{
4058   predicate((n->get_long() != 0)
4059             && ((julong)n->get_long() < 0x80000000ULL)
4060             && is_power_of_2(n->get_long() + 1));
4061   match(ConL);
4062 
4063   op_cost(0);
4064   format %{ %}
4065   interface(CONST_INTER);
4066 %}
4067 
4068 // Scale values for scaled offset addressing modes (up to long but not quad)
4069 operand immIScale()
4070 %{
4071   predicate(0 <= n->get_int() && (n->get_int() <= 3));
4072   match(ConI);
4073 
4074   op_cost(0);
4075   format %{ %}
4076   interface(CONST_INTER);
4077 %}
4078 
4079 // 26 bit signed offset -- for pc-relative branches
4080 operand immI26()
4081 %{
4082   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4083   match(ConI);
4084 
4085   op_cost(0);
4086   format %{ %}
4087   interface(CONST_INTER);
4088 %}
4089 
4090 // 19 bit signed offset -- for pc-relative loads
4091 operand immI19()
4092 %{
4093   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4094   match(ConI);
4095 
4096   op_cost(0);
4097   format %{ %}
4098   interface(CONST_INTER);
4099 %}
4100 
4101 // 12 bit unsigned offset -- for base plus immediate loads
4102 operand immIU12()
4103 %{
4104   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4105   match(ConI);
4106 
4107   op_cost(0);
4108   format %{ %}
4109   interface(CONST_INTER);
4110 %}
4111 
4112 operand immLU12()
4113 %{
4114   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4115   match(ConL);
4116 
4117   op_cost(0);
4118   format %{ %}
4119   interface(CONST_INTER);
4120 %}
4121 
4122 // Offset for scaled or unscaled immediate loads and stores
4123 operand immIOffset()
4124 %{
4125   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4126   match(ConI);
4127 
4128   op_cost(0);
4129   format %{ %}
4130   interface(CONST_INTER);
4131 %}
4132 
4133 operand immIOffset1()
4134 %{
4135   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4136   match(ConI);
4137 
4138   op_cost(0);
4139   format %{ %}
4140   interface(CONST_INTER);
4141 %}
4142 
4143 operand immIOffset2()
4144 %{
4145   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4146   match(ConI);
4147 
4148   op_cost(0);
4149   format %{ %}
4150   interface(CONST_INTER);
4151 %}
4152 
4153 operand immIOffset4()
4154 %{
4155   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4156   match(ConI);
4157 
4158   op_cost(0);
4159   format %{ %}
4160   interface(CONST_INTER);
4161 %}
4162 
4163 operand immIOffset8()
4164 %{
4165   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4166   match(ConI);
4167 
4168   op_cost(0);
4169   format %{ %}
4170   interface(CONST_INTER);
4171 %}
4172 
4173 operand immIOffset16()
4174 %{
4175   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4176   match(ConI);
4177 
4178   op_cost(0);
4179   format %{ %}
4180   interface(CONST_INTER);
4181 %}
4182 
4183 operand immLoffset()
4184 %{
4185   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4186   match(ConL);
4187 
4188   op_cost(0);
4189   format %{ %}
4190   interface(CONST_INTER);
4191 %}
4192 
4193 operand immLoffset1()
4194 %{
4195   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4196   match(ConL);
4197 
4198   op_cost(0);
4199   format %{ %}
4200   interface(CONST_INTER);
4201 %}
4202 
4203 operand immLoffset2()
4204 %{
4205   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4206   match(ConL);
4207 
4208   op_cost(0);
4209   format %{ %}
4210   interface(CONST_INTER);
4211 %}
4212 
4213 operand immLoffset4()
4214 %{
4215   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4216   match(ConL);
4217 
4218   op_cost(0);
4219   format %{ %}
4220   interface(CONST_INTER);
4221 %}
4222 
4223 operand immLoffset8()
4224 %{
4225   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4226   match(ConL);
4227 
4228   op_cost(0);
4229   format %{ %}
4230   interface(CONST_INTER);
4231 %}
4232 
4233 operand immLoffset16()
4234 %{
4235   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4236   match(ConL);
4237 
4238   op_cost(0);
4239   format %{ %}
4240   interface(CONST_INTER);
4241 %}
4242 
4243 // 32 bit integer valid for add sub immediate
4244 operand immIAddSub()
4245 %{
4246   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4247   match(ConI);
4248   op_cost(0);
4249   format %{ %}
4250   interface(CONST_INTER);
4251 %}
4252 
4253 // 32 bit unsigned integer valid for logical immediate
4254 // TODO -- check this is right when e.g the mask is 0x80000000
4255 operand immILog()
4256 %{
4257   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4258   match(ConI);
4259 
4260   op_cost(0);
4261   format %{ %}
4262   interface(CONST_INTER);
4263 %}
4264 
4265 // Integer operands 64 bit
4266 // 64 bit immediate
4267 operand immL()
4268 %{
4269   match(ConL);
4270 
4271   op_cost(0);
4272   format %{ %}
4273   interface(CONST_INTER);
4274 %}
4275 
4276 // 64 bit zero
4277 operand immL0()
4278 %{
4279   predicate(n->get_long() == 0);
4280   match(ConL);
4281 
4282   op_cost(0);
4283   format %{ %}
4284   interface(CONST_INTER);
4285 %}
4286 
4287 // 64 bit unit increment
4288 operand immL_1()
4289 %{
4290   predicate(n->get_long() == 1);
4291   match(ConL);
4292 
4293   op_cost(0);
4294   format %{ %}
4295   interface(CONST_INTER);
4296 %}
4297 
4298 // 64 bit unit decrement
4299 operand immL_M1()
4300 %{
4301   predicate(n->get_long() == -1);
4302   match(ConL);
4303 
4304   op_cost(0);
4305   format %{ %}
4306   interface(CONST_INTER);
4307 %}
4308 
4309 // 32 bit offset of pc in thread anchor
4310 
4311 operand immL_pc_off()
4312 %{
4313   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4314                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4315   match(ConL);
4316 
4317   op_cost(0);
4318   format %{ %}
4319   interface(CONST_INTER);
4320 %}
4321 
4322 // 64 bit integer valid for add sub immediate
4323 operand immLAddSub()
4324 %{
4325   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4326   match(ConL);
4327   op_cost(0);
4328   format %{ %}
4329   interface(CONST_INTER);
4330 %}
4331 
4332 // 64 bit integer valid for logical immediate
4333 operand immLLog()
4334 %{
4335   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4336   match(ConL);
4337   op_cost(0);
4338   format %{ %}
4339   interface(CONST_INTER);
4340 %}
4341 
4342 // Long Immediate: low 32-bit mask
4343 operand immL_32bits()
4344 %{
4345   predicate(n->get_long() == 0xFFFFFFFFL);
4346   match(ConL);
4347   op_cost(0);
4348   format %{ %}
4349   interface(CONST_INTER);
4350 %}
4351 
4352 // Pointer operands
4353 // Pointer Immediate
4354 operand immP()
4355 %{
4356   match(ConP);
4357 
4358   op_cost(0);
4359   format %{ %}
4360   interface(CONST_INTER);
4361 %}
4362 
4363 // NULL Pointer Immediate
4364 operand immP0()
4365 %{
4366   predicate(n->get_ptr() == 0);
4367   match(ConP);
4368 
4369   op_cost(0);
4370   format %{ %}
4371   interface(CONST_INTER);
4372 %}
4373 
4374 // Pointer Immediate One
4375 // this is used in object initialization (initial object header)
4376 operand immP_1()
4377 %{
4378   predicate(n->get_ptr() == 1);
4379   match(ConP);
4380 
4381   op_cost(0);
4382   format %{ %}
4383   interface(CONST_INTER);
4384 %}
4385 
4386 // Card Table Byte Map Base
4387 operand immByteMapBase()
4388 %{
4389   // Get base of card map
4390   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4391             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4392   match(ConP);
4393 
4394   op_cost(0);
4395   format %{ %}
4396   interface(CONST_INTER);
4397 %}
4398 
4399 // Pointer Immediate Minus One
4400 // this is used when we want to write the current PC to the thread anchor
4401 operand immP_M1()
4402 %{
4403   predicate(n->get_ptr() == -1);
4404   match(ConP);
4405 
4406   op_cost(0);
4407   format %{ %}
4408   interface(CONST_INTER);
4409 %}
4410 
4411 // Pointer Immediate Minus Two
4412 // this is used when we want to write the current PC to the thread anchor
4413 operand immP_M2()
4414 %{
4415   predicate(n->get_ptr() == -2);
4416   match(ConP);
4417 
4418   op_cost(0);
4419   format %{ %}
4420   interface(CONST_INTER);
4421 %}
4422 
4423 // Float and Double operands
4424 // Double Immediate
4425 operand immD()
4426 %{
4427   match(ConD);
4428   op_cost(0);
4429   format %{ %}
4430   interface(CONST_INTER);
4431 %}
4432 
4433 // Double Immediate: +0.0d
4434 operand immD0()
4435 %{
4436   predicate(jlong_cast(n->getd()) == 0);
4437   match(ConD);
4438 
4439   op_cost(0);
4440   format %{ %}
4441   interface(CONST_INTER);
4442 %}
4443 
4444 // constant 'double +0.0'.
4445 operand immDPacked()
4446 %{
4447   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4448   match(ConD);
4449   op_cost(0);
4450   format %{ %}
4451   interface(CONST_INTER);
4452 %}
4453 
4454 // Float Immediate
4455 operand immF()
4456 %{
4457   match(ConF);
4458   op_cost(0);
4459   format %{ %}
4460   interface(CONST_INTER);
4461 %}
4462 
4463 // Float Immediate: +0.0f.
4464 operand immF0()
4465 %{
4466   predicate(jint_cast(n->getf()) == 0);
4467   match(ConF);
4468 
4469   op_cost(0);
4470   format %{ %}
4471   interface(CONST_INTER);
4472 %}
4473 
4474 //
4475 operand immFPacked()
4476 %{
4477   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4478   match(ConF);
4479   op_cost(0);
4480   format %{ %}
4481   interface(CONST_INTER);
4482 %}
4483 
4484 // Narrow pointer operands
4485 // Narrow Pointer Immediate
4486 operand immN()
4487 %{
4488   match(ConN);
4489 
4490   op_cost(0);
4491   format %{ %}
4492   interface(CONST_INTER);
4493 %}
4494 
4495 // Narrow NULL Pointer Immediate
4496 operand immN0()
4497 %{
4498   predicate(n->get_narrowcon() == 0);
4499   match(ConN);
4500 
4501   op_cost(0);
4502   format %{ %}
4503   interface(CONST_INTER);
4504 %}
4505 
4506 operand immNKlass()
4507 %{
4508   match(ConNKlass);
4509 
4510   op_cost(0);
4511   format %{ %}
4512   interface(CONST_INTER);
4513 %}
4514 
4515 // Integer 32 bit Register Operands
4516 // Integer 32 bitRegister (excludes SP)
4517 operand iRegI()
4518 %{
4519   constraint(ALLOC_IN_RC(any_reg32));
4520   match(RegI);
4521   match(iRegINoSp);
4522   op_cost(0);
4523   format %{ %}
4524   interface(REG_INTER);
4525 %}
4526 
4527 // Integer 32 bit Register not Special
4528 operand iRegINoSp()
4529 %{
4530   constraint(ALLOC_IN_RC(no_special_reg32));
4531   match(RegI);
4532   op_cost(0);
4533   format %{ %}
4534   interface(REG_INTER);
4535 %}
4536 
4537 // Integer 64 bit Register Operands
4538 // Integer 64 bit Register (includes SP)
4539 operand iRegL()
4540 %{
4541   constraint(ALLOC_IN_RC(any_reg));
4542   match(RegL);
4543   match(iRegLNoSp);
4544   op_cost(0);
4545   format %{ %}
4546   interface(REG_INTER);
4547 %}
4548 
4549 // Integer 64 bit Register not Special
4550 operand iRegLNoSp()
4551 %{
4552   constraint(ALLOC_IN_RC(no_special_reg));
4553   match(RegL);
4554   match(iRegL_R0);
4555   format %{ %}
4556   interface(REG_INTER);
4557 %}
4558 
4559 // Pointer Register Operands
4560 // Pointer Register
4561 operand iRegP()
4562 %{
4563   constraint(ALLOC_IN_RC(ptr_reg));
4564   match(RegP);
4565   match(iRegPNoSp);
4566   match(iRegP_R0);
4567   //match(iRegP_R2);
4568   //match(iRegP_R4);
4569   //match(iRegP_R5);
4570   match(thread_RegP);
4571   op_cost(0);
4572   format %{ %}
4573   interface(REG_INTER);
4574 %}
4575 
4576 // Pointer 64 bit Register not Special
4577 operand iRegPNoSp()
4578 %{
4579   constraint(ALLOC_IN_RC(no_special_ptr_reg));
4580   match(RegP);
4581   // match(iRegP);
4582   // match(iRegP_R0);
4583   // match(iRegP_R2);
4584   // match(iRegP_R4);
4585   // match(iRegP_R5);
4586   // match(thread_RegP);
4587   op_cost(0);
4588   format %{ %}
4589   interface(REG_INTER);
4590 %}
4591 
4592 // Pointer 64 bit Register R0 only
4593 operand iRegP_R0()
4594 %{
4595   constraint(ALLOC_IN_RC(r0_reg));
4596   match(RegP);
4597   // match(iRegP);
4598   match(iRegPNoSp);
4599   op_cost(0);
4600   format %{ %}
4601   interface(REG_INTER);
4602 %}
4603 
4604 // Pointer 64 bit Register R1 only
4605 operand iRegP_R1()
4606 %{
4607   constraint(ALLOC_IN_RC(r1_reg));
4608   match(RegP);
4609   // match(iRegP);
4610   match(iRegPNoSp);
4611   op_cost(0);
4612   format %{ %}
4613   interface(REG_INTER);
4614 %}
4615 
4616 // Pointer 64 bit Register R2 only
4617 operand iRegP_R2()
4618 %{
4619   constraint(ALLOC_IN_RC(r2_reg));
4620   match(RegP);
4621   // match(iRegP);
4622   match(iRegPNoSp);
4623   op_cost(0);
4624   format %{ %}
4625   interface(REG_INTER);
4626 %}
4627 
4628 // Pointer 64 bit Register R3 only
4629 operand iRegP_R3()
4630 %{
4631   constraint(ALLOC_IN_RC(r3_reg));
4632   match(RegP);
4633   // match(iRegP);
4634   match(iRegPNoSp);
4635   op_cost(0);
4636   format %{ %}
4637   interface(REG_INTER);
4638 %}
4639 
4640 // Pointer 64 bit Register R4 only
4641 operand iRegP_R4()
4642 %{
4643   constraint(ALLOC_IN_RC(r4_reg));
4644   match(RegP);
4645   // match(iRegP);
4646   match(iRegPNoSp);
4647   op_cost(0);
4648   format %{ %}
4649   interface(REG_INTER);
4650 %}
4651 
4652 // Pointer 64 bit Register R5 only
4653 operand iRegP_R5()
4654 %{
4655   constraint(ALLOC_IN_RC(r5_reg));
4656   match(RegP);
4657   // match(iRegP);
4658   match(iRegPNoSp);
4659   op_cost(0);
4660   format %{ %}
4661   interface(REG_INTER);
4662 %}
4663 
4664 // Pointer 64 bit Register R10 only
4665 operand iRegP_R10()
4666 %{
4667   constraint(ALLOC_IN_RC(r10_reg));
4668   match(RegP);
4669   // match(iRegP);
4670   match(iRegPNoSp);
4671   op_cost(0);
4672   format %{ %}
4673   interface(REG_INTER);
4674 %}
4675 
4676 // Long 64 bit Register R0 only
4677 operand iRegL_R0()
4678 %{
4679   constraint(ALLOC_IN_RC(r0_reg));
4680   match(RegL);
4681   match(iRegLNoSp);
4682   op_cost(0);
4683   format %{ %}
4684   interface(REG_INTER);
4685 %}
4686 
4687 // Long 64 bit Register R2 only
4688 operand iRegL_R2()
4689 %{
4690   constraint(ALLOC_IN_RC(r2_reg));
4691   match(RegL);
4692   match(iRegLNoSp);
4693   op_cost(0);
4694   format %{ %}
4695   interface(REG_INTER);
4696 %}
4697 
4698 // Long 64 bit Register R3 only
4699 operand iRegL_R3()
4700 %{
4701   constraint(ALLOC_IN_RC(r3_reg));
4702   match(RegL);
4703   match(iRegLNoSp);
4704   op_cost(0);
4705   format %{ %}
4706   interface(REG_INTER);
4707 %}
4708 
4709 // Long 64 bit Register R11 only
4710 operand iRegL_R11()
4711 %{
4712   constraint(ALLOC_IN_RC(r11_reg));
4713   match(RegL);
4714   match(iRegLNoSp);
4715   op_cost(0);
4716   format %{ %}
4717   interface(REG_INTER);
4718 %}
4719 
4720 // Pointer 64 bit Register FP only
4721 operand iRegP_FP()
4722 %{
4723   constraint(ALLOC_IN_RC(fp_reg));
4724   match(RegP);
4725   // match(iRegP);
4726   op_cost(0);
4727   format %{ %}
4728   interface(REG_INTER);
4729 %}
4730 
4731 // Register R0 only
4732 operand iRegI_R0()
4733 %{
4734   constraint(ALLOC_IN_RC(int_r0_reg));
4735   match(RegI);
4736   match(iRegINoSp);
4737   op_cost(0);
4738   format %{ %}
4739   interface(REG_INTER);
4740 %}
4741 
4742 // Register R2 only
4743 operand iRegI_R2()
4744 %{
4745   constraint(ALLOC_IN_RC(int_r2_reg));
4746   match(RegI);
4747   match(iRegINoSp);
4748   op_cost(0);
4749   format %{ %}
4750   interface(REG_INTER);
4751 %}
4752 
4753 // Register R3 only
4754 operand iRegI_R3()
4755 %{
4756   constraint(ALLOC_IN_RC(int_r3_reg));
4757   match(RegI);
4758   match(iRegINoSp);
4759   op_cost(0);
4760   format %{ %}
4761   interface(REG_INTER);
4762 %}
4763 
4764 
4765 // Register R4 only
4766 operand iRegI_R4()
4767 %{
4768   constraint(ALLOC_IN_RC(int_r4_reg));
4769   match(RegI);
4770   match(iRegINoSp);
4771   op_cost(0);
4772   format %{ %}
4773   interface(REG_INTER);
4774 %}
4775 
4776 
4777 // Pointer Register Operands
4778 // Narrow Pointer Register
4779 operand iRegN()
4780 %{
4781   constraint(ALLOC_IN_RC(any_reg32));
4782   match(RegN);
4783   match(iRegNNoSp);
4784   op_cost(0);
4785   format %{ %}
4786   interface(REG_INTER);
4787 %}
4788 
4789 operand iRegN_R0()
4790 %{
4791   constraint(ALLOC_IN_RC(r0_reg));
4792   match(iRegN);
4793   op_cost(0);
4794   format %{ %}
4795   interface(REG_INTER);
4796 %}
4797 
4798 operand iRegN_R2()
4799 %{
4800   constraint(ALLOC_IN_RC(r2_reg));
4801   match(iRegN);
4802   op_cost(0);
4803   format %{ %}
4804   interface(REG_INTER);
4805 %}
4806 
4807 operand iRegN_R3()
4808 %{
4809   constraint(ALLOC_IN_RC(r3_reg));
4810   match(iRegN);
4811   op_cost(0);
4812   format %{ %}
4813   interface(REG_INTER);
4814 %}
4815 
4816 // Integer 64 bit Register not Special
4817 operand iRegNNoSp()
4818 %{
4819   constraint(ALLOC_IN_RC(no_special_reg32));
4820   match(RegN);
4821   op_cost(0);
4822   format %{ %}
4823   interface(REG_INTER);
4824 %}
4825 
4826 // heap base register -- used for encoding immN0
4827 
4828 operand iRegIHeapbase()
4829 %{
4830   constraint(ALLOC_IN_RC(heapbase_reg));
4831   match(RegI);
4832   op_cost(0);
4833   format %{ %}
4834   interface(REG_INTER);
4835 %}
4836 
4837 // Float Register
4838 // Float register operands
4839 operand vRegF()
4840 %{
4841   constraint(ALLOC_IN_RC(float_reg));
4842   match(RegF);
4843 
4844   op_cost(0);
4845   format %{ %}
4846   interface(REG_INTER);
4847 %}
4848 
4849 // Double Register
4850 // Double register operands
4851 operand vRegD()
4852 %{
4853   constraint(ALLOC_IN_RC(double_reg));
4854   match(RegD);
4855 
4856   op_cost(0);
4857   format %{ %}
4858   interface(REG_INTER);
4859 %}
4860 
4861 operand vecD()
4862 %{
4863   constraint(ALLOC_IN_RC(vectord_reg));
4864   match(VecD);
4865 
4866   op_cost(0);
4867   format %{ %}
4868   interface(REG_INTER);
4869 %}
4870 
4871 operand vecX()
4872 %{
4873   constraint(ALLOC_IN_RC(vectorx_reg));
4874   match(VecX);
4875 
4876   op_cost(0);
4877   format %{ %}
4878   interface(REG_INTER);
4879 %}
4880 
4881 operand vRegD_V0()
4882 %{
4883   constraint(ALLOC_IN_RC(v0_reg));
4884   match(RegD);
4885   op_cost(0);
4886   format %{ %}
4887   interface(REG_INTER);
4888 %}
4889 
4890 operand vRegD_V1()
4891 %{
4892   constraint(ALLOC_IN_RC(v1_reg));
4893   match(RegD);
4894   op_cost(0);
4895   format %{ %}
4896   interface(REG_INTER);
4897 %}
4898 
4899 operand vRegD_V2()
4900 %{
4901   constraint(ALLOC_IN_RC(v2_reg));
4902   match(RegD);
4903   op_cost(0);
4904   format %{ %}
4905   interface(REG_INTER);
4906 %}
4907 
4908 operand vRegD_V3()
4909 %{
4910   constraint(ALLOC_IN_RC(v3_reg));
4911   match(RegD);
4912   op_cost(0);
4913   format %{ %}
4914   interface(REG_INTER);
4915 %}
4916 
4917 operand vRegD_V4()
4918 %{
4919   constraint(ALLOC_IN_RC(v4_reg));
4920   match(RegD);
4921   op_cost(0);
4922   format %{ %}
4923   interface(REG_INTER);
4924 %}
4925 
4926 operand vRegD_V5()
4927 %{
4928   constraint(ALLOC_IN_RC(v5_reg));
4929   match(RegD);
4930   op_cost(0);
4931   format %{ %}
4932   interface(REG_INTER);
4933 %}
4934 
4935 operand vRegD_V6()
4936 %{
4937   constraint(ALLOC_IN_RC(v6_reg));
4938   match(RegD);
4939   op_cost(0);
4940   format %{ %}
4941   interface(REG_INTER);
4942 %}
4943 
4944 operand vRegD_V7()
4945 %{
4946   constraint(ALLOC_IN_RC(v7_reg));
4947   match(RegD);
4948   op_cost(0);
4949   format %{ %}
4950   interface(REG_INTER);
4951 %}
4952 
4953 operand vRegD_V8()
4954 %{
4955   constraint(ALLOC_IN_RC(v8_reg));
4956   match(RegD);
4957   op_cost(0);
4958   format %{ %}
4959   interface(REG_INTER);
4960 %}
4961 
4962 operand vRegD_V9()
4963 %{
4964   constraint(ALLOC_IN_RC(v9_reg));
4965   match(RegD);
4966   op_cost(0);
4967   format %{ %}
4968   interface(REG_INTER);
4969 %}
4970 
4971 operand vRegD_V10()
4972 %{
4973   constraint(ALLOC_IN_RC(v10_reg));
4974   match(RegD);
4975   op_cost(0);
4976   format %{ %}
4977   interface(REG_INTER);
4978 %}
4979 
4980 operand vRegD_V11()
4981 %{
4982   constraint(ALLOC_IN_RC(v11_reg));
4983   match(RegD);
4984   op_cost(0);
4985   format %{ %}
4986   interface(REG_INTER);
4987 %}
4988 
4989 operand vRegD_V12()
4990 %{
4991   constraint(ALLOC_IN_RC(v12_reg));
4992   match(RegD);
4993   op_cost(0);
4994   format %{ %}
4995   interface(REG_INTER);
4996 %}
4997 
4998 operand vRegD_V13()
4999 %{
5000   constraint(ALLOC_IN_RC(v13_reg));
5001   match(RegD);
5002   op_cost(0);
5003   format %{ %}
5004   interface(REG_INTER);
5005 %}
5006 
5007 operand vRegD_V14()
5008 %{
5009   constraint(ALLOC_IN_RC(v14_reg));
5010   match(RegD);
5011   op_cost(0);
5012   format %{ %}
5013   interface(REG_INTER);
5014 %}
5015 
5016 operand vRegD_V15()
5017 %{
5018   constraint(ALLOC_IN_RC(v15_reg));
5019   match(RegD);
5020   op_cost(0);
5021   format %{ %}
5022   interface(REG_INTER);
5023 %}
5024 
5025 operand vRegD_V16()
5026 %{
5027   constraint(ALLOC_IN_RC(v16_reg));
5028   match(RegD);
5029   op_cost(0);
5030   format %{ %}
5031   interface(REG_INTER);
5032 %}
5033 
5034 operand vRegD_V17()
5035 %{
5036   constraint(ALLOC_IN_RC(v17_reg));
5037   match(RegD);
5038   op_cost(0);
5039   format %{ %}
5040   interface(REG_INTER);
5041 %}
5042 
5043 operand vRegD_V18()
5044 %{
5045   constraint(ALLOC_IN_RC(v18_reg));
5046   match(RegD);
5047   op_cost(0);
5048   format %{ %}
5049   interface(REG_INTER);
5050 %}
5051 
5052 operand vRegD_V19()
5053 %{
5054   constraint(ALLOC_IN_RC(v19_reg));
5055   match(RegD);
5056   op_cost(0);
5057   format %{ %}
5058   interface(REG_INTER);
5059 %}
5060 
5061 operand vRegD_V20()
5062 %{
5063   constraint(ALLOC_IN_RC(v20_reg));
5064   match(RegD);
5065   op_cost(0);
5066   format %{ %}
5067   interface(REG_INTER);
5068 %}
5069 
5070 operand vRegD_V21()
5071 %{
5072   constraint(ALLOC_IN_RC(v21_reg));
5073   match(RegD);
5074   op_cost(0);
5075   format %{ %}
5076   interface(REG_INTER);
5077 %}
5078 
5079 operand vRegD_V22()
5080 %{
5081   constraint(ALLOC_IN_RC(v22_reg));
5082   match(RegD);
5083   op_cost(0);
5084   format %{ %}
5085   interface(REG_INTER);
5086 %}
5087 
5088 operand vRegD_V23()
5089 %{
5090   constraint(ALLOC_IN_RC(v23_reg));
5091   match(RegD);
5092   op_cost(0);
5093   format %{ %}
5094   interface(REG_INTER);
5095 %}
5096 
5097 operand vRegD_V24()
5098 %{
5099   constraint(ALLOC_IN_RC(v24_reg));
5100   match(RegD);
5101   op_cost(0);
5102   format %{ %}
5103   interface(REG_INTER);
5104 %}
5105 
5106 operand vRegD_V25()
5107 %{
5108   constraint(ALLOC_IN_RC(v25_reg));
5109   match(RegD);
5110   op_cost(0);
5111   format %{ %}
5112   interface(REG_INTER);
5113 %}
5114 
5115 operand vRegD_V26()
5116 %{
5117   constraint(ALLOC_IN_RC(v26_reg));
5118   match(RegD);
5119   op_cost(0);
5120   format %{ %}
5121   interface(REG_INTER);
5122 %}
5123 
5124 operand vRegD_V27()
5125 %{
5126   constraint(ALLOC_IN_RC(v27_reg));
5127   match(RegD);
5128   op_cost(0);
5129   format %{ %}
5130   interface(REG_INTER);
5131 %}
5132 
5133 operand vRegD_V28()
5134 %{
5135   constraint(ALLOC_IN_RC(v28_reg));
5136   match(RegD);
5137   op_cost(0);
5138   format %{ %}
5139   interface(REG_INTER);
5140 %}
5141 
5142 operand vRegD_V29()
5143 %{
5144   constraint(ALLOC_IN_RC(v29_reg));
5145   match(RegD);
5146   op_cost(0);
5147   format %{ %}
5148   interface(REG_INTER);
5149 %}
5150 
5151 operand vRegD_V30()
5152 %{
5153   constraint(ALLOC_IN_RC(v30_reg));
5154   match(RegD);
5155   op_cost(0);
5156   format %{ %}
5157   interface(REG_INTER);
5158 %}
5159 
5160 operand vRegD_V31()
5161 %{
5162   constraint(ALLOC_IN_RC(v31_reg));
5163   match(RegD);
5164   op_cost(0);
5165   format %{ %}
5166   interface(REG_INTER);
5167 %}
5168 
5169 // Flags register, used as output of signed compare instructions
5170 
5171 // note that on AArch64 we also use this register as the output for
5172 // for floating point compare instructions (CmpF CmpD). this ensures
5173 // that ordered inequality tests use GT, GE, LT or LE none of which
5174 // pass through cases where the result is unordered i.e. one or both
5175 // inputs to the compare is a NaN. this means that the ideal code can
5176 // replace e.g. a GT with an LE and not end up capturing the NaN case
5177 // (where the comparison should always fail). EQ and NE tests are
5178 // always generated in ideal code so that unordered folds into the NE
5179 // case, matching the behaviour of AArch64 NE.
5180 //
5181 // This differs from x86 where the outputs of FP compares use a
5182 // special FP flags registers and where compares based on this
5183 // register are distinguished into ordered inequalities (cmpOpUCF) and
5184 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5185 // to explicitly handle the unordered case in branches. x86 also has
5186 // to include extra CMoveX rules to accept a cmpOpUCF input.
5187 
5188 operand rFlagsReg()
5189 %{
5190   constraint(ALLOC_IN_RC(int_flags));
5191   match(RegFlags);
5192 
5193   op_cost(0);
5194   format %{ "RFLAGS" %}
5195   interface(REG_INTER);
5196 %}
5197 
5198 // Flags register, used as output of unsigned compare instructions
5199 operand rFlagsRegU()
5200 %{
5201   constraint(ALLOC_IN_RC(int_flags));
5202   match(RegFlags);
5203 
5204   op_cost(0);
5205   format %{ "RFLAGSU" %}
5206   interface(REG_INTER);
5207 %}
5208 
5209 // Special Registers
5210 
5211 // Method Register
5212 operand inline_cache_RegP(iRegP reg)
5213 %{
5214   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5215   match(reg);
5216   match(iRegPNoSp);
5217   op_cost(0);
5218   format %{ %}
5219   interface(REG_INTER);
5220 %}
5221 
5222 operand interpreter_method_oop_RegP(iRegP reg)
5223 %{
5224   constraint(ALLOC_IN_RC(method_reg)); // interpreter_method_oop_reg
5225   match(reg);
5226   match(iRegPNoSp);
5227   op_cost(0);
5228   format %{ %}
5229   interface(REG_INTER);
5230 %}
5231 
5232 // Thread Register
5233 operand thread_RegP(iRegP reg)
5234 %{
5235   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5236   match(reg);
5237   op_cost(0);
5238   format %{ %}
5239   interface(REG_INTER);
5240 %}
5241 
5242 operand lr_RegP(iRegP reg)
5243 %{
5244   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5245   match(reg);
5246   op_cost(0);
5247   format %{ %}
5248   interface(REG_INTER);
5249 %}
5250 
5251 //----------Memory Operands----------------------------------------------------
5252 
5253 operand indirect(iRegP reg)
5254 %{
5255   constraint(ALLOC_IN_RC(ptr_reg));
5256   match(reg);
5257   op_cost(0);
5258   format %{ "[$reg]" %}
5259   interface(MEMORY_INTER) %{
5260     base($reg);
5261     index(0xffffffff);
5262     scale(0x0);
5263     disp(0x0);
5264   %}
5265 %}
5266 
5267 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5268 %{
5269   constraint(ALLOC_IN_RC(ptr_reg));
5270   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5271   match(AddP reg (LShiftL (ConvI2L ireg) scale));
5272   op_cost(0);
5273   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5274   interface(MEMORY_INTER) %{
5275     base($reg);
5276     index($ireg);
5277     scale($scale);
5278     disp(0x0);
5279   %}
5280 %}
5281 
5282 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5283 %{
5284   constraint(ALLOC_IN_RC(ptr_reg));
5285   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5286   match(AddP reg (LShiftL lreg scale));
5287   op_cost(0);
5288   format %{ "$reg, $lreg lsl($scale)" %}
5289   interface(MEMORY_INTER) %{
5290     base($reg);
5291     index($lreg);
5292     scale($scale);
5293     disp(0x0);
5294   %}
5295 %}
5296 
5297 operand indIndexI2L(iRegP reg, iRegI ireg)
5298 %{
5299   constraint(ALLOC_IN_RC(ptr_reg));
5300   match(AddP reg (ConvI2L ireg));
5301   op_cost(0);
5302   format %{ "$reg, $ireg, 0, I2L" %}
5303   interface(MEMORY_INTER) %{
5304     base($reg);
5305     index($ireg);
5306     scale(0x0);
5307     disp(0x0);
5308   %}
5309 %}
5310 
5311 operand indIndex(iRegP reg, iRegL lreg)
5312 %{
5313   constraint(ALLOC_IN_RC(ptr_reg));
5314   match(AddP reg lreg);
5315   op_cost(0);
5316   format %{ "$reg, $lreg" %}
5317   interface(MEMORY_INTER) %{
5318     base($reg);
5319     index($lreg);
5320     scale(0x0);
5321     disp(0x0);
5322   %}
5323 %}
5324 
5325 operand indOffI(iRegP reg, immIOffset off)
5326 %{
5327   constraint(ALLOC_IN_RC(ptr_reg));
5328   match(AddP reg off);
5329   op_cost(0);
5330   format %{ "[$reg, $off]" %}
5331   interface(MEMORY_INTER) %{
5332     base($reg);
5333     index(0xffffffff);
5334     scale(0x0);
5335     disp($off);
5336   %}
5337 %}
5338 
5339 operand indOffI1(iRegP reg, immIOffset1 off)
5340 %{
5341   constraint(ALLOC_IN_RC(ptr_reg));
5342   match(AddP reg off);
5343   op_cost(0);
5344   format %{ "[$reg, $off]" %}
5345   interface(MEMORY_INTER) %{
5346     base($reg);
5347     index(0xffffffff);
5348     scale(0x0);
5349     disp($off);
5350   %}
5351 %}
5352 
5353 operand indOffI2(iRegP reg, immIOffset2 off)
5354 %{
5355   constraint(ALLOC_IN_RC(ptr_reg));
5356   match(AddP reg off);
5357   op_cost(0);
5358   format %{ "[$reg, $off]" %}
5359   interface(MEMORY_INTER) %{
5360     base($reg);
5361     index(0xffffffff);
5362     scale(0x0);
5363     disp($off);
5364   %}
5365 %}
5366 
5367 operand indOffI4(iRegP reg, immIOffset4 off)
5368 %{
5369   constraint(ALLOC_IN_RC(ptr_reg));
5370   match(AddP reg off);
5371   op_cost(0);
5372   format %{ "[$reg, $off]" %}
5373   interface(MEMORY_INTER) %{
5374     base($reg);
5375     index(0xffffffff);
5376     scale(0x0);
5377     disp($off);
5378   %}
5379 %}
5380 
5381 operand indOffI8(iRegP reg, immIOffset8 off)
5382 %{
5383   constraint(ALLOC_IN_RC(ptr_reg));
5384   match(AddP reg off);
5385   op_cost(0);
5386   format %{ "[$reg, $off]" %}
5387   interface(MEMORY_INTER) %{
5388     base($reg);
5389     index(0xffffffff);
5390     scale(0x0);
5391     disp($off);
5392   %}
5393 %}
5394 
5395 operand indOffI16(iRegP reg, immIOffset16 off)
5396 %{
5397   constraint(ALLOC_IN_RC(ptr_reg));
5398   match(AddP reg off);
5399   op_cost(0);
5400   format %{ "[$reg, $off]" %}
5401   interface(MEMORY_INTER) %{
5402     base($reg);
5403     index(0xffffffff);
5404     scale(0x0);
5405     disp($off);
5406   %}
5407 %}
5408 
5409 operand indOffL(iRegP reg, immLoffset off)
5410 %{
5411   constraint(ALLOC_IN_RC(ptr_reg));
5412   match(AddP reg off);
5413   op_cost(0);
5414   format %{ "[$reg, $off]" %}
5415   interface(MEMORY_INTER) %{
5416     base($reg);
5417     index(0xffffffff);
5418     scale(0x0);
5419     disp($off);
5420   %}
5421 %}
5422 
5423 operand indOffL1(iRegP reg, immLoffset1 off)
5424 %{
5425   constraint(ALLOC_IN_RC(ptr_reg));
5426   match(AddP reg off);
5427   op_cost(0);
5428   format %{ "[$reg, $off]" %}
5429   interface(MEMORY_INTER) %{
5430     base($reg);
5431     index(0xffffffff);
5432     scale(0x0);
5433     disp($off);
5434   %}
5435 %}
5436 
5437 operand indOffL2(iRegP reg, immLoffset2 off)
5438 %{
5439   constraint(ALLOC_IN_RC(ptr_reg));
5440   match(AddP reg off);
5441   op_cost(0);
5442   format %{ "[$reg, $off]" %}
5443   interface(MEMORY_INTER) %{
5444     base($reg);
5445     index(0xffffffff);
5446     scale(0x0);
5447     disp($off);
5448   %}
5449 %}
5450 
5451 operand indOffL4(iRegP reg, immLoffset4 off)
5452 %{
5453   constraint(ALLOC_IN_RC(ptr_reg));
5454   match(AddP reg off);
5455   op_cost(0);
5456   format %{ "[$reg, $off]" %}
5457   interface(MEMORY_INTER) %{
5458     base($reg);
5459     index(0xffffffff);
5460     scale(0x0);
5461     disp($off);
5462   %}
5463 %}
5464 
5465 operand indOffL8(iRegP reg, immLoffset8 off)
5466 %{
5467   constraint(ALLOC_IN_RC(ptr_reg));
5468   match(AddP reg off);
5469   op_cost(0);
5470   format %{ "[$reg, $off]" %}
5471   interface(MEMORY_INTER) %{
5472     base($reg);
5473     index(0xffffffff);
5474     scale(0x0);
5475     disp($off);
5476   %}
5477 %}
5478 
5479 operand indOffL16(iRegP reg, immLoffset16 off)
5480 %{
5481   constraint(ALLOC_IN_RC(ptr_reg));
5482   match(AddP reg off);
5483   op_cost(0);
5484   format %{ "[$reg, $off]" %}
5485   interface(MEMORY_INTER) %{
5486     base($reg);
5487     index(0xffffffff);
5488     scale(0x0);
5489     disp($off);
5490   %}
5491 %}
5492 
5493 operand indirectN(iRegN reg)
5494 %{
5495   predicate(CompressedOops::shift() == 0);
5496   constraint(ALLOC_IN_RC(ptr_reg));
5497   match(DecodeN reg);
5498   op_cost(0);
5499   format %{ "[$reg]\t# narrow" %}
5500   interface(MEMORY_INTER) %{
5501     base($reg);
5502     index(0xffffffff);
5503     scale(0x0);
5504     disp(0x0);
5505   %}
5506 %}
5507 
5508 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5509 %{
5510   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5511   constraint(ALLOC_IN_RC(ptr_reg));
5512   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5513   op_cost(0);
5514   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5515   interface(MEMORY_INTER) %{
5516     base($reg);
5517     index($ireg);
5518     scale($scale);
5519     disp(0x0);
5520   %}
5521 %}
5522 
5523 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5524 %{
5525   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5526   constraint(ALLOC_IN_RC(ptr_reg));
5527   match(AddP (DecodeN reg) (LShiftL lreg scale));
5528   op_cost(0);
5529   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5530   interface(MEMORY_INTER) %{
5531     base($reg);
5532     index($lreg);
5533     scale($scale);
5534     disp(0x0);
5535   %}
5536 %}
5537 
5538 operand indIndexI2LN(iRegN reg, iRegI ireg)
5539 %{
5540   predicate(CompressedOops::shift() == 0);
5541   constraint(ALLOC_IN_RC(ptr_reg));
5542   match(AddP (DecodeN reg) (ConvI2L ireg));
5543   op_cost(0);
5544   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5545   interface(MEMORY_INTER) %{
5546     base($reg);
5547     index($ireg);
5548     scale(0x0);
5549     disp(0x0);
5550   %}
5551 %}
5552 
5553 operand indIndexN(iRegN reg, iRegL lreg)
5554 %{
5555   predicate(CompressedOops::shift() == 0);
5556   constraint(ALLOC_IN_RC(ptr_reg));
5557   match(AddP (DecodeN reg) lreg);
5558   op_cost(0);
5559   format %{ "$reg, $lreg\t# narrow" %}
5560   interface(MEMORY_INTER) %{
5561     base($reg);
5562     index($lreg);
5563     scale(0x0);
5564     disp(0x0);
5565   %}
5566 %}
5567 
5568 operand indOffIN(iRegN reg, immIOffset off)
5569 %{
5570   predicate(CompressedOops::shift() == 0);
5571   constraint(ALLOC_IN_RC(ptr_reg));
5572   match(AddP (DecodeN reg) off);
5573   op_cost(0);
5574   format %{ "[$reg, $off]\t# narrow" %}
5575   interface(MEMORY_INTER) %{
5576     base($reg);
5577     index(0xffffffff);
5578     scale(0x0);
5579     disp($off);
5580   %}
5581 %}
5582 
5583 operand indOffLN(iRegN reg, immLoffset off)
5584 %{
5585   predicate(CompressedOops::shift() == 0);
5586   constraint(ALLOC_IN_RC(ptr_reg));
5587   match(AddP (DecodeN reg) off);
5588   op_cost(0);
5589   format %{ "[$reg, $off]\t# narrow" %}
5590   interface(MEMORY_INTER) %{
5591     base($reg);
5592     index(0xffffffff);
5593     scale(0x0);
5594     disp($off);
5595   %}
5596 %}
5597 
5598 
5599 
5600 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5601 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5602 %{
5603   constraint(ALLOC_IN_RC(ptr_reg));
5604   match(AddP reg off);
5605   op_cost(0);
5606   format %{ "[$reg, $off]" %}
5607   interface(MEMORY_INTER) %{
5608     base($reg);
5609     index(0xffffffff);
5610     scale(0x0);
5611     disp($off);
5612   %}
5613 %}
5614 
5615 //----------Special Memory Operands--------------------------------------------
5616 // Stack Slot Operand - This operand is used for loading and storing temporary
5617 //                      values on the stack where a match requires a value to
5618 //                      flow through memory.
5619 operand stackSlotP(sRegP reg)
5620 %{
5621   constraint(ALLOC_IN_RC(stack_slots));
5622   op_cost(100);
5623   // No match rule because this operand is only generated in matching
5624   // match(RegP);
5625   format %{ "[$reg]" %}
5626   interface(MEMORY_INTER) %{
5627     base(0x1e);  // RSP
5628     index(0x0);  // No Index
5629     scale(0x0);  // No Scale
5630     disp($reg);  // Stack Offset
5631   %}
5632 %}
5633 
5634 operand stackSlotI(sRegI reg)
5635 %{
5636   constraint(ALLOC_IN_RC(stack_slots));
5637   // No match rule because this operand is only generated in matching
5638   // match(RegI);
5639   format %{ "[$reg]" %}
5640   interface(MEMORY_INTER) %{
5641     base(0x1e);  // RSP
5642     index(0x0);  // No Index
5643     scale(0x0);  // No Scale
5644     disp($reg);  // Stack Offset
5645   %}
5646 %}
5647 
5648 operand stackSlotF(sRegF reg)
5649 %{
5650   constraint(ALLOC_IN_RC(stack_slots));
5651   // No match rule because this operand is only generated in matching
5652   // match(RegF);
5653   format %{ "[$reg]" %}
5654   interface(MEMORY_INTER) %{
5655     base(0x1e);  // RSP
5656     index(0x0);  // No Index
5657     scale(0x0);  // No Scale
5658     disp($reg);  // Stack Offset
5659   %}
5660 %}
5661 
5662 operand stackSlotD(sRegD reg)
5663 %{
5664   constraint(ALLOC_IN_RC(stack_slots));
5665   // No match rule because this operand is only generated in matching
5666   // match(RegD);
5667   format %{ "[$reg]" %}
5668   interface(MEMORY_INTER) %{
5669     base(0x1e);  // RSP
5670     index(0x0);  // No Index
5671     scale(0x0);  // No Scale
5672     disp($reg);  // Stack Offset
5673   %}
5674 %}
5675 
5676 operand stackSlotL(sRegL reg)
5677 %{
5678   constraint(ALLOC_IN_RC(stack_slots));
5679   // No match rule because this operand is only generated in matching
5680   // match(RegL);
5681   format %{ "[$reg]" %}
5682   interface(MEMORY_INTER) %{
5683     base(0x1e);  // RSP
5684     index(0x0);  // No Index
5685     scale(0x0);  // No Scale
5686     disp($reg);  // Stack Offset
5687   %}
5688 %}
5689 
5690 // Operands for expressing Control Flow
5691 // NOTE: Label is a predefined operand which should not be redefined in
5692 //       the AD file. It is generically handled within the ADLC.
5693 
5694 //----------Conditional Branch Operands----------------------------------------
5695 // Comparison Op  - This is the operation of the comparison, and is limited to
5696 //                  the following set of codes:
5697 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5698 //
5699 // Other attributes of the comparison, such as unsignedness, are specified
5700 // by the comparison instruction that sets a condition code flags register.
5701 // That result is represented by a flags operand whose subtype is appropriate
5702 // to the unsignedness (etc.) of the comparison.
5703 //
5704 // Later, the instruction which matches both the Comparison Op (a Bool) and
5705 // the flags (produced by the Cmp) specifies the coding of the comparison op
5706 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5707 
5708 // used for signed integral comparisons and fp comparisons
5709 
5710 operand cmpOp()
5711 %{
5712   match(Bool);
5713 
5714   format %{ "" %}
5715   interface(COND_INTER) %{
5716     equal(0x0, "eq");
5717     not_equal(0x1, "ne");
5718     less(0xb, "lt");
5719     greater_equal(0xa, "ge");
5720     less_equal(0xd, "le");
5721     greater(0xc, "gt");
5722     overflow(0x6, "vs");
5723     no_overflow(0x7, "vc");
5724   %}
5725 %}
5726 
5727 // used for unsigned integral comparisons
5728 
5729 operand cmpOpU()
5730 %{
5731   match(Bool);
5732 
5733   format %{ "" %}
5734   interface(COND_INTER) %{
5735     equal(0x0, "eq");
5736     not_equal(0x1, "ne");
5737     less(0x3, "lo");
5738     greater_equal(0x2, "hs");
5739     less_equal(0x9, "ls");
5740     greater(0x8, "hi");
5741     overflow(0x6, "vs");
5742     no_overflow(0x7, "vc");
5743   %}
5744 %}
5745 
5746 // used for certain integral comparisons which can be
5747 // converted to cbxx or tbxx instructions
5748 
5749 operand cmpOpEqNe()
5750 %{
5751   match(Bool);
5752   op_cost(0);
5753   predicate(n->as_Bool()->_test._test == BoolTest::ne
5754             || n->as_Bool()->_test._test == BoolTest::eq);
5755 
5756   format %{ "" %}
5757   interface(COND_INTER) %{
5758     equal(0x0, "eq");
5759     not_equal(0x1, "ne");
5760     less(0xb, "lt");
5761     greater_equal(0xa, "ge");
5762     less_equal(0xd, "le");
5763     greater(0xc, "gt");
5764     overflow(0x6, "vs");
5765     no_overflow(0x7, "vc");
5766   %}
5767 %}
5768 
5769 // used for certain integral comparisons which can be
5770 // converted to cbxx or tbxx instructions
5771 
5772 operand cmpOpLtGe()
5773 %{
5774   match(Bool);
5775   op_cost(0);
5776 
5777   predicate(n->as_Bool()->_test._test == BoolTest::lt
5778             || n->as_Bool()->_test._test == BoolTest::ge);
5779 
5780   format %{ "" %}
5781   interface(COND_INTER) %{
5782     equal(0x0, "eq");
5783     not_equal(0x1, "ne");
5784     less(0xb, "lt");
5785     greater_equal(0xa, "ge");
5786     less_equal(0xd, "le");
5787     greater(0xc, "gt");
5788     overflow(0x6, "vs");
5789     no_overflow(0x7, "vc");
5790   %}
5791 %}
5792 
5793 // used for certain unsigned integral comparisons which can be
5794 // converted to cbxx or tbxx instructions
5795 
5796 operand cmpOpUEqNeLtGe()
5797 %{
5798   match(Bool);
5799   op_cost(0);
5800 
5801   predicate(n->as_Bool()->_test._test == BoolTest::eq
5802             || n->as_Bool()->_test._test == BoolTest::ne
5803             || n->as_Bool()->_test._test == BoolTest::lt
5804             || n->as_Bool()->_test._test == BoolTest::ge);
5805 
5806   format %{ "" %}
5807   interface(COND_INTER) %{
5808     equal(0x0, "eq");
5809     not_equal(0x1, "ne");
5810     less(0xb, "lt");
5811     greater_equal(0xa, "ge");
5812     less_equal(0xd, "le");
5813     greater(0xc, "gt");
5814     overflow(0x6, "vs");
5815     no_overflow(0x7, "vc");
5816   %}
5817 %}
5818 
5819 // Special operand allowing long args to int ops to be truncated for free
5820 
5821 operand iRegL2I(iRegL reg) %{
5822 
5823   op_cost(0);
5824 
5825   match(ConvL2I reg);
5826 
5827   format %{ "l2i($reg)" %}
5828 
5829   interface(REG_INTER)
5830 %}
5831 
5832 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5833 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5834 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5835 
5836 //----------OPERAND CLASSES----------------------------------------------------
5837 // Operand Classes are groups of operands that are used as to simplify
5838 // instruction definitions by not requiring the AD writer to specify
5839 // separate instructions for every form of operand when the
5840 // instruction accepts multiple operand types with the same basic
5841 // encoding and format. The classic case of this is memory operands.
5842 
5843 // memory is used to define read/write location for load/store
5844 // instruction defs. we can turn a memory op into an Address
5845 
5846 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5847                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
5848 
5849 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5850                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
5851 
5852 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5853                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
5854 
5855 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5856                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
5857 
5858 // All of the memory operands. For the pipeline description.
5859 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5860                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5861                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
5862 
5863 
5864 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5865 // operations. it allows the src to be either an iRegI or a (ConvL2I
5866 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5867 // can be elided because the 32-bit instruction will just employ the
5868 // lower 32 bits anyway.
5869 //
5870 // n.b. this does not elide all L2I conversions. if the truncated
5871 // value is consumed by more than one operation then the ConvL2I
5872 // cannot be bundled into the consuming nodes so an l2i gets planted
5873 // (actually a movw $dst $src) and the downstream instructions consume
5874 // the result of the l2i as an iRegI input. That's a shame since the
5875 // movw is actually redundant but its not too costly.
5876 
5877 opclass iRegIorL2I(iRegI, iRegL2I);
5878 
5879 //----------PIPELINE-----------------------------------------------------------
5880 // Rules which define the behavior of the target architectures pipeline.
5881 
5882 // For specific pipelines, eg A53, define the stages of that pipeline
5883 //pipe_desc(ISS, EX1, EX2, WR);
5884 #define ISS S0
5885 #define EX1 S1
5886 #define EX2 S2
5887 #define WR  S3
5888 
5889 // Integer ALU reg operation
5890 pipeline %{
5891 
5892 attributes %{
5893   // ARM instructions are of fixed length
5894   fixed_size_instructions;        // Fixed size instructions TODO does
5895   max_instructions_per_bundle = 2;   // A53 = 2, A57 = 4
5896   // ARM instructions come in 32-bit word units
5897   instruction_unit_size = 4;         // An instruction is 4 bytes long
5898   instruction_fetch_unit_size = 64;  // The processor fetches one line
5899   instruction_fetch_units = 1;       // of 64 bytes
5900 
5901   // List of nop instructions
5902   nops( MachNop );
5903 %}
5904 
5905 // We don't use an actual pipeline model so don't care about resources
5906 // or description. we do use pipeline classes to introduce fixed
5907 // latencies
5908 
5909 //----------RESOURCES----------------------------------------------------------
5910 // Resources are the functional units available to the machine
5911 
5912 resources( INS0, INS1, INS01 = INS0 | INS1,
5913            ALU0, ALU1, ALU = ALU0 | ALU1,
5914            MAC,
5915            DIV,
5916            BRANCH,
5917            LDST,
5918            NEON_FP);
5919 
5920 //----------PIPELINE DESCRIPTION-----------------------------------------------
5921 // Pipeline Description specifies the stages in the machine's pipeline
5922 
5923 // Define the pipeline as a generic 6 stage pipeline
5924 pipe_desc(S0, S1, S2, S3, S4, S5);
5925 
5926 //----------PIPELINE CLASSES---------------------------------------------------
5927 // Pipeline Classes describe the stages in which input and output are
5928 // referenced by the hardware pipeline.
5929 
5930 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5931 %{
5932   single_instruction;
5933   src1   : S1(read);
5934   src2   : S2(read);
5935   dst    : S5(write);
5936   INS01  : ISS;
5937   NEON_FP : S5;
5938 %}
5939 
5940 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5941 %{
5942   single_instruction;
5943   src1   : S1(read);
5944   src2   : S2(read);
5945   dst    : S5(write);
5946   INS01  : ISS;
5947   NEON_FP : S5;
5948 %}
5949 
5950 pipe_class fp_uop_s(vRegF dst, vRegF src)
5951 %{
5952   single_instruction;
5953   src    : S1(read);
5954   dst    : S5(write);
5955   INS01  : ISS;
5956   NEON_FP : S5;
5957 %}
5958 
5959 pipe_class fp_uop_d(vRegD dst, vRegD src)
5960 %{
5961   single_instruction;
5962   src    : S1(read);
5963   dst    : S5(write);
5964   INS01  : ISS;
5965   NEON_FP : S5;
5966 %}
5967 
5968 pipe_class fp_d2f(vRegF dst, vRegD src)
5969 %{
5970   single_instruction;
5971   src    : S1(read);
5972   dst    : S5(write);
5973   INS01  : ISS;
5974   NEON_FP : S5;
5975 %}
5976 
5977 pipe_class fp_f2d(vRegD dst, vRegF src)
5978 %{
5979   single_instruction;
5980   src    : S1(read);
5981   dst    : S5(write);
5982   INS01  : ISS;
5983   NEON_FP : S5;
5984 %}
5985 
5986 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5987 %{
5988   single_instruction;
5989   src    : S1(read);
5990   dst    : S5(write);
5991   INS01  : ISS;
5992   NEON_FP : S5;
5993 %}
5994 
5995 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5996 %{
5997   single_instruction;
5998   src    : S1(read);
5999   dst    : S5(write);
6000   INS01  : ISS;
6001   NEON_FP : S5;
6002 %}
6003 
6004 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6005 %{
6006   single_instruction;
6007   src    : S1(read);
6008   dst    : S5(write);
6009   INS01  : ISS;
6010   NEON_FP : S5;
6011 %}
6012 
6013 pipe_class fp_l2f(vRegF dst, iRegL src)
6014 %{
6015   single_instruction;
6016   src    : S1(read);
6017   dst    : S5(write);
6018   INS01  : ISS;
6019   NEON_FP : S5;
6020 %}
6021 
6022 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6023 %{
6024   single_instruction;
6025   src    : S1(read);
6026   dst    : S5(write);
6027   INS01  : ISS;
6028   NEON_FP : S5;
6029 %}
6030 
6031 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6032 %{
6033   single_instruction;
6034   src    : S1(read);
6035   dst    : S5(write);
6036   INS01  : ISS;
6037   NEON_FP : S5;
6038 %}
6039 
6040 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6041 %{
6042   single_instruction;
6043   src    : S1(read);
6044   dst    : S5(write);
6045   INS01  : ISS;
6046   NEON_FP : S5;
6047 %}
6048 
6049 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6050 %{
6051   single_instruction;
6052   src    : S1(read);
6053   dst    : S5(write);
6054   INS01  : ISS;
6055   NEON_FP : S5;
6056 %}
6057 
6058 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6059 %{
6060   single_instruction;
6061   src1   : S1(read);
6062   src2   : S2(read);
6063   dst    : S5(write);
6064   INS0   : ISS;
6065   NEON_FP : S5;
6066 %}
6067 
6068 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6069 %{
6070   single_instruction;
6071   src1   : S1(read);
6072   src2   : S2(read);
6073   dst    : S5(write);
6074   INS0   : ISS;
6075   NEON_FP : S5;
6076 %}
6077 
6078 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6079 %{
6080   single_instruction;
6081   cr     : S1(read);
6082   src1   : S1(read);
6083   src2   : S1(read);
6084   dst    : S3(write);
6085   INS01  : ISS;
6086   NEON_FP : S3;
6087 %}
6088 
6089 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6090 %{
6091   single_instruction;
6092   cr     : S1(read);
6093   src1   : S1(read);
6094   src2   : S1(read);
6095   dst    : S3(write);
6096   INS01  : ISS;
6097   NEON_FP : S3;
6098 %}
6099 
6100 pipe_class fp_imm_s(vRegF dst)
6101 %{
6102   single_instruction;
6103   dst    : S3(write);
6104   INS01  : ISS;
6105   NEON_FP : S3;
6106 %}
6107 
6108 pipe_class fp_imm_d(vRegD dst)
6109 %{
6110   single_instruction;
6111   dst    : S3(write);
6112   INS01  : ISS;
6113   NEON_FP : S3;
6114 %}
6115 
6116 pipe_class fp_load_constant_s(vRegF dst)
6117 %{
6118   single_instruction;
6119   dst    : S4(write);
6120   INS01  : ISS;
6121   NEON_FP : S4;
6122 %}
6123 
6124 pipe_class fp_load_constant_d(vRegD dst)
6125 %{
6126   single_instruction;
6127   dst    : S4(write);
6128   INS01  : ISS;
6129   NEON_FP : S4;
6130 %}
6131 
6132 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
6133 %{
6134   single_instruction;
6135   dst    : S5(write);
6136   src1   : S1(read);
6137   src2   : S1(read);
6138   INS01  : ISS;
6139   NEON_FP : S5;
6140 %}
6141 
6142 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
6143 %{
6144   single_instruction;
6145   dst    : S5(write);
6146   src1   : S1(read);
6147   src2   : S1(read);
6148   INS0   : ISS;
6149   NEON_FP : S5;
6150 %}
6151 
6152 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
6153 %{
6154   single_instruction;
6155   dst    : S5(write);
6156   src1   : S1(read);
6157   src2   : S1(read);
6158   dst    : S1(read);
6159   INS01  : ISS;
6160   NEON_FP : S5;
6161 %}
6162 
6163 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
6164 %{
6165   single_instruction;
6166   dst    : S5(write);
6167   src1   : S1(read);
6168   src2   : S1(read);
6169   dst    : S1(read);
6170   INS0   : ISS;
6171   NEON_FP : S5;
6172 %}
6173 
6174 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
6175 %{
6176   single_instruction;
6177   dst    : S4(write);
6178   src1   : S2(read);
6179   src2   : S2(read);
6180   INS01  : ISS;
6181   NEON_FP : S4;
6182 %}
6183 
6184 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
6185 %{
6186   single_instruction;
6187   dst    : S4(write);
6188   src1   : S2(read);
6189   src2   : S2(read);
6190   INS0   : ISS;
6191   NEON_FP : S4;
6192 %}
6193 
6194 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
6195 %{
6196   single_instruction;
6197   dst    : S3(write);
6198   src1   : S2(read);
6199   src2   : S2(read);
6200   INS01  : ISS;
6201   NEON_FP : S3;
6202 %}
6203 
6204 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
6205 %{
6206   single_instruction;
6207   dst    : S3(write);
6208   src1   : S2(read);
6209   src2   : S2(read);
6210   INS0   : ISS;
6211   NEON_FP : S3;
6212 %}
6213 
6214 pipe_class vshift64(vecD dst, vecD src, vecX shift)
6215 %{
6216   single_instruction;
6217   dst    : S3(write);
6218   src    : S1(read);
6219   shift  : S1(read);
6220   INS01  : ISS;
6221   NEON_FP : S3;
6222 %}
6223 
6224 pipe_class vshift128(vecX dst, vecX src, vecX shift)
6225 %{
6226   single_instruction;
6227   dst    : S3(write);
6228   src    : S1(read);
6229   shift  : S1(read);
6230   INS0   : ISS;
6231   NEON_FP : S3;
6232 %}
6233 
6234 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
6235 %{
6236   single_instruction;
6237   dst    : S3(write);
6238   src    : S1(read);
6239   INS01  : ISS;
6240   NEON_FP : S3;
6241 %}
6242 
6243 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
6244 %{
6245   single_instruction;
6246   dst    : S3(write);
6247   src    : S1(read);
6248   INS0   : ISS;
6249   NEON_FP : S3;
6250 %}
6251 
6252 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
6253 %{
6254   single_instruction;
6255   dst    : S5(write);
6256   src1   : S1(read);
6257   src2   : S1(read);
6258   INS01  : ISS;
6259   NEON_FP : S5;
6260 %}
6261 
6262 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
6263 %{
6264   single_instruction;
6265   dst    : S5(write);
6266   src1   : S1(read);
6267   src2   : S1(read);
6268   INS0   : ISS;
6269   NEON_FP : S5;
6270 %}
6271 
6272 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
6273 %{
6274   single_instruction;
6275   dst    : S5(write);
6276   src1   : S1(read);
6277   src2   : S1(read);
6278   INS0   : ISS;
6279   NEON_FP : S5;
6280 %}
6281 
6282 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
6283 %{
6284   single_instruction;
6285   dst    : S5(write);
6286   src1   : S1(read);
6287   src2   : S1(read);
6288   INS0   : ISS;
6289   NEON_FP : S5;
6290 %}
6291 
6292 pipe_class vsqrt_fp128(vecX dst, vecX src)
6293 %{
6294   single_instruction;
6295   dst    : S5(write);
6296   src    : S1(read);
6297   INS0   : ISS;
6298   NEON_FP : S5;
6299 %}
6300 
6301 pipe_class vunop_fp64(vecD dst, vecD src)
6302 %{
6303   single_instruction;
6304   dst    : S5(write);
6305   src    : S1(read);
6306   INS01  : ISS;
6307   NEON_FP : S5;
6308 %}
6309 
6310 pipe_class vunop_fp128(vecX dst, vecX src)
6311 %{
6312   single_instruction;
6313   dst    : S5(write);
6314   src    : S1(read);
6315   INS0   : ISS;
6316   NEON_FP : S5;
6317 %}
6318 
6319 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
6320 %{
6321   single_instruction;
6322   dst    : S3(write);
6323   src    : S1(read);
6324   INS01  : ISS;
6325   NEON_FP : S3;
6326 %}
6327 
6328 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
6329 %{
6330   single_instruction;
6331   dst    : S3(write);
6332   src    : S1(read);
6333   INS01  : ISS;
6334   NEON_FP : S3;
6335 %}
6336 
6337 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
6338 %{
6339   single_instruction;
6340   dst    : S3(write);
6341   src    : S1(read);
6342   INS01  : ISS;
6343   NEON_FP : S3;
6344 %}
6345 
6346 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
6347 %{
6348   single_instruction;
6349   dst    : S3(write);
6350   src    : S1(read);
6351   INS01  : ISS;
6352   NEON_FP : S3;
6353 %}
6354 
6355 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
6356 %{
6357   single_instruction;
6358   dst    : S3(write);
6359   src    : S1(read);
6360   INS01  : ISS;
6361   NEON_FP : S3;
6362 %}
6363 
6364 pipe_class vmovi_reg_imm64(vecD dst)
6365 %{
6366   single_instruction;
6367   dst    : S3(write);
6368   INS01  : ISS;
6369   NEON_FP : S3;
6370 %}
6371 
6372 pipe_class vmovi_reg_imm128(vecX dst)
6373 %{
6374   single_instruction;
6375   dst    : S3(write);
6376   INS0   : ISS;
6377   NEON_FP : S3;
6378 %}
6379 
6380 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
6381 %{
6382   single_instruction;
6383   dst    : S5(write);
6384   mem    : ISS(read);
6385   INS01  : ISS;
6386   NEON_FP : S3;
6387 %}
6388 
6389 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
6390 %{
6391   single_instruction;
6392   dst    : S5(write);
6393   mem    : ISS(read);
6394   INS01  : ISS;
6395   NEON_FP : S3;
6396 %}
6397 
6398 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
6399 %{
6400   single_instruction;
6401   mem    : ISS(read);
6402   src    : S2(read);
6403   INS01  : ISS;
6404   NEON_FP : S3;
6405 %}
6406 
6407 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
6408 %{
6409   single_instruction;
6410   mem    : ISS(read);
6411   src    : S2(read);
6412   INS01  : ISS;
6413   NEON_FP : S3;
6414 %}
6415 
6416 //------- Integer ALU operations --------------------------
6417 
6418 // Integer ALU reg-reg operation
6419 // Operands needed in EX1, result generated in EX2
6420 // Eg.  ADD     x0, x1, x2
6421 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6422 %{
6423   single_instruction;
6424   dst    : EX2(write);
6425   src1   : EX1(read);
6426   src2   : EX1(read);
6427   INS01  : ISS; // Dual issue as instruction 0 or 1
6428   ALU    : EX2;
6429 %}
6430 
6431 // Integer ALU reg-reg operation with constant shift
6432 // Shifted register must be available in LATE_ISS instead of EX1
6433 // Eg.  ADD     x0, x1, x2, LSL #2
6434 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6435 %{
6436   single_instruction;
6437   dst    : EX2(write);
6438   src1   : EX1(read);
6439   src2   : ISS(read);
6440   INS01  : ISS;
6441   ALU    : EX2;
6442 %}
6443 
6444 // Integer ALU reg operation with constant shift
6445 // Eg.  LSL     x0, x1, #shift
6446 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6447 %{
6448   single_instruction;
6449   dst    : EX2(write);
6450   src1   : ISS(read);
6451   INS01  : ISS;
6452   ALU    : EX2;
6453 %}
6454 
6455 // Integer ALU reg-reg operation with variable shift
6456 // Both operands must be available in LATE_ISS instead of EX1
6457 // Result is available in EX1 instead of EX2
6458 // Eg.  LSLV    x0, x1, x2
6459 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6460 %{
6461   single_instruction;
6462   dst    : EX1(write);
6463   src1   : ISS(read);
6464   src2   : ISS(read);
6465   INS01  : ISS;
6466   ALU    : EX1;
6467 %}
6468 
6469 // Integer ALU reg-reg operation with extract
6470 // As for _vshift above, but result generated in EX2
6471 // Eg.  EXTR    x0, x1, x2, #N
6472 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6473 %{
6474   single_instruction;
6475   dst    : EX2(write);
6476   src1   : ISS(read);
6477   src2   : ISS(read);
6478   INS1   : ISS; // Can only dual issue as Instruction 1
6479   ALU    : EX1;
6480 %}
6481 
6482 // Integer ALU reg operation
6483 // Eg.  NEG     x0, x1
6484 pipe_class ialu_reg(iRegI dst, iRegI src)
6485 %{
6486   single_instruction;
6487   dst    : EX2(write);
6488   src    : EX1(read);
6489   INS01  : ISS;
6490   ALU    : EX2;
6491 %}
6492 
6493 // Integer ALU reg mmediate operation
6494 // Eg.  ADD     x0, x1, #N
6495 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6496 %{
6497   single_instruction;
6498   dst    : EX2(write);
6499   src1   : EX1(read);
6500   INS01  : ISS;
6501   ALU    : EX2;
6502 %}
6503 
6504 // Integer ALU immediate operation (no source operands)
6505 // Eg.  MOV     x0, #N
6506 pipe_class ialu_imm(iRegI dst)
6507 %{
6508   single_instruction;
6509   dst    : EX1(write);
6510   INS01  : ISS;
6511   ALU    : EX1;
6512 %}
6513 
6514 //------- Compare operation -------------------------------
6515 
6516 // Compare reg-reg
6517 // Eg.  CMP     x0, x1
6518 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6519 %{
6520   single_instruction;
6521 //  fixed_latency(16);
6522   cr     : EX2(write);
6523   op1    : EX1(read);
6524   op2    : EX1(read);
6525   INS01  : ISS;
6526   ALU    : EX2;
6527 %}
6528 
6529 // Compare reg-reg
6530 // Eg.  CMP     x0, #N
6531 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6532 %{
6533   single_instruction;
6534 //  fixed_latency(16);
6535   cr     : EX2(write);
6536   op1    : EX1(read);
6537   INS01  : ISS;
6538   ALU    : EX2;
6539 %}
6540 
6541 //------- Conditional instructions ------------------------
6542 
6543 // Conditional no operands
6544 // Eg.  CSINC   x0, zr, zr, <cond>
6545 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6546 %{
6547   single_instruction;
6548   cr     : EX1(read);
6549   dst    : EX2(write);
6550   INS01  : ISS;
6551   ALU    : EX2;
6552 %}
6553 
6554 // Conditional 2 operand
6555 // EG.  CSEL    X0, X1, X2, <cond>
6556 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6557 %{
6558   single_instruction;
6559   cr     : EX1(read);
6560   src1   : EX1(read);
6561   src2   : EX1(read);
6562   dst    : EX2(write);
6563   INS01  : ISS;
6564   ALU    : EX2;
6565 %}
6566 
6567 // Conditional 2 operand
6568 // EG.  CSEL    X0, X1, X2, <cond>
6569 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6570 %{
6571   single_instruction;
6572   cr     : EX1(read);
6573   src    : EX1(read);
6574   dst    : EX2(write);
6575   INS01  : ISS;
6576   ALU    : EX2;
6577 %}
6578 
6579 //------- Multiply pipeline operations --------------------
6580 
6581 // Multiply reg-reg
6582 // Eg.  MUL     w0, w1, w2
6583 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6584 %{
6585   single_instruction;
6586   dst    : WR(write);
6587   src1   : ISS(read);
6588   src2   : ISS(read);
6589   INS01  : ISS;
6590   MAC    : WR;
6591 %}
6592 
6593 // Multiply accumulate
6594 // Eg.  MADD    w0, w1, w2, w3
6595 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6596 %{
6597   single_instruction;
6598   dst    : WR(write);
6599   src1   : ISS(read);
6600   src2   : ISS(read);
6601   src3   : ISS(read);
6602   INS01  : ISS;
6603   MAC    : WR;
6604 %}
6605 
6606 // Eg.  MUL     w0, w1, w2
6607 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6608 %{
6609   single_instruction;
6610   fixed_latency(3); // Maximum latency for 64 bit mul
6611   dst    : WR(write);
6612   src1   : ISS(read);
6613   src2   : ISS(read);
6614   INS01  : ISS;
6615   MAC    : WR;
6616 %}
6617 
6618 // Multiply accumulate
6619 // Eg.  MADD    w0, w1, w2, w3
6620 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6621 %{
6622   single_instruction;
6623   fixed_latency(3); // Maximum latency for 64 bit mul
6624   dst    : WR(write);
6625   src1   : ISS(read);
6626   src2   : ISS(read);
6627   src3   : ISS(read);
6628   INS01  : ISS;
6629   MAC    : WR;
6630 %}
6631 
6632 //------- Divide pipeline operations --------------------
6633 
6634 // Eg.  SDIV    w0, w1, w2
6635 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6636 %{
6637   single_instruction;
6638   fixed_latency(8); // Maximum latency for 32 bit divide
6639   dst    : WR(write);
6640   src1   : ISS(read);
6641   src2   : ISS(read);
6642   INS0   : ISS; // Can only dual issue as instruction 0
6643   DIV    : WR;
6644 %}
6645 
6646 // Eg.  SDIV    x0, x1, x2
6647 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6648 %{
6649   single_instruction;
6650   fixed_latency(16); // Maximum latency for 64 bit divide
6651   dst    : WR(write);
6652   src1   : ISS(read);
6653   src2   : ISS(read);
6654   INS0   : ISS; // Can only dual issue as instruction 0
6655   DIV    : WR;
6656 %}
6657 
6658 //------- Load pipeline operations ------------------------
6659 
6660 // Load - prefetch
6661 // Eg.  PFRM    <mem>
6662 pipe_class iload_prefetch(memory mem)
6663 %{
6664   single_instruction;
6665   mem    : ISS(read);
6666   INS01  : ISS;
6667   LDST   : WR;
6668 %}
6669 
6670 // Load - reg, mem
6671 // Eg.  LDR     x0, <mem>
6672 pipe_class iload_reg_mem(iRegI dst, memory mem)
6673 %{
6674   single_instruction;
6675   dst    : WR(write);
6676   mem    : ISS(read);
6677   INS01  : ISS;
6678   LDST   : WR;
6679 %}
6680 
6681 // Load - reg, reg
6682 // Eg.  LDR     x0, [sp, x1]
6683 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6684 %{
6685   single_instruction;
6686   dst    : WR(write);
6687   src    : ISS(read);
6688   INS01  : ISS;
6689   LDST   : WR;
6690 %}
6691 
6692 //------- Store pipeline operations -----------------------
6693 
6694 // Store - zr, mem
6695 // Eg.  STR     zr, <mem>
6696 pipe_class istore_mem(memory mem)
6697 %{
6698   single_instruction;
6699   mem    : ISS(read);
6700   INS01  : ISS;
6701   LDST   : WR;
6702 %}
6703 
6704 // Store - reg, mem
6705 // Eg.  STR     x0, <mem>
6706 pipe_class istore_reg_mem(iRegI src, memory mem)
6707 %{
6708   single_instruction;
6709   mem    : ISS(read);
6710   src    : EX2(read);
6711   INS01  : ISS;
6712   LDST   : WR;
6713 %}
6714 
6715 // Store - reg, reg
6716 // Eg. STR      x0, [sp, x1]
6717 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6718 %{
6719   single_instruction;
6720   dst    : ISS(read);
6721   src    : EX2(read);
6722   INS01  : ISS;
6723   LDST   : WR;
6724 %}
6725 
6726 //------- Store pipeline operations -----------------------
6727 
6728 // Branch
6729 pipe_class pipe_branch()
6730 %{
6731   single_instruction;
6732   INS01  : ISS;
6733   BRANCH : EX1;
6734 %}
6735 
6736 // Conditional branch
6737 pipe_class pipe_branch_cond(rFlagsReg cr)
6738 %{
6739   single_instruction;
6740   cr     : EX1(read);
6741   INS01  : ISS;
6742   BRANCH : EX1;
6743 %}
6744 
6745 // Compare & Branch
6746 // EG.  CBZ/CBNZ
6747 pipe_class pipe_cmp_branch(iRegI op1)
6748 %{
6749   single_instruction;
6750   op1    : EX1(read);
6751   INS01  : ISS;
6752   BRANCH : EX1;
6753 %}
6754 
6755 //------- Synchronisation operations ----------------------
6756 
6757 // Any operation requiring serialization.
6758 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
6759 pipe_class pipe_serial()
6760 %{
6761   single_instruction;
6762   force_serialization;
6763   fixed_latency(16);
6764   INS01  : ISS(2); // Cannot dual issue with any other instruction
6765   LDST   : WR;
6766 %}
6767 
6768 // Generic big/slow expanded idiom - also serialized
6769 pipe_class pipe_slow()
6770 %{
6771   instruction_count(10);
6772   multiple_bundles;
6773   force_serialization;
6774   fixed_latency(16);
6775   INS01  : ISS(2); // Cannot dual issue with any other instruction
6776   LDST   : WR;
6777 %}
6778 
6779 // Empty pipeline class
6780 pipe_class pipe_class_empty()
6781 %{
6782   single_instruction;
6783   fixed_latency(0);
6784 %}
6785 
6786 // Default pipeline class.
6787 pipe_class pipe_class_default()
6788 %{
6789   single_instruction;
6790   fixed_latency(2);
6791 %}
6792 
6793 // Pipeline class for compares.
6794 pipe_class pipe_class_compare()
6795 %{
6796   single_instruction;
6797   fixed_latency(16);
6798 %}
6799 
6800 // Pipeline class for memory operations.
6801 pipe_class pipe_class_memory()
6802 %{
6803   single_instruction;
6804   fixed_latency(16);
6805 %}
6806 
6807 // Pipeline class for call.
6808 pipe_class pipe_class_call()
6809 %{
6810   single_instruction;
6811   fixed_latency(100);
6812 %}
6813 
6814 // Define the class for the Nop node.
6815 define %{
6816    MachNop = pipe_class_empty;
6817 %}
6818 
6819 %}
6820 //----------INSTRUCTIONS-------------------------------------------------------
6821 //
6822 // match      -- States which machine-independent subtree may be replaced
6823 //               by this instruction.
6824 // ins_cost   -- The estimated cost of this instruction is used by instruction
6825 //               selection to identify a minimum cost tree of machine
6826 //               instructions that matches a tree of machine-independent
6827 //               instructions.
6828 // format     -- A string providing the disassembly for this instruction.
6829 //               The value of an instruction's operand may be inserted
6830 //               by referring to it with a '$' prefix.
6831 // opcode     -- Three instruction opcodes may be provided.  These are referred
6832 //               to within an encode class as $primary, $secondary, and $tertiary
6833 //               rrspectively.  The primary opcode is commonly used to
6834 //               indicate the type of machine instruction, while secondary
6835 //               and tertiary are often used for prefix options or addressing
6836 //               modes.
6837 // ins_encode -- A list of encode classes with parameters. The encode class
6838 //               name must have been defined in an 'enc_class' specification
6839 //               in the encode section of the architecture description.
6840 
6841 // ============================================================================
6842 // Memory (Load/Store) Instructions
6843 
6844 // Load Instructions
6845 
6846 // Load Byte (8 bit signed)
6847 instruct loadB(iRegINoSp dst, memory1 mem)
6848 %{
6849   match(Set dst (LoadB mem));
6850   predicate(!needs_acquiring_load(n));
6851 
6852   ins_cost(4 * INSN_COST);
6853   format %{ "ldrsbw  $dst, $mem\t# byte" %}
6854 
6855   ins_encode(aarch64_enc_ldrsbw(dst, mem));
6856 
6857   ins_pipe(iload_reg_mem);
6858 %}
6859 
6860 // Load Byte (8 bit signed) into long
6861 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6862 %{
6863   match(Set dst (ConvI2L (LoadB mem)));
6864   predicate(!needs_acquiring_load(n->in(1)));
6865 
6866   ins_cost(4 * INSN_COST);
6867   format %{ "ldrsb  $dst, $mem\t# byte" %}
6868 
6869   ins_encode(aarch64_enc_ldrsb(dst, mem));
6870 
6871   ins_pipe(iload_reg_mem);
6872 %}
6873 
6874 // Load Byte (8 bit unsigned)
6875 instruct loadUB(iRegINoSp dst, memory1 mem)
6876 %{
6877   match(Set dst (LoadUB mem));
6878   predicate(!needs_acquiring_load(n));
6879 
6880   ins_cost(4 * INSN_COST);
6881   format %{ "ldrbw  $dst, $mem\t# byte" %}
6882 
6883   ins_encode(aarch64_enc_ldrb(dst, mem));
6884 
6885   ins_pipe(iload_reg_mem);
6886 %}
6887 
6888 // Load Byte (8 bit unsigned) into long
6889 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6890 %{
6891   match(Set dst (ConvI2L (LoadUB mem)));
6892   predicate(!needs_acquiring_load(n->in(1)));
6893 
6894   ins_cost(4 * INSN_COST);
6895   format %{ "ldrb  $dst, $mem\t# byte" %}
6896 
6897   ins_encode(aarch64_enc_ldrb(dst, mem));
6898 
6899   ins_pipe(iload_reg_mem);
6900 %}
6901 
6902 // Load Short (16 bit signed)
6903 instruct loadS(iRegINoSp dst, memory2 mem)
6904 %{
6905   match(Set dst (LoadS mem));
6906   predicate(!needs_acquiring_load(n));
6907 
6908   ins_cost(4 * INSN_COST);
6909   format %{ "ldrshw  $dst, $mem\t# short" %}
6910 
6911   ins_encode(aarch64_enc_ldrshw(dst, mem));
6912 
6913   ins_pipe(iload_reg_mem);
6914 %}
6915 
6916 // Load Short (16 bit signed) into long
6917 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6918 %{
6919   match(Set dst (ConvI2L (LoadS mem)));
6920   predicate(!needs_acquiring_load(n->in(1)));
6921 
6922   ins_cost(4 * INSN_COST);
6923   format %{ "ldrsh  $dst, $mem\t# short" %}
6924 
6925   ins_encode(aarch64_enc_ldrsh(dst, mem));
6926 
6927   ins_pipe(iload_reg_mem);
6928 %}
6929 
6930 // Load Char (16 bit unsigned)
6931 instruct loadUS(iRegINoSp dst, memory2 mem)
6932 %{
6933   match(Set dst (LoadUS mem));
6934   predicate(!needs_acquiring_load(n));
6935 
6936   ins_cost(4 * INSN_COST);
6937   format %{ "ldrh  $dst, $mem\t# short" %}
6938 
6939   ins_encode(aarch64_enc_ldrh(dst, mem));
6940 
6941   ins_pipe(iload_reg_mem);
6942 %}
6943 
6944 // Load Short/Char (16 bit unsigned) into long
6945 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6946 %{
6947   match(Set dst (ConvI2L (LoadUS mem)));
6948   predicate(!needs_acquiring_load(n->in(1)));
6949 
6950   ins_cost(4 * INSN_COST);
6951   format %{ "ldrh  $dst, $mem\t# short" %}
6952 
6953   ins_encode(aarch64_enc_ldrh(dst, mem));
6954 
6955   ins_pipe(iload_reg_mem);
6956 %}
6957 
6958 // Load Integer (32 bit signed)
6959 instruct loadI(iRegINoSp dst, memory4 mem)
6960 %{
6961   match(Set dst (LoadI mem));
6962   predicate(!needs_acquiring_load(n));
6963 
6964   ins_cost(4 * INSN_COST);
6965   format %{ "ldrw  $dst, $mem\t# int" %}
6966 
6967   ins_encode(aarch64_enc_ldrw(dst, mem));
6968 
6969   ins_pipe(iload_reg_mem);
6970 %}
6971 
6972 // Load Integer (32 bit signed) into long
6973 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6974 %{
6975   match(Set dst (ConvI2L (LoadI mem)));
6976   predicate(!needs_acquiring_load(n->in(1)));
6977 
6978   ins_cost(4 * INSN_COST);
6979   format %{ "ldrsw  $dst, $mem\t# int" %}
6980 
6981   ins_encode(aarch64_enc_ldrsw(dst, mem));
6982 
6983   ins_pipe(iload_reg_mem);
6984 %}
6985 
6986 // Load Integer (32 bit unsigned) into long
6987 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6988 %{
6989   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6990   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6991 
6992   ins_cost(4 * INSN_COST);
6993   format %{ "ldrw  $dst, $mem\t# int" %}
6994 
6995   ins_encode(aarch64_enc_ldrw(dst, mem));
6996 
6997   ins_pipe(iload_reg_mem);
6998 %}
6999 
7000 // Load Long (64 bit signed)
7001 instruct loadL(iRegLNoSp dst, memory8 mem)
7002 %{
7003   match(Set dst (LoadL mem));
7004   predicate(!needs_acquiring_load(n));
7005 
7006   ins_cost(4 * INSN_COST);
7007   format %{ "ldr  $dst, $mem\t# int" %}
7008 
7009   ins_encode(aarch64_enc_ldr(dst, mem));
7010 
7011   ins_pipe(iload_reg_mem);
7012 %}
7013 
7014 // Load Range
7015 instruct loadRange(iRegINoSp dst, memory4 mem)
7016 %{
7017   match(Set dst (LoadRange mem));
7018 
7019   ins_cost(4 * INSN_COST);
7020   format %{ "ldrw  $dst, $mem\t# range" %}
7021 
7022   ins_encode(aarch64_enc_ldrw(dst, mem));
7023 
7024   ins_pipe(iload_reg_mem);
7025 %}
7026 
7027 // Load Pointer
7028 instruct loadP(iRegPNoSp dst, memory8 mem)
7029 %{
7030   match(Set dst (LoadP mem));
7031   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7032 
7033   ins_cost(4 * INSN_COST);
7034   format %{ "ldr  $dst, $mem\t# ptr" %}
7035 
7036   ins_encode(aarch64_enc_ldr(dst, mem));
7037 
7038   ins_pipe(iload_reg_mem);
7039 %}
7040 
7041 // Load Compressed Pointer
7042 instruct loadN(iRegNNoSp dst, memory4 mem)
7043 %{
7044   match(Set dst (LoadN mem));
7045   predicate(!needs_acquiring_load(n));
7046 
7047   ins_cost(4 * INSN_COST);
7048   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
7049 
7050   ins_encode(aarch64_enc_ldrw(dst, mem));
7051 
7052   ins_pipe(iload_reg_mem);
7053 %}
7054 
7055 // Load Klass Pointer
7056 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7057 %{
7058   match(Set dst (LoadKlass mem));
7059   predicate(!needs_acquiring_load(n));
7060 
7061   ins_cost(4 * INSN_COST);
7062   format %{ "ldr  $dst, $mem\t# class" %}
7063 
7064   ins_encode(aarch64_enc_ldr(dst, mem));
7065 
7066   ins_pipe(iload_reg_mem);
7067 %}
7068 
7069 // Load Narrow Klass Pointer
7070 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7071 %{
7072   match(Set dst (LoadNKlass mem));
7073   predicate(!needs_acquiring_load(n));
7074 
7075   ins_cost(4 * INSN_COST);
7076   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
7077 
7078   ins_encode(aarch64_enc_ldrw(dst, mem));
7079 
7080   ins_pipe(iload_reg_mem);
7081 %}
7082 
7083 // Load Float
7084 instruct loadF(vRegF dst, memory4 mem)
7085 %{
7086   match(Set dst (LoadF mem));
7087   predicate(!needs_acquiring_load(n));
7088 
7089   ins_cost(4 * INSN_COST);
7090   format %{ "ldrs  $dst, $mem\t# float" %}
7091 
7092   ins_encode( aarch64_enc_ldrs(dst, mem) );
7093 
7094   ins_pipe(pipe_class_memory);
7095 %}
7096 
7097 // Load Double
7098 instruct loadD(vRegD dst, memory8 mem)
7099 %{
7100   match(Set dst (LoadD mem));
7101   predicate(!needs_acquiring_load(n));
7102 
7103   ins_cost(4 * INSN_COST);
7104   format %{ "ldrd  $dst, $mem\t# double" %}
7105 
7106   ins_encode( aarch64_enc_ldrd(dst, mem) );
7107 
7108   ins_pipe(pipe_class_memory);
7109 %}
7110 
7111 
7112 // Load Int Constant
7113 instruct loadConI(iRegINoSp dst, immI src)
7114 %{
7115   match(Set dst src);
7116 
7117   ins_cost(INSN_COST);
7118   format %{ "mov $dst, $src\t# int" %}
7119 
7120   ins_encode( aarch64_enc_movw_imm(dst, src) );
7121 
7122   ins_pipe(ialu_imm);
7123 %}
7124 
7125 // Load Long Constant
7126 instruct loadConL(iRegLNoSp dst, immL src)
7127 %{
7128   match(Set dst src);
7129 
7130   ins_cost(INSN_COST);
7131   format %{ "mov $dst, $src\t# long" %}
7132 
7133   ins_encode( aarch64_enc_mov_imm(dst, src) );
7134 
7135   ins_pipe(ialu_imm);
7136 %}
7137 
7138 // Load Pointer Constant
7139 
7140 instruct loadConP(iRegPNoSp dst, immP con)
7141 %{
7142   match(Set dst con);
7143 
7144   ins_cost(INSN_COST * 4);
7145   format %{
7146     "mov  $dst, $con\t# ptr\n\t"
7147   %}
7148 
7149   ins_encode(aarch64_enc_mov_p(dst, con));
7150 
7151   ins_pipe(ialu_imm);
7152 %}
7153 
7154 // Load Null Pointer Constant
7155 
7156 instruct loadConP0(iRegPNoSp dst, immP0 con)
7157 %{
7158   match(Set dst con);
7159 
7160   ins_cost(INSN_COST);
7161   format %{ "mov  $dst, $con\t# NULL ptr" %}
7162 
7163   ins_encode(aarch64_enc_mov_p0(dst, con));
7164 
7165   ins_pipe(ialu_imm);
7166 %}
7167 
7168 // Load Pointer Constant One
7169 
7170 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7171 %{
7172   match(Set dst con);
7173 
7174   ins_cost(INSN_COST);
7175   format %{ "mov  $dst, $con\t# NULL ptr" %}
7176 
7177   ins_encode(aarch64_enc_mov_p1(dst, con));
7178 
7179   ins_pipe(ialu_imm);
7180 %}
7181 
7182 // Load Byte Map Base Constant
7183 
7184 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7185 %{
7186   match(Set dst con);
7187 
7188   ins_cost(INSN_COST);
7189   format %{ "adr  $dst, $con\t# Byte Map Base" %}
7190 
7191   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7192 
7193   ins_pipe(ialu_imm);
7194 %}
7195 
7196 // Load Narrow Pointer Constant
7197 
7198 instruct loadConN(iRegNNoSp dst, immN con)
7199 %{
7200   match(Set dst con);
7201 
7202   ins_cost(INSN_COST * 4);
7203   format %{ "mov  $dst, $con\t# compressed ptr" %}
7204 
7205   ins_encode(aarch64_enc_mov_n(dst, con));
7206 
7207   ins_pipe(ialu_imm);
7208 %}
7209 
7210 // Load Narrow Null Pointer Constant
7211 
7212 instruct loadConN0(iRegNNoSp dst, immN0 con)
7213 %{
7214   match(Set dst con);
7215 
7216   ins_cost(INSN_COST);
7217   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
7218 
7219   ins_encode(aarch64_enc_mov_n0(dst, con));
7220 
7221   ins_pipe(ialu_imm);
7222 %}
7223 
7224 // Load Narrow Klass Constant
7225 
7226 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7227 %{
7228   match(Set dst con);
7229 
7230   ins_cost(INSN_COST);
7231   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
7232 
7233   ins_encode(aarch64_enc_mov_nk(dst, con));
7234 
7235   ins_pipe(ialu_imm);
7236 %}
7237 
7238 // Load Packed Float Constant
7239 
7240 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7241   match(Set dst con);
7242   ins_cost(INSN_COST * 4);
7243   format %{ "fmovs  $dst, $con"%}
7244   ins_encode %{
7245     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7246   %}
7247 
7248   ins_pipe(fp_imm_s);
7249 %}
7250 
7251 // Load Float Constant
7252 
7253 instruct loadConF(vRegF dst, immF con) %{
7254   match(Set dst con);
7255 
7256   ins_cost(INSN_COST * 4);
7257 
7258   format %{
7259     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7260   %}
7261 
7262   ins_encode %{
7263     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7264   %}
7265 
7266   ins_pipe(fp_load_constant_s);
7267 %}
7268 
7269 // Load Packed Double Constant
7270 
7271 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7272   match(Set dst con);
7273   ins_cost(INSN_COST);
7274   format %{ "fmovd  $dst, $con"%}
7275   ins_encode %{
7276     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7277   %}
7278 
7279   ins_pipe(fp_imm_d);
7280 %}
7281 
7282 // Load Double Constant
7283 
7284 instruct loadConD(vRegD dst, immD con) %{
7285   match(Set dst con);
7286 
7287   ins_cost(INSN_COST * 5);
7288   format %{
7289     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7290   %}
7291 
7292   ins_encode %{
7293     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7294   %}
7295 
7296   ins_pipe(fp_load_constant_d);
7297 %}
7298 
7299 // Store Instructions
7300 
7301 // Store CMS card-mark Immediate
7302 instruct storeimmCM0(immI0 zero, memory1 mem)
7303 %{
7304   match(Set mem (StoreCM mem zero));
7305 
7306   ins_cost(INSN_COST);
7307   format %{ "storestore (elided)\n\t"
7308             "strb zr, $mem\t# byte" %}
7309 
7310   ins_encode(aarch64_enc_strb0(mem));
7311 
7312   ins_pipe(istore_mem);
7313 %}
7314 
7315 // Store CMS card-mark Immediate with intervening StoreStore
7316 // needed when using CMS with no conditional card marking
7317 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7318 %{
7319   match(Set mem (StoreCM mem zero));
7320 
7321   ins_cost(INSN_COST * 2);
7322   format %{ "storestore\n\t"
7323             "dmb ishst"
7324             "\n\tstrb zr, $mem\t# byte" %}
7325 
7326   ins_encode(aarch64_enc_strb0_ordered(mem));
7327 
7328   ins_pipe(istore_mem);
7329 %}
7330 
7331 // Store Byte
7332 instruct storeB(iRegIorL2I src, memory1 mem)
7333 %{
7334   match(Set mem (StoreB mem src));
7335   predicate(!needs_releasing_store(n));
7336 
7337   ins_cost(INSN_COST);
7338   format %{ "strb  $src, $mem\t# byte" %}
7339 
7340   ins_encode(aarch64_enc_strb(src, mem));
7341 
7342   ins_pipe(istore_reg_mem);
7343 %}
7344 
7345 
7346 instruct storeimmB0(immI0 zero, memory1 mem)
7347 %{
7348   match(Set mem (StoreB mem zero));
7349   predicate(!needs_releasing_store(n));
7350 
7351   ins_cost(INSN_COST);
7352   format %{ "strb rscractch2, $mem\t# byte" %}
7353 
7354   ins_encode(aarch64_enc_strb0(mem));
7355 
7356   ins_pipe(istore_mem);
7357 %}
7358 
7359 // Store Char/Short
7360 instruct storeC(iRegIorL2I src, memory2 mem)
7361 %{
7362   match(Set mem (StoreC mem src));
7363   predicate(!needs_releasing_store(n));
7364 
7365   ins_cost(INSN_COST);
7366   format %{ "strh  $src, $mem\t# short" %}
7367 
7368   ins_encode(aarch64_enc_strh(src, mem));
7369 
7370   ins_pipe(istore_reg_mem);
7371 %}
7372 
7373 instruct storeimmC0(immI0 zero, memory2 mem)
7374 %{
7375   match(Set mem (StoreC mem zero));
7376   predicate(!needs_releasing_store(n));
7377 
7378   ins_cost(INSN_COST);
7379   format %{ "strh  zr, $mem\t# short" %}
7380 
7381   ins_encode(aarch64_enc_strh0(mem));
7382 
7383   ins_pipe(istore_mem);
7384 %}
7385 
7386 // Store Integer
7387 
7388 instruct storeI(iRegIorL2I src, memory4 mem)
7389 %{
7390   match(Set mem(StoreI mem src));
7391   predicate(!needs_releasing_store(n));
7392 
7393   ins_cost(INSN_COST);
7394   format %{ "strw  $src, $mem\t# int" %}
7395 
7396   ins_encode(aarch64_enc_strw(src, mem));
7397 
7398   ins_pipe(istore_reg_mem);
7399 %}
7400 
7401 instruct storeimmI0(immI0 zero, memory4 mem)
7402 %{
7403   match(Set mem(StoreI mem zero));
7404   predicate(!needs_releasing_store(n));
7405 
7406   ins_cost(INSN_COST);
7407   format %{ "strw  zr, $mem\t# int" %}
7408 
7409   ins_encode(aarch64_enc_strw0(mem));
7410 
7411   ins_pipe(istore_mem);
7412 %}
7413 
7414 // Store Long (64 bit signed)
7415 instruct storeL(iRegL src, memory8 mem)
7416 %{
7417   match(Set mem (StoreL mem src));
7418   predicate(!needs_releasing_store(n));
7419 
7420   ins_cost(INSN_COST);
7421   format %{ "str  $src, $mem\t# int" %}
7422 
7423   ins_encode(aarch64_enc_str(src, mem));
7424 
7425   ins_pipe(istore_reg_mem);
7426 %}
7427 
7428 // Store Long (64 bit signed)
7429 instruct storeimmL0(immL0 zero, memory8 mem)
7430 %{
7431   match(Set mem (StoreL mem zero));
7432   predicate(!needs_releasing_store(n));
7433 
7434   ins_cost(INSN_COST);
7435   format %{ "str  zr, $mem\t# int" %}
7436 
7437   ins_encode(aarch64_enc_str0(mem));
7438 
7439   ins_pipe(istore_mem);
7440 %}
7441 
7442 // Store Pointer
7443 instruct storeP(iRegP src, memory8 mem)
7444 %{
7445   match(Set mem (StoreP mem src));
7446   predicate(!needs_releasing_store(n));
7447 
7448   ins_cost(INSN_COST);
7449   format %{ "str  $src, $mem\t# ptr" %}
7450 
7451   ins_encode(aarch64_enc_str(src, mem));
7452 
7453   ins_pipe(istore_reg_mem);
7454 %}
7455 
7456 // Store Pointer
7457 instruct storeimmP0(immP0 zero, memory8 mem)
7458 %{
7459   match(Set mem (StoreP mem zero));
7460   predicate(!needs_releasing_store(n));
7461 
7462   ins_cost(INSN_COST);
7463   format %{ "str zr, $mem\t# ptr" %}
7464 
7465   ins_encode(aarch64_enc_str0(mem));
7466 
7467   ins_pipe(istore_mem);
7468 %}
7469 
7470 // Store Compressed Pointer
7471 instruct storeN(iRegN src, memory4 mem)
7472 %{
7473   match(Set mem (StoreN mem src));
7474   predicate(!needs_releasing_store(n));
7475 
7476   ins_cost(INSN_COST);
7477   format %{ "strw  $src, $mem\t# compressed ptr" %}
7478 
7479   ins_encode(aarch64_enc_strw(src, mem));
7480 
7481   ins_pipe(istore_reg_mem);
7482 %}
7483 
7484 instruct storeImmN0(immN0 zero, memory4 mem)
7485 %{
7486   match(Set mem (StoreN mem zero));
7487   predicate(!needs_releasing_store(n));
7488 
7489   ins_cost(INSN_COST);
7490   format %{ "strw  zr, $mem\t# compressed ptr" %}
7491 
7492   ins_encode(aarch64_enc_strw0(mem));
7493 
7494   ins_pipe(istore_mem);
7495 %}
7496 
7497 // Store Float
7498 instruct storeF(vRegF src, memory4 mem)
7499 %{
7500   match(Set mem (StoreF mem src));
7501   predicate(!needs_releasing_store(n));
7502 
7503   ins_cost(INSN_COST);
7504   format %{ "strs  $src, $mem\t# float" %}
7505 
7506   ins_encode( aarch64_enc_strs(src, mem) );
7507 
7508   ins_pipe(pipe_class_memory);
7509 %}
7510 
7511 // TODO
7512 // implement storeImmF0 and storeFImmPacked
7513 
7514 // Store Double
7515 instruct storeD(vRegD src, memory8 mem)
7516 %{
7517   match(Set mem (StoreD mem src));
7518   predicate(!needs_releasing_store(n));
7519 
7520   ins_cost(INSN_COST);
7521   format %{ "strd  $src, $mem\t# double" %}
7522 
7523   ins_encode( aarch64_enc_strd(src, mem) );
7524 
7525   ins_pipe(pipe_class_memory);
7526 %}
7527 
7528 // Store Compressed Klass Pointer
7529 instruct storeNKlass(iRegN src, memory4 mem)
7530 %{
7531   predicate(!needs_releasing_store(n));
7532   match(Set mem (StoreNKlass mem src));
7533 
7534   ins_cost(INSN_COST);
7535   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
7536 
7537   ins_encode(aarch64_enc_strw(src, mem));
7538 
7539   ins_pipe(istore_reg_mem);
7540 %}
7541 
7542 // TODO
7543 // implement storeImmD0 and storeDImmPacked
7544 
7545 // prefetch instructions
7546 // Must be safe to execute with invalid address (cannot fault).
7547 
7548 instruct prefetchalloc( memory8 mem ) %{
7549   match(PrefetchAllocation mem);
7550 
7551   ins_cost(INSN_COST);
7552   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7553 
7554   ins_encode( aarch64_enc_prefetchw(mem) );
7555 
7556   ins_pipe(iload_prefetch);
7557 %}
7558 
7559 //  ---------------- volatile loads and stores ----------------
7560 
7561 // Load Byte (8 bit signed)
7562 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7563 %{
7564   match(Set dst (LoadB mem));
7565 
7566   ins_cost(VOLATILE_REF_COST);
7567   format %{ "ldarsb  $dst, $mem\t# byte" %}
7568 
7569   ins_encode(aarch64_enc_ldarsb(dst, mem));
7570 
7571   ins_pipe(pipe_serial);
7572 %}
7573 
7574 // Load Byte (8 bit signed) into long
7575 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7576 %{
7577   match(Set dst (ConvI2L (LoadB mem)));
7578 
7579   ins_cost(VOLATILE_REF_COST);
7580   format %{ "ldarsb  $dst, $mem\t# byte" %}
7581 
7582   ins_encode(aarch64_enc_ldarsb(dst, mem));
7583 
7584   ins_pipe(pipe_serial);
7585 %}
7586 
7587 // Load Byte (8 bit unsigned)
7588 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7589 %{
7590   match(Set dst (LoadUB mem));
7591 
7592   ins_cost(VOLATILE_REF_COST);
7593   format %{ "ldarb  $dst, $mem\t# byte" %}
7594 
7595   ins_encode(aarch64_enc_ldarb(dst, mem));
7596 
7597   ins_pipe(pipe_serial);
7598 %}
7599 
7600 // Load Byte (8 bit unsigned) into long
7601 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7602 %{
7603   match(Set dst (ConvI2L (LoadUB mem)));
7604 
7605   ins_cost(VOLATILE_REF_COST);
7606   format %{ "ldarb  $dst, $mem\t# byte" %}
7607 
7608   ins_encode(aarch64_enc_ldarb(dst, mem));
7609 
7610   ins_pipe(pipe_serial);
7611 %}
7612 
7613 // Load Short (16 bit signed)
7614 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7615 %{
7616   match(Set dst (LoadS mem));
7617 
7618   ins_cost(VOLATILE_REF_COST);
7619   format %{ "ldarshw  $dst, $mem\t# short" %}
7620 
7621   ins_encode(aarch64_enc_ldarshw(dst, mem));
7622 
7623   ins_pipe(pipe_serial);
7624 %}
7625 
7626 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7627 %{
7628   match(Set dst (LoadUS mem));
7629 
7630   ins_cost(VOLATILE_REF_COST);
7631   format %{ "ldarhw  $dst, $mem\t# short" %}
7632 
7633   ins_encode(aarch64_enc_ldarhw(dst, mem));
7634 
7635   ins_pipe(pipe_serial);
7636 %}
7637 
7638 // Load Short/Char (16 bit unsigned) into long
7639 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7640 %{
7641   match(Set dst (ConvI2L (LoadUS mem)));
7642 
7643   ins_cost(VOLATILE_REF_COST);
7644   format %{ "ldarh  $dst, $mem\t# short" %}
7645 
7646   ins_encode(aarch64_enc_ldarh(dst, mem));
7647 
7648   ins_pipe(pipe_serial);
7649 %}
7650 
7651 // Load Short/Char (16 bit signed) into long
7652 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7653 %{
7654   match(Set dst (ConvI2L (LoadS mem)));
7655 
7656   ins_cost(VOLATILE_REF_COST);
7657   format %{ "ldarh  $dst, $mem\t# short" %}
7658 
7659   ins_encode(aarch64_enc_ldarsh(dst, mem));
7660 
7661   ins_pipe(pipe_serial);
7662 %}
7663 
7664 // Load Integer (32 bit signed)
7665 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7666 %{
7667   match(Set dst (LoadI mem));
7668 
7669   ins_cost(VOLATILE_REF_COST);
7670   format %{ "ldarw  $dst, $mem\t# int" %}
7671 
7672   ins_encode(aarch64_enc_ldarw(dst, mem));
7673 
7674   ins_pipe(pipe_serial);
7675 %}
7676 
7677 // Load Integer (32 bit unsigned) into long
7678 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7679 %{
7680   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7681 
7682   ins_cost(VOLATILE_REF_COST);
7683   format %{ "ldarw  $dst, $mem\t# int" %}
7684 
7685   ins_encode(aarch64_enc_ldarw(dst, mem));
7686 
7687   ins_pipe(pipe_serial);
7688 %}
7689 
7690 // Load Long (64 bit signed)
7691 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7692 %{
7693   match(Set dst (LoadL mem));
7694 
7695   ins_cost(VOLATILE_REF_COST);
7696   format %{ "ldar  $dst, $mem\t# int" %}
7697 
7698   ins_encode(aarch64_enc_ldar(dst, mem));
7699 
7700   ins_pipe(pipe_serial);
7701 %}
7702 
7703 // Load Pointer
7704 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7705 %{
7706   match(Set dst (LoadP mem));
7707   predicate(n->as_Load()->barrier_data() == 0);
7708 
7709   ins_cost(VOLATILE_REF_COST);
7710   format %{ "ldar  $dst, $mem\t# ptr" %}
7711 
7712   ins_encode(aarch64_enc_ldar(dst, mem));
7713 
7714   ins_pipe(pipe_serial);
7715 %}
7716 
7717 // Load Compressed Pointer
7718 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7719 %{
7720   match(Set dst (LoadN mem));
7721 
7722   ins_cost(VOLATILE_REF_COST);
7723   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
7724 
7725   ins_encode(aarch64_enc_ldarw(dst, mem));
7726 
7727   ins_pipe(pipe_serial);
7728 %}
7729 
7730 // Load Float
7731 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7732 %{
7733   match(Set dst (LoadF mem));
7734 
7735   ins_cost(VOLATILE_REF_COST);
7736   format %{ "ldars  $dst, $mem\t# float" %}
7737 
7738   ins_encode( aarch64_enc_fldars(dst, mem) );
7739 
7740   ins_pipe(pipe_serial);
7741 %}
7742 
7743 // Load Double
7744 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7745 %{
7746   match(Set dst (LoadD mem));
7747 
7748   ins_cost(VOLATILE_REF_COST);
7749   format %{ "ldard  $dst, $mem\t# double" %}
7750 
7751   ins_encode( aarch64_enc_fldard(dst, mem) );
7752 
7753   ins_pipe(pipe_serial);
7754 %}
7755 
7756 // Store Byte
7757 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7758 %{
7759   match(Set mem (StoreB mem src));
7760 
7761   ins_cost(VOLATILE_REF_COST);
7762   format %{ "stlrb  $src, $mem\t# byte" %}
7763 
7764   ins_encode(aarch64_enc_stlrb(src, mem));
7765 
7766   ins_pipe(pipe_class_memory);
7767 %}
7768 
7769 // Store Char/Short
7770 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7771 %{
7772   match(Set mem (StoreC mem src));
7773 
7774   ins_cost(VOLATILE_REF_COST);
7775   format %{ "stlrh  $src, $mem\t# short" %}
7776 
7777   ins_encode(aarch64_enc_stlrh(src, mem));
7778 
7779   ins_pipe(pipe_class_memory);
7780 %}
7781 
7782 // Store Integer
7783 
7784 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7785 %{
7786   match(Set mem(StoreI mem src));
7787 
7788   ins_cost(VOLATILE_REF_COST);
7789   format %{ "stlrw  $src, $mem\t# int" %}
7790 
7791   ins_encode(aarch64_enc_stlrw(src, mem));
7792 
7793   ins_pipe(pipe_class_memory);
7794 %}
7795 
7796 // Store Long (64 bit signed)
7797 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7798 %{
7799   match(Set mem (StoreL mem src));
7800 
7801   ins_cost(VOLATILE_REF_COST);
7802   format %{ "stlr  $src, $mem\t# int" %}
7803 
7804   ins_encode(aarch64_enc_stlr(src, mem));
7805 
7806   ins_pipe(pipe_class_memory);
7807 %}
7808 
7809 // Store Pointer
7810 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7811 %{
7812   match(Set mem (StoreP mem src));
7813 
7814   ins_cost(VOLATILE_REF_COST);
7815   format %{ "stlr  $src, $mem\t# ptr" %}
7816 
7817   ins_encode(aarch64_enc_stlr(src, mem));
7818 
7819   ins_pipe(pipe_class_memory);
7820 %}
7821 
7822 // Store Compressed Pointer
7823 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7824 %{
7825   match(Set mem (StoreN mem src));
7826 
7827   ins_cost(VOLATILE_REF_COST);
7828   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
7829 
7830   ins_encode(aarch64_enc_stlrw(src, mem));
7831 
7832   ins_pipe(pipe_class_memory);
7833 %}
7834 
7835 // Store Float
7836 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7837 %{
7838   match(Set mem (StoreF mem src));
7839 
7840   ins_cost(VOLATILE_REF_COST);
7841   format %{ "stlrs  $src, $mem\t# float" %}
7842 
7843   ins_encode( aarch64_enc_fstlrs(src, mem) );
7844 
7845   ins_pipe(pipe_class_memory);
7846 %}
7847 
7848 // TODO
7849 // implement storeImmF0 and storeFImmPacked
7850 
7851 // Store Double
7852 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7853 %{
7854   match(Set mem (StoreD mem src));
7855 
7856   ins_cost(VOLATILE_REF_COST);
7857   format %{ "stlrd  $src, $mem\t# double" %}
7858 
7859   ins_encode( aarch64_enc_fstlrd(src, mem) );
7860 
7861   ins_pipe(pipe_class_memory);
7862 %}
7863 
7864 //  ---------------- end of volatile loads and stores ----------------
7865 
7866 instruct cacheWB(indirect addr)
7867 %{
7868   predicate(VM_Version::supports_data_cache_line_flush());
7869   match(CacheWB addr);
7870 
7871   ins_cost(100);
7872   format %{"cache wb $addr" %}
7873   ins_encode %{
7874     assert($addr->index_position() < 0, "should be");
7875     assert($addr$$disp == 0, "should be");
7876     __ cache_wb(Address($addr$$base$$Register, 0));
7877   %}
7878   ins_pipe(pipe_slow); // XXX
7879 %}
7880 
7881 instruct cacheWBPreSync()
7882 %{
7883   predicate(VM_Version::supports_data_cache_line_flush());
7884   match(CacheWBPreSync);
7885 
7886   ins_cost(100);
7887   format %{"cache wb presync" %}
7888   ins_encode %{
7889     __ cache_wbsync(true);
7890   %}
7891   ins_pipe(pipe_slow); // XXX
7892 %}
7893 
7894 instruct cacheWBPostSync()
7895 %{
7896   predicate(VM_Version::supports_data_cache_line_flush());
7897   match(CacheWBPostSync);
7898 
7899   ins_cost(100);
7900   format %{"cache wb postsync" %}
7901   ins_encode %{
7902     __ cache_wbsync(false);
7903   %}
7904   ins_pipe(pipe_slow); // XXX
7905 %}
7906 
7907 // ============================================================================
7908 // BSWAP Instructions
7909 
7910 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7911   match(Set dst (ReverseBytesI src));
7912 
7913   ins_cost(INSN_COST);
7914   format %{ "revw  $dst, $src" %}
7915 
7916   ins_encode %{
7917     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7918   %}
7919 
7920   ins_pipe(ialu_reg);
7921 %}
7922 
7923 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7924   match(Set dst (ReverseBytesL src));
7925 
7926   ins_cost(INSN_COST);
7927   format %{ "rev  $dst, $src" %}
7928 
7929   ins_encode %{
7930     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7931   %}
7932 
7933   ins_pipe(ialu_reg);
7934 %}
7935 
7936 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7937   match(Set dst (ReverseBytesUS src));
7938 
7939   ins_cost(INSN_COST);
7940   format %{ "rev16w  $dst, $src" %}
7941 
7942   ins_encode %{
7943     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7944   %}
7945 
7946   ins_pipe(ialu_reg);
7947 %}
7948 
7949 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7950   match(Set dst (ReverseBytesS src));
7951 
7952   ins_cost(INSN_COST);
7953   format %{ "rev16w  $dst, $src\n\t"
7954             "sbfmw $dst, $dst, #0, #15" %}
7955 
7956   ins_encode %{
7957     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7958     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7959   %}
7960 
7961   ins_pipe(ialu_reg);
7962 %}
7963 
7964 // ============================================================================
7965 // Zero Count Instructions
7966 
7967 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7968   match(Set dst (CountLeadingZerosI src));
7969 
7970   ins_cost(INSN_COST);
7971   format %{ "clzw  $dst, $src" %}
7972   ins_encode %{
7973     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7974   %}
7975 
7976   ins_pipe(ialu_reg);
7977 %}
7978 
7979 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7980   match(Set dst (CountLeadingZerosL src));
7981 
7982   ins_cost(INSN_COST);
7983   format %{ "clz   $dst, $src" %}
7984   ins_encode %{
7985     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7986   %}
7987 
7988   ins_pipe(ialu_reg);
7989 %}
7990 
7991 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7992   match(Set dst (CountTrailingZerosI src));
7993 
7994   ins_cost(INSN_COST * 2);
7995   format %{ "rbitw  $dst, $src\n\t"
7996             "clzw   $dst, $dst" %}
7997   ins_encode %{
7998     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7999     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8000   %}
8001 
8002   ins_pipe(ialu_reg);
8003 %}
8004 
8005 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8006   match(Set dst (CountTrailingZerosL src));
8007 
8008   ins_cost(INSN_COST * 2);
8009   format %{ "rbit   $dst, $src\n\t"
8010             "clz    $dst, $dst" %}
8011   ins_encode %{
8012     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8013     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8014   %}
8015 
8016   ins_pipe(ialu_reg);
8017 %}
8018 
8019 //---------- Population Count Instructions -------------------------------------
8020 //
8021 
8022 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8023   predicate(UsePopCountInstruction);
8024   match(Set dst (PopCountI src));
8025   effect(TEMP tmp);
8026   ins_cost(INSN_COST * 13);
8027 
8028   format %{ "movw   $src, $src\n\t"
8029             "mov    $tmp, $src\t# vector (1D)\n\t"
8030             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
8031             "addv   $tmp, $tmp\t# vector (8B)\n\t"
8032             "mov    $dst, $tmp\t# vector (1D)" %}
8033   ins_encode %{
8034     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8035     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
8036     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8037     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8038     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8039   %}
8040 
8041   ins_pipe(pipe_class_default);
8042 %}
8043 
8044 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8045   predicate(UsePopCountInstruction);
8046   match(Set dst (PopCountI (LoadI mem)));
8047   effect(TEMP tmp);
8048   ins_cost(INSN_COST * 13);
8049 
8050   format %{ "ldrs   $tmp, $mem\n\t"
8051             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
8052             "addv   $tmp, $tmp\t# vector (8B)\n\t"
8053             "mov    $dst, $tmp\t# vector (1D)" %}
8054   ins_encode %{
8055     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8056     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8057               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8058     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8059     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8060     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8061   %}
8062 
8063   ins_pipe(pipe_class_default);
8064 %}
8065 
8066 // Note: Long.bitCount(long) returns an int.
8067 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8068   predicate(UsePopCountInstruction);
8069   match(Set dst (PopCountL src));
8070   effect(TEMP tmp);
8071   ins_cost(INSN_COST * 13);
8072 
8073   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
8074             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
8075             "addv   $tmp, $tmp\t# vector (8B)\n\t"
8076             "mov    $dst, $tmp\t# vector (1D)" %}
8077   ins_encode %{
8078     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
8079     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8080     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8081     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8082   %}
8083 
8084   ins_pipe(pipe_class_default);
8085 %}
8086 
8087 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8088   predicate(UsePopCountInstruction);
8089   match(Set dst (PopCountL (LoadL mem)));
8090   effect(TEMP tmp);
8091   ins_cost(INSN_COST * 13);
8092 
8093   format %{ "ldrd   $tmp, $mem\n\t"
8094             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
8095             "addv   $tmp, $tmp\t# vector (8B)\n\t"
8096             "mov    $dst, $tmp\t# vector (1D)" %}
8097   ins_encode %{
8098     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8099     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8100               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8101     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8102     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8103     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8104   %}
8105 
8106   ins_pipe(pipe_class_default);
8107 %}
8108 
8109 // ============================================================================
8110 // MemBar Instruction
8111 
8112 instruct load_fence() %{
8113   match(LoadFence);
8114   ins_cost(VOLATILE_REF_COST);
8115 
8116   format %{ "load_fence" %}
8117 
8118   ins_encode %{
8119     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8120   %}
8121   ins_pipe(pipe_serial);
8122 %}
8123 
8124 instruct unnecessary_membar_acquire() %{
8125   predicate(unnecessary_acquire(n));
8126   match(MemBarAcquire);
8127   ins_cost(0);
8128 
8129   format %{ "membar_acquire (elided)" %}
8130 
8131   ins_encode %{
8132     __ block_comment("membar_acquire (elided)");
8133   %}
8134 
8135   ins_pipe(pipe_class_empty);
8136 %}
8137 
8138 instruct membar_acquire() %{
8139   match(MemBarAcquire);
8140   ins_cost(VOLATILE_REF_COST);
8141 
8142   format %{ "membar_acquire\n\t"
8143             "dmb ish" %}
8144 
8145   ins_encode %{
8146     __ block_comment("membar_acquire");
8147     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8148   %}
8149 
8150   ins_pipe(pipe_serial);
8151 %}
8152 
8153 
8154 instruct membar_acquire_lock() %{
8155   match(MemBarAcquireLock);
8156   ins_cost(VOLATILE_REF_COST);
8157 
8158   format %{ "membar_acquire_lock (elided)" %}
8159 
8160   ins_encode %{
8161     __ block_comment("membar_acquire_lock (elided)");
8162   %}
8163 
8164   ins_pipe(pipe_serial);
8165 %}
8166 
8167 instruct store_fence() %{
8168   match(StoreFence);
8169   ins_cost(VOLATILE_REF_COST);
8170 
8171   format %{ "store_fence" %}
8172 
8173   ins_encode %{
8174     __ membar(Assembler::LoadStore|Assembler::StoreStore);
8175   %}
8176   ins_pipe(pipe_serial);
8177 %}
8178 
8179 instruct unnecessary_membar_release() %{
8180   predicate(unnecessary_release(n));
8181   match(MemBarRelease);
8182   ins_cost(0);
8183 
8184   format %{ "membar_release (elided)" %}
8185 
8186   ins_encode %{
8187     __ block_comment("membar_release (elided)");
8188   %}
8189   ins_pipe(pipe_serial);
8190 %}
8191 
8192 instruct membar_release() %{
8193   match(MemBarRelease);
8194   ins_cost(VOLATILE_REF_COST);
8195 
8196   format %{ "membar_release\n\t"
8197             "dmb ish" %}
8198 
8199   ins_encode %{
8200     __ block_comment("membar_release");
8201     __ membar(Assembler::LoadStore|Assembler::StoreStore);
8202   %}
8203   ins_pipe(pipe_serial);
8204 %}
8205 
8206 instruct membar_storestore() %{
8207   match(MemBarStoreStore);
8208   ins_cost(VOLATILE_REF_COST);
8209 
8210   format %{ "MEMBAR-store-store" %}
8211 
8212   ins_encode %{
8213     __ membar(Assembler::StoreStore);
8214   %}
8215   ins_pipe(pipe_serial);
8216 %}
8217 
8218 instruct membar_release_lock() %{
8219   match(MemBarReleaseLock);
8220   ins_cost(VOLATILE_REF_COST);
8221 
8222   format %{ "membar_release_lock (elided)" %}
8223 
8224   ins_encode %{
8225     __ block_comment("membar_release_lock (elided)");
8226   %}
8227 
8228   ins_pipe(pipe_serial);
8229 %}
8230 
8231 instruct unnecessary_membar_volatile() %{
8232   predicate(unnecessary_volatile(n));
8233   match(MemBarVolatile);
8234   ins_cost(0);
8235 
8236   format %{ "membar_volatile (elided)" %}
8237 
8238   ins_encode %{
8239     __ block_comment("membar_volatile (elided)");
8240   %}
8241 
8242   ins_pipe(pipe_serial);
8243 %}
8244 
8245 instruct membar_volatile() %{
8246   match(MemBarVolatile);
8247   ins_cost(VOLATILE_REF_COST*100);
8248 
8249   format %{ "membar_volatile\n\t"
8250              "dmb ish"%}
8251 
8252   ins_encode %{
8253     __ block_comment("membar_volatile");
8254     __ membar(Assembler::StoreLoad);
8255   %}
8256 
8257   ins_pipe(pipe_serial);
8258 %}
8259 
8260 // ============================================================================
8261 // Cast/Convert Instructions
8262 
8263 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8264   match(Set dst (CastX2P src));
8265 
8266   ins_cost(INSN_COST);
8267   format %{ "mov $dst, $src\t# long -> ptr" %}
8268 
8269   ins_encode %{
8270     if ($dst$$reg != $src$$reg) {
8271       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8272     }
8273   %}
8274 
8275   ins_pipe(ialu_reg);
8276 %}
8277 
8278 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8279   match(Set dst (CastP2X src));
8280 
8281   ins_cost(INSN_COST);
8282   format %{ "mov $dst, $src\t# ptr -> long" %}
8283 
8284   ins_encode %{
8285     if ($dst$$reg != $src$$reg) {
8286       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8287     }
8288   %}
8289 
8290   ins_pipe(ialu_reg);
8291 %}
8292 
8293 // Convert oop into int for vectors alignment masking
8294 instruct convP2I(iRegINoSp dst, iRegP src) %{
8295   match(Set dst (ConvL2I (CastP2X src)));
8296 
8297   ins_cost(INSN_COST);
8298   format %{ "movw $dst, $src\t# ptr -> int" %}
8299   ins_encode %{
8300     __ movw($dst$$Register, $src$$Register);
8301   %}
8302 
8303   ins_pipe(ialu_reg);
8304 %}
8305 
8306 // Convert compressed oop into int for vectors alignment masking
8307 // in case of 32bit oops (heap < 4Gb).
8308 instruct convN2I(iRegINoSp dst, iRegN src)
8309 %{
8310   predicate(CompressedOops::shift() == 0);
8311   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8312 
8313   ins_cost(INSN_COST);
8314   format %{ "mov dst, $src\t# compressed ptr -> int" %}
8315   ins_encode %{
8316     __ movw($dst$$Register, $src$$Register);
8317   %}
8318 
8319   ins_pipe(ialu_reg);
8320 %}
8321 
8322 
8323 // Convert oop pointer into compressed form
8324 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8325   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8326   match(Set dst (EncodeP src));
8327   effect(KILL cr);
8328   ins_cost(INSN_COST * 3);
8329   format %{ "encode_heap_oop $dst, $src" %}
8330   ins_encode %{
8331     Register s = $src$$Register;
8332     Register d = $dst$$Register;
8333     __ encode_heap_oop(d, s);
8334   %}
8335   ins_pipe(ialu_reg);
8336 %}
8337 
8338 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8339   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8340   match(Set dst (EncodeP src));
8341   ins_cost(INSN_COST * 3);
8342   format %{ "encode_heap_oop_not_null $dst, $src" %}
8343   ins_encode %{
8344     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8345   %}
8346   ins_pipe(ialu_reg);
8347 %}
8348 
8349 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8350   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8351             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8352   match(Set dst (DecodeN src));
8353   ins_cost(INSN_COST * 3);
8354   format %{ "decode_heap_oop $dst, $src" %}
8355   ins_encode %{
8356     Register s = $src$$Register;
8357     Register d = $dst$$Register;
8358     __ decode_heap_oop(d, s);
8359   %}
8360   ins_pipe(ialu_reg);
8361 %}
8362 
8363 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8364   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8365             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8366   match(Set dst (DecodeN src));
8367   ins_cost(INSN_COST * 3);
8368   format %{ "decode_heap_oop_not_null $dst, $src" %}
8369   ins_encode %{
8370     Register s = $src$$Register;
8371     Register d = $dst$$Register;
8372     __ decode_heap_oop_not_null(d, s);
8373   %}
8374   ins_pipe(ialu_reg);
8375 %}
8376 
8377 // n.b. AArch64 implementations of encode_klass_not_null and
8378 // decode_klass_not_null do not modify the flags register so, unlike
8379 // Intel, we don't kill CR as a side effect here
8380 
8381 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8382   match(Set dst (EncodePKlass src));
8383 
8384   ins_cost(INSN_COST * 3);
8385   format %{ "encode_klass_not_null $dst,$src" %}
8386 
8387   ins_encode %{
8388     Register src_reg = as_Register($src$$reg);
8389     Register dst_reg = as_Register($dst$$reg);
8390     __ encode_klass_not_null(dst_reg, src_reg);
8391   %}
8392 
8393    ins_pipe(ialu_reg);
8394 %}
8395 
8396 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8397   match(Set dst (DecodeNKlass src));
8398 
8399   ins_cost(INSN_COST * 3);
8400   format %{ "decode_klass_not_null $dst,$src" %}
8401 
8402   ins_encode %{
8403     Register src_reg = as_Register($src$$reg);
8404     Register dst_reg = as_Register($dst$$reg);
8405     if (dst_reg != src_reg) {
8406       __ decode_klass_not_null(dst_reg, src_reg);
8407     } else {
8408       __ decode_klass_not_null(dst_reg);
8409     }
8410   %}
8411 
8412    ins_pipe(ialu_reg);
8413 %}
8414 
8415 instruct checkCastPP(iRegPNoSp dst)
8416 %{
8417   match(Set dst (CheckCastPP dst));
8418 
8419   size(0);
8420   format %{ "# checkcastPP of $dst" %}
8421   ins_encode(/* empty encoding */);
8422   ins_pipe(pipe_class_empty);
8423 %}
8424 
8425 instruct castPP(iRegPNoSp dst)
8426 %{
8427   match(Set dst (CastPP dst));
8428 
8429   size(0);
8430   format %{ "# castPP of $dst" %}
8431   ins_encode(/* empty encoding */);
8432   ins_pipe(pipe_class_empty);
8433 %}
8434 
8435 instruct castII(iRegI dst)
8436 %{
8437   match(Set dst (CastII dst));
8438 
8439   size(0);
8440   format %{ "# castII of $dst" %}
8441   ins_encode(/* empty encoding */);
8442   ins_cost(0);
8443   ins_pipe(pipe_class_empty);
8444 %}
8445 
8446 // ============================================================================
8447 // Atomic operation instructions
8448 //
8449 // Intel and SPARC both implement Ideal Node LoadPLocked and
8450 // Store{PIL}Conditional instructions using a normal load for the
8451 // LoadPLocked and a CAS for the Store{PIL}Conditional.
8452 //
8453 // The ideal code appears only to use LoadPLocked/StorePLocked as a
8454 // pair to lock object allocations from Eden space when not using
8455 // TLABs.
8456 //
8457 // There does not appear to be a Load{IL}Locked Ideal Node and the
8458 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
8459 // and to use StoreIConditional only for 32-bit and StoreLConditional
8460 // only for 64-bit.
8461 //
8462 // We implement LoadPLocked and StorePLocked instructions using,
8463 // respectively the AArch64 hw load-exclusive and store-conditional
8464 // instructions. Whereas we must implement each of
8465 // Store{IL}Conditional using a CAS which employs a pair of
8466 // instructions comprising a load-exclusive followed by a
8467 // store-conditional.
8468 
8469 
8470 // Locked-load (linked load) of the current heap-top
8471 // used when updating the eden heap top
8472 // implemented using ldaxr on AArch64
8473 
8474 instruct loadPLocked(iRegPNoSp dst, indirect mem)
8475 %{
8476   match(Set dst (LoadPLocked mem));
8477 
8478   ins_cost(VOLATILE_REF_COST);
8479 
8480   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
8481 
8482   ins_encode(aarch64_enc_ldaxr(dst, mem));
8483 
8484   ins_pipe(pipe_serial);
8485 %}
8486 
8487 // Conditional-store of the updated heap-top.
8488 // Used during allocation of the shared heap.
8489 // Sets flag (EQ) on success.
8490 // implemented using stlxr on AArch64.
8491 
8492 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
8493 %{
8494   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
8495 
8496   ins_cost(VOLATILE_REF_COST);
8497 
8498  // TODO
8499  // do we need to do a store-conditional release or can we just use a
8500  // plain store-conditional?
8501 
8502   format %{
8503     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
8504     "cmpw rscratch1, zr\t# EQ on successful write"
8505   %}
8506 
8507   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
8508 
8509   ins_pipe(pipe_serial);
8510 %}
8511 
8512 
8513 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
8514 // when attempting to rebias a lock towards the current thread.  We
8515 // must use the acquire form of cmpxchg in order to guarantee acquire
8516 // semantics in this case.
8517 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
8518 %{
8519   match(Set cr (StoreLConditional mem (Binary oldval newval)));
8520 
8521   ins_cost(VOLATILE_REF_COST);
8522 
8523   format %{
8524     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8525     "cmpw rscratch1, zr\t# EQ on successful write"
8526   %}
8527 
8528   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
8529 
8530   ins_pipe(pipe_slow);
8531 %}
8532 
8533 // storeIConditional also has acquire semantics, for no better reason
8534 // than matching storeLConditional.  At the time of writing this
8535 // comment storeIConditional was not used anywhere by AArch64.
8536 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
8537 %{
8538   match(Set cr (StoreIConditional mem (Binary oldval newval)));
8539 
8540   ins_cost(VOLATILE_REF_COST);
8541 
8542   format %{
8543     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8544     "cmpw rscratch1, zr\t# EQ on successful write"
8545   %}
8546 
8547   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
8548 
8549   ins_pipe(pipe_slow);
8550 %}
8551 
8552 // standard CompareAndSwapX when we are using barriers
8553 // these have higher priority than the rules selected by a predicate
8554 
8555 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8556 // can't match them
8557 
8558 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8559 
8560   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8561   ins_cost(2 * VOLATILE_REF_COST);
8562 
8563   effect(KILL cr);
8564 
8565   format %{
8566     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8567     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8568   %}
8569 
8570   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8571             aarch64_enc_cset_eq(res));
8572 
8573   ins_pipe(pipe_slow);
8574 %}
8575 
8576 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8577 
8578   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8579   ins_cost(2 * VOLATILE_REF_COST);
8580 
8581   effect(KILL cr);
8582 
8583   format %{
8584     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8585     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8586   %}
8587 
8588   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8589             aarch64_enc_cset_eq(res));
8590 
8591   ins_pipe(pipe_slow);
8592 %}
8593 
8594 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8595 
8596   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8597   ins_cost(2 * VOLATILE_REF_COST);
8598 
8599   effect(KILL cr);
8600 
8601  format %{
8602     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8603     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8604  %}
8605 
8606  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8607             aarch64_enc_cset_eq(res));
8608 
8609   ins_pipe(pipe_slow);
8610 %}
8611 
8612 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8613 
8614   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8615   ins_cost(2 * VOLATILE_REF_COST);
8616 
8617   effect(KILL cr);
8618 
8619  format %{
8620     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8621     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8622  %}
8623 
8624  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8625             aarch64_enc_cset_eq(res));
8626 
8627   ins_pipe(pipe_slow);
8628 %}
8629 
8630 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8631 
8632   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8633   predicate(n->as_LoadStore()->barrier_data() == 0);
8634   ins_cost(2 * VOLATILE_REF_COST);
8635 
8636   effect(KILL cr);
8637 
8638  format %{
8639     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8640     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8641  %}
8642 
8643  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8644             aarch64_enc_cset_eq(res));
8645 
8646   ins_pipe(pipe_slow);
8647 %}
8648 
8649 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8650 
8651   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8652   ins_cost(2 * VOLATILE_REF_COST);
8653 
8654   effect(KILL cr);
8655 
8656  format %{
8657     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8658     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8659  %}
8660 
8661  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8662             aarch64_enc_cset_eq(res));
8663 
8664   ins_pipe(pipe_slow);
8665 %}
8666 
8667 // alternative CompareAndSwapX when we are eliding barriers
8668 
8669 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8670 
8671   predicate(needs_acquiring_load_exclusive(n));
8672   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8673   ins_cost(VOLATILE_REF_COST);
8674 
8675   effect(KILL cr);
8676 
8677   format %{
8678     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8679     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8680   %}
8681 
8682   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8683             aarch64_enc_cset_eq(res));
8684 
8685   ins_pipe(pipe_slow);
8686 %}
8687 
8688 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8689 
8690   predicate(needs_acquiring_load_exclusive(n));
8691   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8692   ins_cost(VOLATILE_REF_COST);
8693 
8694   effect(KILL cr);
8695 
8696   format %{
8697     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8698     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8699   %}
8700 
8701   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8702             aarch64_enc_cset_eq(res));
8703 
8704   ins_pipe(pipe_slow);
8705 %}
8706 
8707 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8708 
8709   predicate(needs_acquiring_load_exclusive(n));
8710   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8711   ins_cost(VOLATILE_REF_COST);
8712 
8713   effect(KILL cr);
8714 
8715  format %{
8716     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8717     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8718  %}
8719 
8720  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8721             aarch64_enc_cset_eq(res));
8722 
8723   ins_pipe(pipe_slow);
8724 %}
8725 
8726 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8727 
8728   predicate(needs_acquiring_load_exclusive(n));
8729   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8730   ins_cost(VOLATILE_REF_COST);
8731 
8732   effect(KILL cr);
8733 
8734  format %{
8735     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8736     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8737  %}
8738 
8739  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8740             aarch64_enc_cset_eq(res));
8741 
8742   ins_pipe(pipe_slow);
8743 %}
8744 
8745 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8746 
8747   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8748   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8749   ins_cost(VOLATILE_REF_COST);
8750 
8751   effect(KILL cr);
8752 
8753  format %{
8754     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8755     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8756  %}
8757 
8758  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8759             aarch64_enc_cset_eq(res));
8760 
8761   ins_pipe(pipe_slow);
8762 %}
8763 
8764 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8765 
8766   predicate(needs_acquiring_load_exclusive(n));
8767   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8768   ins_cost(VOLATILE_REF_COST);
8769 
8770   effect(KILL cr);
8771 
8772  format %{
8773     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8774     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8775  %}
8776 
8777  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8778             aarch64_enc_cset_eq(res));
8779 
8780   ins_pipe(pipe_slow);
8781 %}
8782 
8783 
8784 // ---------------------------------------------------------------------
8785 
8786 
8787 // BEGIN This section of the file is automatically generated. Do not edit --------------
8788 
8789 // Sundry CAS operations.  Note that release is always true,
8790 // regardless of the memory ordering of the CAS.  This is because we
8791 // need the volatile case to be sequentially consistent but there is
8792 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
8793 // can't check the type of memory ordering here, so we always emit a
8794 // STLXR.
8795 
8796 // This section is generated from aarch64_ad_cas.m4
8797 
8798 
8799 
8800 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8801   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8802   ins_cost(2 * VOLATILE_REF_COST);
8803   effect(TEMP_DEF res, KILL cr);
8804   format %{
8805     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8806   %}
8807   ins_encode %{
8808     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8809                Assembler::byte, /*acquire*/ false, /*release*/ true,
8810                /*weak*/ false, $res$$Register);
8811     __ sxtbw($res$$Register, $res$$Register);
8812   %}
8813   ins_pipe(pipe_slow);
8814 %}
8815 
8816 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8817   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8818   ins_cost(2 * VOLATILE_REF_COST);
8819   effect(TEMP_DEF res, KILL cr);
8820   format %{
8821     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8822   %}
8823   ins_encode %{
8824     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8825                Assembler::halfword, /*acquire*/ false, /*release*/ true,
8826                /*weak*/ false, $res$$Register);
8827     __ sxthw($res$$Register, $res$$Register);
8828   %}
8829   ins_pipe(pipe_slow);
8830 %}
8831 
8832 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8833   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8834   ins_cost(2 * VOLATILE_REF_COST);
8835   effect(TEMP_DEF res, KILL cr);
8836   format %{
8837     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8838   %}
8839   ins_encode %{
8840     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8841                Assembler::word, /*acquire*/ false, /*release*/ true,
8842                /*weak*/ false, $res$$Register);
8843   %}
8844   ins_pipe(pipe_slow);
8845 %}
8846 
8847 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8848   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8849   ins_cost(2 * VOLATILE_REF_COST);
8850   effect(TEMP_DEF res, KILL cr);
8851   format %{
8852     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8853   %}
8854   ins_encode %{
8855     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8856                Assembler::xword, /*acquire*/ false, /*release*/ true,
8857                /*weak*/ false, $res$$Register);
8858   %}
8859   ins_pipe(pipe_slow);
8860 %}
8861 
8862 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8863   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8864   ins_cost(2 * VOLATILE_REF_COST);
8865   effect(TEMP_DEF res, KILL cr);
8866   format %{
8867     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8868   %}
8869   ins_encode %{
8870     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8871                Assembler::word, /*acquire*/ false, /*release*/ true,
8872                /*weak*/ false, $res$$Register);
8873   %}
8874   ins_pipe(pipe_slow);
8875 %}
8876 
8877 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8878   predicate(n->as_LoadStore()->barrier_data() == 0);
8879   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8880   ins_cost(2 * VOLATILE_REF_COST);
8881   effect(TEMP_DEF res, KILL cr);
8882   format %{
8883     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8884   %}
8885   ins_encode %{
8886     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8887                Assembler::xword, /*acquire*/ false, /*release*/ true,
8888                /*weak*/ false, $res$$Register);
8889   %}
8890   ins_pipe(pipe_slow);
8891 %}
8892 
8893 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8894   predicate(needs_acquiring_load_exclusive(n));
8895   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8896   ins_cost(VOLATILE_REF_COST);
8897   effect(TEMP_DEF res, KILL cr);
8898   format %{
8899     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8900   %}
8901   ins_encode %{
8902     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8903                Assembler::byte, /*acquire*/ true, /*release*/ true,
8904                /*weak*/ false, $res$$Register);
8905     __ sxtbw($res$$Register, $res$$Register);
8906   %}
8907   ins_pipe(pipe_slow);
8908 %}
8909 
8910 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8911   predicate(needs_acquiring_load_exclusive(n));
8912   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8913   ins_cost(VOLATILE_REF_COST);
8914   effect(TEMP_DEF res, KILL cr);
8915   format %{
8916     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8917   %}
8918   ins_encode %{
8919     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8920                Assembler::halfword, /*acquire*/ true, /*release*/ true,
8921                /*weak*/ false, $res$$Register);
8922     __ sxthw($res$$Register, $res$$Register);
8923   %}
8924   ins_pipe(pipe_slow);
8925 %}
8926 
8927 
8928 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8929   predicate(needs_acquiring_load_exclusive(n));
8930   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8931   ins_cost(VOLATILE_REF_COST);
8932   effect(TEMP_DEF res, KILL cr);
8933   format %{
8934     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8935   %}
8936   ins_encode %{
8937     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8938                Assembler::word, /*acquire*/ true, /*release*/ true,
8939                /*weak*/ false, $res$$Register);
8940   %}
8941   ins_pipe(pipe_slow);
8942 %}
8943 
8944 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8945   predicate(needs_acquiring_load_exclusive(n));
8946   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8947   ins_cost(VOLATILE_REF_COST);
8948   effect(TEMP_DEF res, KILL cr);
8949   format %{
8950     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8951   %}
8952   ins_encode %{
8953     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8954                Assembler::xword, /*acquire*/ true, /*release*/ true,
8955                /*weak*/ false, $res$$Register);
8956   %}
8957   ins_pipe(pipe_slow);
8958 %}
8959 
8960 
8961 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8962   predicate(needs_acquiring_load_exclusive(n));
8963   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8964   ins_cost(VOLATILE_REF_COST);
8965   effect(TEMP_DEF res, KILL cr);
8966   format %{
8967     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8968   %}
8969   ins_encode %{
8970     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8971                Assembler::word, /*acquire*/ true, /*release*/ true,
8972                /*weak*/ false, $res$$Register);
8973   %}
8974   ins_pipe(pipe_slow);
8975 %}
8976 
8977 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8978   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8979   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8980   ins_cost(VOLATILE_REF_COST);
8981   effect(TEMP_DEF res, KILL cr);
8982   format %{
8983     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8984   %}
8985   ins_encode %{
8986     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8987                Assembler::xword, /*acquire*/ true, /*release*/ true,
8988                /*weak*/ false, $res$$Register);
8989   %}
8990   ins_pipe(pipe_slow);
8991 %}
8992 
8993 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8994   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8995   ins_cost(2 * VOLATILE_REF_COST);
8996   effect(KILL cr);
8997   format %{
8998     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8999     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9000   %}
9001   ins_encode %{
9002     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9003                Assembler::byte, /*acquire*/ false, /*release*/ true,
9004                /*weak*/ true, noreg);
9005     __ csetw($res$$Register, Assembler::EQ);
9006   %}
9007   ins_pipe(pipe_slow);
9008 %}
9009 
9010 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9011   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9012   ins_cost(2 * VOLATILE_REF_COST);
9013   effect(KILL cr);
9014   format %{
9015     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9016     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9017   %}
9018   ins_encode %{
9019     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9020                Assembler::halfword, /*acquire*/ false, /*release*/ true,
9021                /*weak*/ true, noreg);
9022     __ csetw($res$$Register, Assembler::EQ);
9023   %}
9024   ins_pipe(pipe_slow);
9025 %}
9026 
9027 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9028   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9029   ins_cost(2 * VOLATILE_REF_COST);
9030   effect(KILL cr);
9031   format %{
9032     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9033     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9034   %}
9035   ins_encode %{
9036     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9037                Assembler::word, /*acquire*/ false, /*release*/ true,
9038                /*weak*/ true, noreg);
9039     __ csetw($res$$Register, Assembler::EQ);
9040   %}
9041   ins_pipe(pipe_slow);
9042 %}
9043 
9044 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9045   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9046   ins_cost(2 * VOLATILE_REF_COST);
9047   effect(KILL cr);
9048   format %{
9049     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9050     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9051   %}
9052   ins_encode %{
9053     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9054                Assembler::xword, /*acquire*/ false, /*release*/ true,
9055                /*weak*/ true, noreg);
9056     __ csetw($res$$Register, Assembler::EQ);
9057   %}
9058   ins_pipe(pipe_slow);
9059 %}
9060 
9061 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9062   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9063   ins_cost(2 * VOLATILE_REF_COST);
9064   effect(KILL cr);
9065   format %{
9066     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9067     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9068   %}
9069   ins_encode %{
9070     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9071                Assembler::word, /*acquire*/ false, /*release*/ true,
9072                /*weak*/ true, noreg);
9073     __ csetw($res$$Register, Assembler::EQ);
9074   %}
9075   ins_pipe(pipe_slow);
9076 %}
9077 
9078 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9079   predicate(n->as_LoadStore()->barrier_data() == 0);
9080   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9081   ins_cost(2 * VOLATILE_REF_COST);
9082   effect(KILL cr);
9083   format %{
9084     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9085     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9086   %}
9087   ins_encode %{
9088     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9089                Assembler::xword, /*acquire*/ false, /*release*/ true,
9090                /*weak*/ true, noreg);
9091     __ csetw($res$$Register, Assembler::EQ);
9092   %}
9093   ins_pipe(pipe_slow);
9094 %}
9095 
9096 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9097   predicate(needs_acquiring_load_exclusive(n));
9098   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9099   ins_cost(VOLATILE_REF_COST);
9100   effect(KILL cr);
9101   format %{
9102     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9103     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9104   %}
9105   ins_encode %{
9106     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9107                Assembler::byte, /*acquire*/ true, /*release*/ true,
9108                /*weak*/ true, noreg);
9109     __ csetw($res$$Register, Assembler::EQ);
9110   %}
9111   ins_pipe(pipe_slow);
9112 %}
9113 
9114 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9115   predicate(needs_acquiring_load_exclusive(n));
9116   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9117   ins_cost(VOLATILE_REF_COST);
9118   effect(KILL cr);
9119   format %{
9120     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9121     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9122   %}
9123   ins_encode %{
9124     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9125                Assembler::halfword, /*acquire*/ true, /*release*/ true,
9126                /*weak*/ true, noreg);
9127     __ csetw($res$$Register, Assembler::EQ);
9128   %}
9129   ins_pipe(pipe_slow);
9130 %}
9131 
9132 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9133   predicate(needs_acquiring_load_exclusive(n));
9134   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9135   ins_cost(VOLATILE_REF_COST);
9136   effect(KILL cr);
9137   format %{
9138     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9139     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9140   %}
9141   ins_encode %{
9142     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9143                Assembler::word, /*acquire*/ true, /*release*/ true,
9144                /*weak*/ true, noreg);
9145     __ csetw($res$$Register, Assembler::EQ);
9146   %}
9147   ins_pipe(pipe_slow);
9148 %}
9149 
9150 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9151   predicate(needs_acquiring_load_exclusive(n));
9152   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9153   ins_cost(VOLATILE_REF_COST);
9154   effect(KILL cr);
9155   format %{
9156     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9157     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9158   %}
9159   ins_encode %{
9160     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9161                Assembler::xword, /*acquire*/ true, /*release*/ true,
9162                /*weak*/ true, noreg);
9163     __ csetw($res$$Register, Assembler::EQ);
9164   %}
9165   ins_pipe(pipe_slow);
9166 %}
9167 
9168 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9169   predicate(needs_acquiring_load_exclusive(n));
9170   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9171   ins_cost(VOLATILE_REF_COST);
9172   effect(KILL cr);
9173   format %{
9174     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9175     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9176   %}
9177   ins_encode %{
9178     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9179                Assembler::word, /*acquire*/ true, /*release*/ true,
9180                /*weak*/ true, noreg);
9181     __ csetw($res$$Register, Assembler::EQ);
9182   %}
9183   ins_pipe(pipe_slow);
9184 %}
9185 
9186 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9187   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9188   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9189   ins_cost(VOLATILE_REF_COST);
9190   effect(KILL cr);
9191   format %{
9192     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9193     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9194   %}
9195   ins_encode %{
9196     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9197                Assembler::xword, /*acquire*/ true, /*release*/ true,
9198                /*weak*/ true, noreg);
9199     __ csetw($res$$Register, Assembler::EQ);
9200   %}
9201   ins_pipe(pipe_slow);
9202 %}
9203 
9204 // END This section of the file is automatically generated. Do not edit --------------
9205 // ---------------------------------------------------------------------
9206 
9207 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9208   match(Set prev (GetAndSetI mem newv));
9209   ins_cost(2 * VOLATILE_REF_COST);
9210   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
9211   ins_encode %{
9212     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9213   %}
9214   ins_pipe(pipe_serial);
9215 %}
9216 
9217 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9218   match(Set prev (GetAndSetL mem newv));
9219   ins_cost(2 * VOLATILE_REF_COST);
9220   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
9221   ins_encode %{
9222     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9223   %}
9224   ins_pipe(pipe_serial);
9225 %}
9226 
9227 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9228   match(Set prev (GetAndSetN mem newv));
9229   ins_cost(2 * VOLATILE_REF_COST);
9230   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9231   ins_encode %{
9232     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9233   %}
9234   ins_pipe(pipe_serial);
9235 %}
9236 
9237 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9238   predicate(n->as_LoadStore()->barrier_data() == 0);
9239   match(Set prev (GetAndSetP mem newv));
9240   ins_cost(2 * VOLATILE_REF_COST);
9241   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
9242   ins_encode %{
9243     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9244   %}
9245   ins_pipe(pipe_serial);
9246 %}
9247 
9248 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9249   predicate(needs_acquiring_load_exclusive(n));
9250   match(Set prev (GetAndSetI mem newv));
9251   ins_cost(VOLATILE_REF_COST);
9252   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
9253   ins_encode %{
9254     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9255   %}
9256   ins_pipe(pipe_serial);
9257 %}
9258 
9259 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9260   predicate(needs_acquiring_load_exclusive(n));
9261   match(Set prev (GetAndSetL mem newv));
9262   ins_cost(VOLATILE_REF_COST);
9263   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
9264   ins_encode %{
9265     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9266   %}
9267   ins_pipe(pipe_serial);
9268 %}
9269 
9270 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9271   predicate(needs_acquiring_load_exclusive(n));
9272   match(Set prev (GetAndSetN mem newv));
9273   ins_cost(VOLATILE_REF_COST);
9274   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9275   ins_encode %{
9276     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9277   %}
9278   ins_pipe(pipe_serial);
9279 %}
9280 
9281 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9282   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9283   match(Set prev (GetAndSetP mem newv));
9284   ins_cost(VOLATILE_REF_COST);
9285   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
9286   ins_encode %{
9287     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9288   %}
9289   ins_pipe(pipe_serial);
9290 %}
9291 
9292 
9293 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9294   match(Set newval (GetAndAddL mem incr));
9295   ins_cost(2 * VOLATILE_REF_COST + 1);
9296   format %{ "get_and_addL $newval, [$mem], $incr" %}
9297   ins_encode %{
9298     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9299   %}
9300   ins_pipe(pipe_serial);
9301 %}
9302 
9303 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9304   predicate(n->as_LoadStore()->result_not_used());
9305   match(Set dummy (GetAndAddL mem incr));
9306   ins_cost(2 * VOLATILE_REF_COST);
9307   format %{ "get_and_addL [$mem], $incr" %}
9308   ins_encode %{
9309     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9310   %}
9311   ins_pipe(pipe_serial);
9312 %}
9313 
9314 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9315   match(Set newval (GetAndAddL mem incr));
9316   ins_cost(2 * VOLATILE_REF_COST + 1);
9317   format %{ "get_and_addL $newval, [$mem], $incr" %}
9318   ins_encode %{
9319     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9320   %}
9321   ins_pipe(pipe_serial);
9322 %}
9323 
9324 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9325   predicate(n->as_LoadStore()->result_not_used());
9326   match(Set dummy (GetAndAddL mem incr));
9327   ins_cost(2 * VOLATILE_REF_COST);
9328   format %{ "get_and_addL [$mem], $incr" %}
9329   ins_encode %{
9330     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9331   %}
9332   ins_pipe(pipe_serial);
9333 %}
9334 
9335 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9336   match(Set newval (GetAndAddI mem incr));
9337   ins_cost(2 * VOLATILE_REF_COST + 1);
9338   format %{ "get_and_addI $newval, [$mem], $incr" %}
9339   ins_encode %{
9340     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9341   %}
9342   ins_pipe(pipe_serial);
9343 %}
9344 
9345 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9346   predicate(n->as_LoadStore()->result_not_used());
9347   match(Set dummy (GetAndAddI mem incr));
9348   ins_cost(2 * VOLATILE_REF_COST);
9349   format %{ "get_and_addI [$mem], $incr" %}
9350   ins_encode %{
9351     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9352   %}
9353   ins_pipe(pipe_serial);
9354 %}
9355 
9356 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9357   match(Set newval (GetAndAddI mem incr));
9358   ins_cost(2 * VOLATILE_REF_COST + 1);
9359   format %{ "get_and_addI $newval, [$mem], $incr" %}
9360   ins_encode %{
9361     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9362   %}
9363   ins_pipe(pipe_serial);
9364 %}
9365 
9366 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9367   predicate(n->as_LoadStore()->result_not_used());
9368   match(Set dummy (GetAndAddI mem incr));
9369   ins_cost(2 * VOLATILE_REF_COST);
9370   format %{ "get_and_addI [$mem], $incr" %}
9371   ins_encode %{
9372     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9373   %}
9374   ins_pipe(pipe_serial);
9375 %}
9376 
9377 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9378   predicate(needs_acquiring_load_exclusive(n));
9379   match(Set newval (GetAndAddL mem incr));
9380   ins_cost(VOLATILE_REF_COST + 1);
9381   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9382   ins_encode %{
9383     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9384   %}
9385   ins_pipe(pipe_serial);
9386 %}
9387 
9388 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9389   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9390   match(Set dummy (GetAndAddL mem incr));
9391   ins_cost(VOLATILE_REF_COST);
9392   format %{ "get_and_addL_acq [$mem], $incr" %}
9393   ins_encode %{
9394     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9395   %}
9396   ins_pipe(pipe_serial);
9397 %}
9398 
9399 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9400   predicate(needs_acquiring_load_exclusive(n));
9401   match(Set newval (GetAndAddL mem incr));
9402   ins_cost(VOLATILE_REF_COST + 1);
9403   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9404   ins_encode %{
9405     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9406   %}
9407   ins_pipe(pipe_serial);
9408 %}
9409 
9410 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9411   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9412   match(Set dummy (GetAndAddL mem incr));
9413   ins_cost(VOLATILE_REF_COST);
9414   format %{ "get_and_addL_acq [$mem], $incr" %}
9415   ins_encode %{
9416     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9417   %}
9418   ins_pipe(pipe_serial);
9419 %}
9420 
9421 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9422   predicate(needs_acquiring_load_exclusive(n));
9423   match(Set newval (GetAndAddI mem incr));
9424   ins_cost(VOLATILE_REF_COST + 1);
9425   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9426   ins_encode %{
9427     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9428   %}
9429   ins_pipe(pipe_serial);
9430 %}
9431 
9432 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9433   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9434   match(Set dummy (GetAndAddI mem incr));
9435   ins_cost(VOLATILE_REF_COST);
9436   format %{ "get_and_addI_acq [$mem], $incr" %}
9437   ins_encode %{
9438     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9439   %}
9440   ins_pipe(pipe_serial);
9441 %}
9442 
9443 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9444   predicate(needs_acquiring_load_exclusive(n));
9445   match(Set newval (GetAndAddI mem incr));
9446   ins_cost(VOLATILE_REF_COST + 1);
9447   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9448   ins_encode %{
9449     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9450   %}
9451   ins_pipe(pipe_serial);
9452 %}
9453 
9454 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9455   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9456   match(Set dummy (GetAndAddI mem incr));
9457   ins_cost(VOLATILE_REF_COST);
9458   format %{ "get_and_addI_acq [$mem], $incr" %}
9459   ins_encode %{
9460     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9461   %}
9462   ins_pipe(pipe_serial);
9463 %}
9464 
9465 // Manifest a CmpL result in an integer register.
9466 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9467 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9468 %{
9469   match(Set dst (CmpL3 src1 src2));
9470   effect(KILL flags);
9471 
9472   ins_cost(INSN_COST * 6);
9473   format %{
9474       "cmp $src1, $src2"
9475       "csetw $dst, ne"
9476       "cnegw $dst, lt"
9477   %}
9478   // format %{ "CmpL3 $dst, $src1, $src2" %}
9479   ins_encode %{
9480     __ cmp($src1$$Register, $src2$$Register);
9481     __ csetw($dst$$Register, Assembler::NE);
9482     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9483   %}
9484 
9485   ins_pipe(pipe_class_default);
9486 %}
9487 
9488 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9489 %{
9490   match(Set dst (CmpL3 src1 src2));
9491   effect(KILL flags);
9492 
9493   ins_cost(INSN_COST * 6);
9494   format %{
9495       "cmp $src1, $src2"
9496       "csetw $dst, ne"
9497       "cnegw $dst, lt"
9498   %}
9499   ins_encode %{
9500     int32_t con = (int32_t)$src2$$constant;
9501      if (con < 0) {
9502       __ adds(zr, $src1$$Register, -con);
9503     } else {
9504       __ subs(zr, $src1$$Register, con);
9505     }
9506     __ csetw($dst$$Register, Assembler::NE);
9507     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9508   %}
9509 
9510   ins_pipe(pipe_class_default);
9511 %}
9512 
9513 // ============================================================================
9514 // Conditional Move Instructions
9515 
9516 // n.b. we have identical rules for both a signed compare op (cmpOp)
9517 // and an unsigned compare op (cmpOpU). it would be nice if we could
9518 // define an op class which merged both inputs and use it to type the
9519 // argument to a single rule. unfortunatelyt his fails because the
9520 // opclass does not live up to the COND_INTER interface of its
9521 // component operands. When the generic code tries to negate the
9522 // operand it ends up running the generci Machoper::negate method
9523 // which throws a ShouldNotHappen. So, we have to provide two flavours
9524 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9525 
9526 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9527   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9528 
9529   ins_cost(INSN_COST * 2);
9530   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
9531 
9532   ins_encode %{
9533     __ cselw(as_Register($dst$$reg),
9534              as_Register($src2$$reg),
9535              as_Register($src1$$reg),
9536              (Assembler::Condition)$cmp$$cmpcode);
9537   %}
9538 
9539   ins_pipe(icond_reg_reg);
9540 %}
9541 
9542 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9543   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9544 
9545   ins_cost(INSN_COST * 2);
9546   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
9547 
9548   ins_encode %{
9549     __ cselw(as_Register($dst$$reg),
9550              as_Register($src2$$reg),
9551              as_Register($src1$$reg),
9552              (Assembler::Condition)$cmp$$cmpcode);
9553   %}
9554 
9555   ins_pipe(icond_reg_reg);
9556 %}
9557 
9558 // special cases where one arg is zero
9559 
9560 // n.b. this is selected in preference to the rule above because it
9561 // avoids loading constant 0 into a source register
9562 
9563 // TODO
9564 // we ought only to be able to cull one of these variants as the ideal
9565 // transforms ought always to order the zero consistently (to left/right?)
9566 
9567 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9568   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9569 
9570   ins_cost(INSN_COST * 2);
9571   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
9572 
9573   ins_encode %{
9574     __ cselw(as_Register($dst$$reg),
9575              as_Register($src$$reg),
9576              zr,
9577              (Assembler::Condition)$cmp$$cmpcode);
9578   %}
9579 
9580   ins_pipe(icond_reg);
9581 %}
9582 
9583 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9584   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9585 
9586   ins_cost(INSN_COST * 2);
9587   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
9588 
9589   ins_encode %{
9590     __ cselw(as_Register($dst$$reg),
9591              as_Register($src$$reg),
9592              zr,
9593              (Assembler::Condition)$cmp$$cmpcode);
9594   %}
9595 
9596   ins_pipe(icond_reg);
9597 %}
9598 
9599 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9600   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9601 
9602   ins_cost(INSN_COST * 2);
9603   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
9604 
9605   ins_encode %{
9606     __ cselw(as_Register($dst$$reg),
9607              zr,
9608              as_Register($src$$reg),
9609              (Assembler::Condition)$cmp$$cmpcode);
9610   %}
9611 
9612   ins_pipe(icond_reg);
9613 %}
9614 
9615 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9616   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9617 
9618   ins_cost(INSN_COST * 2);
9619   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
9620 
9621   ins_encode %{
9622     __ cselw(as_Register($dst$$reg),
9623              zr,
9624              as_Register($src$$reg),
9625              (Assembler::Condition)$cmp$$cmpcode);
9626   %}
9627 
9628   ins_pipe(icond_reg);
9629 %}
9630 
9631 // special case for creating a boolean 0 or 1
9632 
9633 // n.b. this is selected in preference to the rule above because it
9634 // avoids loading constants 0 and 1 into a source register
9635 
9636 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9637   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9638 
9639   ins_cost(INSN_COST * 2);
9640   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
9641 
9642   ins_encode %{
9643     // equivalently
9644     // cset(as_Register($dst$$reg),
9645     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
9646     __ csincw(as_Register($dst$$reg),
9647              zr,
9648              zr,
9649              (Assembler::Condition)$cmp$$cmpcode);
9650   %}
9651 
9652   ins_pipe(icond_none);
9653 %}
9654 
9655 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9656   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9657 
9658   ins_cost(INSN_COST * 2);
9659   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
9660 
9661   ins_encode %{
9662     // equivalently
9663     // cset(as_Register($dst$$reg),
9664     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
9665     __ csincw(as_Register($dst$$reg),
9666              zr,
9667              zr,
9668              (Assembler::Condition)$cmp$$cmpcode);
9669   %}
9670 
9671   ins_pipe(icond_none);
9672 %}
9673 
9674 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9675   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9676 
9677   ins_cost(INSN_COST * 2);
9678   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
9679 
9680   ins_encode %{
9681     __ csel(as_Register($dst$$reg),
9682             as_Register($src2$$reg),
9683             as_Register($src1$$reg),
9684             (Assembler::Condition)$cmp$$cmpcode);
9685   %}
9686 
9687   ins_pipe(icond_reg_reg);
9688 %}
9689 
9690 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9691   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9692 
9693   ins_cost(INSN_COST * 2);
9694   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
9695 
9696   ins_encode %{
9697     __ csel(as_Register($dst$$reg),
9698             as_Register($src2$$reg),
9699             as_Register($src1$$reg),
9700             (Assembler::Condition)$cmp$$cmpcode);
9701   %}
9702 
9703   ins_pipe(icond_reg_reg);
9704 %}
9705 
9706 // special cases where one arg is zero
9707 
9708 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9709   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9710 
9711   ins_cost(INSN_COST * 2);
9712   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
9713 
9714   ins_encode %{
9715     __ csel(as_Register($dst$$reg),
9716             zr,
9717             as_Register($src$$reg),
9718             (Assembler::Condition)$cmp$$cmpcode);
9719   %}
9720 
9721   ins_pipe(icond_reg);
9722 %}
9723 
9724 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9725   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9726 
9727   ins_cost(INSN_COST * 2);
9728   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
9729 
9730   ins_encode %{
9731     __ csel(as_Register($dst$$reg),
9732             zr,
9733             as_Register($src$$reg),
9734             (Assembler::Condition)$cmp$$cmpcode);
9735   %}
9736 
9737   ins_pipe(icond_reg);
9738 %}
9739 
9740 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9741   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9742 
9743   ins_cost(INSN_COST * 2);
9744   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
9745 
9746   ins_encode %{
9747     __ csel(as_Register($dst$$reg),
9748             as_Register($src$$reg),
9749             zr,
9750             (Assembler::Condition)$cmp$$cmpcode);
9751   %}
9752 
9753   ins_pipe(icond_reg);
9754 %}
9755 
9756 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9757   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9758 
9759   ins_cost(INSN_COST * 2);
9760   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
9761 
9762   ins_encode %{
9763     __ csel(as_Register($dst$$reg),
9764             as_Register($src$$reg),
9765             zr,
9766             (Assembler::Condition)$cmp$$cmpcode);
9767   %}
9768 
9769   ins_pipe(icond_reg);
9770 %}
9771 
9772 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9773   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9774 
9775   ins_cost(INSN_COST * 2);
9776   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
9777 
9778   ins_encode %{
9779     __ csel(as_Register($dst$$reg),
9780             as_Register($src2$$reg),
9781             as_Register($src1$$reg),
9782             (Assembler::Condition)$cmp$$cmpcode);
9783   %}
9784 
9785   ins_pipe(icond_reg_reg);
9786 %}
9787 
9788 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9789   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9790 
9791   ins_cost(INSN_COST * 2);
9792   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
9793 
9794   ins_encode %{
9795     __ csel(as_Register($dst$$reg),
9796             as_Register($src2$$reg),
9797             as_Register($src1$$reg),
9798             (Assembler::Condition)$cmp$$cmpcode);
9799   %}
9800 
9801   ins_pipe(icond_reg_reg);
9802 %}
9803 
9804 // special cases where one arg is zero
9805 
9806 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9807   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9808 
9809   ins_cost(INSN_COST * 2);
9810   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
9811 
9812   ins_encode %{
9813     __ csel(as_Register($dst$$reg),
9814             zr,
9815             as_Register($src$$reg),
9816             (Assembler::Condition)$cmp$$cmpcode);
9817   %}
9818 
9819   ins_pipe(icond_reg);
9820 %}
9821 
9822 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9823   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9824 
9825   ins_cost(INSN_COST * 2);
9826   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
9827 
9828   ins_encode %{
9829     __ csel(as_Register($dst$$reg),
9830             zr,
9831             as_Register($src$$reg),
9832             (Assembler::Condition)$cmp$$cmpcode);
9833   %}
9834 
9835   ins_pipe(icond_reg);
9836 %}
9837 
9838 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9839   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9840 
9841   ins_cost(INSN_COST * 2);
9842   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
9843 
9844   ins_encode %{
9845     __ csel(as_Register($dst$$reg),
9846             as_Register($src$$reg),
9847             zr,
9848             (Assembler::Condition)$cmp$$cmpcode);
9849   %}
9850 
9851   ins_pipe(icond_reg);
9852 %}
9853 
9854 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9855   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9856 
9857   ins_cost(INSN_COST * 2);
9858   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
9859 
9860   ins_encode %{
9861     __ csel(as_Register($dst$$reg),
9862             as_Register($src$$reg),
9863             zr,
9864             (Assembler::Condition)$cmp$$cmpcode);
9865   %}
9866 
9867   ins_pipe(icond_reg);
9868 %}
9869 
9870 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9871   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9872 
9873   ins_cost(INSN_COST * 2);
9874   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
9875 
9876   ins_encode %{
9877     __ cselw(as_Register($dst$$reg),
9878              as_Register($src2$$reg),
9879              as_Register($src1$$reg),
9880              (Assembler::Condition)$cmp$$cmpcode);
9881   %}
9882 
9883   ins_pipe(icond_reg_reg);
9884 %}
9885 
9886 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9887   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9888 
9889   ins_cost(INSN_COST * 2);
9890   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
9891 
9892   ins_encode %{
9893     __ cselw(as_Register($dst$$reg),
9894              as_Register($src2$$reg),
9895              as_Register($src1$$reg),
9896              (Assembler::Condition)$cmp$$cmpcode);
9897   %}
9898 
9899   ins_pipe(icond_reg_reg);
9900 %}
9901 
9902 // special cases where one arg is zero
9903 
9904 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9905   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9906 
9907   ins_cost(INSN_COST * 2);
9908   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
9909 
9910   ins_encode %{
9911     __ cselw(as_Register($dst$$reg),
9912              zr,
9913              as_Register($src$$reg),
9914              (Assembler::Condition)$cmp$$cmpcode);
9915   %}
9916 
9917   ins_pipe(icond_reg);
9918 %}
9919 
9920 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9921   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9922 
9923   ins_cost(INSN_COST * 2);
9924   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
9925 
9926   ins_encode %{
9927     __ cselw(as_Register($dst$$reg),
9928              zr,
9929              as_Register($src$$reg),
9930              (Assembler::Condition)$cmp$$cmpcode);
9931   %}
9932 
9933   ins_pipe(icond_reg);
9934 %}
9935 
9936 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9937   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9938 
9939   ins_cost(INSN_COST * 2);
9940   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
9941 
9942   ins_encode %{
9943     __ cselw(as_Register($dst$$reg),
9944              as_Register($src$$reg),
9945              zr,
9946              (Assembler::Condition)$cmp$$cmpcode);
9947   %}
9948 
9949   ins_pipe(icond_reg);
9950 %}
9951 
9952 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9953   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9954 
9955   ins_cost(INSN_COST * 2);
9956   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
9957 
9958   ins_encode %{
9959     __ cselw(as_Register($dst$$reg),
9960              as_Register($src$$reg),
9961              zr,
9962              (Assembler::Condition)$cmp$$cmpcode);
9963   %}
9964 
9965   ins_pipe(icond_reg);
9966 %}
9967 
9968 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
9969 %{
9970   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9971 
9972   ins_cost(INSN_COST * 3);
9973 
9974   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9975   ins_encode %{
9976     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9977     __ fcsels(as_FloatRegister($dst$$reg),
9978               as_FloatRegister($src2$$reg),
9979               as_FloatRegister($src1$$reg),
9980               cond);
9981   %}
9982 
9983   ins_pipe(fp_cond_reg_reg_s);
9984 %}
9985 
9986 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
9987 %{
9988   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9989 
9990   ins_cost(INSN_COST * 3);
9991 
9992   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9993   ins_encode %{
9994     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9995     __ fcsels(as_FloatRegister($dst$$reg),
9996               as_FloatRegister($src2$$reg),
9997               as_FloatRegister($src1$$reg),
9998               cond);
9999   %}
10000 
10001   ins_pipe(fp_cond_reg_reg_s);
10002 %}
10003 
10004 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10005 %{
10006   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10007 
10008   ins_cost(INSN_COST * 3);
10009 
10010   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10011   ins_encode %{
10012     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10013     __ fcseld(as_FloatRegister($dst$$reg),
10014               as_FloatRegister($src2$$reg),
10015               as_FloatRegister($src1$$reg),
10016               cond);
10017   %}
10018 
10019   ins_pipe(fp_cond_reg_reg_d);
10020 %}
10021 
10022 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10023 %{
10024   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10025 
10026   ins_cost(INSN_COST * 3);
10027 
10028   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10029   ins_encode %{
10030     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10031     __ fcseld(as_FloatRegister($dst$$reg),
10032               as_FloatRegister($src2$$reg),
10033               as_FloatRegister($src1$$reg),
10034               cond);
10035   %}
10036 
10037   ins_pipe(fp_cond_reg_reg_d);
10038 %}
10039 
10040 // ============================================================================
10041 // Arithmetic Instructions
10042 //
10043 
10044 // Integer Addition
10045 
10046 // TODO
10047 // these currently employ operations which do not set CR and hence are
10048 // not flagged as killing CR but we would like to isolate the cases
10049 // where we want to set flags from those where we don't. need to work
10050 // out how to do that.
10051 
10052 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10053   match(Set dst (AddI src1 src2));
10054 
10055   ins_cost(INSN_COST);
10056   format %{ "addw  $dst, $src1, $src2" %}
10057 
10058   ins_encode %{
10059     __ addw(as_Register($dst$$reg),
10060             as_Register($src1$$reg),
10061             as_Register($src2$$reg));
10062   %}
10063 
10064   ins_pipe(ialu_reg_reg);
10065 %}
10066 
10067 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10068   match(Set dst (AddI src1 src2));
10069 
10070   ins_cost(INSN_COST);
10071   format %{ "addw $dst, $src1, $src2" %}
10072 
10073   // use opcode to indicate that this is an add not a sub
10074   opcode(0x0);
10075 
10076   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10077 
10078   ins_pipe(ialu_reg_imm);
10079 %}
10080 
10081 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10082   match(Set dst (AddI (ConvL2I src1) src2));
10083 
10084   ins_cost(INSN_COST);
10085   format %{ "addw $dst, $src1, $src2" %}
10086 
10087   // use opcode to indicate that this is an add not a sub
10088   opcode(0x0);
10089 
10090   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10091 
10092   ins_pipe(ialu_reg_imm);
10093 %}
10094 
10095 // Pointer Addition
10096 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10097   match(Set dst (AddP src1 src2));
10098 
10099   ins_cost(INSN_COST);
10100   format %{ "add $dst, $src1, $src2\t# ptr" %}
10101 
10102   ins_encode %{
10103     __ add(as_Register($dst$$reg),
10104            as_Register($src1$$reg),
10105            as_Register($src2$$reg));
10106   %}
10107 
10108   ins_pipe(ialu_reg_reg);
10109 %}
10110 
10111 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10112   match(Set dst (AddP src1 (ConvI2L src2)));
10113 
10114   ins_cost(1.9 * INSN_COST);
10115   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10116 
10117   ins_encode %{
10118     __ add(as_Register($dst$$reg),
10119            as_Register($src1$$reg),
10120            as_Register($src2$$reg), ext::sxtw);
10121   %}
10122 
10123   ins_pipe(ialu_reg_reg);
10124 %}
10125 
10126 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10127   match(Set dst (AddP src1 (LShiftL src2 scale)));
10128 
10129   ins_cost(1.9 * INSN_COST);
10130   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10131 
10132   ins_encode %{
10133     __ lea(as_Register($dst$$reg),
10134            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10135                    Address::lsl($scale$$constant)));
10136   %}
10137 
10138   ins_pipe(ialu_reg_reg_shift);
10139 %}
10140 
10141 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10142   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10143 
10144   ins_cost(1.9 * INSN_COST);
10145   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10146 
10147   ins_encode %{
10148     __ lea(as_Register($dst$$reg),
10149            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10150                    Address::sxtw($scale$$constant)));
10151   %}
10152 
10153   ins_pipe(ialu_reg_reg_shift);
10154 %}
10155 
10156 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10157   match(Set dst (LShiftL (ConvI2L src) scale));
10158 
10159   ins_cost(INSN_COST);
10160   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10161 
10162   ins_encode %{
10163     __ sbfiz(as_Register($dst$$reg),
10164           as_Register($src$$reg),
10165           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10166   %}
10167 
10168   ins_pipe(ialu_reg_shift);
10169 %}
10170 
10171 // Pointer Immediate Addition
10172 // n.b. this needs to be more expensive than using an indirect memory
10173 // operand
10174 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10175   match(Set dst (AddP src1 src2));
10176 
10177   ins_cost(INSN_COST);
10178   format %{ "add $dst, $src1, $src2\t# ptr" %}
10179 
10180   // use opcode to indicate that this is an add not a sub
10181   opcode(0x0);
10182 
10183   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10184 
10185   ins_pipe(ialu_reg_imm);
10186 %}
10187 
10188 // Long Addition
10189 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10190 
10191   match(Set dst (AddL src1 src2));
10192 
10193   ins_cost(INSN_COST);
10194   format %{ "add  $dst, $src1, $src2" %}
10195 
10196   ins_encode %{
10197     __ add(as_Register($dst$$reg),
10198            as_Register($src1$$reg),
10199            as_Register($src2$$reg));
10200   %}
10201 
10202   ins_pipe(ialu_reg_reg);
10203 %}
10204 
10205 // No constant pool entries requiredLong Immediate Addition.
10206 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10207   match(Set dst (AddL src1 src2));
10208 
10209   ins_cost(INSN_COST);
10210   format %{ "add $dst, $src1, $src2" %}
10211 
10212   // use opcode to indicate that this is an add not a sub
10213   opcode(0x0);
10214 
10215   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10216 
10217   ins_pipe(ialu_reg_imm);
10218 %}
10219 
10220 // Integer Subtraction
10221 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10222   match(Set dst (SubI src1 src2));
10223 
10224   ins_cost(INSN_COST);
10225   format %{ "subw  $dst, $src1, $src2" %}
10226 
10227   ins_encode %{
10228     __ subw(as_Register($dst$$reg),
10229             as_Register($src1$$reg),
10230             as_Register($src2$$reg));
10231   %}
10232 
10233   ins_pipe(ialu_reg_reg);
10234 %}
10235 
10236 // Immediate Subtraction
10237 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10238   match(Set dst (SubI src1 src2));
10239 
10240   ins_cost(INSN_COST);
10241   format %{ "subw $dst, $src1, $src2" %}
10242 
10243   // use opcode to indicate that this is a sub not an add
10244   opcode(0x1);
10245 
10246   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10247 
10248   ins_pipe(ialu_reg_imm);
10249 %}
10250 
10251 // Long Subtraction
10252 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10253 
10254   match(Set dst (SubL src1 src2));
10255 
10256   ins_cost(INSN_COST);
10257   format %{ "sub  $dst, $src1, $src2" %}
10258 
10259   ins_encode %{
10260     __ sub(as_Register($dst$$reg),
10261            as_Register($src1$$reg),
10262            as_Register($src2$$reg));
10263   %}
10264 
10265   ins_pipe(ialu_reg_reg);
10266 %}
10267 
10268 // No constant pool entries requiredLong Immediate Subtraction.
10269 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10270   match(Set dst (SubL src1 src2));
10271 
10272   ins_cost(INSN_COST);
10273   format %{ "sub$dst, $src1, $src2" %}
10274 
10275   // use opcode to indicate that this is a sub not an add
10276   opcode(0x1);
10277 
10278   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10279 
10280   ins_pipe(ialu_reg_imm);
10281 %}
10282 
10283 // Integer Negation (special case for sub)
10284 
10285 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10286   match(Set dst (SubI zero src));
10287 
10288   ins_cost(INSN_COST);
10289   format %{ "negw $dst, $src\t# int" %}
10290 
10291   ins_encode %{
10292     __ negw(as_Register($dst$$reg),
10293             as_Register($src$$reg));
10294   %}
10295 
10296   ins_pipe(ialu_reg);
10297 %}
10298 
10299 // Long Negation
10300 
10301 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10302   match(Set dst (SubL zero src));
10303 
10304   ins_cost(INSN_COST);
10305   format %{ "neg $dst, $src\t# long" %}
10306 
10307   ins_encode %{
10308     __ neg(as_Register($dst$$reg),
10309            as_Register($src$$reg));
10310   %}
10311 
10312   ins_pipe(ialu_reg);
10313 %}
10314 
10315 // Integer Multiply
10316 
10317 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10318   match(Set dst (MulI src1 src2));
10319 
10320   ins_cost(INSN_COST * 3);
10321   format %{ "mulw  $dst, $src1, $src2" %}
10322 
10323   ins_encode %{
10324     __ mulw(as_Register($dst$$reg),
10325             as_Register($src1$$reg),
10326             as_Register($src2$$reg));
10327   %}
10328 
10329   ins_pipe(imul_reg_reg);
10330 %}
10331 
10332 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10333   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10334 
10335   ins_cost(INSN_COST * 3);
10336   format %{ "smull  $dst, $src1, $src2" %}
10337 
10338   ins_encode %{
10339     __ smull(as_Register($dst$$reg),
10340              as_Register($src1$$reg),
10341              as_Register($src2$$reg));
10342   %}
10343 
10344   ins_pipe(imul_reg_reg);
10345 %}
10346 
10347 // Long Multiply
10348 
10349 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10350   match(Set dst (MulL src1 src2));
10351 
10352   ins_cost(INSN_COST * 5);
10353   format %{ "mul  $dst, $src1, $src2" %}
10354 
10355   ins_encode %{
10356     __ mul(as_Register($dst$$reg),
10357            as_Register($src1$$reg),
10358            as_Register($src2$$reg));
10359   %}
10360 
10361   ins_pipe(lmul_reg_reg);
10362 %}
10363 
10364 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10365 %{
10366   match(Set dst (MulHiL src1 src2));
10367 
10368   ins_cost(INSN_COST * 7);
10369   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
10370 
10371   ins_encode %{
10372     __ smulh(as_Register($dst$$reg),
10373              as_Register($src1$$reg),
10374              as_Register($src2$$reg));
10375   %}
10376 
10377   ins_pipe(lmul_reg_reg);
10378 %}
10379 
10380 // Combined Integer Multiply & Add/Sub
10381 
10382 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10383   match(Set dst (AddI src3 (MulI src1 src2)));
10384 
10385   ins_cost(INSN_COST * 3);
10386   format %{ "madd  $dst, $src1, $src2, $src3" %}
10387 
10388   ins_encode %{
10389     __ maddw(as_Register($dst$$reg),
10390              as_Register($src1$$reg),
10391              as_Register($src2$$reg),
10392              as_Register($src3$$reg));
10393   %}
10394 
10395   ins_pipe(imac_reg_reg);
10396 %}
10397 
10398 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10399   match(Set dst (SubI src3 (MulI src1 src2)));
10400 
10401   ins_cost(INSN_COST * 3);
10402   format %{ "msub  $dst, $src1, $src2, $src3" %}
10403 
10404   ins_encode %{
10405     __ msubw(as_Register($dst$$reg),
10406              as_Register($src1$$reg),
10407              as_Register($src2$$reg),
10408              as_Register($src3$$reg));
10409   %}
10410 
10411   ins_pipe(imac_reg_reg);
10412 %}
10413 
10414 // Combined Integer Multiply & Neg
10415 
10416 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10417   match(Set dst (MulI (SubI zero src1) src2));
10418   match(Set dst (MulI src1 (SubI zero src2)));
10419 
10420   ins_cost(INSN_COST * 3);
10421   format %{ "mneg  $dst, $src1, $src2" %}
10422 
10423   ins_encode %{
10424     __ mnegw(as_Register($dst$$reg),
10425              as_Register($src1$$reg),
10426              as_Register($src2$$reg));
10427   %}
10428 
10429   ins_pipe(imac_reg_reg);
10430 %}
10431 
10432 // Combined Long Multiply & Add/Sub
10433 
10434 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10435   match(Set dst (AddL src3 (MulL src1 src2)));
10436 
10437   ins_cost(INSN_COST * 5);
10438   format %{ "madd  $dst, $src1, $src2, $src3" %}
10439 
10440   ins_encode %{
10441     __ madd(as_Register($dst$$reg),
10442             as_Register($src1$$reg),
10443             as_Register($src2$$reg),
10444             as_Register($src3$$reg));
10445   %}
10446 
10447   ins_pipe(lmac_reg_reg);
10448 %}
10449 
10450 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10451   match(Set dst (SubL src3 (MulL src1 src2)));
10452 
10453   ins_cost(INSN_COST * 5);
10454   format %{ "msub  $dst, $src1, $src2, $src3" %}
10455 
10456   ins_encode %{
10457     __ msub(as_Register($dst$$reg),
10458             as_Register($src1$$reg),
10459             as_Register($src2$$reg),
10460             as_Register($src3$$reg));
10461   %}
10462 
10463   ins_pipe(lmac_reg_reg);
10464 %}
10465 
10466 // Combined Long Multiply & Neg
10467 
10468 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10469   match(Set dst (MulL (SubL zero src1) src2));
10470   match(Set dst (MulL src1 (SubL zero src2)));
10471 
10472   ins_cost(INSN_COST * 5);
10473   format %{ "mneg  $dst, $src1, $src2" %}
10474 
10475   ins_encode %{
10476     __ mneg(as_Register($dst$$reg),
10477             as_Register($src1$$reg),
10478             as_Register($src2$$reg));
10479   %}
10480 
10481   ins_pipe(lmac_reg_reg);
10482 %}
10483 
10484 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10485 
10486 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10487   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10488 
10489   ins_cost(INSN_COST * 3);
10490   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10491 
10492   ins_encode %{
10493     __ smaddl(as_Register($dst$$reg),
10494               as_Register($src1$$reg),
10495               as_Register($src2$$reg),
10496               as_Register($src3$$reg));
10497   %}
10498 
10499   ins_pipe(imac_reg_reg);
10500 %}
10501 
10502 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10503   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10504 
10505   ins_cost(INSN_COST * 3);
10506   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10507 
10508   ins_encode %{
10509     __ smsubl(as_Register($dst$$reg),
10510               as_Register($src1$$reg),
10511               as_Register($src2$$reg),
10512               as_Register($src3$$reg));
10513   %}
10514 
10515   ins_pipe(imac_reg_reg);
10516 %}
10517 
10518 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10519   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10520   match(Set dst (MulL (ConvI2L src1) (SubL zero (ConvI2L src2))));
10521 
10522   ins_cost(INSN_COST * 3);
10523   format %{ "smnegl  $dst, $src1, $src2" %}
10524 
10525   ins_encode %{
10526     __ smnegl(as_Register($dst$$reg),
10527               as_Register($src1$$reg),
10528               as_Register($src2$$reg));
10529   %}
10530 
10531   ins_pipe(imac_reg_reg);
10532 %}
10533 
10534 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10535 
10536 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10537   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10538 
10539   ins_cost(INSN_COST * 5);
10540   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10541             "maddw $dst, $src3, $src4, rscratch1" %}
10542 
10543   ins_encode %{
10544     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10545     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10546 
10547   ins_pipe(imac_reg_reg);
10548 %}
10549 
10550 // Integer Divide
10551 
10552 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10553   match(Set dst (DivI src1 src2));
10554 
10555   ins_cost(INSN_COST * 19);
10556   format %{ "sdivw  $dst, $src1, $src2" %}
10557 
10558   ins_encode(aarch64_enc_divw(dst, src1, src2));
10559   ins_pipe(idiv_reg_reg);
10560 %}
10561 
10562 // Long Divide
10563 
10564 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10565   match(Set dst (DivL src1 src2));
10566 
10567   ins_cost(INSN_COST * 35);
10568   format %{ "sdiv   $dst, $src1, $src2" %}
10569 
10570   ins_encode(aarch64_enc_div(dst, src1, src2));
10571   ins_pipe(ldiv_reg_reg);
10572 %}
10573 
10574 // Integer Remainder
10575 
10576 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10577   match(Set dst (ModI src1 src2));
10578 
10579   ins_cost(INSN_COST * 22);
10580   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10581             "msubw($dst, rscratch1, $src2, $src1" %}
10582 
10583   ins_encode(aarch64_enc_modw(dst, src1, src2));
10584   ins_pipe(idiv_reg_reg);
10585 %}
10586 
10587 // Long Remainder
10588 
10589 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10590   match(Set dst (ModL src1 src2));
10591 
10592   ins_cost(INSN_COST * 38);
10593   format %{ "sdiv   rscratch1, $src1, $src2\n"
10594             "msub($dst, rscratch1, $src2, $src1" %}
10595 
10596   ins_encode(aarch64_enc_mod(dst, src1, src2));
10597   ins_pipe(ldiv_reg_reg);
10598 %}
10599 
10600 // Integer Shifts
10601 
10602 // Shift Left Register
10603 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10604   match(Set dst (LShiftI src1 src2));
10605 
10606   ins_cost(INSN_COST * 2);
10607   format %{ "lslvw  $dst, $src1, $src2" %}
10608 
10609   ins_encode %{
10610     __ lslvw(as_Register($dst$$reg),
10611              as_Register($src1$$reg),
10612              as_Register($src2$$reg));
10613   %}
10614 
10615   ins_pipe(ialu_reg_reg_vshift);
10616 %}
10617 
10618 // Shift Left Immediate
10619 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10620   match(Set dst (LShiftI src1 src2));
10621 
10622   ins_cost(INSN_COST);
10623   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10624 
10625   ins_encode %{
10626     __ lslw(as_Register($dst$$reg),
10627             as_Register($src1$$reg),
10628             $src2$$constant & 0x1f);
10629   %}
10630 
10631   ins_pipe(ialu_reg_shift);
10632 %}
10633 
10634 // Shift Right Logical Register
10635 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10636   match(Set dst (URShiftI src1 src2));
10637 
10638   ins_cost(INSN_COST * 2);
10639   format %{ "lsrvw  $dst, $src1, $src2" %}
10640 
10641   ins_encode %{
10642     __ lsrvw(as_Register($dst$$reg),
10643              as_Register($src1$$reg),
10644              as_Register($src2$$reg));
10645   %}
10646 
10647   ins_pipe(ialu_reg_reg_vshift);
10648 %}
10649 
10650 // Shift Right Logical Immediate
10651 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10652   match(Set dst (URShiftI src1 src2));
10653 
10654   ins_cost(INSN_COST);
10655   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10656 
10657   ins_encode %{
10658     __ lsrw(as_Register($dst$$reg),
10659             as_Register($src1$$reg),
10660             $src2$$constant & 0x1f);
10661   %}
10662 
10663   ins_pipe(ialu_reg_shift);
10664 %}
10665 
10666 // Shift Right Arithmetic Register
10667 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10668   match(Set dst (RShiftI src1 src2));
10669 
10670   ins_cost(INSN_COST * 2);
10671   format %{ "asrvw  $dst, $src1, $src2" %}
10672 
10673   ins_encode %{
10674     __ asrvw(as_Register($dst$$reg),
10675              as_Register($src1$$reg),
10676              as_Register($src2$$reg));
10677   %}
10678 
10679   ins_pipe(ialu_reg_reg_vshift);
10680 %}
10681 
10682 // Shift Right Arithmetic Immediate
10683 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10684   match(Set dst (RShiftI src1 src2));
10685 
10686   ins_cost(INSN_COST);
10687   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10688 
10689   ins_encode %{
10690     __ asrw(as_Register($dst$$reg),
10691             as_Register($src1$$reg),
10692             $src2$$constant & 0x1f);
10693   %}
10694 
10695   ins_pipe(ialu_reg_shift);
10696 %}
10697 
10698 // Combined Int Mask and Right Shift (using UBFM)
10699 // TODO
10700 
10701 // Long Shifts
10702 
10703 // Shift Left Register
10704 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10705   match(Set dst (LShiftL src1 src2));
10706 
10707   ins_cost(INSN_COST * 2);
10708   format %{ "lslv  $dst, $src1, $src2" %}
10709 
10710   ins_encode %{
10711     __ lslv(as_Register($dst$$reg),
10712             as_Register($src1$$reg),
10713             as_Register($src2$$reg));
10714   %}
10715 
10716   ins_pipe(ialu_reg_reg_vshift);
10717 %}
10718 
10719 // Shift Left Immediate
10720 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10721   match(Set dst (LShiftL src1 src2));
10722 
10723   ins_cost(INSN_COST);
10724   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10725 
10726   ins_encode %{
10727     __ lsl(as_Register($dst$$reg),
10728             as_Register($src1$$reg),
10729             $src2$$constant & 0x3f);
10730   %}
10731 
10732   ins_pipe(ialu_reg_shift);
10733 %}
10734 
10735 // Shift Right Logical Register
10736 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10737   match(Set dst (URShiftL src1 src2));
10738 
10739   ins_cost(INSN_COST * 2);
10740   format %{ "lsrv  $dst, $src1, $src2" %}
10741 
10742   ins_encode %{
10743     __ lsrv(as_Register($dst$$reg),
10744             as_Register($src1$$reg),
10745             as_Register($src2$$reg));
10746   %}
10747 
10748   ins_pipe(ialu_reg_reg_vshift);
10749 %}
10750 
10751 // Shift Right Logical Immediate
10752 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10753   match(Set dst (URShiftL src1 src2));
10754 
10755   ins_cost(INSN_COST);
10756   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10757 
10758   ins_encode %{
10759     __ lsr(as_Register($dst$$reg),
10760            as_Register($src1$$reg),
10761            $src2$$constant & 0x3f);
10762   %}
10763 
10764   ins_pipe(ialu_reg_shift);
10765 %}
10766 
10767 // A special-case pattern for card table stores.
10768 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10769   match(Set dst (URShiftL (CastP2X src1) src2));
10770 
10771   ins_cost(INSN_COST);
10772   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10773 
10774   ins_encode %{
10775     __ lsr(as_Register($dst$$reg),
10776            as_Register($src1$$reg),
10777            $src2$$constant & 0x3f);
10778   %}
10779 
10780   ins_pipe(ialu_reg_shift);
10781 %}
10782 
10783 // Shift Right Arithmetic Register
10784 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10785   match(Set dst (RShiftL src1 src2));
10786 
10787   ins_cost(INSN_COST * 2);
10788   format %{ "asrv  $dst, $src1, $src2" %}
10789 
10790   ins_encode %{
10791     __ asrv(as_Register($dst$$reg),
10792             as_Register($src1$$reg),
10793             as_Register($src2$$reg));
10794   %}
10795 
10796   ins_pipe(ialu_reg_reg_vshift);
10797 %}
10798 
10799 // Shift Right Arithmetic Immediate
10800 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10801   match(Set dst (RShiftL src1 src2));
10802 
10803   ins_cost(INSN_COST);
10804   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10805 
10806   ins_encode %{
10807     __ asr(as_Register($dst$$reg),
10808            as_Register($src1$$reg),
10809            $src2$$constant & 0x3f);
10810   %}
10811 
10812   ins_pipe(ialu_reg_shift);
10813 %}
10814 
10815 // BEGIN This section of the file is automatically generated. Do not edit --------------
10816 
10817 
10818 // This pattern is automatically generated from aarch64_ad.m4
10819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10820 instruct regL_not_reg(iRegLNoSp dst,
10821                          iRegL src1, immL_M1 m1,
10822                          rFlagsReg cr) %{
10823   match(Set dst (XorL src1 m1));
10824   ins_cost(INSN_COST);
10825   format %{ "eon  $dst, $src1, zr" %}
10826 
10827   ins_encode %{
10828     __ eon(as_Register($dst$$reg),
10829               as_Register($src1$$reg),
10830               zr,
10831               Assembler::LSL, 0);
10832   %}
10833 
10834   ins_pipe(ialu_reg);
10835 %}
10836 
10837 // This pattern is automatically generated from aarch64_ad.m4
10838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10839 instruct regI_not_reg(iRegINoSp dst,
10840                          iRegIorL2I src1, immI_M1 m1,
10841                          rFlagsReg cr) %{
10842   match(Set dst (XorI src1 m1));
10843   ins_cost(INSN_COST);
10844   format %{ "eonw  $dst, $src1, zr" %}
10845 
10846   ins_encode %{
10847     __ eonw(as_Register($dst$$reg),
10848               as_Register($src1$$reg),
10849               zr,
10850               Assembler::LSL, 0);
10851   %}
10852 
10853   ins_pipe(ialu_reg);
10854 %}
10855 
10856 // This pattern is automatically generated from aarch64_ad.m4
10857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10858 instruct AndI_reg_not_reg(iRegINoSp dst,
10859                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10860                          rFlagsReg cr) %{
10861   match(Set dst (AndI src1 (XorI src2 m1)));
10862   ins_cost(INSN_COST);
10863   format %{ "bicw  $dst, $src1, $src2" %}
10864 
10865   ins_encode %{
10866     __ bicw(as_Register($dst$$reg),
10867               as_Register($src1$$reg),
10868               as_Register($src2$$reg),
10869               Assembler::LSL, 0);
10870   %}
10871 
10872   ins_pipe(ialu_reg_reg);
10873 %}
10874 
10875 // This pattern is automatically generated from aarch64_ad.m4
10876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10877 instruct AndL_reg_not_reg(iRegLNoSp dst,
10878                          iRegL src1, iRegL src2, immL_M1 m1,
10879                          rFlagsReg cr) %{
10880   match(Set dst (AndL src1 (XorL src2 m1)));
10881   ins_cost(INSN_COST);
10882   format %{ "bic  $dst, $src1, $src2" %}
10883 
10884   ins_encode %{
10885     __ bic(as_Register($dst$$reg),
10886               as_Register($src1$$reg),
10887               as_Register($src2$$reg),
10888               Assembler::LSL, 0);
10889   %}
10890 
10891   ins_pipe(ialu_reg_reg);
10892 %}
10893 
10894 // This pattern is automatically generated from aarch64_ad.m4
10895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10896 instruct OrI_reg_not_reg(iRegINoSp dst,
10897                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10898                          rFlagsReg cr) %{
10899   match(Set dst (OrI src1 (XorI src2 m1)));
10900   ins_cost(INSN_COST);
10901   format %{ "ornw  $dst, $src1, $src2" %}
10902 
10903   ins_encode %{
10904     __ ornw(as_Register($dst$$reg),
10905               as_Register($src1$$reg),
10906               as_Register($src2$$reg),
10907               Assembler::LSL, 0);
10908   %}
10909 
10910   ins_pipe(ialu_reg_reg);
10911 %}
10912 
10913 // This pattern is automatically generated from aarch64_ad.m4
10914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10915 instruct OrL_reg_not_reg(iRegLNoSp dst,
10916                          iRegL src1, iRegL src2, immL_M1 m1,
10917                          rFlagsReg cr) %{
10918   match(Set dst (OrL src1 (XorL src2 m1)));
10919   ins_cost(INSN_COST);
10920   format %{ "orn  $dst, $src1, $src2" %}
10921 
10922   ins_encode %{
10923     __ orn(as_Register($dst$$reg),
10924               as_Register($src1$$reg),
10925               as_Register($src2$$reg),
10926               Assembler::LSL, 0);
10927   %}
10928 
10929   ins_pipe(ialu_reg_reg);
10930 %}
10931 
10932 // This pattern is automatically generated from aarch64_ad.m4
10933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10934 instruct XorI_reg_not_reg(iRegINoSp dst,
10935                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10936                          rFlagsReg cr) %{
10937   match(Set dst (XorI m1 (XorI src2 src1)));
10938   ins_cost(INSN_COST);
10939   format %{ "eonw  $dst, $src1, $src2" %}
10940 
10941   ins_encode %{
10942     __ eonw(as_Register($dst$$reg),
10943               as_Register($src1$$reg),
10944               as_Register($src2$$reg),
10945               Assembler::LSL, 0);
10946   %}
10947 
10948   ins_pipe(ialu_reg_reg);
10949 %}
10950 
10951 // This pattern is automatically generated from aarch64_ad.m4
10952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10953 instruct XorL_reg_not_reg(iRegLNoSp dst,
10954                          iRegL src1, iRegL src2, immL_M1 m1,
10955                          rFlagsReg cr) %{
10956   match(Set dst (XorL m1 (XorL src2 src1)));
10957   ins_cost(INSN_COST);
10958   format %{ "eon  $dst, $src1, $src2" %}
10959 
10960   ins_encode %{
10961     __ eon(as_Register($dst$$reg),
10962               as_Register($src1$$reg),
10963               as_Register($src2$$reg),
10964               Assembler::LSL, 0);
10965   %}
10966 
10967   ins_pipe(ialu_reg_reg);
10968 %}
10969 
10970 // This pattern is automatically generated from aarch64_ad.m4
10971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10972 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10973                          iRegIorL2I src1, iRegIorL2I src2,
10974                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10975   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10976   ins_cost(1.9 * INSN_COST);
10977   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
10978 
10979   ins_encode %{
10980     __ bicw(as_Register($dst$$reg),
10981               as_Register($src1$$reg),
10982               as_Register($src2$$reg),
10983               Assembler::LSR,
10984               $src3$$constant & 0x1f);
10985   %}
10986 
10987   ins_pipe(ialu_reg_reg_shift);
10988 %}
10989 
10990 // This pattern is automatically generated from aarch64_ad.m4
10991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10992 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10993                          iRegL src1, iRegL src2,
10994                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10995   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10996   ins_cost(1.9 * INSN_COST);
10997   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
10998 
10999   ins_encode %{
11000     __ bic(as_Register($dst$$reg),
11001               as_Register($src1$$reg),
11002               as_Register($src2$$reg),
11003               Assembler::LSR,
11004               $src3$$constant & 0x3f);
11005   %}
11006 
11007   ins_pipe(ialu_reg_reg_shift);
11008 %}
11009 
11010 // This pattern is automatically generated from aarch64_ad.m4
11011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11012 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11013                          iRegIorL2I src1, iRegIorL2I src2,
11014                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11015   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11016   ins_cost(1.9 * INSN_COST);
11017   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11018 
11019   ins_encode %{
11020     __ bicw(as_Register($dst$$reg),
11021               as_Register($src1$$reg),
11022               as_Register($src2$$reg),
11023               Assembler::ASR,
11024               $src3$$constant & 0x1f);
11025   %}
11026 
11027   ins_pipe(ialu_reg_reg_shift);
11028 %}
11029 
11030 // This pattern is automatically generated from aarch64_ad.m4
11031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11032 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11033                          iRegL src1, iRegL src2,
11034                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11035   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11036   ins_cost(1.9 * INSN_COST);
11037   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11038 
11039   ins_encode %{
11040     __ bic(as_Register($dst$$reg),
11041               as_Register($src1$$reg),
11042               as_Register($src2$$reg),
11043               Assembler::ASR,
11044               $src3$$constant & 0x3f);
11045   %}
11046 
11047   ins_pipe(ialu_reg_reg_shift);
11048 %}
11049 
11050 // This pattern is automatically generated from aarch64_ad.m4
11051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11052 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11053                          iRegIorL2I src1, iRegIorL2I src2,
11054                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11055   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11056   ins_cost(1.9 * INSN_COST);
11057   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11058 
11059   ins_encode %{
11060     __ bicw(as_Register($dst$$reg),
11061               as_Register($src1$$reg),
11062               as_Register($src2$$reg),
11063               Assembler::LSL,
11064               $src3$$constant & 0x1f);
11065   %}
11066 
11067   ins_pipe(ialu_reg_reg_shift);
11068 %}
11069 
11070 // This pattern is automatically generated from aarch64_ad.m4
11071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11072 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11073                          iRegL src1, iRegL src2,
11074                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11075   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11076   ins_cost(1.9 * INSN_COST);
11077   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11078 
11079   ins_encode %{
11080     __ bic(as_Register($dst$$reg),
11081               as_Register($src1$$reg),
11082               as_Register($src2$$reg),
11083               Assembler::LSL,
11084               $src3$$constant & 0x3f);
11085   %}
11086 
11087   ins_pipe(ialu_reg_reg_shift);
11088 %}
11089 
11090 // This pattern is automatically generated from aarch64_ad.m4
11091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11092 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11093                          iRegIorL2I src1, iRegIorL2I src2,
11094                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11095   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11096   ins_cost(1.9 * INSN_COST);
11097   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11098 
11099   ins_encode %{
11100     __ eonw(as_Register($dst$$reg),
11101               as_Register($src1$$reg),
11102               as_Register($src2$$reg),
11103               Assembler::LSR,
11104               $src3$$constant & 0x1f);
11105   %}
11106 
11107   ins_pipe(ialu_reg_reg_shift);
11108 %}
11109 
11110 // This pattern is automatically generated from aarch64_ad.m4
11111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11112 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11113                          iRegL src1, iRegL src2,
11114                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11115   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11116   ins_cost(1.9 * INSN_COST);
11117   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11118 
11119   ins_encode %{
11120     __ eon(as_Register($dst$$reg),
11121               as_Register($src1$$reg),
11122               as_Register($src2$$reg),
11123               Assembler::LSR,
11124               $src3$$constant & 0x3f);
11125   %}
11126 
11127   ins_pipe(ialu_reg_reg_shift);
11128 %}
11129 
11130 // This pattern is automatically generated from aarch64_ad.m4
11131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11132 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11133                          iRegIorL2I src1, iRegIorL2I src2,
11134                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11135   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11136   ins_cost(1.9 * INSN_COST);
11137   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11138 
11139   ins_encode %{
11140     __ eonw(as_Register($dst$$reg),
11141               as_Register($src1$$reg),
11142               as_Register($src2$$reg),
11143               Assembler::ASR,
11144               $src3$$constant & 0x1f);
11145   %}
11146 
11147   ins_pipe(ialu_reg_reg_shift);
11148 %}
11149 
11150 // This pattern is automatically generated from aarch64_ad.m4
11151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11152 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11153                          iRegL src1, iRegL src2,
11154                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11155   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11156   ins_cost(1.9 * INSN_COST);
11157   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11158 
11159   ins_encode %{
11160     __ eon(as_Register($dst$$reg),
11161               as_Register($src1$$reg),
11162               as_Register($src2$$reg),
11163               Assembler::ASR,
11164               $src3$$constant & 0x3f);
11165   %}
11166 
11167   ins_pipe(ialu_reg_reg_shift);
11168 %}
11169 
11170 // This pattern is automatically generated from aarch64_ad.m4
11171 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11172 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11173                          iRegIorL2I src1, iRegIorL2I src2,
11174                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11175   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11176   ins_cost(1.9 * INSN_COST);
11177   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11178 
11179   ins_encode %{
11180     __ eonw(as_Register($dst$$reg),
11181               as_Register($src1$$reg),
11182               as_Register($src2$$reg),
11183               Assembler::LSL,
11184               $src3$$constant & 0x1f);
11185   %}
11186 
11187   ins_pipe(ialu_reg_reg_shift);
11188 %}
11189 
11190 // This pattern is automatically generated from aarch64_ad.m4
11191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11192 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11193                          iRegL src1, iRegL src2,
11194                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11195   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11196   ins_cost(1.9 * INSN_COST);
11197   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11198 
11199   ins_encode %{
11200     __ eon(as_Register($dst$$reg),
11201               as_Register($src1$$reg),
11202               as_Register($src2$$reg),
11203               Assembler::LSL,
11204               $src3$$constant & 0x3f);
11205   %}
11206 
11207   ins_pipe(ialu_reg_reg_shift);
11208 %}
11209 
11210 // This pattern is automatically generated from aarch64_ad.m4
11211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11212 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11213                          iRegIorL2I src1, iRegIorL2I src2,
11214                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11215   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11216   ins_cost(1.9 * INSN_COST);
11217   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11218 
11219   ins_encode %{
11220     __ ornw(as_Register($dst$$reg),
11221               as_Register($src1$$reg),
11222               as_Register($src2$$reg),
11223               Assembler::LSR,
11224               $src3$$constant & 0x1f);
11225   %}
11226 
11227   ins_pipe(ialu_reg_reg_shift);
11228 %}
11229 
11230 // This pattern is automatically generated from aarch64_ad.m4
11231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11232 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11233                          iRegL src1, iRegL src2,
11234                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11235   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11236   ins_cost(1.9 * INSN_COST);
11237   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11238 
11239   ins_encode %{
11240     __ orn(as_Register($dst$$reg),
11241               as_Register($src1$$reg),
11242               as_Register($src2$$reg),
11243               Assembler::LSR,
11244               $src3$$constant & 0x3f);
11245   %}
11246 
11247   ins_pipe(ialu_reg_reg_shift);
11248 %}
11249 
11250 // This pattern is automatically generated from aarch64_ad.m4
11251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11252 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11253                          iRegIorL2I src1, iRegIorL2I src2,
11254                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11255   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11256   ins_cost(1.9 * INSN_COST);
11257   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11258 
11259   ins_encode %{
11260     __ ornw(as_Register($dst$$reg),
11261               as_Register($src1$$reg),
11262               as_Register($src2$$reg),
11263               Assembler::ASR,
11264               $src3$$constant & 0x1f);
11265   %}
11266 
11267   ins_pipe(ialu_reg_reg_shift);
11268 %}
11269 
11270 // This pattern is automatically generated from aarch64_ad.m4
11271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11272 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11273                          iRegL src1, iRegL src2,
11274                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11275   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11276   ins_cost(1.9 * INSN_COST);
11277   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11278 
11279   ins_encode %{
11280     __ orn(as_Register($dst$$reg),
11281               as_Register($src1$$reg),
11282               as_Register($src2$$reg),
11283               Assembler::ASR,
11284               $src3$$constant & 0x3f);
11285   %}
11286 
11287   ins_pipe(ialu_reg_reg_shift);
11288 %}
11289 
11290 // This pattern is automatically generated from aarch64_ad.m4
11291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11292 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11293                          iRegIorL2I src1, iRegIorL2I src2,
11294                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11295   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11296   ins_cost(1.9 * INSN_COST);
11297   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11298 
11299   ins_encode %{
11300     __ ornw(as_Register($dst$$reg),
11301               as_Register($src1$$reg),
11302               as_Register($src2$$reg),
11303               Assembler::LSL,
11304               $src3$$constant & 0x1f);
11305   %}
11306 
11307   ins_pipe(ialu_reg_reg_shift);
11308 %}
11309 
11310 // This pattern is automatically generated from aarch64_ad.m4
11311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11312 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11313                          iRegL src1, iRegL src2,
11314                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11315   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11316   ins_cost(1.9 * INSN_COST);
11317   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11318 
11319   ins_encode %{
11320     __ orn(as_Register($dst$$reg),
11321               as_Register($src1$$reg),
11322               as_Register($src2$$reg),
11323               Assembler::LSL,
11324               $src3$$constant & 0x3f);
11325   %}
11326 
11327   ins_pipe(ialu_reg_reg_shift);
11328 %}
11329 
11330 // This pattern is automatically generated from aarch64_ad.m4
11331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11332 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11333                          iRegIorL2I src1, iRegIorL2I src2,
11334                          immI src3, rFlagsReg cr) %{
11335   match(Set dst (AndI src1 (URShiftI src2 src3)));
11336 
11337   ins_cost(1.9 * INSN_COST);
11338   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11339 
11340   ins_encode %{
11341     __ andw(as_Register($dst$$reg),
11342               as_Register($src1$$reg),
11343               as_Register($src2$$reg),
11344               Assembler::LSR,
11345               $src3$$constant & 0x1f);
11346   %}
11347 
11348   ins_pipe(ialu_reg_reg_shift);
11349 %}
11350 
11351 // This pattern is automatically generated from aarch64_ad.m4
11352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11353 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11354                          iRegL src1, iRegL src2,
11355                          immI src3, rFlagsReg cr) %{
11356   match(Set dst (AndL src1 (URShiftL src2 src3)));
11357 
11358   ins_cost(1.9 * INSN_COST);
11359   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11360 
11361   ins_encode %{
11362     __ andr(as_Register($dst$$reg),
11363               as_Register($src1$$reg),
11364               as_Register($src2$$reg),
11365               Assembler::LSR,
11366               $src3$$constant & 0x3f);
11367   %}
11368 
11369   ins_pipe(ialu_reg_reg_shift);
11370 %}
11371 
11372 // This pattern is automatically generated from aarch64_ad.m4
11373 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11374 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11375                          iRegIorL2I src1, iRegIorL2I src2,
11376                          immI src3, rFlagsReg cr) %{
11377   match(Set dst (AndI src1 (RShiftI src2 src3)));
11378 
11379   ins_cost(1.9 * INSN_COST);
11380   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11381 
11382   ins_encode %{
11383     __ andw(as_Register($dst$$reg),
11384               as_Register($src1$$reg),
11385               as_Register($src2$$reg),
11386               Assembler::ASR,
11387               $src3$$constant & 0x1f);
11388   %}
11389 
11390   ins_pipe(ialu_reg_reg_shift);
11391 %}
11392 
11393 // This pattern is automatically generated from aarch64_ad.m4
11394 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11395 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11396                          iRegL src1, iRegL src2,
11397                          immI src3, rFlagsReg cr) %{
11398   match(Set dst (AndL src1 (RShiftL src2 src3)));
11399 
11400   ins_cost(1.9 * INSN_COST);
11401   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11402 
11403   ins_encode %{
11404     __ andr(as_Register($dst$$reg),
11405               as_Register($src1$$reg),
11406               as_Register($src2$$reg),
11407               Assembler::ASR,
11408               $src3$$constant & 0x3f);
11409   %}
11410 
11411   ins_pipe(ialu_reg_reg_shift);
11412 %}
11413 
11414 // This pattern is automatically generated from aarch64_ad.m4
11415 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11416 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11417                          iRegIorL2I src1, iRegIorL2I src2,
11418                          immI src3, rFlagsReg cr) %{
11419   match(Set dst (AndI src1 (LShiftI src2 src3)));
11420 
11421   ins_cost(1.9 * INSN_COST);
11422   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11423 
11424   ins_encode %{
11425     __ andw(as_Register($dst$$reg),
11426               as_Register($src1$$reg),
11427               as_Register($src2$$reg),
11428               Assembler::LSL,
11429               $src3$$constant & 0x1f);
11430   %}
11431 
11432   ins_pipe(ialu_reg_reg_shift);
11433 %}
11434 
11435 // This pattern is automatically generated from aarch64_ad.m4
11436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11437 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11438                          iRegL src1, iRegL src2,
11439                          immI src3, rFlagsReg cr) %{
11440   match(Set dst (AndL src1 (LShiftL src2 src3)));
11441 
11442   ins_cost(1.9 * INSN_COST);
11443   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11444 
11445   ins_encode %{
11446     __ andr(as_Register($dst$$reg),
11447               as_Register($src1$$reg),
11448               as_Register($src2$$reg),
11449               Assembler::LSL,
11450               $src3$$constant & 0x3f);
11451   %}
11452 
11453   ins_pipe(ialu_reg_reg_shift);
11454 %}
11455 
11456 // This pattern is automatically generated from aarch64_ad.m4
11457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11458 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11459                          iRegIorL2I src1, iRegIorL2I src2,
11460                          immI src3, rFlagsReg cr) %{
11461   match(Set dst (XorI src1 (URShiftI src2 src3)));
11462 
11463   ins_cost(1.9 * INSN_COST);
11464   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
11465 
11466   ins_encode %{
11467     __ eorw(as_Register($dst$$reg),
11468               as_Register($src1$$reg),
11469               as_Register($src2$$reg),
11470               Assembler::LSR,
11471               $src3$$constant & 0x1f);
11472   %}
11473 
11474   ins_pipe(ialu_reg_reg_shift);
11475 %}
11476 
11477 // This pattern is automatically generated from aarch64_ad.m4
11478 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11479 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11480                          iRegL src1, iRegL src2,
11481                          immI src3, rFlagsReg cr) %{
11482   match(Set dst (XorL src1 (URShiftL src2 src3)));
11483 
11484   ins_cost(1.9 * INSN_COST);
11485   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
11486 
11487   ins_encode %{
11488     __ eor(as_Register($dst$$reg),
11489               as_Register($src1$$reg),
11490               as_Register($src2$$reg),
11491               Assembler::LSR,
11492               $src3$$constant & 0x3f);
11493   %}
11494 
11495   ins_pipe(ialu_reg_reg_shift);
11496 %}
11497 
11498 // This pattern is automatically generated from aarch64_ad.m4
11499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11500 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11501                          iRegIorL2I src1, iRegIorL2I src2,
11502                          immI src3, rFlagsReg cr) %{
11503   match(Set dst (XorI src1 (RShiftI src2 src3)));
11504 
11505   ins_cost(1.9 * INSN_COST);
11506   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
11507 
11508   ins_encode %{
11509     __ eorw(as_Register($dst$$reg),
11510               as_Register($src1$$reg),
11511               as_Register($src2$$reg),
11512               Assembler::ASR,
11513               $src3$$constant & 0x1f);
11514   %}
11515 
11516   ins_pipe(ialu_reg_reg_shift);
11517 %}
11518 
11519 // This pattern is automatically generated from aarch64_ad.m4
11520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11521 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11522                          iRegL src1, iRegL src2,
11523                          immI src3, rFlagsReg cr) %{
11524   match(Set dst (XorL src1 (RShiftL src2 src3)));
11525 
11526   ins_cost(1.9 * INSN_COST);
11527   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
11528 
11529   ins_encode %{
11530     __ eor(as_Register($dst$$reg),
11531               as_Register($src1$$reg),
11532               as_Register($src2$$reg),
11533               Assembler::ASR,
11534               $src3$$constant & 0x3f);
11535   %}
11536 
11537   ins_pipe(ialu_reg_reg_shift);
11538 %}
11539 
11540 // This pattern is automatically generated from aarch64_ad.m4
11541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11542 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11543                          iRegIorL2I src1, iRegIorL2I src2,
11544                          immI src3, rFlagsReg cr) %{
11545   match(Set dst (XorI src1 (LShiftI src2 src3)));
11546 
11547   ins_cost(1.9 * INSN_COST);
11548   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
11549 
11550   ins_encode %{
11551     __ eorw(as_Register($dst$$reg),
11552               as_Register($src1$$reg),
11553               as_Register($src2$$reg),
11554               Assembler::LSL,
11555               $src3$$constant & 0x1f);
11556   %}
11557 
11558   ins_pipe(ialu_reg_reg_shift);
11559 %}
11560 
11561 // This pattern is automatically generated from aarch64_ad.m4
11562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11563 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11564                          iRegL src1, iRegL src2,
11565                          immI src3, rFlagsReg cr) %{
11566   match(Set dst (XorL src1 (LShiftL src2 src3)));
11567 
11568   ins_cost(1.9 * INSN_COST);
11569   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
11570 
11571   ins_encode %{
11572     __ eor(as_Register($dst$$reg),
11573               as_Register($src1$$reg),
11574               as_Register($src2$$reg),
11575               Assembler::LSL,
11576               $src3$$constant & 0x3f);
11577   %}
11578 
11579   ins_pipe(ialu_reg_reg_shift);
11580 %}
11581 
11582 // This pattern is automatically generated from aarch64_ad.m4
11583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11584 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11585                          iRegIorL2I src1, iRegIorL2I src2,
11586                          immI src3, rFlagsReg cr) %{
11587   match(Set dst (OrI src1 (URShiftI src2 src3)));
11588 
11589   ins_cost(1.9 * INSN_COST);
11590   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
11591 
11592   ins_encode %{
11593     __ orrw(as_Register($dst$$reg),
11594               as_Register($src1$$reg),
11595               as_Register($src2$$reg),
11596               Assembler::LSR,
11597               $src3$$constant & 0x1f);
11598   %}
11599 
11600   ins_pipe(ialu_reg_reg_shift);
11601 %}
11602 
11603 // This pattern is automatically generated from aarch64_ad.m4
11604 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11605 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11606                          iRegL src1, iRegL src2,
11607                          immI src3, rFlagsReg cr) %{
11608   match(Set dst (OrL src1 (URShiftL src2 src3)));
11609 
11610   ins_cost(1.9 * INSN_COST);
11611   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
11612 
11613   ins_encode %{
11614     __ orr(as_Register($dst$$reg),
11615               as_Register($src1$$reg),
11616               as_Register($src2$$reg),
11617               Assembler::LSR,
11618               $src3$$constant & 0x3f);
11619   %}
11620 
11621   ins_pipe(ialu_reg_reg_shift);
11622 %}
11623 
11624 // This pattern is automatically generated from aarch64_ad.m4
11625 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11626 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11627                          iRegIorL2I src1, iRegIorL2I src2,
11628                          immI src3, rFlagsReg cr) %{
11629   match(Set dst (OrI src1 (RShiftI src2 src3)));
11630 
11631   ins_cost(1.9 * INSN_COST);
11632   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
11633 
11634   ins_encode %{
11635     __ orrw(as_Register($dst$$reg),
11636               as_Register($src1$$reg),
11637               as_Register($src2$$reg),
11638               Assembler::ASR,
11639               $src3$$constant & 0x1f);
11640   %}
11641 
11642   ins_pipe(ialu_reg_reg_shift);
11643 %}
11644 
11645 // This pattern is automatically generated from aarch64_ad.m4
11646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11647 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11648                          iRegL src1, iRegL src2,
11649                          immI src3, rFlagsReg cr) %{
11650   match(Set dst (OrL src1 (RShiftL src2 src3)));
11651 
11652   ins_cost(1.9 * INSN_COST);
11653   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
11654 
11655   ins_encode %{
11656     __ orr(as_Register($dst$$reg),
11657               as_Register($src1$$reg),
11658               as_Register($src2$$reg),
11659               Assembler::ASR,
11660               $src3$$constant & 0x3f);
11661   %}
11662 
11663   ins_pipe(ialu_reg_reg_shift);
11664 %}
11665 
11666 // This pattern is automatically generated from aarch64_ad.m4
11667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11668 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11669                          iRegIorL2I src1, iRegIorL2I src2,
11670                          immI src3, rFlagsReg cr) %{
11671   match(Set dst (OrI src1 (LShiftI src2 src3)));
11672 
11673   ins_cost(1.9 * INSN_COST);
11674   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
11675 
11676   ins_encode %{
11677     __ orrw(as_Register($dst$$reg),
11678               as_Register($src1$$reg),
11679               as_Register($src2$$reg),
11680               Assembler::LSL,
11681               $src3$$constant & 0x1f);
11682   %}
11683 
11684   ins_pipe(ialu_reg_reg_shift);
11685 %}
11686 
11687 // This pattern is automatically generated from aarch64_ad.m4
11688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11689 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11690                          iRegL src1, iRegL src2,
11691                          immI src3, rFlagsReg cr) %{
11692   match(Set dst (OrL src1 (LShiftL src2 src3)));
11693 
11694   ins_cost(1.9 * INSN_COST);
11695   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
11696 
11697   ins_encode %{
11698     __ orr(as_Register($dst$$reg),
11699               as_Register($src1$$reg),
11700               as_Register($src2$$reg),
11701               Assembler::LSL,
11702               $src3$$constant & 0x3f);
11703   %}
11704 
11705   ins_pipe(ialu_reg_reg_shift);
11706 %}
11707 
11708 // This pattern is automatically generated from aarch64_ad.m4
11709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11710 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11711                          iRegIorL2I src1, iRegIorL2I src2,
11712                          immI src3, rFlagsReg cr) %{
11713   match(Set dst (AddI src1 (URShiftI src2 src3)));
11714 
11715   ins_cost(1.9 * INSN_COST);
11716   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
11717 
11718   ins_encode %{
11719     __ addw(as_Register($dst$$reg),
11720               as_Register($src1$$reg),
11721               as_Register($src2$$reg),
11722               Assembler::LSR,
11723               $src3$$constant & 0x1f);
11724   %}
11725 
11726   ins_pipe(ialu_reg_reg_shift);
11727 %}
11728 
11729 // This pattern is automatically generated from aarch64_ad.m4
11730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11731 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11732                          iRegL src1, iRegL src2,
11733                          immI src3, rFlagsReg cr) %{
11734   match(Set dst (AddL src1 (URShiftL src2 src3)));
11735 
11736   ins_cost(1.9 * INSN_COST);
11737   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
11738 
11739   ins_encode %{
11740     __ add(as_Register($dst$$reg),
11741               as_Register($src1$$reg),
11742               as_Register($src2$$reg),
11743               Assembler::LSR,
11744               $src3$$constant & 0x3f);
11745   %}
11746 
11747   ins_pipe(ialu_reg_reg_shift);
11748 %}
11749 
11750 // This pattern is automatically generated from aarch64_ad.m4
11751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11752 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11753                          iRegIorL2I src1, iRegIorL2I src2,
11754                          immI src3, rFlagsReg cr) %{
11755   match(Set dst (AddI src1 (RShiftI src2 src3)));
11756 
11757   ins_cost(1.9 * INSN_COST);
11758   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
11759 
11760   ins_encode %{
11761     __ addw(as_Register($dst$$reg),
11762               as_Register($src1$$reg),
11763               as_Register($src2$$reg),
11764               Assembler::ASR,
11765               $src3$$constant & 0x1f);
11766   %}
11767 
11768   ins_pipe(ialu_reg_reg_shift);
11769 %}
11770 
11771 // This pattern is automatically generated from aarch64_ad.m4
11772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11773 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11774                          iRegL src1, iRegL src2,
11775                          immI src3, rFlagsReg cr) %{
11776   match(Set dst (AddL src1 (RShiftL src2 src3)));
11777 
11778   ins_cost(1.9 * INSN_COST);
11779   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
11780 
11781   ins_encode %{
11782     __ add(as_Register($dst$$reg),
11783               as_Register($src1$$reg),
11784               as_Register($src2$$reg),
11785               Assembler::ASR,
11786               $src3$$constant & 0x3f);
11787   %}
11788 
11789   ins_pipe(ialu_reg_reg_shift);
11790 %}
11791 
11792 // This pattern is automatically generated from aarch64_ad.m4
11793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11794 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11795                          iRegIorL2I src1, iRegIorL2I src2,
11796                          immI src3, rFlagsReg cr) %{
11797   match(Set dst (AddI src1 (LShiftI src2 src3)));
11798 
11799   ins_cost(1.9 * INSN_COST);
11800   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
11801 
11802   ins_encode %{
11803     __ addw(as_Register($dst$$reg),
11804               as_Register($src1$$reg),
11805               as_Register($src2$$reg),
11806               Assembler::LSL,
11807               $src3$$constant & 0x1f);
11808   %}
11809 
11810   ins_pipe(ialu_reg_reg_shift);
11811 %}
11812 
11813 // This pattern is automatically generated from aarch64_ad.m4
11814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11815 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11816                          iRegL src1, iRegL src2,
11817                          immI src3, rFlagsReg cr) %{
11818   match(Set dst (AddL src1 (LShiftL src2 src3)));
11819 
11820   ins_cost(1.9 * INSN_COST);
11821   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
11822 
11823   ins_encode %{
11824     __ add(as_Register($dst$$reg),
11825               as_Register($src1$$reg),
11826               as_Register($src2$$reg),
11827               Assembler::LSL,
11828               $src3$$constant & 0x3f);
11829   %}
11830 
11831   ins_pipe(ialu_reg_reg_shift);
11832 %}
11833 
11834 // This pattern is automatically generated from aarch64_ad.m4
11835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11836 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11837                          iRegIorL2I src1, iRegIorL2I src2,
11838                          immI src3, rFlagsReg cr) %{
11839   match(Set dst (SubI src1 (URShiftI src2 src3)));
11840 
11841   ins_cost(1.9 * INSN_COST);
11842   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
11843 
11844   ins_encode %{
11845     __ subw(as_Register($dst$$reg),
11846               as_Register($src1$$reg),
11847               as_Register($src2$$reg),
11848               Assembler::LSR,
11849               $src3$$constant & 0x1f);
11850   %}
11851 
11852   ins_pipe(ialu_reg_reg_shift);
11853 %}
11854 
11855 // This pattern is automatically generated from aarch64_ad.m4
11856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11857 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11858                          iRegL src1, iRegL src2,
11859                          immI src3, rFlagsReg cr) %{
11860   match(Set dst (SubL src1 (URShiftL src2 src3)));
11861 
11862   ins_cost(1.9 * INSN_COST);
11863   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
11864 
11865   ins_encode %{
11866     __ sub(as_Register($dst$$reg),
11867               as_Register($src1$$reg),
11868               as_Register($src2$$reg),
11869               Assembler::LSR,
11870               $src3$$constant & 0x3f);
11871   %}
11872 
11873   ins_pipe(ialu_reg_reg_shift);
11874 %}
11875 
11876 // This pattern is automatically generated from aarch64_ad.m4
11877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11878 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11879                          iRegIorL2I src1, iRegIorL2I src2,
11880                          immI src3, rFlagsReg cr) %{
11881   match(Set dst (SubI src1 (RShiftI src2 src3)));
11882 
11883   ins_cost(1.9 * INSN_COST);
11884   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
11885 
11886   ins_encode %{
11887     __ subw(as_Register($dst$$reg),
11888               as_Register($src1$$reg),
11889               as_Register($src2$$reg),
11890               Assembler::ASR,
11891               $src3$$constant & 0x1f);
11892   %}
11893 
11894   ins_pipe(ialu_reg_reg_shift);
11895 %}
11896 
11897 // This pattern is automatically generated from aarch64_ad.m4
11898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11899 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11900                          iRegL src1, iRegL src2,
11901                          immI src3, rFlagsReg cr) %{
11902   match(Set dst (SubL src1 (RShiftL src2 src3)));
11903 
11904   ins_cost(1.9 * INSN_COST);
11905   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
11906 
11907   ins_encode %{
11908     __ sub(as_Register($dst$$reg),
11909               as_Register($src1$$reg),
11910               as_Register($src2$$reg),
11911               Assembler::ASR,
11912               $src3$$constant & 0x3f);
11913   %}
11914 
11915   ins_pipe(ialu_reg_reg_shift);
11916 %}
11917 
11918 // This pattern is automatically generated from aarch64_ad.m4
11919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11920 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11921                          iRegIorL2I src1, iRegIorL2I src2,
11922                          immI src3, rFlagsReg cr) %{
11923   match(Set dst (SubI src1 (LShiftI src2 src3)));
11924 
11925   ins_cost(1.9 * INSN_COST);
11926   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
11927 
11928   ins_encode %{
11929     __ subw(as_Register($dst$$reg),
11930               as_Register($src1$$reg),
11931               as_Register($src2$$reg),
11932               Assembler::LSL,
11933               $src3$$constant & 0x1f);
11934   %}
11935 
11936   ins_pipe(ialu_reg_reg_shift);
11937 %}
11938 
11939 // This pattern is automatically generated from aarch64_ad.m4
11940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11941 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11942                          iRegL src1, iRegL src2,
11943                          immI src3, rFlagsReg cr) %{
11944   match(Set dst (SubL src1 (LShiftL src2 src3)));
11945 
11946   ins_cost(1.9 * INSN_COST);
11947   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
11948 
11949   ins_encode %{
11950     __ sub(as_Register($dst$$reg),
11951               as_Register($src1$$reg),
11952               as_Register($src2$$reg),
11953               Assembler::LSL,
11954               $src3$$constant & 0x3f);
11955   %}
11956 
11957   ins_pipe(ialu_reg_reg_shift);
11958 %}
11959 
11960  
11961 // This pattern is automatically generated from aarch64_ad.m4
11962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11963 
11964 // Shift Left followed by Shift Right.
11965 // This idiom is used by the compiler for the i2b bytecode etc.
11966 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11967 %{
11968   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11969   ins_cost(INSN_COST * 2);
11970   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11971   ins_encode %{
11972     int lshift = $lshift_count$$constant & 63;
11973     int rshift = $rshift_count$$constant & 63;
11974     int s = 63 - lshift;
11975     int r = (rshift - lshift) & 63;
11976     __ sbfm(as_Register($dst$$reg),
11977             as_Register($src$$reg),
11978             r, s);
11979   %}
11980 
11981   ins_pipe(ialu_reg_shift);
11982 %}
11983 
11984 // This pattern is automatically generated from aarch64_ad.m4
11985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11986 
11987 // Shift Left followed by Shift Right.
11988 // This idiom is used by the compiler for the i2b bytecode etc.
11989 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11990 %{
11991   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11992   ins_cost(INSN_COST * 2);
11993   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11994   ins_encode %{
11995     int lshift = $lshift_count$$constant & 31;
11996     int rshift = $rshift_count$$constant & 31;
11997     int s = 31 - lshift;
11998     int r = (rshift - lshift) & 31;
11999     __ sbfmw(as_Register($dst$$reg),
12000             as_Register($src$$reg),
12001             r, s);
12002   %}
12003 
12004   ins_pipe(ialu_reg_shift);
12005 %}
12006 
12007 // This pattern is automatically generated from aarch64_ad.m4
12008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12009 
12010 // Shift Left followed by Shift Right.
12011 // This idiom is used by the compiler for the i2b bytecode etc.
12012 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12013 %{
12014   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12015   ins_cost(INSN_COST * 2);
12016   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12017   ins_encode %{
12018     int lshift = $lshift_count$$constant & 63;
12019     int rshift = $rshift_count$$constant & 63;
12020     int s = 63 - lshift;
12021     int r = (rshift - lshift) & 63;
12022     __ ubfm(as_Register($dst$$reg),
12023             as_Register($src$$reg),
12024             r, s);
12025   %}
12026 
12027   ins_pipe(ialu_reg_shift);
12028 %}
12029 
12030 // This pattern is automatically generated from aarch64_ad.m4
12031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12032 
12033 // Shift Left followed by Shift Right.
12034 // This idiom is used by the compiler for the i2b bytecode etc.
12035 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12036 %{
12037   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12038   ins_cost(INSN_COST * 2);
12039   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12040   ins_encode %{
12041     int lshift = $lshift_count$$constant & 31;
12042     int rshift = $rshift_count$$constant & 31;
12043     int s = 31 - lshift;
12044     int r = (rshift - lshift) & 31;
12045     __ ubfmw(as_Register($dst$$reg),
12046             as_Register($src$$reg),
12047             r, s);
12048   %}
12049 
12050   ins_pipe(ialu_reg_shift);
12051 %}
12052 
12053 // Bitfield extract with shift & mask
12054 
12055 // This pattern is automatically generated from aarch64_ad.m4
12056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12057 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12058 %{
12059   match(Set dst (AndI (URShiftI src rshift) mask));
12060   // Make sure we are not going to exceed what ubfxw can do.
12061   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12062 
12063   ins_cost(INSN_COST);
12064   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12065   ins_encode %{
12066     int rshift = $rshift$$constant & 31;
12067     intptr_t mask = $mask$$constant;
12068     int width = exact_log2(mask+1);
12069     __ ubfxw(as_Register($dst$$reg),
12070             as_Register($src$$reg), rshift, width);
12071   %}
12072   ins_pipe(ialu_reg_shift);
12073 %}
12074 
12075 // This pattern is automatically generated from aarch64_ad.m4
12076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12077 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12078 %{
12079   match(Set dst (AndL (URShiftL src rshift) mask));
12080   // Make sure we are not going to exceed what ubfx can do.
12081   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12082 
12083   ins_cost(INSN_COST);
12084   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12085   ins_encode %{
12086     int rshift = $rshift$$constant & 63;
12087     intptr_t mask = $mask$$constant;
12088     int width = exact_log2_long(mask+1);
12089     __ ubfx(as_Register($dst$$reg),
12090             as_Register($src$$reg), rshift, width);
12091   %}
12092   ins_pipe(ialu_reg_shift);
12093 %}
12094 
12095 
12096 // This pattern is automatically generated from aarch64_ad.m4
12097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12098 
12099 // We can use ubfx when extending an And with a mask when we know mask
12100 // is positive.  We know that because immI_bitmask guarantees it.
12101 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12102 %{
12103   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12104   // Make sure we are not going to exceed what ubfxw can do.
12105   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12106 
12107   ins_cost(INSN_COST * 2);
12108   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12109   ins_encode %{
12110     int rshift = $rshift$$constant & 31;
12111     intptr_t mask = $mask$$constant;
12112     int width = exact_log2(mask+1);
12113     __ ubfx(as_Register($dst$$reg),
12114             as_Register($src$$reg), rshift, width);
12115   %}
12116   ins_pipe(ialu_reg_shift);
12117 %}
12118 
12119 
12120 // This pattern is automatically generated from aarch64_ad.m4
12121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12122 
12123 // We can use ubfiz when masking by a positive number and then left shifting the result.
12124 // We know that the mask is positive because immI_bitmask guarantees it.
12125 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12126 %{
12127   match(Set dst (LShiftI (AndI src mask) lshift));
12128   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12129 
12130   ins_cost(INSN_COST);
12131   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12132   ins_encode %{
12133     int lshift = $lshift$$constant & 31;
12134     intptr_t mask = $mask$$constant;
12135     int width = exact_log2(mask+1);
12136     __ ubfizw(as_Register($dst$$reg),
12137           as_Register($src$$reg), lshift, width);
12138   %}
12139   ins_pipe(ialu_reg_shift);
12140 %}
12141 
12142 // This pattern is automatically generated from aarch64_ad.m4
12143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12144 
12145 // We can use ubfiz when masking by a positive number and then left shifting the result.
12146 // We know that the mask is positive because immL_bitmask guarantees it.
12147 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12148 %{
12149   match(Set dst (LShiftL (AndL src mask) lshift));
12150   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12151 
12152   ins_cost(INSN_COST);
12153   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12154   ins_encode %{
12155     int lshift = $lshift$$constant & 63;
12156     intptr_t mask = $mask$$constant;
12157     int width = exact_log2_long(mask+1);
12158     __ ubfiz(as_Register($dst$$reg),
12159           as_Register($src$$reg), lshift, width);
12160   %}
12161   ins_pipe(ialu_reg_shift);
12162 %}
12163 
12164 // This pattern is automatically generated from aarch64_ad.m4
12165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12166 
12167 // We can use ubfiz when masking by a positive number and then left shifting the result.
12168 // We know that the mask is positive because immI_bitmask guarantees it.
12169 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12170 %{
12171   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12172   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12173 
12174   ins_cost(INSN_COST);
12175   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12176   ins_encode %{
12177     int lshift = $lshift$$constant & 31;
12178     intptr_t mask = $mask$$constant;
12179     int width = exact_log2(mask+1);
12180     __ ubfizw(as_Register($dst$$reg),
12181           as_Register($src$$reg), lshift, width);
12182   %}
12183   ins_pipe(ialu_reg_shift);
12184 %}
12185 
12186 // This pattern is automatically generated from aarch64_ad.m4
12187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12188 
12189 // We can use ubfiz when masking by a positive number and then left shifting the result.
12190 // We know that the mask is positive because immL_bitmask guarantees it.
12191 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12192 %{
12193   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12194   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12195 
12196   ins_cost(INSN_COST);
12197   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12198   ins_encode %{
12199     int lshift = $lshift$$constant & 63;
12200     intptr_t mask = $mask$$constant;
12201     int width = exact_log2_long(mask+1);
12202     __ ubfiz(as_Register($dst$$reg),
12203           as_Register($src$$reg), lshift, width);
12204   %}
12205   ins_pipe(ialu_reg_shift);
12206 %}
12207 
12208 
12209 // This pattern is automatically generated from aarch64_ad.m4
12210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12211 
12212 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12213 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12214 %{
12215   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12216   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12217 
12218   ins_cost(INSN_COST);
12219   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12220   ins_encode %{
12221     int lshift = $lshift$$constant & 63;
12222     intptr_t mask = $mask$$constant;
12223     int width = exact_log2(mask+1);
12224     __ ubfiz(as_Register($dst$$reg),
12225              as_Register($src$$reg), lshift, width);
12226   %}
12227   ins_pipe(ialu_reg_shift);
12228 %}
12229 
12230 // This pattern is automatically generated from aarch64_ad.m4
12231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12232 
12233 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12234 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12235 %{
12236   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12237   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12238 
12239   ins_cost(INSN_COST);
12240   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12241   ins_encode %{
12242     int lshift = $lshift$$constant & 31;
12243     intptr_t mask = $mask$$constant;
12244     int width = exact_log2(mask+1);
12245     __ ubfiz(as_Register($dst$$reg),
12246              as_Register($src$$reg), lshift, width);
12247   %}
12248   ins_pipe(ialu_reg_shift);
12249 %}
12250 
12251 // This pattern is automatically generated from aarch64_ad.m4
12252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12253 
12254 // Can skip int2long conversions after AND with small bitmask
12255 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12256 %{
12257   match(Set dst (ConvI2L (AndI src msk)));
12258   ins_cost(INSN_COST);
12259   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12260   ins_encode %{
12261     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12262   %}
12263   ins_pipe(ialu_reg_shift);
12264 %}
12265 
12266 
12267 // Rotations 
12268 // This pattern is automatically generated from aarch64_ad.m4
12269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12270 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12271 %{
12272   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12273   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12274 
12275   ins_cost(INSN_COST);
12276   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12277 
12278   ins_encode %{
12279     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12280             $rshift$$constant & 63);
12281   %}
12282   ins_pipe(ialu_reg_reg_extr);
12283 %}
12284 
12285 
12286 // This pattern is automatically generated from aarch64_ad.m4
12287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12288 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12289 %{
12290   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12291   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12292 
12293   ins_cost(INSN_COST);
12294   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12295 
12296   ins_encode %{
12297     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12298             $rshift$$constant & 31);
12299   %}
12300   ins_pipe(ialu_reg_reg_extr);
12301 %}
12302 
12303 
12304 // This pattern is automatically generated from aarch64_ad.m4
12305 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12306 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12307 %{
12308   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12309   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12310 
12311   ins_cost(INSN_COST);
12312   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12313 
12314   ins_encode %{
12315     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12316             $rshift$$constant & 63);
12317   %}
12318   ins_pipe(ialu_reg_reg_extr);
12319 %}
12320 
12321 
12322 // This pattern is automatically generated from aarch64_ad.m4
12323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12324 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12325 %{
12326   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12327   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12328 
12329   ins_cost(INSN_COST);
12330   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12331 
12332   ins_encode %{
12333     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12334             $rshift$$constant & 31);
12335   %}
12336   ins_pipe(ialu_reg_reg_extr);
12337 %}
12338 
12339 
12340 // This pattern is automatically generated from aarch64_ad.m4
12341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12342 
12343 // rol expander
12344 instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
12345 %{
12346   effect(DEF dst, USE src, USE shift);
12347 
12348   format %{ "rol    $dst, $src, $shift" %}
12349   ins_cost(INSN_COST * 3);
12350   ins_encode %{
12351     __ subw(rscratch1, zr, as_Register($shift$$reg));
12352     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
12353             rscratch1);
12354     %}
12355   ins_pipe(ialu_reg_reg_vshift);
12356 %}
12357 
12358 // This pattern is automatically generated from aarch64_ad.m4
12359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12360 
12361 // rol expander
12362 instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
12363 %{
12364   effect(DEF dst, USE src, USE shift);
12365 
12366   format %{ "rol    $dst, $src, $shift" %}
12367   ins_cost(INSN_COST * 3);
12368   ins_encode %{
12369     __ subw(rscratch1, zr, as_Register($shift$$reg));
12370     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
12371             rscratch1);
12372     %}
12373   ins_pipe(ialu_reg_reg_vshift);
12374 %}
12375 
12376 // This pattern is automatically generated from aarch64_ad.m4
12377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12378 instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
12379 %{
12380   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
12381 
12382   expand %{
12383     rolL_rReg(dst, src, shift, cr);
12384   %}
12385 %}
12386 
12387 // This pattern is automatically generated from aarch64_ad.m4
12388 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12389 instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
12390 %{
12391   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
12392 
12393   expand %{
12394     rolL_rReg(dst, src, shift, cr);
12395   %}
12396 %}
12397 
12398 // This pattern is automatically generated from aarch64_ad.m4
12399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12400 instruct rolI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
12401 %{
12402   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
12403 
12404   expand %{
12405     rolI_rReg(dst, src, shift, cr);
12406   %}
12407 %}
12408 
12409 // This pattern is automatically generated from aarch64_ad.m4
12410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12411 instruct rolI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
12412 %{
12413   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
12414 
12415   expand %{
12416     rolI_rReg(dst, src, shift, cr);
12417   %}
12418 %}
12419 
12420 // This pattern is automatically generated from aarch64_ad.m4
12421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12422 
12423 // ror expander
12424 instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
12425 %{
12426   effect(DEF dst, USE src, USE shift);
12427 
12428   format %{ "ror    $dst, $src, $shift" %}
12429   ins_cost(INSN_COST);
12430   ins_encode %{
12431     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
12432             as_Register($shift$$reg));
12433     %}
12434   ins_pipe(ialu_reg_reg_vshift);
12435 %}
12436 
12437 // This pattern is automatically generated from aarch64_ad.m4
12438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12439 
12440 // ror expander
12441 instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
12442 %{
12443   effect(DEF dst, USE src, USE shift);
12444 
12445   format %{ "ror    $dst, $src, $shift" %}
12446   ins_cost(INSN_COST);
12447   ins_encode %{
12448     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
12449             as_Register($shift$$reg));
12450     %}
12451   ins_pipe(ialu_reg_reg_vshift);
12452 %}
12453 
12454 // This pattern is automatically generated from aarch64_ad.m4
12455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12456 instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
12457 %{
12458   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
12459 
12460   expand %{
12461     rorL_rReg(dst, src, shift, cr);
12462   %}
12463 %}
12464 
12465 // This pattern is automatically generated from aarch64_ad.m4
12466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12467 instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
12468 %{
12469   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
12470 
12471   expand %{
12472     rorL_rReg(dst, src, shift, cr);
12473   %}
12474 %}
12475 
12476 // This pattern is automatically generated from aarch64_ad.m4
12477 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12478 instruct rorI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
12479 %{
12480   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
12481 
12482   expand %{
12483     rorI_rReg(dst, src, shift, cr);
12484   %}
12485 %}
12486 
12487 // This pattern is automatically generated from aarch64_ad.m4
12488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12489 instruct rorI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
12490 %{
12491   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
12492 
12493   expand %{
12494     rorI_rReg(dst, src, shift, cr);
12495   %}
12496 %}
12497 
12498 
12499 // Add/subtract (extended)
12500 
12501 // This pattern is automatically generated from aarch64_ad.m4
12502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12503 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12504 %{
12505   match(Set dst (AddL src1 (ConvI2L src2)));
12506   ins_cost(INSN_COST);
12507   format %{ "add  $dst, $src1, $src2, sxtw" %}
12508 
12509    ins_encode %{
12510      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12511             as_Register($src2$$reg), ext::sxtw);
12512    %}
12513   ins_pipe(ialu_reg_reg);
12514 %}
12515 
12516 // This pattern is automatically generated from aarch64_ad.m4
12517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12518 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12519 %{
12520   match(Set dst (SubL src1 (ConvI2L src2)));
12521   ins_cost(INSN_COST);
12522   format %{ "sub  $dst, $src1, $src2, sxtw" %}
12523 
12524    ins_encode %{
12525      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12526             as_Register($src2$$reg), ext::sxtw);
12527    %}
12528   ins_pipe(ialu_reg_reg);
12529 %}
12530 
12531 // This pattern is automatically generated from aarch64_ad.m4
12532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12533 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12534 %{
12535   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12536   ins_cost(INSN_COST);
12537   format %{ "add  $dst, $src1, $src2, sxth" %}
12538 
12539    ins_encode %{
12540      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12541             as_Register($src2$$reg), ext::sxth);
12542    %}
12543   ins_pipe(ialu_reg_reg);
12544 %}
12545 
12546 // This pattern is automatically generated from aarch64_ad.m4
12547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12548 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12549 %{
12550   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12551   ins_cost(INSN_COST);
12552   format %{ "add  $dst, $src1, $src2, sxtb" %}
12553 
12554    ins_encode %{
12555      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12556             as_Register($src2$$reg), ext::sxtb);
12557    %}
12558   ins_pipe(ialu_reg_reg);
12559 %}
12560 
12561 // This pattern is automatically generated from aarch64_ad.m4
12562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12563 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12564 %{
12565   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12566   ins_cost(INSN_COST);
12567   format %{ "add  $dst, $src1, $src2, uxtb" %}
12568 
12569    ins_encode %{
12570      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12571             as_Register($src2$$reg), ext::uxtb);
12572    %}
12573   ins_pipe(ialu_reg_reg);
12574 %}
12575 
12576 // This pattern is automatically generated from aarch64_ad.m4
12577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12578 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12579 %{
12580   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12581   ins_cost(INSN_COST);
12582   format %{ "add  $dst, $src1, $src2, sxth" %}
12583 
12584    ins_encode %{
12585      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12586             as_Register($src2$$reg), ext::sxth);
12587    %}
12588   ins_pipe(ialu_reg_reg);
12589 %}
12590 
12591 // This pattern is automatically generated from aarch64_ad.m4
12592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12593 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12594 %{
12595   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12596   ins_cost(INSN_COST);
12597   format %{ "add  $dst, $src1, $src2, sxtw" %}
12598 
12599    ins_encode %{
12600      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12601             as_Register($src2$$reg), ext::sxtw);
12602    %}
12603   ins_pipe(ialu_reg_reg);
12604 %}
12605 
12606 // This pattern is automatically generated from aarch64_ad.m4
12607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12608 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12609 %{
12610   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12611   ins_cost(INSN_COST);
12612   format %{ "add  $dst, $src1, $src2, sxtb" %}
12613 
12614    ins_encode %{
12615      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12616             as_Register($src2$$reg), ext::sxtb);
12617    %}
12618   ins_pipe(ialu_reg_reg);
12619 %}
12620 
12621 // This pattern is automatically generated from aarch64_ad.m4
12622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12623 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12624 %{
12625   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12626   ins_cost(INSN_COST);
12627   format %{ "add  $dst, $src1, $src2, uxtb" %}
12628 
12629    ins_encode %{
12630      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12631             as_Register($src2$$reg), ext::uxtb);
12632    %}
12633   ins_pipe(ialu_reg_reg);
12634 %}
12635 
12636 // This pattern is automatically generated from aarch64_ad.m4
12637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12638 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12639 %{
12640   match(Set dst (AddI src1 (AndI src2 mask)));
12641   ins_cost(INSN_COST);
12642   format %{ "addw  $dst, $src1, $src2, uxtb" %}
12643 
12644    ins_encode %{
12645      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12646             as_Register($src2$$reg), ext::uxtb);
12647    %}
12648   ins_pipe(ialu_reg_reg);
12649 %}
12650 
12651 // This pattern is automatically generated from aarch64_ad.m4
12652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12653 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12654 %{
12655   match(Set dst (AddI src1 (AndI src2 mask)));
12656   ins_cost(INSN_COST);
12657   format %{ "addw  $dst, $src1, $src2, uxth" %}
12658 
12659    ins_encode %{
12660      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12661             as_Register($src2$$reg), ext::uxth);
12662    %}
12663   ins_pipe(ialu_reg_reg);
12664 %}
12665 
12666 // This pattern is automatically generated from aarch64_ad.m4
12667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12668 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12669 %{
12670   match(Set dst (AddL src1 (AndL src2 mask)));
12671   ins_cost(INSN_COST);
12672   format %{ "add  $dst, $src1, $src2, uxtb" %}
12673 
12674    ins_encode %{
12675      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12676             as_Register($src2$$reg), ext::uxtb);
12677    %}
12678   ins_pipe(ialu_reg_reg);
12679 %}
12680 
12681 // This pattern is automatically generated from aarch64_ad.m4
12682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12683 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12684 %{
12685   match(Set dst (AddL src1 (AndL src2 mask)));
12686   ins_cost(INSN_COST);
12687   format %{ "add  $dst, $src1, $src2, uxth" %}
12688 
12689    ins_encode %{
12690      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12691             as_Register($src2$$reg), ext::uxth);
12692    %}
12693   ins_pipe(ialu_reg_reg);
12694 %}
12695 
12696 // This pattern is automatically generated from aarch64_ad.m4
12697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12698 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12699 %{
12700   match(Set dst (AddL src1 (AndL src2 mask)));
12701   ins_cost(INSN_COST);
12702   format %{ "add  $dst, $src1, $src2, uxtw" %}
12703 
12704    ins_encode %{
12705      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12706             as_Register($src2$$reg), ext::uxtw);
12707    %}
12708   ins_pipe(ialu_reg_reg);
12709 %}
12710 
12711 // This pattern is automatically generated from aarch64_ad.m4
12712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12713 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12714 %{
12715   match(Set dst (SubI src1 (AndI src2 mask)));
12716   ins_cost(INSN_COST);
12717   format %{ "subw  $dst, $src1, $src2, uxtb" %}
12718 
12719    ins_encode %{
12720      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12721             as_Register($src2$$reg), ext::uxtb);
12722    %}
12723   ins_pipe(ialu_reg_reg);
12724 %}
12725 
12726 // This pattern is automatically generated from aarch64_ad.m4
12727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12728 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12729 %{
12730   match(Set dst (SubI src1 (AndI src2 mask)));
12731   ins_cost(INSN_COST);
12732   format %{ "subw  $dst, $src1, $src2, uxth" %}
12733 
12734    ins_encode %{
12735      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12736             as_Register($src2$$reg), ext::uxth);
12737    %}
12738   ins_pipe(ialu_reg_reg);
12739 %}
12740 
12741 // This pattern is automatically generated from aarch64_ad.m4
12742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12743 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12744 %{
12745   match(Set dst (SubL src1 (AndL src2 mask)));
12746   ins_cost(INSN_COST);
12747   format %{ "sub  $dst, $src1, $src2, uxtb" %}
12748 
12749    ins_encode %{
12750      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12751             as_Register($src2$$reg), ext::uxtb);
12752    %}
12753   ins_pipe(ialu_reg_reg);
12754 %}
12755 
12756 // This pattern is automatically generated from aarch64_ad.m4
12757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12758 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12759 %{
12760   match(Set dst (SubL src1 (AndL src2 mask)));
12761   ins_cost(INSN_COST);
12762   format %{ "sub  $dst, $src1, $src2, uxth" %}
12763 
12764    ins_encode %{
12765      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12766             as_Register($src2$$reg), ext::uxth);
12767    %}
12768   ins_pipe(ialu_reg_reg);
12769 %}
12770 
12771 // This pattern is automatically generated from aarch64_ad.m4
12772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12773 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12774 %{
12775   match(Set dst (SubL src1 (AndL src2 mask)));
12776   ins_cost(INSN_COST);
12777   format %{ "sub  $dst, $src1, $src2, uxtw" %}
12778 
12779    ins_encode %{
12780      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12781             as_Register($src2$$reg), ext::uxtw);
12782    %}
12783   ins_pipe(ialu_reg_reg);
12784 %}
12785 
12786 
12787 // This pattern is automatically generated from aarch64_ad.m4
12788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12789 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12790 %{
12791   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12792   ins_cost(1.9 * INSN_COST);
12793   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
12794 
12795    ins_encode %{
12796      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12797             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12798    %}
12799   ins_pipe(ialu_reg_reg_shift);
12800 %}
12801 
12802 // This pattern is automatically generated from aarch64_ad.m4
12803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12804 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12805 %{
12806   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12807   ins_cost(1.9 * INSN_COST);
12808   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
12809 
12810    ins_encode %{
12811      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12812             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12813    %}
12814   ins_pipe(ialu_reg_reg_shift);
12815 %}
12816 
12817 // This pattern is automatically generated from aarch64_ad.m4
12818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12819 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12820 %{
12821   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12822   ins_cost(1.9 * INSN_COST);
12823   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
12824 
12825    ins_encode %{
12826      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12827             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12828    %}
12829   ins_pipe(ialu_reg_reg_shift);
12830 %}
12831 
12832 // This pattern is automatically generated from aarch64_ad.m4
12833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12834 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12835 %{
12836   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12837   ins_cost(1.9 * INSN_COST);
12838   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
12839 
12840    ins_encode %{
12841      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12842             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12843    %}
12844   ins_pipe(ialu_reg_reg_shift);
12845 %}
12846 
12847 // This pattern is automatically generated from aarch64_ad.m4
12848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12849 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12850 %{
12851   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12852   ins_cost(1.9 * INSN_COST);
12853   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
12854 
12855    ins_encode %{
12856      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12857             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12858    %}
12859   ins_pipe(ialu_reg_reg_shift);
12860 %}
12861 
12862 // This pattern is automatically generated from aarch64_ad.m4
12863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12864 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12865 %{
12866   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12867   ins_cost(1.9 * INSN_COST);
12868   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
12869 
12870    ins_encode %{
12871      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12872             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12873    %}
12874   ins_pipe(ialu_reg_reg_shift);
12875 %}
12876 
12877 // This pattern is automatically generated from aarch64_ad.m4
12878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12879 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12880 %{
12881   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12882   ins_cost(1.9 * INSN_COST);
12883   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
12884 
12885    ins_encode %{
12886      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12887             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12888    %}
12889   ins_pipe(ialu_reg_reg_shift);
12890 %}
12891 
12892 // This pattern is automatically generated from aarch64_ad.m4
12893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12894 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12895 %{
12896   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12897   ins_cost(1.9 * INSN_COST);
12898   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
12899 
12900    ins_encode %{
12901      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12902             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12903    %}
12904   ins_pipe(ialu_reg_reg_shift);
12905 %}
12906 
12907 // This pattern is automatically generated from aarch64_ad.m4
12908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12909 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12910 %{
12911   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12912   ins_cost(1.9 * INSN_COST);
12913   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
12914 
12915    ins_encode %{
12916      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12917             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12918    %}
12919   ins_pipe(ialu_reg_reg_shift);
12920 %}
12921 
12922 // This pattern is automatically generated from aarch64_ad.m4
12923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12924 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12925 %{
12926   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12927   ins_cost(1.9 * INSN_COST);
12928   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
12929 
12930    ins_encode %{
12931      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12932             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12933    %}
12934   ins_pipe(ialu_reg_reg_shift);
12935 %}
12936 
12937 // This pattern is automatically generated from aarch64_ad.m4
12938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12939 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12940 %{
12941   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12942   ins_cost(1.9 * INSN_COST);
12943   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
12944 
12945    ins_encode %{
12946      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12947             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12948    %}
12949   ins_pipe(ialu_reg_reg_shift);
12950 %}
12951 
12952 // This pattern is automatically generated from aarch64_ad.m4
12953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12954 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12955 %{
12956   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12957   ins_cost(1.9 * INSN_COST);
12958   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
12959 
12960    ins_encode %{
12961      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12962             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12963    %}
12964   ins_pipe(ialu_reg_reg_shift);
12965 %}
12966 
12967 // This pattern is automatically generated from aarch64_ad.m4
12968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12969 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12970 %{
12971   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12972   ins_cost(1.9 * INSN_COST);
12973   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
12974 
12975    ins_encode %{
12976      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12977             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12978    %}
12979   ins_pipe(ialu_reg_reg_shift);
12980 %}
12981 
12982 // This pattern is automatically generated from aarch64_ad.m4
12983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12984 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12985 %{
12986   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12987   ins_cost(1.9 * INSN_COST);
12988   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
12989 
12990    ins_encode %{
12991      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12992             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12993    %}
12994   ins_pipe(ialu_reg_reg_shift);
12995 %}
12996 
12997 // This pattern is automatically generated from aarch64_ad.m4
12998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12999 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13000 %{
13001   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13002   ins_cost(1.9 * INSN_COST);
13003   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13004 
13005    ins_encode %{
13006      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13007             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13008    %}
13009   ins_pipe(ialu_reg_reg_shift);
13010 %}
13011 
13012 // This pattern is automatically generated from aarch64_ad.m4
13013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13014 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13015 %{
13016   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13017   ins_cost(1.9 * INSN_COST);
13018   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13019 
13020    ins_encode %{
13021      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13022             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13023    %}
13024   ins_pipe(ialu_reg_reg_shift);
13025 %}
13026 
13027 // This pattern is automatically generated from aarch64_ad.m4
13028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13029 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13030 %{
13031   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13032   ins_cost(1.9 * INSN_COST);
13033   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13034 
13035    ins_encode %{
13036      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13037             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13038    %}
13039   ins_pipe(ialu_reg_reg_shift);
13040 %}
13041 
13042 // This pattern is automatically generated from aarch64_ad.m4
13043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13044 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13045 %{
13046   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13047   ins_cost(1.9 * INSN_COST);
13048   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13049 
13050    ins_encode %{
13051      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13052             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13053    %}
13054   ins_pipe(ialu_reg_reg_shift);
13055 %}
13056 
13057 // This pattern is automatically generated from aarch64_ad.m4
13058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13059 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13060 %{
13061   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13062   ins_cost(1.9 * INSN_COST);
13063   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13064 
13065    ins_encode %{
13066      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13067             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13068    %}
13069   ins_pipe(ialu_reg_reg_shift);
13070 %}
13071 
13072 // This pattern is automatically generated from aarch64_ad.m4
13073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13074 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13075 %{
13076   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13077   ins_cost(1.9 * INSN_COST);
13078   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13079 
13080    ins_encode %{
13081      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13082             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13083    %}
13084   ins_pipe(ialu_reg_reg_shift);
13085 %}
13086 
13087 // This pattern is automatically generated from aarch64_ad.m4
13088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13089 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13090 %{
13091   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13092   ins_cost(1.9 * INSN_COST);
13093   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13094 
13095    ins_encode %{
13096      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13097             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13098    %}
13099   ins_pipe(ialu_reg_reg_shift);
13100 %}
13101 
13102 // This pattern is automatically generated from aarch64_ad.m4
13103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13104 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13105 %{
13106   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13107   ins_cost(1.9 * INSN_COST);
13108   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13109 
13110    ins_encode %{
13111      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13112             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13113    %}
13114   ins_pipe(ialu_reg_reg_shift);
13115 %}
13116 
13117 
13118 
13119 // END This section of the file is automatically generated. Do not edit --------------
13120 
13121 
13122 // ============================================================================
13123 // Floating Point Arithmetic Instructions
13124 
13125 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13126   match(Set dst (AddF src1 src2));
13127 
13128   ins_cost(INSN_COST * 5);
13129   format %{ "fadds   $dst, $src1, $src2" %}
13130 
13131   ins_encode %{
13132     __ fadds(as_FloatRegister($dst$$reg),
13133              as_FloatRegister($src1$$reg),
13134              as_FloatRegister($src2$$reg));
13135   %}
13136 
13137   ins_pipe(fp_dop_reg_reg_s);
13138 %}
13139 
13140 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13141   match(Set dst (AddD src1 src2));
13142 
13143   ins_cost(INSN_COST * 5);
13144   format %{ "faddd   $dst, $src1, $src2" %}
13145 
13146   ins_encode %{
13147     __ faddd(as_FloatRegister($dst$$reg),
13148              as_FloatRegister($src1$$reg),
13149              as_FloatRegister($src2$$reg));
13150   %}
13151 
13152   ins_pipe(fp_dop_reg_reg_d);
13153 %}
13154 
13155 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13156   match(Set dst (SubF src1 src2));
13157 
13158   ins_cost(INSN_COST * 5);
13159   format %{ "fsubs   $dst, $src1, $src2" %}
13160 
13161   ins_encode %{
13162     __ fsubs(as_FloatRegister($dst$$reg),
13163              as_FloatRegister($src1$$reg),
13164              as_FloatRegister($src2$$reg));
13165   %}
13166 
13167   ins_pipe(fp_dop_reg_reg_s);
13168 %}
13169 
13170 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13171   match(Set dst (SubD src1 src2));
13172 
13173   ins_cost(INSN_COST * 5);
13174   format %{ "fsubd   $dst, $src1, $src2" %}
13175 
13176   ins_encode %{
13177     __ fsubd(as_FloatRegister($dst$$reg),
13178              as_FloatRegister($src1$$reg),
13179              as_FloatRegister($src2$$reg));
13180   %}
13181 
13182   ins_pipe(fp_dop_reg_reg_d);
13183 %}
13184 
13185 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13186   match(Set dst (MulF src1 src2));
13187 
13188   ins_cost(INSN_COST * 6);
13189   format %{ "fmuls   $dst, $src1, $src2" %}
13190 
13191   ins_encode %{
13192     __ fmuls(as_FloatRegister($dst$$reg),
13193              as_FloatRegister($src1$$reg),
13194              as_FloatRegister($src2$$reg));
13195   %}
13196 
13197   ins_pipe(fp_dop_reg_reg_s);
13198 %}
13199 
13200 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13201   match(Set dst (MulD src1 src2));
13202 
13203   ins_cost(INSN_COST * 6);
13204   format %{ "fmuld   $dst, $src1, $src2" %}
13205 
13206   ins_encode %{
13207     __ fmuld(as_FloatRegister($dst$$reg),
13208              as_FloatRegister($src1$$reg),
13209              as_FloatRegister($src2$$reg));
13210   %}
13211 
13212   ins_pipe(fp_dop_reg_reg_d);
13213 %}
13214 
13215 // src1 * src2 + src3
13216 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13217   predicate(UseFMA);
13218   match(Set dst (FmaF src3 (Binary src1 src2)));
13219 
13220   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13221 
13222   ins_encode %{
13223     __ fmadds(as_FloatRegister($dst$$reg),
13224              as_FloatRegister($src1$$reg),
13225              as_FloatRegister($src2$$reg),
13226              as_FloatRegister($src3$$reg));
13227   %}
13228 
13229   ins_pipe(pipe_class_default);
13230 %}
13231 
13232 // src1 * src2 + src3
13233 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13234   predicate(UseFMA);
13235   match(Set dst (FmaD src3 (Binary src1 src2)));
13236 
13237   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13238 
13239   ins_encode %{
13240     __ fmaddd(as_FloatRegister($dst$$reg),
13241              as_FloatRegister($src1$$reg),
13242              as_FloatRegister($src2$$reg),
13243              as_FloatRegister($src3$$reg));
13244   %}
13245 
13246   ins_pipe(pipe_class_default);
13247 %}
13248 
13249 // -src1 * src2 + src3
13250 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13251   predicate(UseFMA);
13252   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13253   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13254 
13255   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13256 
13257   ins_encode %{
13258     __ fmsubs(as_FloatRegister($dst$$reg),
13259               as_FloatRegister($src1$$reg),
13260               as_FloatRegister($src2$$reg),
13261               as_FloatRegister($src3$$reg));
13262   %}
13263 
13264   ins_pipe(pipe_class_default);
13265 %}
13266 
13267 // -src1 * src2 + src3
13268 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13269   predicate(UseFMA);
13270   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13271   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13272 
13273   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13274 
13275   ins_encode %{
13276     __ fmsubd(as_FloatRegister($dst$$reg),
13277               as_FloatRegister($src1$$reg),
13278               as_FloatRegister($src2$$reg),
13279               as_FloatRegister($src3$$reg));
13280   %}
13281 
13282   ins_pipe(pipe_class_default);
13283 %}
13284 
13285 // -src1 * src2 - src3
13286 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13287   predicate(UseFMA);
13288   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13289   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13290 
13291   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13292 
13293   ins_encode %{
13294     __ fnmadds(as_FloatRegister($dst$$reg),
13295                as_FloatRegister($src1$$reg),
13296                as_FloatRegister($src2$$reg),
13297                as_FloatRegister($src3$$reg));
13298   %}
13299 
13300   ins_pipe(pipe_class_default);
13301 %}
13302 
13303 // -src1 * src2 - src3
13304 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13305   predicate(UseFMA);
13306   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
13307   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13308 
13309   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
13310 
13311   ins_encode %{
13312     __ fnmaddd(as_FloatRegister($dst$$reg),
13313                as_FloatRegister($src1$$reg),
13314                as_FloatRegister($src2$$reg),
13315                as_FloatRegister($src3$$reg));
13316   %}
13317 
13318   ins_pipe(pipe_class_default);
13319 %}
13320 
13321 // src1 * src2 - src3
13322 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13323   predicate(UseFMA);
13324   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13325 
13326   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
13327 
13328   ins_encode %{
13329     __ fnmsubs(as_FloatRegister($dst$$reg),
13330                as_FloatRegister($src1$$reg),
13331                as_FloatRegister($src2$$reg),
13332                as_FloatRegister($src3$$reg));
13333   %}
13334 
13335   ins_pipe(pipe_class_default);
13336 %}
13337 
13338 // src1 * src2 - src3
13339 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13340   predicate(UseFMA);
13341   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13342 
13343   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
13344 
13345   ins_encode %{
13346   // n.b. insn name should be fnmsubd
13347     __ fnmsub(as_FloatRegister($dst$$reg),
13348               as_FloatRegister($src1$$reg),
13349               as_FloatRegister($src2$$reg),
13350               as_FloatRegister($src3$$reg));
13351   %}
13352 
13353   ins_pipe(pipe_class_default);
13354 %}
13355 
13356 
13357 // Math.max(FF)F
13358 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13359   match(Set dst (MaxF src1 src2));
13360 
13361   format %{ "fmaxs   $dst, $src1, $src2" %}
13362   ins_encode %{
13363     __ fmaxs(as_FloatRegister($dst$$reg),
13364              as_FloatRegister($src1$$reg),
13365              as_FloatRegister($src2$$reg));
13366   %}
13367 
13368   ins_pipe(fp_dop_reg_reg_s);
13369 %}
13370 
13371 // Math.min(FF)F
13372 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13373   match(Set dst (MinF src1 src2));
13374 
13375   format %{ "fmins   $dst, $src1, $src2" %}
13376   ins_encode %{
13377     __ fmins(as_FloatRegister($dst$$reg),
13378              as_FloatRegister($src1$$reg),
13379              as_FloatRegister($src2$$reg));
13380   %}
13381 
13382   ins_pipe(fp_dop_reg_reg_s);
13383 %}
13384 
13385 // Math.max(DD)D
13386 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13387   match(Set dst (MaxD src1 src2));
13388 
13389   format %{ "fmaxd   $dst, $src1, $src2" %}
13390   ins_encode %{
13391     __ fmaxd(as_FloatRegister($dst$$reg),
13392              as_FloatRegister($src1$$reg),
13393              as_FloatRegister($src2$$reg));
13394   %}
13395 
13396   ins_pipe(fp_dop_reg_reg_d);
13397 %}
13398 
13399 // Math.min(DD)D
13400 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13401   match(Set dst (MinD src1 src2));
13402 
13403   format %{ "fmind   $dst, $src1, $src2" %}
13404   ins_encode %{
13405     __ fmind(as_FloatRegister($dst$$reg),
13406              as_FloatRegister($src1$$reg),
13407              as_FloatRegister($src2$$reg));
13408   %}
13409 
13410   ins_pipe(fp_dop_reg_reg_d);
13411 %}
13412 
13413 
13414 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13415   match(Set dst (DivF src1  src2));
13416 
13417   ins_cost(INSN_COST * 18);
13418   format %{ "fdivs   $dst, $src1, $src2" %}
13419 
13420   ins_encode %{
13421     __ fdivs(as_FloatRegister($dst$$reg),
13422              as_FloatRegister($src1$$reg),
13423              as_FloatRegister($src2$$reg));
13424   %}
13425 
13426   ins_pipe(fp_div_s);
13427 %}
13428 
13429 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13430   match(Set dst (DivD src1  src2));
13431 
13432   ins_cost(INSN_COST * 32);
13433   format %{ "fdivd   $dst, $src1, $src2" %}
13434 
13435   ins_encode %{
13436     __ fdivd(as_FloatRegister($dst$$reg),
13437              as_FloatRegister($src1$$reg),
13438              as_FloatRegister($src2$$reg));
13439   %}
13440 
13441   ins_pipe(fp_div_d);
13442 %}
13443 
13444 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13445   match(Set dst (NegF src));
13446 
13447   ins_cost(INSN_COST * 3);
13448   format %{ "fneg   $dst, $src" %}
13449 
13450   ins_encode %{
13451     __ fnegs(as_FloatRegister($dst$$reg),
13452              as_FloatRegister($src$$reg));
13453   %}
13454 
13455   ins_pipe(fp_uop_s);
13456 %}
13457 
13458 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13459   match(Set dst (NegD src));
13460 
13461   ins_cost(INSN_COST * 3);
13462   format %{ "fnegd   $dst, $src" %}
13463 
13464   ins_encode %{
13465     __ fnegd(as_FloatRegister($dst$$reg),
13466              as_FloatRegister($src$$reg));
13467   %}
13468 
13469   ins_pipe(fp_uop_d);
13470 %}
13471 
13472 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13473 %{
13474   match(Set dst (AbsI src));
13475 
13476   effect(KILL cr);
13477   ins_cost(INSN_COST * 2);
13478   format %{ "cmpw  $src, zr\n\t"
13479             "cnegw $dst, $src, Assembler::LT\t# int abs"
13480   %}
13481 
13482   ins_encode %{
13483     __ cmpw(as_Register($src$$reg), zr);
13484     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13485   %}
13486   ins_pipe(pipe_class_default);
13487 %}
13488 
13489 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13490 %{
13491   match(Set dst (AbsL src));
13492 
13493   effect(KILL cr);
13494   ins_cost(INSN_COST * 2);
13495   format %{ "cmp  $src, zr\n\t"
13496             "cneg $dst, $src, Assembler::LT\t# long abs"
13497   %}
13498 
13499   ins_encode %{
13500     __ cmp(as_Register($src$$reg), zr);
13501     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13502   %}
13503   ins_pipe(pipe_class_default);
13504 %}
13505 
13506 instruct absF_reg(vRegF dst, vRegF src) %{
13507   match(Set dst (AbsF src));
13508 
13509   ins_cost(INSN_COST * 3);
13510   format %{ "fabss   $dst, $src" %}
13511   ins_encode %{
13512     __ fabss(as_FloatRegister($dst$$reg),
13513              as_FloatRegister($src$$reg));
13514   %}
13515 
13516   ins_pipe(fp_uop_s);
13517 %}
13518 
13519 instruct absD_reg(vRegD dst, vRegD src) %{
13520   match(Set dst (AbsD src));
13521 
13522   ins_cost(INSN_COST * 3);
13523   format %{ "fabsd   $dst, $src" %}
13524   ins_encode %{
13525     __ fabsd(as_FloatRegister($dst$$reg),
13526              as_FloatRegister($src$$reg));
13527   %}
13528 
13529   ins_pipe(fp_uop_d);
13530 %}
13531 
13532 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13533   match(Set dst (SqrtD src));
13534 
13535   ins_cost(INSN_COST * 50);
13536   format %{ "fsqrtd  $dst, $src" %}
13537   ins_encode %{
13538     __ fsqrtd(as_FloatRegister($dst$$reg),
13539              as_FloatRegister($src$$reg));
13540   %}
13541 
13542   ins_pipe(fp_div_s);
13543 %}
13544 
13545 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13546   match(Set dst (SqrtF src));
13547 
13548   ins_cost(INSN_COST * 50);
13549   format %{ "fsqrts  $dst, $src" %}
13550   ins_encode %{
13551     __ fsqrts(as_FloatRegister($dst$$reg),
13552              as_FloatRegister($src$$reg));
13553   %}
13554 
13555   ins_pipe(fp_div_d);
13556 %}
13557 
13558 // Math.rint, floor, ceil
13559 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13560   match(Set dst (RoundDoubleMode src rmode));
13561   format %{ "frint  $dst, $src, $rmode" %}
13562   ins_encode %{
13563     switch ($rmode$$constant) {
13564       case RoundDoubleModeNode::rmode_rint:
13565         __ frintnd(as_FloatRegister($dst$$reg),
13566                    as_FloatRegister($src$$reg));
13567         break;
13568       case RoundDoubleModeNode::rmode_floor:
13569         __ frintmd(as_FloatRegister($dst$$reg),
13570                    as_FloatRegister($src$$reg));
13571         break;
13572       case RoundDoubleModeNode::rmode_ceil:
13573         __ frintpd(as_FloatRegister($dst$$reg),
13574                    as_FloatRegister($src$$reg));
13575         break;
13576     }
13577   %}
13578   ins_pipe(fp_uop_d);
13579 %}
13580 
13581 // ============================================================================
13582 // Logical Instructions
13583 
13584 // Integer Logical Instructions
13585 
13586 // And Instructions
13587 
13588 
13589 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13590   match(Set dst (AndI src1 src2));
13591 
13592   format %{ "andw  $dst, $src1, $src2\t# int" %}
13593 
13594   ins_cost(INSN_COST);
13595   ins_encode %{
13596     __ andw(as_Register($dst$$reg),
13597             as_Register($src1$$reg),
13598             as_Register($src2$$reg));
13599   %}
13600 
13601   ins_pipe(ialu_reg_reg);
13602 %}
13603 
13604 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13605   match(Set dst (AndI src1 src2));
13606 
13607   format %{ "andsw  $dst, $src1, $src2\t# int" %}
13608 
13609   ins_cost(INSN_COST);
13610   ins_encode %{
13611     __ andw(as_Register($dst$$reg),
13612             as_Register($src1$$reg),
13613             (uint64_t)($src2$$constant));
13614   %}
13615 
13616   ins_pipe(ialu_reg_imm);
13617 %}
13618 
13619 // Or Instructions
13620 
13621 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13622   match(Set dst (OrI src1 src2));
13623 
13624   format %{ "orrw  $dst, $src1, $src2\t# int" %}
13625 
13626   ins_cost(INSN_COST);
13627   ins_encode %{
13628     __ orrw(as_Register($dst$$reg),
13629             as_Register($src1$$reg),
13630             as_Register($src2$$reg));
13631   %}
13632 
13633   ins_pipe(ialu_reg_reg);
13634 %}
13635 
13636 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13637   match(Set dst (OrI src1 src2));
13638 
13639   format %{ "orrw  $dst, $src1, $src2\t# int" %}
13640 
13641   ins_cost(INSN_COST);
13642   ins_encode %{
13643     __ orrw(as_Register($dst$$reg),
13644             as_Register($src1$$reg),
13645             (uint64_t)($src2$$constant));
13646   %}
13647 
13648   ins_pipe(ialu_reg_imm);
13649 %}
13650 
13651 // Xor Instructions
13652 
13653 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13654   match(Set dst (XorI src1 src2));
13655 
13656   format %{ "eorw  $dst, $src1, $src2\t# int" %}
13657 
13658   ins_cost(INSN_COST);
13659   ins_encode %{
13660     __ eorw(as_Register($dst$$reg),
13661             as_Register($src1$$reg),
13662             as_Register($src2$$reg));
13663   %}
13664 
13665   ins_pipe(ialu_reg_reg);
13666 %}
13667 
13668 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13669   match(Set dst (XorI src1 src2));
13670 
13671   format %{ "eorw  $dst, $src1, $src2\t# int" %}
13672 
13673   ins_cost(INSN_COST);
13674   ins_encode %{
13675     __ eorw(as_Register($dst$$reg),
13676             as_Register($src1$$reg),
13677             (uint64_t)($src2$$constant));
13678   %}
13679 
13680   ins_pipe(ialu_reg_imm);
13681 %}
13682 
13683 // Long Logical Instructions
13684 // TODO
13685 
13686 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13687   match(Set dst (AndL src1 src2));
13688 
13689   format %{ "and  $dst, $src1, $src2\t# int" %}
13690 
13691   ins_cost(INSN_COST);
13692   ins_encode %{
13693     __ andr(as_Register($dst$$reg),
13694             as_Register($src1$$reg),
13695             as_Register($src2$$reg));
13696   %}
13697 
13698   ins_pipe(ialu_reg_reg);
13699 %}
13700 
13701 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13702   match(Set dst (AndL src1 src2));
13703 
13704   format %{ "and  $dst, $src1, $src2\t# int" %}
13705 
13706   ins_cost(INSN_COST);
13707   ins_encode %{
13708     __ andr(as_Register($dst$$reg),
13709             as_Register($src1$$reg),
13710             (uint64_t)($src2$$constant));
13711   %}
13712 
13713   ins_pipe(ialu_reg_imm);
13714 %}
13715 
13716 // Or Instructions
13717 
13718 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13719   match(Set dst (OrL src1 src2));
13720 
13721   format %{ "orr  $dst, $src1, $src2\t# int" %}
13722 
13723   ins_cost(INSN_COST);
13724   ins_encode %{
13725     __ orr(as_Register($dst$$reg),
13726            as_Register($src1$$reg),
13727            as_Register($src2$$reg));
13728   %}
13729 
13730   ins_pipe(ialu_reg_reg);
13731 %}
13732 
13733 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13734   match(Set dst (OrL src1 src2));
13735 
13736   format %{ "orr  $dst, $src1, $src2\t# int" %}
13737 
13738   ins_cost(INSN_COST);
13739   ins_encode %{
13740     __ orr(as_Register($dst$$reg),
13741            as_Register($src1$$reg),
13742            (uint64_t)($src2$$constant));
13743   %}
13744 
13745   ins_pipe(ialu_reg_imm);
13746 %}
13747 
13748 // Xor Instructions
13749 
13750 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13751   match(Set dst (XorL src1 src2));
13752 
13753   format %{ "eor  $dst, $src1, $src2\t# int" %}
13754 
13755   ins_cost(INSN_COST);
13756   ins_encode %{
13757     __ eor(as_Register($dst$$reg),
13758            as_Register($src1$$reg),
13759            as_Register($src2$$reg));
13760   %}
13761 
13762   ins_pipe(ialu_reg_reg);
13763 %}
13764 
13765 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13766   match(Set dst (XorL src1 src2));
13767 
13768   ins_cost(INSN_COST);
13769   format %{ "eor  $dst, $src1, $src2\t# int" %}
13770 
13771   ins_encode %{
13772     __ eor(as_Register($dst$$reg),
13773            as_Register($src1$$reg),
13774            (uint64_t)($src2$$constant));
13775   %}
13776 
13777   ins_pipe(ialu_reg_imm);
13778 %}
13779 
13780 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13781 %{
13782   match(Set dst (ConvI2L src));
13783 
13784   ins_cost(INSN_COST);
13785   format %{ "sxtw  $dst, $src\t# i2l" %}
13786   ins_encode %{
13787     __ sbfm($dst$$Register, $src$$Register, 0, 31);
13788   %}
13789   ins_pipe(ialu_reg_shift);
13790 %}
13791 
13792 // this pattern occurs in bigmath arithmetic
13793 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13794 %{
13795   match(Set dst (AndL (ConvI2L src) mask));
13796 
13797   ins_cost(INSN_COST);
13798   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
13799   ins_encode %{
13800     __ ubfm($dst$$Register, $src$$Register, 0, 31);
13801   %}
13802 
13803   ins_pipe(ialu_reg_shift);
13804 %}
13805 
13806 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13807   match(Set dst (ConvL2I src));
13808 
13809   ins_cost(INSN_COST);
13810   format %{ "movw  $dst, $src \t// l2i" %}
13811 
13812   ins_encode %{
13813     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13814   %}
13815 
13816   ins_pipe(ialu_reg);
13817 %}
13818 
13819 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13820 %{
13821   match(Set dst (Conv2B src));
13822   effect(KILL cr);
13823 
13824   format %{
13825     "cmpw $src, zr\n\t"
13826     "cset $dst, ne"
13827   %}
13828 
13829   ins_encode %{
13830     __ cmpw(as_Register($src$$reg), zr);
13831     __ cset(as_Register($dst$$reg), Assembler::NE);
13832   %}
13833 
13834   ins_pipe(ialu_reg);
13835 %}
13836 
13837 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
13838 %{
13839   match(Set dst (Conv2B src));
13840   effect(KILL cr);
13841 
13842   format %{
13843     "cmp  $src, zr\n\t"
13844     "cset $dst, ne"
13845   %}
13846 
13847   ins_encode %{
13848     __ cmp(as_Register($src$$reg), zr);
13849     __ cset(as_Register($dst$$reg), Assembler::NE);
13850   %}
13851 
13852   ins_pipe(ialu_reg);
13853 %}
13854 
13855 instruct convD2F_reg(vRegF dst, vRegD src) %{
13856   match(Set dst (ConvD2F src));
13857 
13858   ins_cost(INSN_COST * 5);
13859   format %{ "fcvtd  $dst, $src \t// d2f" %}
13860 
13861   ins_encode %{
13862     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13863   %}
13864 
13865   ins_pipe(fp_d2f);
13866 %}
13867 
13868 instruct convF2D_reg(vRegD dst, vRegF src) %{
13869   match(Set dst (ConvF2D src));
13870 
13871   ins_cost(INSN_COST * 5);
13872   format %{ "fcvts  $dst, $src \t// f2d" %}
13873 
13874   ins_encode %{
13875     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13876   %}
13877 
13878   ins_pipe(fp_f2d);
13879 %}
13880 
13881 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13882   match(Set dst (ConvF2I src));
13883 
13884   ins_cost(INSN_COST * 5);
13885   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
13886 
13887   ins_encode %{
13888     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13889   %}
13890 
13891   ins_pipe(fp_f2i);
13892 %}
13893 
13894 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13895   match(Set dst (ConvF2L src));
13896 
13897   ins_cost(INSN_COST * 5);
13898   format %{ "fcvtzs  $dst, $src \t// f2l" %}
13899 
13900   ins_encode %{
13901     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13902   %}
13903 
13904   ins_pipe(fp_f2l);
13905 %}
13906 
13907 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13908   match(Set dst (ConvI2F src));
13909 
13910   ins_cost(INSN_COST * 5);
13911   format %{ "scvtfws  $dst, $src \t// i2f" %}
13912 
13913   ins_encode %{
13914     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13915   %}
13916 
13917   ins_pipe(fp_i2f);
13918 %}
13919 
13920 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13921   match(Set dst (ConvL2F src));
13922 
13923   ins_cost(INSN_COST * 5);
13924   format %{ "scvtfs  $dst, $src \t// l2f" %}
13925 
13926   ins_encode %{
13927     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13928   %}
13929 
13930   ins_pipe(fp_l2f);
13931 %}
13932 
13933 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13934   match(Set dst (ConvD2I src));
13935 
13936   ins_cost(INSN_COST * 5);
13937   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
13938 
13939   ins_encode %{
13940     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13941   %}
13942 
13943   ins_pipe(fp_d2i);
13944 %}
13945 
13946 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13947   match(Set dst (ConvD2L src));
13948 
13949   ins_cost(INSN_COST * 5);
13950   format %{ "fcvtzd  $dst, $src \t// d2l" %}
13951 
13952   ins_encode %{
13953     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13954   %}
13955 
13956   ins_pipe(fp_d2l);
13957 %}
13958 
13959 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13960   match(Set dst (ConvI2D src));
13961 
13962   ins_cost(INSN_COST * 5);
13963   format %{ "scvtfwd  $dst, $src \t// i2d" %}
13964 
13965   ins_encode %{
13966     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13967   %}
13968 
13969   ins_pipe(fp_i2d);
13970 %}
13971 
13972 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13973   match(Set dst (ConvL2D src));
13974 
13975   ins_cost(INSN_COST * 5);
13976   format %{ "scvtfd  $dst, $src \t// l2d" %}
13977 
13978   ins_encode %{
13979     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13980   %}
13981 
13982   ins_pipe(fp_l2d);
13983 %}
13984 
13985 // stack <-> reg and reg <-> reg shuffles with no conversion
13986 
13987 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13988 
13989   match(Set dst (MoveF2I src));
13990 
13991   effect(DEF dst, USE src);
13992 
13993   ins_cost(4 * INSN_COST);
13994 
13995   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13996 
13997   ins_encode %{
13998     __ ldrw($dst$$Register, Address(sp, $src$$disp));
13999   %}
14000 
14001   ins_pipe(iload_reg_reg);
14002 
14003 %}
14004 
14005 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14006 
14007   match(Set dst (MoveI2F src));
14008 
14009   effect(DEF dst, USE src);
14010 
14011   ins_cost(4 * INSN_COST);
14012 
14013   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14014 
14015   ins_encode %{
14016     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14017   %}
14018 
14019   ins_pipe(pipe_class_memory);
14020 
14021 %}
14022 
14023 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14024 
14025   match(Set dst (MoveD2L src));
14026 
14027   effect(DEF dst, USE src);
14028 
14029   ins_cost(4 * INSN_COST);
14030 
14031   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14032 
14033   ins_encode %{
14034     __ ldr($dst$$Register, Address(sp, $src$$disp));
14035   %}
14036 
14037   ins_pipe(iload_reg_reg);
14038 
14039 %}
14040 
14041 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14042 
14043   match(Set dst (MoveL2D src));
14044 
14045   effect(DEF dst, USE src);
14046 
14047   ins_cost(4 * INSN_COST);
14048 
14049   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14050 
14051   ins_encode %{
14052     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14053   %}
14054 
14055   ins_pipe(pipe_class_memory);
14056 
14057 %}
14058 
14059 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14060 
14061   match(Set dst (MoveF2I src));
14062 
14063   effect(DEF dst, USE src);
14064 
14065   ins_cost(INSN_COST);
14066 
14067   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14068 
14069   ins_encode %{
14070     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14071   %}
14072 
14073   ins_pipe(pipe_class_memory);
14074 
14075 %}
14076 
14077 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14078 
14079   match(Set dst (MoveI2F src));
14080 
14081   effect(DEF dst, USE src);
14082 
14083   ins_cost(INSN_COST);
14084 
14085   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14086 
14087   ins_encode %{
14088     __ strw($src$$Register, Address(sp, $dst$$disp));
14089   %}
14090 
14091   ins_pipe(istore_reg_reg);
14092 
14093 %}
14094 
14095 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14096 
14097   match(Set dst (MoveD2L src));
14098 
14099   effect(DEF dst, USE src);
14100 
14101   ins_cost(INSN_COST);
14102 
14103   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14104 
14105   ins_encode %{
14106     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14107   %}
14108 
14109   ins_pipe(pipe_class_memory);
14110 
14111 %}
14112 
14113 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14114 
14115   match(Set dst (MoveL2D src));
14116 
14117   effect(DEF dst, USE src);
14118 
14119   ins_cost(INSN_COST);
14120 
14121   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14122 
14123   ins_encode %{
14124     __ str($src$$Register, Address(sp, $dst$$disp));
14125   %}
14126 
14127   ins_pipe(istore_reg_reg);
14128 
14129 %}
14130 
14131 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14132 
14133   match(Set dst (MoveF2I src));
14134 
14135   effect(DEF dst, USE src);
14136 
14137   ins_cost(INSN_COST);
14138 
14139   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14140 
14141   ins_encode %{
14142     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14143   %}
14144 
14145   ins_pipe(fp_f2i);
14146 
14147 %}
14148 
14149 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14150 
14151   match(Set dst (MoveI2F src));
14152 
14153   effect(DEF dst, USE src);
14154 
14155   ins_cost(INSN_COST);
14156 
14157   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14158 
14159   ins_encode %{
14160     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14161   %}
14162 
14163   ins_pipe(fp_i2f);
14164 
14165 %}
14166 
14167 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14168 
14169   match(Set dst (MoveD2L src));
14170 
14171   effect(DEF dst, USE src);
14172 
14173   ins_cost(INSN_COST);
14174 
14175   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14176 
14177   ins_encode %{
14178     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14179   %}
14180 
14181   ins_pipe(fp_d2l);
14182 
14183 %}
14184 
14185 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14186 
14187   match(Set dst (MoveL2D src));
14188 
14189   effect(DEF dst, USE src);
14190 
14191   ins_cost(INSN_COST);
14192 
14193   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14194 
14195   ins_encode %{
14196     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14197   %}
14198 
14199   ins_pipe(fp_l2d);
14200 
14201 %}
14202 
14203 // ============================================================================
14204 // clearing of an array
14205 
14206 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14207 %{
14208   match(Set dummy (ClearArray cnt base));
14209   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14210 
14211   ins_cost(4 * INSN_COST);
14212   format %{ "ClearArray $cnt, $base" %}
14213 
14214   ins_encode %{
14215     __ zero_words($base$$Register, $cnt$$Register);
14216   %}
14217 
14218   ins_pipe(pipe_class_memory);
14219 %}
14220 
14221 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14222 %{
14223   predicate((uint64_t)n->in(2)->get_long()
14224             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14225   match(Set dummy (ClearArray cnt base));
14226   effect(USE_KILL base);
14227 
14228   ins_cost(4 * INSN_COST);
14229   format %{ "ClearArray $cnt, $base" %}
14230 
14231   ins_encode %{
14232     __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14233   %}
14234 
14235   ins_pipe(pipe_class_memory);
14236 %}
14237 
14238 // ============================================================================
14239 // Overflow Math Instructions
14240 
14241 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14242 %{
14243   match(Set cr (OverflowAddI op1 op2));
14244 
14245   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14246   ins_cost(INSN_COST);
14247   ins_encode %{
14248     __ cmnw($op1$$Register, $op2$$Register);
14249   %}
14250 
14251   ins_pipe(icmp_reg_reg);
14252 %}
14253 
14254 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14255 %{
14256   match(Set cr (OverflowAddI op1 op2));
14257 
14258   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14259   ins_cost(INSN_COST);
14260   ins_encode %{
14261     __ cmnw($op1$$Register, $op2$$constant);
14262   %}
14263 
14264   ins_pipe(icmp_reg_imm);
14265 %}
14266 
14267 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14268 %{
14269   match(Set cr (OverflowAddL op1 op2));
14270 
14271   format %{ "cmn   $op1, $op2\t# overflow check long" %}
14272   ins_cost(INSN_COST);
14273   ins_encode %{
14274     __ cmn($op1$$Register, $op2$$Register);
14275   %}
14276 
14277   ins_pipe(icmp_reg_reg);
14278 %}
14279 
14280 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14281 %{
14282   match(Set cr (OverflowAddL op1 op2));
14283 
14284   format %{ "cmn   $op1, $op2\t# overflow check long" %}
14285   ins_cost(INSN_COST);
14286   ins_encode %{
14287     __ cmn($op1$$Register, $op2$$constant);
14288   %}
14289 
14290   ins_pipe(icmp_reg_imm);
14291 %}
14292 
14293 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14294 %{
14295   match(Set cr (OverflowSubI op1 op2));
14296 
14297   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
14298   ins_cost(INSN_COST);
14299   ins_encode %{
14300     __ cmpw($op1$$Register, $op2$$Register);
14301   %}
14302 
14303   ins_pipe(icmp_reg_reg);
14304 %}
14305 
14306 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14307 %{
14308   match(Set cr (OverflowSubI op1 op2));
14309 
14310   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
14311   ins_cost(INSN_COST);
14312   ins_encode %{
14313     __ cmpw($op1$$Register, $op2$$constant);
14314   %}
14315 
14316   ins_pipe(icmp_reg_imm);
14317 %}
14318 
14319 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14320 %{
14321   match(Set cr (OverflowSubL op1 op2));
14322 
14323   format %{ "cmp   $op1, $op2\t# overflow check long" %}
14324   ins_cost(INSN_COST);
14325   ins_encode %{
14326     __ cmp($op1$$Register, $op2$$Register);
14327   %}
14328 
14329   ins_pipe(icmp_reg_reg);
14330 %}
14331 
14332 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14333 %{
14334   match(Set cr (OverflowSubL op1 op2));
14335 
14336   format %{ "cmp   $op1, $op2\t# overflow check long" %}
14337   ins_cost(INSN_COST);
14338   ins_encode %{
14339     __ subs(zr, $op1$$Register, $op2$$constant);
14340   %}
14341 
14342   ins_pipe(icmp_reg_imm);
14343 %}
14344 
14345 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14346 %{
14347   match(Set cr (OverflowSubI zero op1));
14348 
14349   format %{ "cmpw  zr, $op1\t# overflow check int" %}
14350   ins_cost(INSN_COST);
14351   ins_encode %{
14352     __ cmpw(zr, $op1$$Register);
14353   %}
14354 
14355   ins_pipe(icmp_reg_imm);
14356 %}
14357 
14358 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14359 %{
14360   match(Set cr (OverflowSubL zero op1));
14361 
14362   format %{ "cmp   zr, $op1\t# overflow check long" %}
14363   ins_cost(INSN_COST);
14364   ins_encode %{
14365     __ cmp(zr, $op1$$Register);
14366   %}
14367 
14368   ins_pipe(icmp_reg_imm);
14369 %}
14370 
14371 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14372 %{
14373   match(Set cr (OverflowMulI op1 op2));
14374 
14375   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14376             "cmp   rscratch1, rscratch1, sxtw\n\t"
14377             "movw  rscratch1, #0x80000000\n\t"
14378             "cselw rscratch1, rscratch1, zr, NE\n\t"
14379             "cmpw  rscratch1, #1" %}
14380   ins_cost(5 * INSN_COST);
14381   ins_encode %{
14382     __ smull(rscratch1, $op1$$Register, $op2$$Register);
14383     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
14384     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
14385     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14386     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
14387   %}
14388 
14389   ins_pipe(pipe_slow);
14390 %}
14391 
14392 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14393 %{
14394   match(If cmp (OverflowMulI op1 op2));
14395   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14396             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14397   effect(USE labl, KILL cr);
14398 
14399   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14400             "cmp   rscratch1, rscratch1, sxtw\n\t"
14401             "b$cmp   $labl" %}
14402   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14403   ins_encode %{
14404     Label* L = $labl$$label;
14405     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14406     __ smull(rscratch1, $op1$$Register, $op2$$Register);
14407     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
14408     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14409   %}
14410 
14411   ins_pipe(pipe_serial);
14412 %}
14413 
14414 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14415 %{
14416   match(Set cr (OverflowMulL op1 op2));
14417 
14418   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
14419             "smulh rscratch2, $op1, $op2\n\t"
14420             "cmp   rscratch2, rscratch1, ASR #63\n\t"
14421             "movw  rscratch1, #0x80000000\n\t"
14422             "cselw rscratch1, rscratch1, zr, NE\n\t"
14423             "cmpw  rscratch1, #1" %}
14424   ins_cost(6 * INSN_COST);
14425   ins_encode %{
14426     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
14427     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14428     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
14429     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
14430     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14431     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
14432   %}
14433 
14434   ins_pipe(pipe_slow);
14435 %}
14436 
14437 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14438 %{
14439   match(If cmp (OverflowMulL op1 op2));
14440   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14441             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14442   effect(USE labl, KILL cr);
14443 
14444   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
14445             "smulh rscratch2, $op1, $op2\n\t"
14446             "cmp   rscratch2, rscratch1, ASR #63\n\t"
14447             "b$cmp $labl" %}
14448   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14449   ins_encode %{
14450     Label* L = $labl$$label;
14451     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14452     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
14453     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14454     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
14455     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14456   %}
14457 
14458   ins_pipe(pipe_serial);
14459 %}
14460 
14461 // ============================================================================
14462 // Compare Instructions
14463 
14464 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14465 %{
14466   match(Set cr (CmpI op1 op2));
14467 
14468   effect(DEF cr, USE op1, USE op2);
14469 
14470   ins_cost(INSN_COST);
14471   format %{ "cmpw  $op1, $op2" %}
14472 
14473   ins_encode(aarch64_enc_cmpw(op1, op2));
14474 
14475   ins_pipe(icmp_reg_reg);
14476 %}
14477 
14478 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14479 %{
14480   match(Set cr (CmpI op1 zero));
14481 
14482   effect(DEF cr, USE op1);
14483 
14484   ins_cost(INSN_COST);
14485   format %{ "cmpw $op1, 0" %}
14486 
14487   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14488 
14489   ins_pipe(icmp_reg_imm);
14490 %}
14491 
14492 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14493 %{
14494   match(Set cr (CmpI op1 op2));
14495 
14496   effect(DEF cr, USE op1);
14497 
14498   ins_cost(INSN_COST);
14499   format %{ "cmpw  $op1, $op2" %}
14500 
14501   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14502 
14503   ins_pipe(icmp_reg_imm);
14504 %}
14505 
14506 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14507 %{
14508   match(Set cr (CmpI op1 op2));
14509 
14510   effect(DEF cr, USE op1);
14511 
14512   ins_cost(INSN_COST * 2);
14513   format %{ "cmpw  $op1, $op2" %}
14514 
14515   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14516 
14517   ins_pipe(icmp_reg_imm);
14518 %}
14519 
14520 // Unsigned compare Instructions; really, same as signed compare
14521 // except it should only be used to feed an If or a CMovI which takes a
14522 // cmpOpU.
14523 
14524 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14525 %{
14526   match(Set cr (CmpU op1 op2));
14527 
14528   effect(DEF cr, USE op1, USE op2);
14529 
14530   ins_cost(INSN_COST);
14531   format %{ "cmpw  $op1, $op2\t# unsigned" %}
14532 
14533   ins_encode(aarch64_enc_cmpw(op1, op2));
14534 
14535   ins_pipe(icmp_reg_reg);
14536 %}
14537 
14538 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14539 %{
14540   match(Set cr (CmpU op1 zero));
14541 
14542   effect(DEF cr, USE op1);
14543 
14544   ins_cost(INSN_COST);
14545   format %{ "cmpw $op1, #0\t# unsigned" %}
14546 
14547   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14548 
14549   ins_pipe(icmp_reg_imm);
14550 %}
14551 
14552 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14553 %{
14554   match(Set cr (CmpU op1 op2));
14555 
14556   effect(DEF cr, USE op1);
14557 
14558   ins_cost(INSN_COST);
14559   format %{ "cmpw  $op1, $op2\t# unsigned" %}
14560 
14561   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14562 
14563   ins_pipe(icmp_reg_imm);
14564 %}
14565 
14566 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14567 %{
14568   match(Set cr (CmpU op1 op2));
14569 
14570   effect(DEF cr, USE op1);
14571 
14572   ins_cost(INSN_COST * 2);
14573   format %{ "cmpw  $op1, $op2\t# unsigned" %}
14574 
14575   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14576 
14577   ins_pipe(icmp_reg_imm);
14578 %}
14579 
14580 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14581 %{
14582   match(Set cr (CmpL op1 op2));
14583 
14584   effect(DEF cr, USE op1, USE op2);
14585 
14586   ins_cost(INSN_COST);
14587   format %{ "cmp  $op1, $op2" %}
14588 
14589   ins_encode(aarch64_enc_cmp(op1, op2));
14590 
14591   ins_pipe(icmp_reg_reg);
14592 %}
14593 
14594 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14595 %{
14596   match(Set cr (CmpL op1 zero));
14597 
14598   effect(DEF cr, USE op1);
14599 
14600   ins_cost(INSN_COST);
14601   format %{ "tst  $op1" %}
14602 
14603   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14604 
14605   ins_pipe(icmp_reg_imm);
14606 %}
14607 
14608 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14609 %{
14610   match(Set cr (CmpL op1 op2));
14611 
14612   effect(DEF cr, USE op1);
14613 
14614   ins_cost(INSN_COST);
14615   format %{ "cmp  $op1, $op2" %}
14616 
14617   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14618 
14619   ins_pipe(icmp_reg_imm);
14620 %}
14621 
14622 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14623 %{
14624   match(Set cr (CmpL op1 op2));
14625 
14626   effect(DEF cr, USE op1);
14627 
14628   ins_cost(INSN_COST * 2);
14629   format %{ "cmp  $op1, $op2" %}
14630 
14631   ins_encode(aarch64_enc_cmp_imm(op1, op2));
14632 
14633   ins_pipe(icmp_reg_imm);
14634 %}
14635 
14636 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14637 %{
14638   match(Set cr (CmpUL op1 op2));
14639 
14640   effect(DEF cr, USE op1, USE op2);
14641 
14642   ins_cost(INSN_COST);
14643   format %{ "cmp  $op1, $op2" %}
14644 
14645   ins_encode(aarch64_enc_cmp(op1, op2));
14646 
14647   ins_pipe(icmp_reg_reg);
14648 %}
14649 
14650 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14651 %{
14652   match(Set cr (CmpUL op1 zero));
14653 
14654   effect(DEF cr, USE op1);
14655 
14656   ins_cost(INSN_COST);
14657   format %{ "tst  $op1" %}
14658 
14659   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14660 
14661   ins_pipe(icmp_reg_imm);
14662 %}
14663 
14664 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14665 %{
14666   match(Set cr (CmpUL op1 op2));
14667 
14668   effect(DEF cr, USE op1);
14669 
14670   ins_cost(INSN_COST);
14671   format %{ "cmp  $op1, $op2" %}
14672 
14673   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14674 
14675   ins_pipe(icmp_reg_imm);
14676 %}
14677 
14678 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14679 %{
14680   match(Set cr (CmpUL op1 op2));
14681 
14682   effect(DEF cr, USE op1);
14683 
14684   ins_cost(INSN_COST * 2);
14685   format %{ "cmp  $op1, $op2" %}
14686 
14687   ins_encode(aarch64_enc_cmp_imm(op1, op2));
14688 
14689   ins_pipe(icmp_reg_imm);
14690 %}
14691 
14692 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14693 %{
14694   match(Set cr (CmpP op1 op2));
14695 
14696   effect(DEF cr, USE op1, USE op2);
14697 
14698   ins_cost(INSN_COST);
14699   format %{ "cmp  $op1, $op2\t // ptr" %}
14700 
14701   ins_encode(aarch64_enc_cmpp(op1, op2));
14702 
14703   ins_pipe(icmp_reg_reg);
14704 %}
14705 
14706 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14707 %{
14708   match(Set cr (CmpN op1 op2));
14709 
14710   effect(DEF cr, USE op1, USE op2);
14711 
14712   ins_cost(INSN_COST);
14713   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
14714 
14715   ins_encode(aarch64_enc_cmpn(op1, op2));
14716 
14717   ins_pipe(icmp_reg_reg);
14718 %}
14719 
14720 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14721 %{
14722   match(Set cr (CmpP op1 zero));
14723 
14724   effect(DEF cr, USE op1, USE zero);
14725 
14726   ins_cost(INSN_COST);
14727   format %{ "cmp  $op1, 0\t // ptr" %}
14728 
14729   ins_encode(aarch64_enc_testp(op1));
14730 
14731   ins_pipe(icmp_reg_imm);
14732 %}
14733 
14734 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14735 %{
14736   match(Set cr (CmpN op1 zero));
14737 
14738   effect(DEF cr, USE op1, USE zero);
14739 
14740   ins_cost(INSN_COST);
14741   format %{ "cmp  $op1, 0\t // compressed ptr" %}
14742 
14743   ins_encode(aarch64_enc_testn(op1));
14744 
14745   ins_pipe(icmp_reg_imm);
14746 %}
14747 
14748 // FP comparisons
14749 //
14750 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14751 // using normal cmpOp. See declaration of rFlagsReg for details.
14752 
14753 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14754 %{
14755   match(Set cr (CmpF src1 src2));
14756 
14757   ins_cost(3 * INSN_COST);
14758   format %{ "fcmps $src1, $src2" %}
14759 
14760   ins_encode %{
14761     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14762   %}
14763 
14764   ins_pipe(pipe_class_compare);
14765 %}
14766 
14767 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14768 %{
14769   match(Set cr (CmpF src1 src2));
14770 
14771   ins_cost(3 * INSN_COST);
14772   format %{ "fcmps $src1, 0.0" %}
14773 
14774   ins_encode %{
14775     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14776   %}
14777 
14778   ins_pipe(pipe_class_compare);
14779 %}
14780 // FROM HERE
14781 
14782 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14783 %{
14784   match(Set cr (CmpD src1 src2));
14785 
14786   ins_cost(3 * INSN_COST);
14787   format %{ "fcmpd $src1, $src2" %}
14788 
14789   ins_encode %{
14790     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14791   %}
14792 
14793   ins_pipe(pipe_class_compare);
14794 %}
14795 
14796 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14797 %{
14798   match(Set cr (CmpD src1 src2));
14799 
14800   ins_cost(3 * INSN_COST);
14801   format %{ "fcmpd $src1, 0.0" %}
14802 
14803   ins_encode %{
14804     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14805   %}
14806 
14807   ins_pipe(pipe_class_compare);
14808 %}
14809 
14810 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14811 %{
14812   match(Set dst (CmpF3 src1 src2));
14813   effect(KILL cr);
14814 
14815   ins_cost(5 * INSN_COST);
14816   format %{ "fcmps $src1, $src2\n\t"
14817             "csinvw($dst, zr, zr, eq\n\t"
14818             "csnegw($dst, $dst, $dst, lt)"
14819   %}
14820 
14821   ins_encode %{
14822     Label done;
14823     FloatRegister s1 = as_FloatRegister($src1$$reg);
14824     FloatRegister s2 = as_FloatRegister($src2$$reg);
14825     Register d = as_Register($dst$$reg);
14826     __ fcmps(s1, s2);
14827     // installs 0 if EQ else -1
14828     __ csinvw(d, zr, zr, Assembler::EQ);
14829     // keeps -1 if less or unordered else installs 1
14830     __ csnegw(d, d, d, Assembler::LT);
14831     __ bind(done);
14832   %}
14833 
14834   ins_pipe(pipe_class_default);
14835 
14836 %}
14837 
14838 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14839 %{
14840   match(Set dst (CmpD3 src1 src2));
14841   effect(KILL cr);
14842 
14843   ins_cost(5 * INSN_COST);
14844   format %{ "fcmpd $src1, $src2\n\t"
14845             "csinvw($dst, zr, zr, eq\n\t"
14846             "csnegw($dst, $dst, $dst, lt)"
14847   %}
14848 
14849   ins_encode %{
14850     Label done;
14851     FloatRegister s1 = as_FloatRegister($src1$$reg);
14852     FloatRegister s2 = as_FloatRegister($src2$$reg);
14853     Register d = as_Register($dst$$reg);
14854     __ fcmpd(s1, s2);
14855     // installs 0 if EQ else -1
14856     __ csinvw(d, zr, zr, Assembler::EQ);
14857     // keeps -1 if less or unordered else installs 1
14858     __ csnegw(d, d, d, Assembler::LT);
14859     __ bind(done);
14860   %}
14861   ins_pipe(pipe_class_default);
14862 
14863 %}
14864 
14865 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14866 %{
14867   match(Set dst (CmpF3 src1 zero));
14868   effect(KILL cr);
14869 
14870   ins_cost(5 * INSN_COST);
14871   format %{ "fcmps $src1, 0.0\n\t"
14872             "csinvw($dst, zr, zr, eq\n\t"
14873             "csnegw($dst, $dst, $dst, lt)"
14874   %}
14875 
14876   ins_encode %{
14877     Label done;
14878     FloatRegister s1 = as_FloatRegister($src1$$reg);
14879     Register d = as_Register($dst$$reg);
14880     __ fcmps(s1, 0.0);
14881     // installs 0 if EQ else -1
14882     __ csinvw(d, zr, zr, Assembler::EQ);
14883     // keeps -1 if less or unordered else installs 1
14884     __ csnegw(d, d, d, Assembler::LT);
14885     __ bind(done);
14886   %}
14887 
14888   ins_pipe(pipe_class_default);
14889 
14890 %}
14891 
14892 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14893 %{
14894   match(Set dst (CmpD3 src1 zero));
14895   effect(KILL cr);
14896 
14897   ins_cost(5 * INSN_COST);
14898   format %{ "fcmpd $src1, 0.0\n\t"
14899             "csinvw($dst, zr, zr, eq\n\t"
14900             "csnegw($dst, $dst, $dst, lt)"
14901   %}
14902 
14903   ins_encode %{
14904     Label done;
14905     FloatRegister s1 = as_FloatRegister($src1$$reg);
14906     Register d = as_Register($dst$$reg);
14907     __ fcmpd(s1, 0.0);
14908     // installs 0 if EQ else -1
14909     __ csinvw(d, zr, zr, Assembler::EQ);
14910     // keeps -1 if less or unordered else installs 1
14911     __ csnegw(d, d, d, Assembler::LT);
14912     __ bind(done);
14913   %}
14914   ins_pipe(pipe_class_default);
14915 
14916 %}
14917 
14918 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14919 %{
14920   match(Set dst (CmpLTMask p q));
14921   effect(KILL cr);
14922 
14923   ins_cost(3 * INSN_COST);
14924 
14925   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14926             "csetw $dst, lt\n\t"
14927             "subw $dst, zr, $dst"
14928   %}
14929 
14930   ins_encode %{
14931     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14932     __ csetw(as_Register($dst$$reg), Assembler::LT);
14933     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14934   %}
14935 
14936   ins_pipe(ialu_reg_reg);
14937 %}
14938 
14939 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14940 %{
14941   match(Set dst (CmpLTMask src zero));
14942   effect(KILL cr);
14943 
14944   ins_cost(INSN_COST);
14945 
14946   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14947 
14948   ins_encode %{
14949     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14950   %}
14951 
14952   ins_pipe(ialu_reg_shift);
14953 %}
14954 
14955 // ============================================================================
14956 // Max and Min
14957 
14958 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14959 %{
14960   effect( DEF dst, USE src1, USE src2, USE cr );
14961 
14962   ins_cost(INSN_COST * 2);
14963   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
14964 
14965   ins_encode %{
14966     __ cselw(as_Register($dst$$reg),
14967              as_Register($src1$$reg),
14968              as_Register($src2$$reg),
14969              Assembler::LT);
14970   %}
14971 
14972   ins_pipe(icond_reg_reg);
14973 %}
14974 
14975 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
14976 %{
14977   match(Set dst (MinI src1 src2));
14978   ins_cost(INSN_COST * 3);
14979 
14980   expand %{
14981     rFlagsReg cr;
14982     compI_reg_reg(cr, src1, src2);
14983     cmovI_reg_reg_lt(dst, src1, src2, cr);
14984   %}
14985 
14986 %}
14987 // FROM HERE
14988 
14989 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14990 %{
14991   effect( DEF dst, USE src1, USE src2, USE cr );
14992 
14993   ins_cost(INSN_COST * 2);
14994   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
14995 
14996   ins_encode %{
14997     __ cselw(as_Register($dst$$reg),
14998              as_Register($src1$$reg),
14999              as_Register($src2$$reg),
15000              Assembler::GT);
15001   %}
15002 
15003   ins_pipe(icond_reg_reg);
15004 %}
15005 
15006 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15007 %{
15008   match(Set dst (MaxI src1 src2));
15009   ins_cost(INSN_COST * 3);
15010   expand %{
15011     rFlagsReg cr;
15012     compI_reg_reg(cr, src1, src2);
15013     cmovI_reg_reg_gt(dst, src1, src2, cr);
15014   %}
15015 %}
15016 
15017 // ============================================================================
15018 // Branch Instructions
15019 
15020 // Direct Branch.
15021 instruct branch(label lbl)
15022 %{
15023   match(Goto);
15024 
15025   effect(USE lbl);
15026 
15027   ins_cost(BRANCH_COST);
15028   format %{ "b  $lbl" %}
15029 
15030   ins_encode(aarch64_enc_b(lbl));
15031 
15032   ins_pipe(pipe_branch);
15033 %}
15034 
15035 // Conditional Near Branch
15036 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15037 %{
15038   // Same match rule as `branchConFar'.
15039   match(If cmp cr);
15040 
15041   effect(USE lbl);
15042 
15043   ins_cost(BRANCH_COST);
15044   // If set to 1 this indicates that the current instruction is a
15045   // short variant of a long branch. This avoids using this
15046   // instruction in first-pass matching. It will then only be used in
15047   // the `Shorten_branches' pass.
15048   // ins_short_branch(1);
15049   format %{ "b$cmp  $lbl" %}
15050 
15051   ins_encode(aarch64_enc_br_con(cmp, lbl));
15052 
15053   ins_pipe(pipe_branch_cond);
15054 %}
15055 
15056 // Conditional Near Branch Unsigned
15057 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15058 %{
15059   // Same match rule as `branchConFar'.
15060   match(If cmp cr);
15061 
15062   effect(USE lbl);
15063 
15064   ins_cost(BRANCH_COST);
15065   // If set to 1 this indicates that the current instruction is a
15066   // short variant of a long branch. This avoids using this
15067   // instruction in first-pass matching. It will then only be used in
15068   // the `Shorten_branches' pass.
15069   // ins_short_branch(1);
15070   format %{ "b$cmp  $lbl\t# unsigned" %}
15071 
15072   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15073 
15074   ins_pipe(pipe_branch_cond);
15075 %}
15076 
15077 // Make use of CBZ and CBNZ.  These instructions, as well as being
15078 // shorter than (cmp; branch), have the additional benefit of not
15079 // killing the flags.
15080 
15081 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15082   match(If cmp (CmpI op1 op2));
15083   effect(USE labl);
15084 
15085   ins_cost(BRANCH_COST);
15086   format %{ "cbw$cmp   $op1, $labl" %}
15087   ins_encode %{
15088     Label* L = $labl$$label;
15089     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15090     if (cond == Assembler::EQ)
15091       __ cbzw($op1$$Register, *L);
15092     else
15093       __ cbnzw($op1$$Register, *L);
15094   %}
15095   ins_pipe(pipe_cmp_branch);
15096 %}
15097 
15098 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15099   match(If cmp (CmpL op1 op2));
15100   effect(USE labl);
15101 
15102   ins_cost(BRANCH_COST);
15103   format %{ "cb$cmp   $op1, $labl" %}
15104   ins_encode %{
15105     Label* L = $labl$$label;
15106     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15107     if (cond == Assembler::EQ)
15108       __ cbz($op1$$Register, *L);
15109     else
15110       __ cbnz($op1$$Register, *L);
15111   %}
15112   ins_pipe(pipe_cmp_branch);
15113 %}
15114 
15115 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15116   match(If cmp (CmpP op1 op2));
15117   effect(USE labl);
15118 
15119   ins_cost(BRANCH_COST);
15120   format %{ "cb$cmp   $op1, $labl" %}
15121   ins_encode %{
15122     Label* L = $labl$$label;
15123     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15124     if (cond == Assembler::EQ)
15125       __ cbz($op1$$Register, *L);
15126     else
15127       __ cbnz($op1$$Register, *L);
15128   %}
15129   ins_pipe(pipe_cmp_branch);
15130 %}
15131 
15132 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15133   match(If cmp (CmpN op1 op2));
15134   effect(USE labl);
15135 
15136   ins_cost(BRANCH_COST);
15137   format %{ "cbw$cmp   $op1, $labl" %}
15138   ins_encode %{
15139     Label* L = $labl$$label;
15140     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15141     if (cond == Assembler::EQ)
15142       __ cbzw($op1$$Register, *L);
15143     else
15144       __ cbnzw($op1$$Register, *L);
15145   %}
15146   ins_pipe(pipe_cmp_branch);
15147 %}
15148 
15149 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15150   match(If cmp (CmpP (DecodeN oop) zero));
15151   effect(USE labl);
15152 
15153   ins_cost(BRANCH_COST);
15154   format %{ "cb$cmp   $oop, $labl" %}
15155   ins_encode %{
15156     Label* L = $labl$$label;
15157     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15158     if (cond == Assembler::EQ)
15159       __ cbzw($oop$$Register, *L);
15160     else
15161       __ cbnzw($oop$$Register, *L);
15162   %}
15163   ins_pipe(pipe_cmp_branch);
15164 %}
15165 
15166 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
15167   match(If cmp (CmpU op1 op2));
15168   effect(USE labl);
15169 
15170   ins_cost(BRANCH_COST);
15171   format %{ "cbw$cmp   $op1, $labl" %}
15172   ins_encode %{
15173     Label* L = $labl$$label;
15174     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15175     if (cond == Assembler::EQ || cond == Assembler::LS)
15176       __ cbzw($op1$$Register, *L);
15177     else
15178       __ cbnzw($op1$$Register, *L);
15179   %}
15180   ins_pipe(pipe_cmp_branch);
15181 %}
15182 
15183 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
15184   match(If cmp (CmpUL op1 op2));
15185   effect(USE labl);
15186 
15187   ins_cost(BRANCH_COST);
15188   format %{ "cb$cmp   $op1, $labl" %}
15189   ins_encode %{
15190     Label* L = $labl$$label;
15191     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15192     if (cond == Assembler::EQ || cond == Assembler::LS)
15193       __ cbz($op1$$Register, *L);
15194     else
15195       __ cbnz($op1$$Register, *L);
15196   %}
15197   ins_pipe(pipe_cmp_branch);
15198 %}
15199 
15200 // Test bit and Branch
15201 
15202 // Patterns for short (< 32KiB) variants
15203 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15204   match(If cmp (CmpL op1 op2));
15205   effect(USE labl);
15206 
15207   ins_cost(BRANCH_COST);
15208   format %{ "cb$cmp   $op1, $labl # long" %}
15209   ins_encode %{
15210     Label* L = $labl$$label;
15211     Assembler::Condition cond =
15212       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15213     __ tbr(cond, $op1$$Register, 63, *L);
15214   %}
15215   ins_pipe(pipe_cmp_branch);
15216   ins_short_branch(1);
15217 %}
15218 
15219 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15220   match(If cmp (CmpI op1 op2));
15221   effect(USE labl);
15222 
15223   ins_cost(BRANCH_COST);
15224   format %{ "cb$cmp   $op1, $labl # int" %}
15225   ins_encode %{
15226     Label* L = $labl$$label;
15227     Assembler::Condition cond =
15228       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15229     __ tbr(cond, $op1$$Register, 31, *L);
15230   %}
15231   ins_pipe(pipe_cmp_branch);
15232   ins_short_branch(1);
15233 %}
15234 
15235 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15236   match(If cmp (CmpL (AndL op1 op2) op3));
15237   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15238   effect(USE labl);
15239 
15240   ins_cost(BRANCH_COST);
15241   format %{ "tb$cmp   $op1, $op2, $labl" %}
15242   ins_encode %{
15243     Label* L = $labl$$label;
15244     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15245     int bit = exact_log2_long($op2$$constant);
15246     __ tbr(cond, $op1$$Register, bit, *L);
15247   %}
15248   ins_pipe(pipe_cmp_branch);
15249   ins_short_branch(1);
15250 %}
15251 
15252 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15253   match(If cmp (CmpI (AndI op1 op2) op3));
15254   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15255   effect(USE labl);
15256 
15257   ins_cost(BRANCH_COST);
15258   format %{ "tb$cmp   $op1, $op2, $labl" %}
15259   ins_encode %{
15260     Label* L = $labl$$label;
15261     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15262     int bit = exact_log2((juint)$op2$$constant);
15263     __ tbr(cond, $op1$$Register, bit, *L);
15264   %}
15265   ins_pipe(pipe_cmp_branch);
15266   ins_short_branch(1);
15267 %}
15268 
15269 // And far variants
15270 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15271   match(If cmp (CmpL op1 op2));
15272   effect(USE labl);
15273 
15274   ins_cost(BRANCH_COST);
15275   format %{ "cb$cmp   $op1, $labl # long" %}
15276   ins_encode %{
15277     Label* L = $labl$$label;
15278     Assembler::Condition cond =
15279       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15280     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15281   %}
15282   ins_pipe(pipe_cmp_branch);
15283 %}
15284 
15285 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15286   match(If cmp (CmpI op1 op2));
15287   effect(USE labl);
15288 
15289   ins_cost(BRANCH_COST);
15290   format %{ "cb$cmp   $op1, $labl # int" %}
15291   ins_encode %{
15292     Label* L = $labl$$label;
15293     Assembler::Condition cond =
15294       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15295     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15296   %}
15297   ins_pipe(pipe_cmp_branch);
15298 %}
15299 
15300 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15301   match(If cmp (CmpL (AndL op1 op2) op3));
15302   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15303   effect(USE labl);
15304 
15305   ins_cost(BRANCH_COST);
15306   format %{ "tb$cmp   $op1, $op2, $labl" %}
15307   ins_encode %{
15308     Label* L = $labl$$label;
15309     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15310     int bit = exact_log2_long($op2$$constant);
15311     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15312   %}
15313   ins_pipe(pipe_cmp_branch);
15314 %}
15315 
15316 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15317   match(If cmp (CmpI (AndI op1 op2) op3));
15318   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15319   effect(USE labl);
15320 
15321   ins_cost(BRANCH_COST);
15322   format %{ "tb$cmp   $op1, $op2, $labl" %}
15323   ins_encode %{
15324     Label* L = $labl$$label;
15325     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15326     int bit = exact_log2((juint)$op2$$constant);
15327     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15328   %}
15329   ins_pipe(pipe_cmp_branch);
15330 %}
15331 
15332 // Test bits
15333 
15334 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15335   match(Set cr (CmpL (AndL op1 op2) op3));
15336   predicate(Assembler::operand_valid_for_logical_immediate
15337             (/*is_32*/false, n->in(1)->in(2)->get_long()));
15338 
15339   ins_cost(INSN_COST);
15340   format %{ "tst $op1, $op2 # long" %}
15341   ins_encode %{
15342     __ tst($op1$$Register, $op2$$constant);
15343   %}
15344   ins_pipe(ialu_reg_reg);
15345 %}
15346 
15347 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15348   match(Set cr (CmpI (AndI op1 op2) op3));
15349   predicate(Assembler::operand_valid_for_logical_immediate
15350             (/*is_32*/true, n->in(1)->in(2)->get_int()));
15351 
15352   ins_cost(INSN_COST);
15353   format %{ "tst $op1, $op2 # int" %}
15354   ins_encode %{
15355     __ tstw($op1$$Register, $op2$$constant);
15356   %}
15357   ins_pipe(ialu_reg_reg);
15358 %}
15359 
15360 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15361   match(Set cr (CmpL (AndL op1 op2) op3));
15362 
15363   ins_cost(INSN_COST);
15364   format %{ "tst $op1, $op2 # long" %}
15365   ins_encode %{
15366     __ tst($op1$$Register, $op2$$Register);
15367   %}
15368   ins_pipe(ialu_reg_reg);
15369 %}
15370 
15371 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15372   match(Set cr (CmpI (AndI op1 op2) op3));
15373 
15374   ins_cost(INSN_COST);
15375   format %{ "tstw $op1, $op2 # int" %}
15376   ins_encode %{
15377     __ tstw($op1$$Register, $op2$$Register);
15378   %}
15379   ins_pipe(ialu_reg_reg);
15380 %}
15381 
15382 
15383 // Conditional Far Branch
15384 // Conditional Far Branch Unsigned
15385 // TODO: fixme
15386 
15387 // counted loop end branch near
15388 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15389 %{
15390   match(CountedLoopEnd cmp cr);
15391 
15392   effect(USE lbl);
15393 
15394   ins_cost(BRANCH_COST);
15395   // short variant.
15396   // ins_short_branch(1);
15397   format %{ "b$cmp $lbl \t// counted loop end" %}
15398 
15399   ins_encode(aarch64_enc_br_con(cmp, lbl));
15400 
15401   ins_pipe(pipe_branch);
15402 %}
15403 
15404 // counted loop end branch near Unsigned
15405 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15406 %{
15407   match(CountedLoopEnd cmp cr);
15408 
15409   effect(USE lbl);
15410 
15411   ins_cost(BRANCH_COST);
15412   // short variant.
15413   // ins_short_branch(1);
15414   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
15415 
15416   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15417 
15418   ins_pipe(pipe_branch);
15419 %}
15420 
15421 // counted loop end branch far
15422 // counted loop end branch far unsigned
15423 // TODO: fixme
15424 
15425 // ============================================================================
15426 // inlined locking and unlocking
15427 
15428 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
15429 %{
15430   match(Set cr (FastLock object box));
15431   effect(TEMP tmp, TEMP tmp2);
15432 
15433   // TODO
15434   // identify correct cost
15435   ins_cost(5 * INSN_COST);
15436   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
15437 
15438   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
15439 
15440   ins_pipe(pipe_serial);
15441 %}
15442 
15443 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
15444 %{
15445   match(Set cr (FastUnlock object box));
15446   effect(TEMP tmp, TEMP tmp2);
15447 
15448   ins_cost(5 * INSN_COST);
15449   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
15450 
15451   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
15452 
15453   ins_pipe(pipe_serial);
15454 %}
15455 
15456 
15457 // ============================================================================
15458 // Safepoint Instructions
15459 
15460 // TODO
15461 // provide a near and far version of this code
15462 
15463 instruct safePoint(rFlagsReg cr, iRegP poll)
15464 %{
15465   match(SafePoint poll);
15466   effect(KILL cr);
15467 
15468   format %{
15469     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15470   %}
15471   ins_encode %{
15472     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15473   %}
15474   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15475 %}
15476 
15477 
15478 // ============================================================================
15479 // Procedure Call/Return Instructions
15480 
15481 // Call Java Static Instruction
15482 
15483 instruct CallStaticJavaDirect(method meth)
15484 %{
15485   match(CallStaticJava);
15486 
15487   effect(USE meth);
15488 
15489   ins_cost(CALL_COST);
15490 
15491   format %{ "call,static $meth \t// ==> " %}
15492 
15493   ins_encode( aarch64_enc_java_static_call(meth),
15494               aarch64_enc_call_epilog );
15495 
15496   ins_pipe(pipe_class_call);
15497 %}
15498 
15499 // TO HERE
15500 
15501 // Call Java Dynamic Instruction
15502 instruct CallDynamicJavaDirect(method meth)
15503 %{
15504   match(CallDynamicJava);
15505 
15506   effect(USE meth);
15507 
15508   ins_cost(CALL_COST);
15509 
15510   format %{ "CALL,dynamic $meth \t// ==> " %}
15511 
15512   ins_encode( aarch64_enc_java_dynamic_call(meth),
15513                aarch64_enc_call_epilog );
15514 
15515   ins_pipe(pipe_class_call);
15516 %}
15517 
15518 // Call Runtime Instruction
15519 
15520 instruct CallRuntimeDirect(method meth)
15521 %{
15522   match(CallRuntime);
15523 
15524   effect(USE meth);
15525 
15526   ins_cost(CALL_COST);
15527 
15528   format %{ "CALL, runtime $meth" %}
15529 
15530   ins_encode( aarch64_enc_java_to_runtime(meth) );
15531 
15532   ins_pipe(pipe_class_call);
15533 %}
15534 
15535 // Call Runtime Instruction
15536 
15537 instruct CallLeafDirect(method meth)
15538 %{
15539   match(CallLeaf);
15540 
15541   effect(USE meth);
15542 
15543   ins_cost(CALL_COST);
15544 
15545   format %{ "CALL, runtime leaf $meth" %}
15546 
15547   ins_encode( aarch64_enc_java_to_runtime(meth) );
15548 
15549   ins_pipe(pipe_class_call);
15550 %}
15551 
15552 // Call Runtime Instruction
15553 
15554 instruct CallLeafNoFPDirect(method meth)
15555 %{
15556   match(CallLeafNoFP);
15557 
15558   effect(USE meth);
15559 
15560   ins_cost(CALL_COST);
15561 
15562   format %{ "CALL, runtime leaf nofp $meth" %}
15563 
15564   ins_encode( aarch64_enc_java_to_runtime(meth) );
15565 
15566   ins_pipe(pipe_class_call);
15567 %}
15568 
15569 // Tail Call; Jump from runtime stub to Java code.
15570 // Also known as an 'interprocedural jump'.
15571 // Target of jump will eventually return to caller.
15572 // TailJump below removes the return address.
15573 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
15574 %{
15575   match(TailCall jump_target method_ptr);
15576 
15577   ins_cost(CALL_COST);
15578 
15579   format %{ "br $jump_target\t# $method_ptr holds method" %}
15580 
15581   ins_encode(aarch64_enc_tail_call(jump_target));
15582 
15583   ins_pipe(pipe_class_call);
15584 %}
15585 
15586 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
15587 %{
15588   match(TailJump jump_target ex_oop);
15589 
15590   ins_cost(CALL_COST);
15591 
15592   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15593 
15594   ins_encode(aarch64_enc_tail_jmp(jump_target));
15595 
15596   ins_pipe(pipe_class_call);
15597 %}
15598 
15599 // Create exception oop: created by stack-crawling runtime code.
15600 // Created exception is now available to this handler, and is setup
15601 // just prior to jumping to this handler. No code emitted.
15602 // TODO check
15603 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15604 instruct CreateException(iRegP_R0 ex_oop)
15605 %{
15606   match(Set ex_oop (CreateEx));
15607 
15608   format %{ " -- \t// exception oop; no code emitted" %}
15609 
15610   size(0);
15611 
15612   ins_encode( /*empty*/ );
15613 
15614   ins_pipe(pipe_class_empty);
15615 %}
15616 
15617 // Rethrow exception: The exception oop will come in the first
15618 // argument position. Then JUMP (not call) to the rethrow stub code.
15619 instruct RethrowException() %{
15620   match(Rethrow);
15621   ins_cost(CALL_COST);
15622 
15623   format %{ "b rethrow_stub" %}
15624 
15625   ins_encode( aarch64_enc_rethrow() );
15626 
15627   ins_pipe(pipe_class_call);
15628 %}
15629 
15630 
15631 // Return Instruction
15632 // epilog node loads ret address into lr as part of frame pop
15633 instruct Ret()
15634 %{
15635   match(Return);
15636 
15637   format %{ "ret\t// return register" %}
15638 
15639   ins_encode( aarch64_enc_ret() );
15640 
15641   ins_pipe(pipe_branch);
15642 %}
15643 
15644 // Die now.
15645 instruct ShouldNotReachHere() %{
15646   match(Halt);
15647 
15648   ins_cost(CALL_COST);
15649   format %{ "ShouldNotReachHere" %}
15650 
15651   ins_encode %{
15652     if (is_reachable()) {
15653       __ stop(_halt_reason);
15654     }
15655   %}
15656 
15657   ins_pipe(pipe_class_default);
15658 %}
15659 
15660 // ============================================================================
15661 // Partial Subtype Check
15662 //
15663 // superklass array for an instance of the superklass.  Set a hidden
15664 // internal cache on a hit (cache is checked with exposed code in
15665 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
15666 // encoding ALSO sets flags.
15667 
15668 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15669 %{
15670   match(Set result (PartialSubtypeCheck sub super));
15671   effect(KILL cr, KILL temp);
15672 
15673   ins_cost(1100);  // slightly larger than the next version
15674   format %{ "partialSubtypeCheck $result, $sub, $super" %}
15675 
15676   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15677 
15678   opcode(0x1); // Force zero of result reg on hit
15679 
15680   ins_pipe(pipe_class_memory);
15681 %}
15682 
15683 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
15684 %{
15685   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
15686   effect(KILL temp, KILL result);
15687 
15688   ins_cost(1100);  // slightly larger than the next version
15689   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
15690 
15691   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15692 
15693   opcode(0x0); // Don't zero result reg on hit
15694 
15695   ins_pipe(pipe_class_memory);
15696 %}
15697 
15698 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15699                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15700 %{
15701   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
15702   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15703   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15704 
15705   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
15706   ins_encode %{
15707     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15708     __ string_compare($str1$$Register, $str2$$Register,
15709                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
15710                       $tmp1$$Register, $tmp2$$Register,
15711                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
15712   %}
15713   ins_pipe(pipe_class_memory);
15714 %}
15715 
15716 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15717                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15718 %{
15719   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
15720   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15721   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15722 
15723   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
15724   ins_encode %{
15725     __ string_compare($str1$$Register, $str2$$Register,
15726                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
15727                       $tmp1$$Register, $tmp2$$Register,
15728                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
15729   %}
15730   ins_pipe(pipe_class_memory);
15731 %}
15732 
15733 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15734                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15735                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15736 %{
15737   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15738   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15739   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15740          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15741 
15742   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15743   ins_encode %{
15744     __ string_compare($str1$$Register, $str2$$Register,
15745                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
15746                       $tmp1$$Register, $tmp2$$Register,
15747                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15748                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
15749   %}
15750   ins_pipe(pipe_class_memory);
15751 %}
15752 
15753 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15754                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15755                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15756 %{
15757   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
15758   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15759   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15760          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15761 
15762   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15763   ins_encode %{
15764     __ string_compare($str1$$Register, $str2$$Register,
15765                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
15766                       $tmp1$$Register, $tmp2$$Register,
15767                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15768                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
15769   %}
15770   ins_pipe(pipe_class_memory);
15771 %}
15772 
15773 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15774        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15775        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
15776 %{
15777   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15778   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15779   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15780          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
15781   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
15782 
15783   ins_encode %{
15784     __ string_indexof($str1$$Register, $str2$$Register,
15785                       $cnt1$$Register, $cnt2$$Register,
15786                       $tmp1$$Register, $tmp2$$Register,
15787                       $tmp3$$Register, $tmp4$$Register,
15788                       $tmp5$$Register, $tmp6$$Register,
15789                       -1, $result$$Register, StrIntrinsicNode::UU);
15790   %}
15791   ins_pipe(pipe_class_memory);
15792 %}
15793 
15794 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15795        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15796        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
15797 %{
15798   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15799   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15800   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15801          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
15802   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
15803 
15804   ins_encode %{
15805     __ string_indexof($str1$$Register, $str2$$Register,
15806                       $cnt1$$Register, $cnt2$$Register,
15807                       $tmp1$$Register, $tmp2$$Register,
15808                       $tmp3$$Register, $tmp4$$Register,
15809                       $tmp5$$Register, $tmp6$$Register,
15810                       -1, $result$$Register, StrIntrinsicNode::LL);
15811   %}
15812   ins_pipe(pipe_class_memory);
15813 %}
15814 
15815 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15816        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15817        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
15818 %{
15819   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15820   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15821   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15822          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
15823   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
15824 
15825   ins_encode %{
15826     __ string_indexof($str1$$Register, $str2$$Register,
15827                       $cnt1$$Register, $cnt2$$Register,
15828                       $tmp1$$Register, $tmp2$$Register,
15829                       $tmp3$$Register, $tmp4$$Register,
15830                       $tmp5$$Register, $tmp6$$Register,
15831                       -1, $result$$Register, StrIntrinsicNode::UL);
15832   %}
15833   ins_pipe(pipe_class_memory);
15834 %}
15835 
15836 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15837                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15838                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15839 %{
15840   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15841   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15842   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15843          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15844   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
15845 
15846   ins_encode %{
15847     int icnt2 = (int)$int_cnt2$$constant;
15848     __ string_indexof($str1$$Register, $str2$$Register,
15849                       $cnt1$$Register, zr,
15850                       $tmp1$$Register, $tmp2$$Register,
15851                       $tmp3$$Register, $tmp4$$Register, zr, zr,
15852                       icnt2, $result$$Register, StrIntrinsicNode::UU);
15853   %}
15854   ins_pipe(pipe_class_memory);
15855 %}
15856 
15857 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15858                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15859                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15860 %{
15861   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15862   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15863   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15864          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15865   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
15866 
15867   ins_encode %{
15868     int icnt2 = (int)$int_cnt2$$constant;
15869     __ string_indexof($str1$$Register, $str2$$Register,
15870                       $cnt1$$Register, zr,
15871                       $tmp1$$Register, $tmp2$$Register,
15872                       $tmp3$$Register, $tmp4$$Register, zr, zr,
15873                       icnt2, $result$$Register, StrIntrinsicNode::LL);
15874   %}
15875   ins_pipe(pipe_class_memory);
15876 %}
15877 
15878 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15879                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15880                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15881 %{
15882   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15883   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15884   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15885          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15886   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
15887 
15888   ins_encode %{
15889     int icnt2 = (int)$int_cnt2$$constant;
15890     __ string_indexof($str1$$Register, $str2$$Register,
15891                       $cnt1$$Register, zr,
15892                       $tmp1$$Register, $tmp2$$Register,
15893                       $tmp3$$Register, $tmp4$$Register, zr, zr,
15894                       icnt2, $result$$Register, StrIntrinsicNode::UL);
15895   %}
15896   ins_pipe(pipe_class_memory);
15897 %}
15898 
15899 instruct string_indexofU_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15900                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15901                               iRegINoSp tmp3, rFlagsReg cr)
15902 %{
15903   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15904   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15905          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15906 
15907   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15908 
15909   ins_encode %{
15910     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15911                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
15912                            $tmp3$$Register);
15913   %}
15914   ins_pipe(pipe_class_memory);
15915 %}
15916 
15917 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15918                         iRegI_R0 result, rFlagsReg cr)
15919 %{
15920   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
15921   match(Set result (StrEquals (Binary str1 str2) cnt));
15922   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15923 
15924   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15925   ins_encode %{
15926     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15927     __ string_equals($str1$$Register, $str2$$Register,
15928                      $result$$Register, $cnt$$Register, 1);
15929   %}
15930   ins_pipe(pipe_class_memory);
15931 %}
15932 
15933 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15934                         iRegI_R0 result, rFlagsReg cr)
15935 %{
15936   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
15937   match(Set result (StrEquals (Binary str1 str2) cnt));
15938   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15939 
15940   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15941   ins_encode %{
15942     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15943     __ string_equals($str1$$Register, $str2$$Register,
15944                      $result$$Register, $cnt$$Register, 2);
15945   %}
15946   ins_pipe(pipe_class_memory);
15947 %}
15948 
15949 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
15950                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
15951                        iRegP_R10 tmp, rFlagsReg cr)
15952 %{
15953   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15954   match(Set result (AryEq ary1 ary2));
15955   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15956 
15957   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
15958   ins_encode %{
15959     __ arrays_equals($ary1$$Register, $ary2$$Register,
15960                      $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15961                      $result$$Register, $tmp$$Register, 1);
15962     %}
15963   ins_pipe(pipe_class_memory);
15964 %}
15965 
15966 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
15967                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
15968                        iRegP_R10 tmp, rFlagsReg cr)
15969 %{
15970   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15971   match(Set result (AryEq ary1 ary2));
15972   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15973 
15974   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
15975   ins_encode %{
15976     __ arrays_equals($ary1$$Register, $ary2$$Register,
15977                      $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15978                      $result$$Register, $tmp$$Register, 2);
15979   %}
15980   ins_pipe(pipe_class_memory);
15981 %}
15982 
15983 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
15984 %{
15985   match(Set result (HasNegatives ary1 len));
15986   effect(USE_KILL ary1, USE_KILL len, KILL cr);
15987   format %{ "has negatives byte[] $ary1,$len -> $result" %}
15988   ins_encode %{
15989     __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
15990   %}
15991   ins_pipe( pipe_slow );
15992 %}
15993 
15994 // fast char[] to byte[] compression
15995 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
15996                          vRegD_V0 tmp1, vRegD_V1 tmp2,
15997                          vRegD_V2 tmp3, vRegD_V3 tmp4,
15998                          iRegI_R0 result, rFlagsReg cr)
15999 %{
16000   match(Set result (StrCompressedCopy src (Binary dst len)));
16001   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16002 
16003   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
16004   ins_encode %{
16005     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16006                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16007                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
16008                            $result$$Register);
16009   %}
16010   ins_pipe( pipe_slow );
16011 %}
16012 
16013 // fast byte[] to char[] inflation
16014 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16015                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16016 %{
16017   match(Set dummy (StrInflatedCopy src (Binary dst len)));
16018   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16019 
16020   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
16021   ins_encode %{
16022     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16023                           $tmp1$$FloatRegister, $tmp2$$FloatRegister, $tmp3$$FloatRegister, $tmp4$$Register);
16024   %}
16025   ins_pipe(pipe_class_memory);
16026 %}
16027 
16028 // encode char[] to byte[] in ISO_8859_1
16029 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16030                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
16031                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
16032                           iRegI_R0 result, rFlagsReg cr)
16033 %{
16034   match(Set result (EncodeISOArray src (Binary dst len)));
16035   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
16036          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
16037 
16038   format %{ "Encode array $src,$dst,$len -> $result" %}
16039   ins_encode %{
16040     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16041          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
16042          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
16043   %}
16044   ins_pipe( pipe_class_memory );
16045 %}
16046 
16047 // ============================================================================
16048 // This name is KNOWN by the ADLC and cannot be changed.
16049 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16050 // for this guy.
16051 instruct tlsLoadP(thread_RegP dst)
16052 %{
16053   match(Set dst (ThreadLocal));
16054 
16055   ins_cost(0);
16056 
16057   format %{ " -- \t// $dst=Thread::current(), empty" %}
16058 
16059   size(0);
16060 
16061   ins_encode( /*empty*/ );
16062 
16063   ins_pipe(pipe_class_empty);
16064 %}
16065 
16066 // ====================VECTOR INSTRUCTIONS=====================================
16067 
16068 // Load vector (32 bits)
16069 instruct loadV4(vecD dst, vmem4 mem)
16070 %{
16071   predicate(n->as_LoadVector()->memory_size() == 4);
16072   match(Set dst (LoadVector mem));
16073   ins_cost(4 * INSN_COST);
16074   format %{ "ldrs   $dst,$mem\t# vector (32 bits)" %}
16075   ins_encode( aarch64_enc_ldrvS(dst, mem) );
16076   ins_pipe(vload_reg_mem64);
16077 %}
16078 
16079 // Load vector (64 bits)
16080 instruct loadV8(vecD dst, vmem8 mem)
16081 %{
16082   predicate(n->as_LoadVector()->memory_size() == 8);
16083   match(Set dst (LoadVector mem));
16084   ins_cost(4 * INSN_COST);
16085   format %{ "ldrd   $dst,$mem\t# vector (64 bits)" %}
16086   ins_encode( aarch64_enc_ldrvD(dst, mem) );
16087   ins_pipe(vload_reg_mem64);
16088 %}
16089 
16090 // Load Vector (128 bits)
16091 instruct loadV16(vecX dst, vmem16 mem)
16092 %{
16093   predicate(n->as_LoadVector()->memory_size() == 16);
16094   match(Set dst (LoadVector mem));
16095   ins_cost(4 * INSN_COST);
16096   format %{ "ldrq   $dst,$mem\t# vector (128 bits)" %}
16097   ins_encode( aarch64_enc_ldrvQ(dst, mem) );
16098   ins_pipe(vload_reg_mem128);
16099 %}
16100 
16101 // Store Vector (32 bits)
16102 instruct storeV4(vecD src, vmem4 mem)
16103 %{
16104   predicate(n->as_StoreVector()->memory_size() == 4);
16105   match(Set mem (StoreVector mem src));
16106   ins_cost(4 * INSN_COST);
16107   format %{ "strs   $mem,$src\t# vector (32 bits)" %}
16108   ins_encode( aarch64_enc_strvS(src, mem) );
16109   ins_pipe(vstore_reg_mem64);
16110 %}
16111 
16112 // Store Vector (64 bits)
16113 instruct storeV8(vecD src, vmem8 mem)
16114 %{
16115   predicate(n->as_StoreVector()->memory_size() == 8);
16116   match(Set mem (StoreVector mem src));
16117   ins_cost(4 * INSN_COST);
16118   format %{ "strd   $mem,$src\t# vector (64 bits)" %}
16119   ins_encode( aarch64_enc_strvD(src, mem) );
16120   ins_pipe(vstore_reg_mem64);
16121 %}
16122 
16123 // Store Vector (128 bits)
16124 instruct storeV16(vecX src, vmem16 mem)
16125 %{
16126   predicate(n->as_StoreVector()->memory_size() == 16);
16127   match(Set mem (StoreVector mem src));
16128   ins_cost(4 * INSN_COST);
16129   format %{ "strq   $mem,$src\t# vector (128 bits)" %}
16130   ins_encode( aarch64_enc_strvQ(src, mem) );
16131   ins_pipe(vstore_reg_mem128);
16132 %}
16133 
16134 instruct replicate8B(vecD dst, iRegIorL2I src)
16135 %{
16136   predicate(n->as_Vector()->length() == 4 ||
16137             n->as_Vector()->length() == 8);
16138   match(Set dst (ReplicateB src));
16139   ins_cost(INSN_COST);
16140   format %{ "dup  $dst, $src\t# vector (8B)" %}
16141   ins_encode %{
16142     __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($src$$reg));
16143   %}
16144   ins_pipe(vdup_reg_reg64);
16145 %}
16146 
16147 instruct replicate16B(vecX dst, iRegIorL2I src)
16148 %{
16149   predicate(n->as_Vector()->length() == 16);
16150   match(Set dst (ReplicateB src));
16151   ins_cost(INSN_COST);
16152   format %{ "dup  $dst, $src\t# vector (16B)" %}
16153   ins_encode %{
16154     __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($src$$reg));
16155   %}
16156   ins_pipe(vdup_reg_reg128);
16157 %}
16158 
16159 instruct replicate8B_imm(vecD dst, immI con)
16160 %{
16161   predicate(n->as_Vector()->length() == 4 ||
16162             n->as_Vector()->length() == 8);
16163   match(Set dst (ReplicateB con));
16164   ins_cost(INSN_COST);
16165   format %{ "movi  $dst, $con\t# vector(8B)" %}
16166   ins_encode %{
16167     __ mov(as_FloatRegister($dst$$reg), __ T8B, $con$$constant & 0xff);
16168   %}
16169   ins_pipe(vmovi_reg_imm64);
16170 %}
16171 
16172 instruct replicate16B_imm(vecX dst, immI con)
16173 %{
16174   predicate(n->as_Vector()->length() == 16);
16175   match(Set dst (ReplicateB con));
16176   ins_cost(INSN_COST);
16177   format %{ "movi  $dst, $con\t# vector(16B)" %}
16178   ins_encode %{
16179     __ mov(as_FloatRegister($dst$$reg), __ T16B, $con$$constant & 0xff);
16180   %}
16181   ins_pipe(vmovi_reg_imm128);
16182 %}
16183 
16184 instruct replicate4S(vecD dst, iRegIorL2I src)
16185 %{
16186   predicate(n->as_Vector()->length() == 2 ||
16187             n->as_Vector()->length() == 4);
16188   match(Set dst (ReplicateS src));
16189   ins_cost(INSN_COST);
16190   format %{ "dup  $dst, $src\t# vector (4S)" %}
16191   ins_encode %{
16192     __ dup(as_FloatRegister($dst$$reg), __ T4H, as_Register($src$$reg));
16193   %}
16194   ins_pipe(vdup_reg_reg64);
16195 %}
16196 
16197 instruct replicate8S(vecX dst, iRegIorL2I src)
16198 %{
16199   predicate(n->as_Vector()->length() == 8);
16200   match(Set dst (ReplicateS src));
16201   ins_cost(INSN_COST);
16202   format %{ "dup  $dst, $src\t# vector (8S)" %}
16203   ins_encode %{
16204     __ dup(as_FloatRegister($dst$$reg), __ T8H, as_Register($src$$reg));
16205   %}
16206   ins_pipe(vdup_reg_reg128);
16207 %}
16208 
16209 instruct replicate4S_imm(vecD dst, immI con)
16210 %{
16211   predicate(n->as_Vector()->length() == 2 ||
16212             n->as_Vector()->length() == 4);
16213   match(Set dst (ReplicateS con));
16214   ins_cost(INSN_COST);
16215   format %{ "movi  $dst, $con\t# vector(4H)" %}
16216   ins_encode %{
16217     __ mov(as_FloatRegister($dst$$reg), __ T4H, $con$$constant & 0xffff);
16218   %}
16219   ins_pipe(vmovi_reg_imm64);
16220 %}
16221 
16222 instruct replicate8S_imm(vecX dst, immI con)
16223 %{
16224   predicate(n->as_Vector()->length() == 8);
16225   match(Set dst (ReplicateS con));
16226   ins_cost(INSN_COST);
16227   format %{ "movi  $dst, $con\t# vector(8H)" %}
16228   ins_encode %{
16229     __ mov(as_FloatRegister($dst$$reg), __ T8H, $con$$constant & 0xffff);
16230   %}
16231   ins_pipe(vmovi_reg_imm128);
16232 %}
16233 
16234 instruct replicate2I(vecD dst, iRegIorL2I src)
16235 %{
16236   predicate(n->as_Vector()->length() == 2);
16237   match(Set dst (ReplicateI src));
16238   ins_cost(INSN_COST);
16239   format %{ "dup  $dst, $src\t# vector (2I)" %}
16240   ins_encode %{
16241     __ dup(as_FloatRegister($dst$$reg), __ T2S, as_Register($src$$reg));
16242   %}
16243   ins_pipe(vdup_reg_reg64);
16244 %}
16245 
16246 instruct replicate4I(vecX dst, iRegIorL2I src)
16247 %{
16248   predicate(n->as_Vector()->length() == 4);
16249   match(Set dst (ReplicateI src));
16250   ins_cost(INSN_COST);
16251   format %{ "dup  $dst, $src\t# vector (4I)" %}
16252   ins_encode %{
16253     __ dup(as_FloatRegister($dst$$reg), __ T4S, as_Register($src$$reg));
16254   %}
16255   ins_pipe(vdup_reg_reg128);
16256 %}
16257 
16258 instruct replicate2I_imm(vecD dst, immI con)
16259 %{
16260   predicate(n->as_Vector()->length() == 2);
16261   match(Set dst (ReplicateI con));
16262   ins_cost(INSN_COST);
16263   format %{ "movi  $dst, $con\t# vector(2I)" %}
16264   ins_encode %{
16265     __ mov(as_FloatRegister($dst$$reg), __ T2S, $con$$constant);
16266   %}
16267   ins_pipe(vmovi_reg_imm64);
16268 %}
16269 
16270 instruct replicate4I_imm(vecX dst, immI con)
16271 %{
16272   predicate(n->as_Vector()->length() == 4);
16273   match(Set dst (ReplicateI con));
16274   ins_cost(INSN_COST);
16275   format %{ "movi  $dst, $con\t# vector(4I)" %}
16276   ins_encode %{
16277     __ mov(as_FloatRegister($dst$$reg), __ T4S, $con$$constant);
16278   %}
16279   ins_pipe(vmovi_reg_imm128);
16280 %}
16281 
16282 instruct replicate2L(vecX dst, iRegL src)
16283 %{
16284   predicate(n->as_Vector()->length() == 2);
16285   match(Set dst (ReplicateL src));
16286   ins_cost(INSN_COST);
16287   format %{ "dup  $dst, $src\t# vector (2L)" %}
16288   ins_encode %{
16289     __ dup(as_FloatRegister($dst$$reg), __ T2D, as_Register($src$$reg));
16290   %}
16291   ins_pipe(vdup_reg_reg128);
16292 %}
16293 
16294 instruct replicate2L_zero(vecX dst, immI0 zero)
16295 %{
16296   predicate(n->as_Vector()->length() == 2);
16297   match(Set dst (ReplicateI zero));
16298   ins_cost(INSN_COST);
16299   format %{ "movi  $dst, $zero\t# vector(4I)" %}
16300   ins_encode %{
16301     __ eor(as_FloatRegister($dst$$reg), __ T16B,
16302            as_FloatRegister($dst$$reg),
16303            as_FloatRegister($dst$$reg));
16304   %}
16305   ins_pipe(vmovi_reg_imm128);
16306 %}
16307 
16308 instruct replicate2F(vecD dst, vRegF src)
16309 %{
16310   predicate(n->as_Vector()->length() == 2);
16311   match(Set dst (ReplicateF src));
16312   ins_cost(INSN_COST);
16313   format %{ "dup  $dst, $src\t# vector (2F)" %}
16314   ins_encode %{
16315     __ dup(as_FloatRegister($dst$$reg), __ T2S,
16316            as_FloatRegister($src$$reg));
16317   %}
16318   ins_pipe(vdup_reg_freg64);
16319 %}
16320 
16321 instruct replicate4F(vecX dst, vRegF src)
16322 %{
16323   predicate(n->as_Vector()->length() == 4);
16324   match(Set dst (ReplicateF src));
16325   ins_cost(INSN_COST);
16326   format %{ "dup  $dst, $src\t# vector (4F)" %}
16327   ins_encode %{
16328     __ dup(as_FloatRegister($dst$$reg), __ T4S,
16329            as_FloatRegister($src$$reg));
16330   %}
16331   ins_pipe(vdup_reg_freg128);
16332 %}
16333 
16334 instruct replicate2D(vecX dst, vRegD src)
16335 %{
16336   predicate(n->as_Vector()->length() == 2);
16337   match(Set dst (ReplicateD src));
16338   ins_cost(INSN_COST);
16339   format %{ "dup  $dst, $src\t# vector (2D)" %}
16340   ins_encode %{
16341     __ dup(as_FloatRegister($dst$$reg), __ T2D,
16342            as_FloatRegister($src$$reg));
16343   %}
16344   ins_pipe(vdup_reg_dreg128);
16345 %}
16346 
16347 // ====================REDUCTION ARITHMETIC====================================
16348 
16349 instruct reduce_add2I(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, iRegINoSp tmp, iRegINoSp tmp2)
16350 %{
16351   match(Set dst (AddReductionVI isrc vsrc));
16352   ins_cost(INSN_COST);
16353   effect(TEMP tmp, TEMP tmp2);
16354   format %{ "umov  $tmp, $vsrc, S, 0\n\t"
16355             "umov  $tmp2, $vsrc, S, 1\n\t"
16356             "addw  $tmp, $isrc, $tmp\n\t"
16357             "addw  $dst, $tmp, $tmp2\t# add reduction2I"
16358   %}
16359   ins_encode %{
16360     __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 0);
16361     __ umov($tmp2$$Register, as_FloatRegister($vsrc$$reg), __ S, 1);
16362     __ addw($tmp$$Register, $isrc$$Register, $tmp$$Register);
16363     __ addw($dst$$Register, $tmp$$Register, $tmp2$$Register);
16364   %}
16365   ins_pipe(pipe_class_default);
16366 %}
16367 
16368 instruct reduce_add4I(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX vtmp, iRegINoSp itmp)
16369 %{
16370   match(Set dst (AddReductionVI isrc vsrc));
16371   ins_cost(INSN_COST);
16372   effect(TEMP vtmp, TEMP itmp);
16373   format %{ "addv  $vtmp, T4S, $vsrc\n\t"
16374             "umov  $itmp, $vtmp, S, 0\n\t"
16375             "addw  $dst, $itmp, $isrc\t# add reduction4I"
16376   %}
16377   ins_encode %{
16378     __ addv(as_FloatRegister($vtmp$$reg), __ T4S,
16379             as_FloatRegister($vsrc$$reg));
16380     __ umov($itmp$$Register, as_FloatRegister($vtmp$$reg), __ S, 0);
16381     __ addw($dst$$Register, $itmp$$Register, $isrc$$Register);
16382   %}
16383   ins_pipe(pipe_class_default);
16384 %}
16385 
16386 instruct reduce_mul2I(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, iRegINoSp tmp)
16387 %{
16388   match(Set dst (MulReductionVI isrc vsrc));
16389   ins_cost(INSN_COST);
16390   effect(TEMP tmp, TEMP dst);
16391   format %{ "umov  $tmp, $vsrc, S, 0\n\t"
16392             "mul   $dst, $tmp, $isrc\n\t"
16393             "umov  $tmp, $vsrc, S, 1\n\t"
16394             "mul   $dst, $tmp, $dst\t# mul reduction2I"
16395   %}
16396   ins_encode %{
16397     __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 0);
16398     __ mul($dst$$Register, $tmp$$Register, $isrc$$Register);
16399     __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 1);
16400     __ mul($dst$$Register, $tmp$$Register, $dst$$Register);
16401   %}
16402   ins_pipe(pipe_class_default);
16403 %}
16404 
16405 instruct reduce_mul4I(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX vtmp, iRegINoSp itmp)
16406 %{
16407   match(Set dst (MulReductionVI isrc vsrc));
16408   ins_cost(INSN_COST);
16409   effect(TEMP vtmp, TEMP itmp, TEMP dst);
16410   format %{ "ins   $vtmp, D, $vsrc, 0, 1\n\t"
16411             "mulv  $vtmp, T2S, $vtmp, $vsrc\n\t"
16412             "umov  $itmp, $vtmp, S, 0\n\t"
16413             "mul   $dst, $itmp, $isrc\n\t"
16414             "umov  $itmp, $vtmp, S, 1\n\t"
16415             "mul   $dst, $itmp, $dst\t# mul reduction4I"
16416   %}
16417   ins_encode %{
16418     __ ins(as_FloatRegister($vtmp$$reg), __ D,
16419            as_FloatRegister($vsrc$$reg), 0, 1);
16420     __ mulv(as_FloatRegister($vtmp$$reg), __ T2S,
16421             as_FloatRegister($vtmp$$reg), as_FloatRegister($vsrc$$reg));
16422     __ umov($itmp$$Register, as_FloatRegister($vtmp$$reg), __ S, 0);
16423     __ mul($dst$$Register, $itmp$$Register, $isrc$$Register);
16424     __ umov($itmp$$Register, as_FloatRegister($vtmp$$reg), __ S, 1);
16425     __ mul($dst$$Register, $itmp$$Register, $dst$$Register);
16426   %}
16427   ins_pipe(pipe_class_default);
16428 %}
16429 
16430 instruct reduce_add2F(vRegF dst, vRegF fsrc, vecD vsrc, vecD tmp)
16431 %{
16432   match(Set dst (AddReductionVF fsrc vsrc));
16433   ins_cost(INSN_COST);
16434   effect(TEMP tmp, TEMP dst);
16435   format %{ "fadds $dst, $fsrc, $vsrc\n\t"
16436             "ins   $tmp, S, $vsrc, 0, 1\n\t"
16437             "fadds $dst, $dst, $tmp\t# add reduction2F"
16438   %}
16439   ins_encode %{
16440     __ fadds(as_FloatRegister($dst$$reg),
16441              as_FloatRegister($fsrc$$reg), as_FloatRegister($vsrc$$reg));
16442     __ ins(as_FloatRegister($tmp$$reg), __ S,
16443            as_FloatRegister($vsrc$$reg), 0, 1);
16444     __ fadds(as_FloatRegister($dst$$reg),
16445              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16446   %}
16447   ins_pipe(pipe_class_default);
16448 %}
16449 
16450 instruct reduce_add4F(vRegF dst, vRegF fsrc, vecX vsrc, vecX tmp)
16451 %{
16452   match(Set dst (AddReductionVF fsrc vsrc));
16453   ins_cost(INSN_COST);
16454   effect(TEMP tmp, TEMP dst);
16455   format %{ "fadds $dst, $fsrc, $vsrc\n\t"
16456             "ins   $tmp, S, $vsrc, 0, 1\n\t"
16457             "fadds $dst, $dst, $tmp\n\t"
16458             "ins   $tmp, S, $vsrc, 0, 2\n\t"
16459             "fadds $dst, $dst, $tmp\n\t"
16460             "ins   $tmp, S, $vsrc, 0, 3\n\t"
16461             "fadds $dst, $dst, $tmp\t# add reduction4F"
16462   %}
16463   ins_encode %{
16464     __ fadds(as_FloatRegister($dst$$reg),
16465              as_FloatRegister($fsrc$$reg), as_FloatRegister($vsrc$$reg));
16466     __ ins(as_FloatRegister($tmp$$reg), __ S,
16467            as_FloatRegister($vsrc$$reg), 0, 1);
16468     __ fadds(as_FloatRegister($dst$$reg),
16469              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16470     __ ins(as_FloatRegister($tmp$$reg), __ S,
16471            as_FloatRegister($vsrc$$reg), 0, 2);
16472     __ fadds(as_FloatRegister($dst$$reg),
16473              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16474     __ ins(as_FloatRegister($tmp$$reg), __ S,
16475            as_FloatRegister($vsrc$$reg), 0, 3);
16476     __ fadds(as_FloatRegister($dst$$reg),
16477              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16478   %}
16479   ins_pipe(pipe_class_default);
16480 %}
16481 
16482 instruct reduce_mul2F(vRegF dst, vRegF fsrc, vecD vsrc, vecD tmp)
16483 %{
16484   match(Set dst (MulReductionVF fsrc vsrc));
16485   ins_cost(INSN_COST);
16486   effect(TEMP tmp, TEMP dst);
16487   format %{ "fmuls $dst, $fsrc, $vsrc\n\t"
16488             "ins   $tmp, S, $vsrc, 0, 1\n\t"
16489             "fmuls $dst, $dst, $tmp\t# mul reduction2F"
16490   %}
16491   ins_encode %{
16492     __ fmuls(as_FloatRegister($dst$$reg),
16493              as_FloatRegister($fsrc$$reg), as_FloatRegister($vsrc$$reg));
16494     __ ins(as_FloatRegister($tmp$$reg), __ S,
16495            as_FloatRegister($vsrc$$reg), 0, 1);
16496     __ fmuls(as_FloatRegister($dst$$reg),
16497              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16498   %}
16499   ins_pipe(pipe_class_default);
16500 %}
16501 
16502 instruct reduce_mul4F(vRegF dst, vRegF fsrc, vecX vsrc, vecX tmp)
16503 %{
16504   match(Set dst (MulReductionVF fsrc vsrc));
16505   ins_cost(INSN_COST);
16506   effect(TEMP tmp, TEMP dst);
16507   format %{ "fmuls $dst, $fsrc, $vsrc\n\t"
16508             "ins   $tmp, S, $vsrc, 0, 1\n\t"
16509             "fmuls $dst, $dst, $tmp\n\t"
16510             "ins   $tmp, S, $vsrc, 0, 2\n\t"
16511             "fmuls $dst, $dst, $tmp\n\t"
16512             "ins   $tmp, S, $vsrc, 0, 3\n\t"
16513             "fmuls $dst, $dst, $tmp\t# mul reduction4F"
16514   %}
16515   ins_encode %{
16516     __ fmuls(as_FloatRegister($dst$$reg),
16517              as_FloatRegister($fsrc$$reg), as_FloatRegister($vsrc$$reg));
16518     __ ins(as_FloatRegister($tmp$$reg), __ S,
16519            as_FloatRegister($vsrc$$reg), 0, 1);
16520     __ fmuls(as_FloatRegister($dst$$reg),
16521              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16522     __ ins(as_FloatRegister($tmp$$reg), __ S,
16523            as_FloatRegister($vsrc$$reg), 0, 2);
16524     __ fmuls(as_FloatRegister($dst$$reg),
16525              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16526     __ ins(as_FloatRegister($tmp$$reg), __ S,
16527            as_FloatRegister($vsrc$$reg), 0, 3);
16528     __ fmuls(as_FloatRegister($dst$$reg),
16529              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16530   %}
16531   ins_pipe(pipe_class_default);
16532 %}
16533 
16534 instruct reduce_add2D(vRegD dst, vRegD dsrc, vecX vsrc, vecX tmp)
16535 %{
16536   match(Set dst (AddReductionVD dsrc vsrc));
16537   ins_cost(INSN_COST);
16538   effect(TEMP tmp, TEMP dst);
16539   format %{ "faddd $dst, $dsrc, $vsrc\n\t"
16540             "ins   $tmp, D, $vsrc, 0, 1\n\t"
16541             "faddd $dst, $dst, $tmp\t# add reduction2D"
16542   %}
16543   ins_encode %{
16544     __ faddd(as_FloatRegister($dst$$reg),
16545              as_FloatRegister($dsrc$$reg), as_FloatRegister($vsrc$$reg));
16546     __ ins(as_FloatRegister($tmp$$reg), __ D,
16547            as_FloatRegister($vsrc$$reg), 0, 1);
16548     __ faddd(as_FloatRegister($dst$$reg),
16549              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16550   %}
16551   ins_pipe(pipe_class_default);
16552 %}
16553 
16554 instruct reduce_mul2D(vRegD dst, vRegD dsrc, vecX vsrc, vecX tmp)
16555 %{
16556   match(Set dst (MulReductionVD dsrc vsrc));
16557   ins_cost(INSN_COST);
16558   effect(TEMP tmp, TEMP dst);
16559   format %{ "fmuld $dst, $dsrc, $vsrc\n\t"
16560             "ins   $tmp, D, $vsrc, 0, 1\n\t"
16561             "fmuld $dst, $dst, $tmp\t# mul reduction2D"
16562   %}
16563   ins_encode %{
16564     __ fmuld(as_FloatRegister($dst$$reg),
16565              as_FloatRegister($dsrc$$reg), as_FloatRegister($vsrc$$reg));
16566     __ ins(as_FloatRegister($tmp$$reg), __ D,
16567            as_FloatRegister($vsrc$$reg), 0, 1);
16568     __ fmuld(as_FloatRegister($dst$$reg),
16569              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16570   %}
16571   ins_pipe(pipe_class_default);
16572 %}
16573 
16574 instruct reduce_max2F(vRegF dst, vRegF fsrc, vecD vsrc, vecD tmp) %{
16575   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
16576   match(Set dst (MaxReductionV fsrc vsrc));
16577   ins_cost(INSN_COST);
16578   effect(TEMP_DEF dst, TEMP tmp);
16579   format %{ "fmaxs $dst, $fsrc, $vsrc\n\t"
16580             "ins   $tmp, S, $vsrc, 0, 1\n\t"
16581             "fmaxs $dst, $dst, $tmp\t# max reduction2F" %}
16582   ins_encode %{
16583     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($fsrc$$reg), as_FloatRegister($vsrc$$reg));
16584     __ ins(as_FloatRegister($tmp$$reg), __ S, as_FloatRegister($vsrc$$reg), 0, 1);
16585     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16586   %}
16587   ins_pipe(pipe_class_default);
16588 %}
16589 
16590 instruct reduce_max4F(vRegF dst, vRegF fsrc, vecX vsrc) %{
16591   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
16592   match(Set dst (MaxReductionV fsrc vsrc));
16593   ins_cost(INSN_COST);
16594   effect(TEMP_DEF dst);
16595   format %{ "fmaxv $dst, T4S, $vsrc\n\t"
16596             "fmaxs $dst, $dst, $fsrc\t# max reduction4F" %}
16597   ins_encode %{
16598     __ fmaxv(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($vsrc$$reg));
16599     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($fsrc$$reg));
16600   %}
16601   ins_pipe(pipe_class_default);
16602 %}
16603 
16604 instruct reduce_max2D(vRegD dst, vRegD dsrc, vecX vsrc, vecX tmp) %{
16605   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
16606   match(Set dst (MaxReductionV dsrc vsrc));
16607   ins_cost(INSN_COST);
16608   effect(TEMP_DEF dst, TEMP tmp);
16609   format %{ "fmaxd $dst, $dsrc, $vsrc\n\t"
16610             "ins   $tmp, D, $vsrc, 0, 1\n\t"
16611             "fmaxd $dst, $dst, $tmp\t# max reduction2D" %}
16612   ins_encode %{
16613     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dsrc$$reg), as_FloatRegister($vsrc$$reg));
16614     __ ins(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($vsrc$$reg), 0, 1);
16615     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16616   %}
16617   ins_pipe(pipe_class_default);
16618 %}
16619 
16620 instruct reduce_min2F(vRegF dst, vRegF fsrc, vecD vsrc, vecD tmp) %{
16621   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
16622   match(Set dst (MinReductionV fsrc vsrc));
16623   ins_cost(INSN_COST);
16624   effect(TEMP_DEF dst, TEMP tmp);
16625   format %{ "fmins $dst, $fsrc, $vsrc\n\t"
16626             "ins   $tmp, S, $vsrc, 0, 1\n\t"
16627             "fmins $dst, $dst, $tmp\t# min reduction2F" %}
16628   ins_encode %{
16629     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($fsrc$$reg), as_FloatRegister($vsrc$$reg));
16630     __ ins(as_FloatRegister($tmp$$reg), __ S, as_FloatRegister($vsrc$$reg), 0, 1);
16631     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16632   %}
16633   ins_pipe(pipe_class_default);
16634 %}
16635 
16636 instruct reduce_min4F(vRegF dst, vRegF fsrc, vecX vsrc) %{
16637   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
16638   match(Set dst (MinReductionV fsrc vsrc));
16639   ins_cost(INSN_COST);
16640   effect(TEMP_DEF dst);
16641   format %{ "fminv $dst, T4S, $vsrc\n\t"
16642             "fmins $dst, $dst, $fsrc\t# min reduction4F" %}
16643   ins_encode %{
16644     __ fminv(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($vsrc$$reg));
16645     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($fsrc$$reg));
16646   %}
16647   ins_pipe(pipe_class_default);
16648 %}
16649 
16650 instruct reduce_min2D(vRegD dst, vRegD dsrc, vecX vsrc, vecX tmp) %{
16651   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
16652   match(Set dst (MinReductionV dsrc vsrc));
16653   ins_cost(INSN_COST);
16654   effect(TEMP_DEF dst, TEMP tmp);
16655   format %{ "fmind $dst, $dsrc, $vsrc\n\t"
16656             "ins   $tmp, D, $vsrc, 0, 1\n\t"
16657             "fmind $dst, $dst, $tmp\t# min reduction2D" %}
16658   ins_encode %{
16659     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dsrc$$reg), as_FloatRegister($vsrc$$reg));
16660     __ ins(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($vsrc$$reg), 0, 1);
16661     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16662   %}
16663   ins_pipe(pipe_class_default);
16664 %}
16665 
16666 // ====================VECTOR ARITHMETIC=======================================
16667 
16668 // --------------------------------- ADD --------------------------------------
16669 
16670 instruct vadd8B(vecD dst, vecD src1, vecD src2)
16671 %{
16672   predicate(n->as_Vector()->length() == 4 ||
16673             n->as_Vector()->length() == 8);
16674   match(Set dst (AddVB src1 src2));
16675   ins_cost(INSN_COST);
16676   format %{ "addv  $dst,$src1,$src2\t# vector (8B)" %}
16677   ins_encode %{
16678     __ addv(as_FloatRegister($dst$$reg), __ T8B,
16679             as_FloatRegister($src1$$reg),
16680             as_FloatRegister($src2$$reg));
16681   %}
16682   ins_pipe(vdop64);
16683 %}
16684 
16685 instruct vadd16B(vecX dst, vecX src1, vecX src2)
16686 %{
16687   predicate(n->as_Vector()->length() == 16);
16688   match(Set dst (AddVB src1 src2));
16689   ins_cost(INSN_COST);
16690   format %{ "addv  $dst,$src1,$src2\t# vector (16B)" %}
16691   ins_encode %{
16692     __ addv(as_FloatRegister($dst$$reg), __ T16B,
16693             as_FloatRegister($src1$$reg),
16694             as_FloatRegister($src2$$reg));
16695   %}
16696   ins_pipe(vdop128);
16697 %}
16698 
16699 instruct vadd4S(vecD dst, vecD src1, vecD src2)
16700 %{
16701   predicate(n->as_Vector()->length() == 2 ||
16702             n->as_Vector()->length() == 4);
16703   match(Set dst (AddVS src1 src2));
16704   ins_cost(INSN_COST);
16705   format %{ "addv  $dst,$src1,$src2\t# vector (4H)" %}
16706   ins_encode %{
16707     __ addv(as_FloatRegister($dst$$reg), __ T4H,
16708             as_FloatRegister($src1$$reg),
16709             as_FloatRegister($src2$$reg));
16710   %}
16711   ins_pipe(vdop64);
16712 %}
16713 
16714 instruct vadd8S(vecX dst, vecX src1, vecX src2)
16715 %{
16716   predicate(n->as_Vector()->length() == 8);
16717   match(Set dst (AddVS src1 src2));
16718   ins_cost(INSN_COST);
16719   format %{ "addv  $dst,$src1,$src2\t# vector (8H)" %}
16720   ins_encode %{
16721     __ addv(as_FloatRegister($dst$$reg), __ T8H,
16722             as_FloatRegister($src1$$reg),
16723             as_FloatRegister($src2$$reg));
16724   %}
16725   ins_pipe(vdop128);
16726 %}
16727 
16728 instruct vadd2I(vecD dst, vecD src1, vecD src2)
16729 %{
16730   predicate(n->as_Vector()->length() == 2);
16731   match(Set dst (AddVI src1 src2));
16732   ins_cost(INSN_COST);
16733   format %{ "addv  $dst,$src1,$src2\t# vector (2S)" %}
16734   ins_encode %{
16735     __ addv(as_FloatRegister($dst$$reg), __ T2S,
16736             as_FloatRegister($src1$$reg),
16737             as_FloatRegister($src2$$reg));
16738   %}
16739   ins_pipe(vdop64);
16740 %}
16741 
16742 instruct vadd4I(vecX dst, vecX src1, vecX src2)
16743 %{
16744   predicate(n->as_Vector()->length() == 4);
16745   match(Set dst (AddVI src1 src2));
16746   ins_cost(INSN_COST);
16747   format %{ "addv  $dst,$src1,$src2\t# vector (4S)" %}
16748   ins_encode %{
16749     __ addv(as_FloatRegister($dst$$reg), __ T4S,
16750             as_FloatRegister($src1$$reg),
16751             as_FloatRegister($src2$$reg));
16752   %}
16753   ins_pipe(vdop128);
16754 %}
16755 
16756 instruct vadd2L(vecX dst, vecX src1, vecX src2)
16757 %{
16758   predicate(n->as_Vector()->length() == 2);
16759   match(Set dst (AddVL src1 src2));
16760   ins_cost(INSN_COST);
16761   format %{ "addv  $dst,$src1,$src2\t# vector (2L)" %}
16762   ins_encode %{
16763     __ addv(as_FloatRegister($dst$$reg), __ T2D,
16764             as_FloatRegister($src1$$reg),
16765             as_FloatRegister($src2$$reg));
16766   %}
16767   ins_pipe(vdop128);
16768 %}
16769 
16770 instruct vadd2F(vecD dst, vecD src1, vecD src2)
16771 %{
16772   predicate(n->as_Vector()->length() == 2);
16773   match(Set dst (AddVF src1 src2));
16774   ins_cost(INSN_COST);
16775   format %{ "fadd  $dst,$src1,$src2\t# vector (2S)" %}
16776   ins_encode %{
16777     __ fadd(as_FloatRegister($dst$$reg), __ T2S,
16778             as_FloatRegister($src1$$reg),
16779             as_FloatRegister($src2$$reg));
16780   %}
16781   ins_pipe(vdop_fp64);
16782 %}
16783 
16784 instruct vadd4F(vecX dst, vecX src1, vecX src2)
16785 %{
16786   predicate(n->as_Vector()->length() == 4);
16787   match(Set dst (AddVF src1 src2));
16788   ins_cost(INSN_COST);
16789   format %{ "fadd  $dst,$src1,$src2\t# vector (4S)" %}
16790   ins_encode %{
16791     __ fadd(as_FloatRegister($dst$$reg), __ T4S,
16792             as_FloatRegister($src1$$reg),
16793             as_FloatRegister($src2$$reg));
16794   %}
16795   ins_pipe(vdop_fp128);
16796 %}
16797 
16798 instruct vadd2D(vecX dst, vecX src1, vecX src2)
16799 %{
16800   match(Set dst (AddVD src1 src2));
16801   ins_cost(INSN_COST);
16802   format %{ "fadd  $dst,$src1,$src2\t# vector (2D)" %}
16803   ins_encode %{
16804     __ fadd(as_FloatRegister($dst$$reg), __ T2D,
16805             as_FloatRegister($src1$$reg),
16806             as_FloatRegister($src2$$reg));
16807   %}
16808   ins_pipe(vdop_fp128);
16809 %}
16810 
16811 // --------------------------------- SUB --------------------------------------
16812 
16813 instruct vsub8B(vecD dst, vecD src1, vecD src2)
16814 %{
16815   predicate(n->as_Vector()->length() == 4 ||
16816             n->as_Vector()->length() == 8);
16817   match(Set dst (SubVB src1 src2));
16818   ins_cost(INSN_COST);
16819   format %{ "subv  $dst,$src1,$src2\t# vector (8B)" %}
16820   ins_encode %{
16821     __ subv(as_FloatRegister($dst$$reg), __ T8B,
16822             as_FloatRegister($src1$$reg),
16823             as_FloatRegister($src2$$reg));
16824   %}
16825   ins_pipe(vdop64);
16826 %}
16827 
16828 instruct vsub16B(vecX dst, vecX src1, vecX src2)
16829 %{
16830   predicate(n->as_Vector()->length() == 16);
16831   match(Set dst (SubVB src1 src2));
16832   ins_cost(INSN_COST);
16833   format %{ "subv  $dst,$src1,$src2\t# vector (16B)" %}
16834   ins_encode %{
16835     __ subv(as_FloatRegister($dst$$reg), __ T16B,
16836             as_FloatRegister($src1$$reg),
16837             as_FloatRegister($src2$$reg));
16838   %}
16839   ins_pipe(vdop128);
16840 %}
16841 
16842 instruct vsub4S(vecD dst, vecD src1, vecD src2)
16843 %{
16844   predicate(n->as_Vector()->length() == 2 ||
16845             n->as_Vector()->length() == 4);
16846   match(Set dst (SubVS src1 src2));
16847   ins_cost(INSN_COST);
16848   format %{ "subv  $dst,$src1,$src2\t# vector (4H)" %}
16849   ins_encode %{
16850     __ subv(as_FloatRegister($dst$$reg), __ T4H,
16851             as_FloatRegister($src1$$reg),
16852             as_FloatRegister($src2$$reg));
16853   %}
16854   ins_pipe(vdop64);
16855 %}
16856 
16857 instruct vsub8S(vecX dst, vecX src1, vecX src2)
16858 %{
16859   predicate(n->as_Vector()->length() == 8);
16860   match(Set dst (SubVS src1 src2));
16861   ins_cost(INSN_COST);
16862   format %{ "subv  $dst,$src1,$src2\t# vector (8H)" %}
16863   ins_encode %{
16864     __ subv(as_FloatRegister($dst$$reg), __ T8H,
16865             as_FloatRegister($src1$$reg),
16866             as_FloatRegister($src2$$reg));
16867   %}
16868   ins_pipe(vdop128);
16869 %}
16870 
16871 instruct vsub2I(vecD dst, vecD src1, vecD src2)
16872 %{
16873   predicate(n->as_Vector()->length() == 2);
16874   match(Set dst (SubVI src1 src2));
16875   ins_cost(INSN_COST);
16876   format %{ "subv  $dst,$src1,$src2\t# vector (2S)" %}
16877   ins_encode %{
16878     __ subv(as_FloatRegister($dst$$reg), __ T2S,
16879             as_FloatRegister($src1$$reg),
16880             as_FloatRegister($src2$$reg));
16881   %}
16882   ins_pipe(vdop64);
16883 %}
16884 
16885 instruct vsub4I(vecX dst, vecX src1, vecX src2)
16886 %{
16887   predicate(n->as_Vector()->length() == 4);
16888   match(Set dst (SubVI src1 src2));
16889   ins_cost(INSN_COST);
16890   format %{ "subv  $dst,$src1,$src2\t# vector (4S)" %}
16891   ins_encode %{
16892     __ subv(as_FloatRegister($dst$$reg), __ T4S,
16893             as_FloatRegister($src1$$reg),
16894             as_FloatRegister($src2$$reg));
16895   %}
16896   ins_pipe(vdop128);
16897 %}
16898 
16899 instruct vsub2L(vecX dst, vecX src1, vecX src2)
16900 %{
16901   predicate(n->as_Vector()->length() == 2);
16902   match(Set dst (SubVL src1 src2));
16903   ins_cost(INSN_COST);
16904   format %{ "subv  $dst,$src1,$src2\t# vector (2L)" %}
16905   ins_encode %{
16906     __ subv(as_FloatRegister($dst$$reg), __ T2D,
16907             as_FloatRegister($src1$$reg),
16908             as_FloatRegister($src2$$reg));
16909   %}
16910   ins_pipe(vdop128);
16911 %}
16912 
16913 instruct vsub2F(vecD dst, vecD src1, vecD src2)
16914 %{
16915   predicate(n->as_Vector()->length() == 2);
16916   match(Set dst (SubVF src1 src2));
16917   ins_cost(INSN_COST);
16918   format %{ "fsub  $dst,$src1,$src2\t# vector (2S)" %}
16919   ins_encode %{
16920     __ fsub(as_FloatRegister($dst$$reg), __ T2S,
16921             as_FloatRegister($src1$$reg),
16922             as_FloatRegister($src2$$reg));
16923   %}
16924   ins_pipe(vdop_fp64);
16925 %}
16926 
16927 instruct vsub4F(vecX dst, vecX src1, vecX src2)
16928 %{
16929   predicate(n->as_Vector()->length() == 4);
16930   match(Set dst (SubVF src1 src2));
16931   ins_cost(INSN_COST);
16932   format %{ "fsub  $dst,$src1,$src2\t# vector (4S)" %}
16933   ins_encode %{
16934     __ fsub(as_FloatRegister($dst$$reg), __ T4S,
16935             as_FloatRegister($src1$$reg),
16936             as_FloatRegister($src2$$reg));
16937   %}
16938   ins_pipe(vdop_fp128);
16939 %}
16940 
16941 instruct vsub2D(vecX dst, vecX src1, vecX src2)
16942 %{
16943   predicate(n->as_Vector()->length() == 2);
16944   match(Set dst (SubVD src1 src2));
16945   ins_cost(INSN_COST);
16946   format %{ "fsub  $dst,$src1,$src2\t# vector (2D)" %}
16947   ins_encode %{
16948     __ fsub(as_FloatRegister($dst$$reg), __ T2D,
16949             as_FloatRegister($src1$$reg),
16950             as_FloatRegister($src2$$reg));
16951   %}
16952   ins_pipe(vdop_fp128);
16953 %}
16954 
16955 // --------------------------------- MUL --------------------------------------
16956 
16957 instruct vmul8B(vecD dst, vecD src1, vecD src2)
16958 %{
16959   predicate(n->as_Vector()->length() == 4 ||
16960             n->as_Vector()->length() == 8);
16961   match(Set dst (MulVB src1 src2));
16962   ins_cost(INSN_COST);
16963   format %{ "mulv  $dst,$src1,$src2\t# vector (8B)" %}
16964   ins_encode %{
16965     __ mulv(as_FloatRegister($dst$$reg), __ T8B,
16966             as_FloatRegister($src1$$reg),
16967             as_FloatRegister($src2$$reg));
16968   %}
16969   ins_pipe(vmul64);
16970 %}
16971 
16972 instruct vmul16B(vecX dst, vecX src1, vecX src2)
16973 %{
16974   predicate(n->as_Vector()->length() == 16);
16975   match(Set dst (MulVB src1 src2));
16976   ins_cost(INSN_COST);
16977   format %{ "mulv  $dst,$src1,$src2\t# vector (16B)" %}
16978   ins_encode %{
16979     __ mulv(as_FloatRegister($dst$$reg), __ T16B,
16980             as_FloatRegister($src1$$reg),
16981             as_FloatRegister($src2$$reg));
16982   %}
16983   ins_pipe(vmul128);
16984 %}
16985 
16986 instruct vmul4S(vecD dst, vecD src1, vecD src2)
16987 %{
16988   predicate(n->as_Vector()->length() == 2 ||
16989             n->as_Vector()->length() == 4);
16990   match(Set dst (MulVS src1 src2));
16991   ins_cost(INSN_COST);
16992   format %{ "mulv  $dst,$src1,$src2\t# vector (4H)" %}
16993   ins_encode %{
16994     __ mulv(as_FloatRegister($dst$$reg), __ T4H,
16995             as_FloatRegister($src1$$reg),
16996             as_FloatRegister($src2$$reg));
16997   %}
16998   ins_pipe(vmul64);
16999 %}
17000 
17001 instruct vmul8S(vecX dst, vecX src1, vecX src2)
17002 %{
17003   predicate(n->as_Vector()->length() == 8);
17004   match(Set dst (MulVS src1 src2));
17005   ins_cost(INSN_COST);
17006   format %{ "mulv  $dst,$src1,$src2\t# vector (8H)" %}
17007   ins_encode %{
17008     __ mulv(as_FloatRegister($dst$$reg), __ T8H,
17009             as_FloatRegister($src1$$reg),
17010             as_FloatRegister($src2$$reg));
17011   %}
17012   ins_pipe(vmul128);
17013 %}
17014 
17015 instruct vmul2I(vecD dst, vecD src1, vecD src2)
17016 %{
17017   predicate(n->as_Vector()->length() == 2);
17018   match(Set dst (MulVI src1 src2));
17019   ins_cost(INSN_COST);
17020   format %{ "mulv  $dst,$src1,$src2\t# vector (2S)" %}
17021   ins_encode %{
17022     __ mulv(as_FloatRegister($dst$$reg), __ T2S,
17023             as_FloatRegister($src1$$reg),
17024             as_FloatRegister($src2$$reg));
17025   %}
17026   ins_pipe(vmul64);
17027 %}
17028 
17029 instruct vmul4I(vecX dst, vecX src1, vecX src2)
17030 %{
17031   predicate(n->as_Vector()->length() == 4);
17032   match(Set dst (MulVI src1 src2));
17033   ins_cost(INSN_COST);
17034   format %{ "mulv  $dst,$src1,$src2\t# vector (4S)" %}
17035   ins_encode %{
17036     __ mulv(as_FloatRegister($dst$$reg), __ T4S,
17037             as_FloatRegister($src1$$reg),
17038             as_FloatRegister($src2$$reg));
17039   %}
17040   ins_pipe(vmul128);
17041 %}
17042 
17043 instruct vmul2F(vecD dst, vecD src1, vecD src2)
17044 %{
17045   predicate(n->as_Vector()->length() == 2);
17046   match(Set dst (MulVF src1 src2));
17047   ins_cost(INSN_COST);
17048   format %{ "fmul  $dst,$src1,$src2\t# vector (2S)" %}
17049   ins_encode %{
17050     __ fmul(as_FloatRegister($dst$$reg), __ T2S,
17051             as_FloatRegister($src1$$reg),
17052             as_FloatRegister($src2$$reg));
17053   %}
17054   ins_pipe(vmuldiv_fp64);
17055 %}
17056 
17057 instruct vmul4F(vecX dst, vecX src1, vecX src2)
17058 %{
17059   predicate(n->as_Vector()->length() == 4);
17060   match(Set dst (MulVF src1 src2));
17061   ins_cost(INSN_COST);
17062   format %{ "fmul  $dst,$src1,$src2\t# vector (4S)" %}
17063   ins_encode %{
17064     __ fmul(as_FloatRegister($dst$$reg), __ T4S,
17065             as_FloatRegister($src1$$reg),
17066             as_FloatRegister($src2$$reg));
17067   %}
17068   ins_pipe(vmuldiv_fp128);
17069 %}
17070 
17071 instruct vmul2D(vecX dst, vecX src1, vecX src2)
17072 %{
17073   predicate(n->as_Vector()->length() == 2);
17074   match(Set dst (MulVD src1 src2));
17075   ins_cost(INSN_COST);
17076   format %{ "fmul  $dst,$src1,$src2\t# vector (2D)" %}
17077   ins_encode %{
17078     __ fmul(as_FloatRegister($dst$$reg), __ T2D,
17079             as_FloatRegister($src1$$reg),
17080             as_FloatRegister($src2$$reg));
17081   %}
17082   ins_pipe(vmuldiv_fp128);
17083 %}
17084 
17085 // --------------------------------- MLA --------------------------------------
17086 
17087 instruct vmla4S(vecD dst, vecD src1, vecD src2)
17088 %{
17089   predicate(n->as_Vector()->length() == 2 ||
17090             n->as_Vector()->length() == 4);
17091   match(Set dst (AddVS dst (MulVS src1 src2)));
17092   ins_cost(INSN_COST);
17093   format %{ "mlav  $dst,$src1,$src2\t# vector (4H)" %}
17094   ins_encode %{
17095     __ mlav(as_FloatRegister($dst$$reg), __ T4H,
17096             as_FloatRegister($src1$$reg),
17097             as_FloatRegister($src2$$reg));
17098   %}
17099   ins_pipe(vmla64);
17100 %}
17101 
17102 instruct vmla8S(vecX dst, vecX src1, vecX src2)
17103 %{
17104   predicate(n->as_Vector()->length() == 8);
17105   match(Set dst (AddVS dst (MulVS src1 src2)));
17106   ins_cost(INSN_COST);
17107   format %{ "mlav  $dst,$src1,$src2\t# vector (8H)" %}
17108   ins_encode %{
17109     __ mlav(as_FloatRegister($dst$$reg), __ T8H,
17110             as_FloatRegister($src1$$reg),
17111             as_FloatRegister($src2$$reg));
17112   %}
17113   ins_pipe(vmla128);
17114 %}
17115 
17116 instruct vmla2I(vecD dst, vecD src1, vecD src2)
17117 %{
17118   predicate(n->as_Vector()->length() == 2);
17119   match(Set dst (AddVI dst (MulVI src1 src2)));
17120   ins_cost(INSN_COST);
17121   format %{ "mlav  $dst,$src1,$src2\t# vector (2S)" %}
17122   ins_encode %{
17123     __ mlav(as_FloatRegister($dst$$reg), __ T2S,
17124             as_FloatRegister($src1$$reg),
17125             as_FloatRegister($src2$$reg));
17126   %}
17127   ins_pipe(vmla64);
17128 %}
17129 
17130 instruct vmla4I(vecX dst, vecX src1, vecX src2)
17131 %{
17132   predicate(n->as_Vector()->length() == 4);
17133   match(Set dst (AddVI dst (MulVI src1 src2)));
17134   ins_cost(INSN_COST);
17135   format %{ "mlav  $dst,$src1,$src2\t# vector (4S)" %}
17136   ins_encode %{
17137     __ mlav(as_FloatRegister($dst$$reg), __ T4S,
17138             as_FloatRegister($src1$$reg),
17139             as_FloatRegister($src2$$reg));
17140   %}
17141   ins_pipe(vmla128);
17142 %}
17143 
17144 // dst + src1 * src2
17145 instruct vmla2F(vecD dst, vecD src1, vecD src2) %{
17146   predicate(UseFMA && n->as_Vector()->length() == 2);
17147   match(Set dst (FmaVF  dst (Binary src1 src2)));
17148   format %{ "fmla  $dst,$src1,$src2\t# vector (2S)" %}
17149   ins_cost(INSN_COST);
17150   ins_encode %{
17151     __ fmla(as_FloatRegister($dst$$reg), __ T2S,
17152             as_FloatRegister($src1$$reg),
17153             as_FloatRegister($src2$$reg));
17154   %}
17155   ins_pipe(vmuldiv_fp64);
17156 %}
17157 
17158 // dst + src1 * src2
17159 instruct vmla4F(vecX dst, vecX src1, vecX src2) %{
17160   predicate(UseFMA && n->as_Vector()->length() == 4);
17161   match(Set dst (FmaVF  dst (Binary src1 src2)));
17162   format %{ "fmla  $dst,$src1,$src2\t# vector (4S)" %}
17163   ins_cost(INSN_COST);
17164   ins_encode %{
17165     __ fmla(as_FloatRegister($dst$$reg), __ T4S,
17166             as_FloatRegister($src1$$reg),
17167             as_FloatRegister($src2$$reg));
17168   %}
17169   ins_pipe(vmuldiv_fp128);
17170 %}
17171 
17172 // dst + src1 * src2
17173 instruct vmla2D(vecX dst, vecX src1, vecX src2) %{
17174   predicate(UseFMA && n->as_Vector()->length() == 2);
17175   match(Set dst (FmaVD  dst (Binary src1 src2)));
17176   format %{ "fmla  $dst,$src1,$src2\t# vector (2D)" %}
17177   ins_cost(INSN_COST);
17178   ins_encode %{
17179     __ fmla(as_FloatRegister($dst$$reg), __ T2D,
17180             as_FloatRegister($src1$$reg),
17181             as_FloatRegister($src2$$reg));
17182   %}
17183   ins_pipe(vmuldiv_fp128);
17184 %}
17185 
17186 // --------------------------------- MLS --------------------------------------
17187 
17188 instruct vmls4S(vecD dst, vecD src1, vecD src2)
17189 %{
17190   predicate(n->as_Vector()->length() == 2 ||
17191             n->as_Vector()->length() == 4);
17192   match(Set dst (SubVS dst (MulVS src1 src2)));
17193   ins_cost(INSN_COST);
17194   format %{ "mlsv  $dst,$src1,$src2\t# vector (4H)" %}
17195   ins_encode %{
17196     __ mlsv(as_FloatRegister($dst$$reg), __ T4H,
17197             as_FloatRegister($src1$$reg),
17198             as_FloatRegister($src2$$reg));
17199   %}
17200   ins_pipe(vmla64);
17201 %}
17202 
17203 instruct vmls8S(vecX dst, vecX src1, vecX src2)
17204 %{
17205   predicate(n->as_Vector()->length() == 8);
17206   match(Set dst (SubVS dst (MulVS src1 src2)));
17207   ins_cost(INSN_COST);
17208   format %{ "mlsv  $dst,$src1,$src2\t# vector (8H)" %}
17209   ins_encode %{
17210     __ mlsv(as_FloatRegister($dst$$reg), __ T8H,
17211             as_FloatRegister($src1$$reg),
17212             as_FloatRegister($src2$$reg));
17213   %}
17214   ins_pipe(vmla128);
17215 %}
17216 
17217 instruct vmls2I(vecD dst, vecD src1, vecD src2)
17218 %{
17219   predicate(n->as_Vector()->length() == 2);
17220   match(Set dst (SubVI dst (MulVI src1 src2)));
17221   ins_cost(INSN_COST);
17222   format %{ "mlsv  $dst,$src1,$src2\t# vector (2S)" %}
17223   ins_encode %{
17224     __ mlsv(as_FloatRegister($dst$$reg), __ T2S,
17225             as_FloatRegister($src1$$reg),
17226             as_FloatRegister($src2$$reg));
17227   %}
17228   ins_pipe(vmla64);
17229 %}
17230 
17231 instruct vmls4I(vecX dst, vecX src1, vecX src2)
17232 %{
17233   predicate(n->as_Vector()->length() == 4);
17234   match(Set dst (SubVI dst (MulVI src1 src2)));
17235   ins_cost(INSN_COST);
17236   format %{ "mlsv  $dst,$src1,$src2\t# vector (4S)" %}
17237   ins_encode %{
17238     __ mlsv(as_FloatRegister($dst$$reg), __ T4S,
17239             as_FloatRegister($src1$$reg),
17240             as_FloatRegister($src2$$reg));
17241   %}
17242   ins_pipe(vmla128);
17243 %}
17244 
17245 // dst - src1 * src2
17246 instruct vmls2F(vecD dst, vecD src1, vecD src2) %{
17247   predicate(UseFMA && n->as_Vector()->length() == 2);
17248   match(Set dst (FmaVF  dst (Binary (NegVF src1) src2)));
17249   match(Set dst (FmaVF  dst (Binary src1 (NegVF src2))));
17250   format %{ "fmls  $dst,$src1,$src2\t# vector (2S)" %}
17251   ins_cost(INSN_COST);
17252   ins_encode %{
17253     __ fmls(as_FloatRegister($dst$$reg), __ T2S,
17254             as_FloatRegister($src1$$reg),
17255             as_FloatRegister($src2$$reg));
17256   %}
17257   ins_pipe(vmuldiv_fp64);
17258 %}
17259 
17260 // dst - src1 * src2
17261 instruct vmls4F(vecX dst, vecX src1, vecX src2) %{
17262   predicate(UseFMA && n->as_Vector()->length() == 4);
17263   match(Set dst (FmaVF  dst (Binary (NegVF src1) src2)));
17264   match(Set dst (FmaVF  dst (Binary src1 (NegVF src2))));
17265   format %{ "fmls  $dst,$src1,$src2\t# vector (4S)" %}
17266   ins_cost(INSN_COST);
17267   ins_encode %{
17268     __ fmls(as_FloatRegister($dst$$reg), __ T4S,
17269             as_FloatRegister($src1$$reg),
17270             as_FloatRegister($src2$$reg));
17271   %}
17272   ins_pipe(vmuldiv_fp128);
17273 %}
17274 
17275 // dst - src1 * src2
17276 instruct vmls2D(vecX dst, vecX src1, vecX src2) %{
17277   predicate(UseFMA && n->as_Vector()->length() == 2);
17278   match(Set dst (FmaVD  dst (Binary (NegVD src1) src2)));
17279   match(Set dst (FmaVD  dst (Binary src1 (NegVD src2))));
17280   format %{ "fmls  $dst,$src1,$src2\t# vector (2D)" %}
17281   ins_cost(INSN_COST);
17282   ins_encode %{
17283     __ fmls(as_FloatRegister($dst$$reg), __ T2D,
17284             as_FloatRegister($src1$$reg),
17285             as_FloatRegister($src2$$reg));
17286   %}
17287   ins_pipe(vmuldiv_fp128);
17288 %}
17289 
17290 // --------------- Vector Multiply-Add Shorts into Integer --------------------
17291 
17292 instruct vmuladdS2I(vecX dst, vecX src1, vecX src2, vecX tmp) %{
17293   predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
17294   match(Set dst (MulAddVS2VI src1 src2));
17295   ins_cost(INSN_COST);
17296   effect(TEMP_DEF dst, TEMP tmp);
17297   format %{ "smullv  $tmp, $src1, $src2\t# vector (4H)\n\t"
17298             "smullv  $dst, $src1, $src2\t# vector (8H)\n\t"
17299             "addpv   $dst, $tmp, $dst\t# vector (4S)\n\t" %}
17300   ins_encode %{
17301     __ smullv(as_FloatRegister($tmp$$reg), __ T4H,
17302               as_FloatRegister($src1$$reg),
17303               as_FloatRegister($src2$$reg));
17304     __ smullv(as_FloatRegister($dst$$reg), __ T8H,
17305               as_FloatRegister($src1$$reg),
17306               as_FloatRegister($src2$$reg));
17307     __ addpv(as_FloatRegister($dst$$reg), __ T4S,
17308              as_FloatRegister($tmp$$reg),
17309              as_FloatRegister($dst$$reg));
17310   %}
17311   ins_pipe(vmuldiv_fp128);
17312 %}
17313 
17314 // --------------------------------- DIV --------------------------------------
17315 
17316 instruct vdiv2F(vecD dst, vecD src1, vecD src2)
17317 %{
17318   predicate(n->as_Vector()->length() == 2);
17319   match(Set dst (DivVF src1 src2));
17320   ins_cost(INSN_COST);
17321   format %{ "fdiv  $dst,$src1,$src2\t# vector (2S)" %}
17322   ins_encode %{
17323     __ fdiv(as_FloatRegister($dst$$reg), __ T2S,
17324             as_FloatRegister($src1$$reg),
17325             as_FloatRegister($src2$$reg));
17326   %}
17327   ins_pipe(vmuldiv_fp64);
17328 %}
17329 
17330 instruct vdiv4F(vecX dst, vecX src1, vecX src2)
17331 %{
17332   predicate(n->as_Vector()->length() == 4);
17333   match(Set dst (DivVF src1 src2));
17334   ins_cost(INSN_COST);
17335   format %{ "fdiv  $dst,$src1,$src2\t# vector (4S)" %}
17336   ins_encode %{
17337     __ fdiv(as_FloatRegister($dst$$reg), __ T4S,
17338             as_FloatRegister($src1$$reg),
17339             as_FloatRegister($src2$$reg));
17340   %}
17341   ins_pipe(vmuldiv_fp128);
17342 %}
17343 
17344 instruct vdiv2D(vecX dst, vecX src1, vecX src2)
17345 %{
17346   predicate(n->as_Vector()->length() == 2);
17347   match(Set dst (DivVD src1 src2));
17348   ins_cost(INSN_COST);
17349   format %{ "fdiv  $dst,$src1,$src2\t# vector (2D)" %}
17350   ins_encode %{
17351     __ fdiv(as_FloatRegister($dst$$reg), __ T2D,
17352             as_FloatRegister($src1$$reg),
17353             as_FloatRegister($src2$$reg));
17354   %}
17355   ins_pipe(vmuldiv_fp128);
17356 %}
17357 
17358 // --------------------------------- SQRT -------------------------------------
17359 
17360 instruct vsqrt2F(vecD dst, vecD src)
17361 %{
17362   predicate(n->as_Vector()->length() == 2);
17363   match(Set dst (SqrtVF src));
17364   format %{ "fsqrt  $dst, $src\t# vector (2F)" %}
17365   ins_encode %{
17366     __ fsqrt(as_FloatRegister($dst$$reg), __ T2S, as_FloatRegister($src$$reg));
17367   %}
17368   ins_pipe(vunop_fp64);
17369 %}
17370 
17371 instruct vsqrt4F(vecX dst, vecX src)
17372 %{
17373   predicate(n->as_Vector()->length() == 4);
17374   match(Set dst (SqrtVF src));
17375   format %{ "fsqrt  $dst, $src\t# vector (4F)" %}
17376   ins_encode %{
17377     __ fsqrt(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src$$reg));
17378   %}
17379   ins_pipe(vsqrt_fp128);
17380 %}
17381 
17382 instruct vsqrt2D(vecX dst, vecX src)
17383 %{
17384   predicate(n->as_Vector()->length() == 2);
17385   match(Set dst (SqrtVD src));
17386   format %{ "fsqrt  $dst, $src\t# vector (2D)" %}
17387   ins_encode %{
17388     __ fsqrt(as_FloatRegister($dst$$reg), __ T2D,
17389              as_FloatRegister($src$$reg));
17390   %}
17391   ins_pipe(vsqrt_fp128);
17392 %}
17393 
17394 // --------------------------------- ABS --------------------------------------
17395 
17396 instruct vabs8B(vecD dst, vecD src)
17397 %{
17398   predicate(n->as_Vector()->length() == 4 ||
17399             n->as_Vector()->length() == 8);
17400   match(Set dst (AbsVB src));
17401   ins_cost(INSN_COST);
17402   format %{ "abs  $dst, $src\t# vector (8B)" %}
17403   ins_encode %{
17404     __ absr(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($src$$reg));
17405   %}
17406   ins_pipe(vlogical64);
17407 %}
17408 
17409 instruct vabs16B(vecX dst, vecX src)
17410 %{
17411   predicate(n->as_Vector()->length() == 16);
17412   match(Set dst (AbsVB src));
17413   ins_cost(INSN_COST);
17414   format %{ "abs  $dst, $src\t# vector (16B)" %}
17415   ins_encode %{
17416     __ absr(as_FloatRegister($dst$$reg), __ T16B, as_FloatRegister($src$$reg));
17417   %}
17418   ins_pipe(vlogical128);
17419 %}
17420 
17421 instruct vabs4S(vecD dst, vecD src)
17422 %{
17423   predicate(n->as_Vector()->length() == 4);
17424   match(Set dst (AbsVS src));
17425   ins_cost(INSN_COST);
17426   format %{ "abs  $dst, $src\t# vector (4H)" %}
17427   ins_encode %{
17428     __ absr(as_FloatRegister($dst$$reg), __ T4H, as_FloatRegister($src$$reg));
17429   %}
17430   ins_pipe(vlogical64);
17431 %}
17432 
17433 instruct vabs8S(vecX dst, vecX src)
17434 %{
17435   predicate(n->as_Vector()->length() == 8);
17436   match(Set dst (AbsVS src));
17437   ins_cost(INSN_COST);
17438   format %{ "abs  $dst, $src\t# vector (8H)" %}
17439   ins_encode %{
17440     __ absr(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg));
17441   %}
17442   ins_pipe(vlogical128);
17443 %}
17444 
17445 instruct vabs2I(vecD dst, vecD src)
17446 %{
17447   predicate(n->as_Vector()->length() == 2);
17448   match(Set dst (AbsVI src));
17449   ins_cost(INSN_COST);
17450   format %{ "abs  $dst, $src\t# vector (2S)" %}
17451   ins_encode %{
17452     __ absr(as_FloatRegister($dst$$reg), __ T2S, as_FloatRegister($src$$reg));
17453   %}
17454   ins_pipe(vlogical64);
17455 %}
17456 
17457 instruct vabs4I(vecX dst, vecX src)
17458 %{
17459   predicate(n->as_Vector()->length() == 4);
17460   match(Set dst (AbsVI src));
17461   ins_cost(INSN_COST);
17462   format %{ "abs  $dst, $src\t# vector (4S)" %}
17463   ins_encode %{
17464     __ absr(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src$$reg));
17465   %}
17466   ins_pipe(vlogical128);
17467 %}
17468 
17469 instruct vabs2L(vecX dst, vecX src)
17470 %{
17471   predicate(n->as_Vector()->length() == 2);
17472   match(Set dst (AbsVL src));
17473   ins_cost(INSN_COST);
17474   format %{ "abs  $dst, $src\t# vector (2D)" %}
17475   ins_encode %{
17476     __ absr(as_FloatRegister($dst$$reg), __ T2D, as_FloatRegister($src$$reg));
17477   %}
17478   ins_pipe(vlogical128);
17479 %}
17480 
17481 instruct vabs2F(vecD dst, vecD src)
17482 %{
17483   predicate(n->as_Vector()->length() == 2);
17484   match(Set dst (AbsVF src));
17485   ins_cost(INSN_COST * 3);
17486   format %{ "fabs  $dst,$src\t# vector (2S)" %}
17487   ins_encode %{
17488     __ fabs(as_FloatRegister($dst$$reg), __ T2S,
17489             as_FloatRegister($src$$reg));
17490   %}
17491   ins_pipe(vunop_fp64);
17492 %}
17493 
17494 instruct vabs4F(vecX dst, vecX src)
17495 %{
17496   predicate(n->as_Vector()->length() == 4);
17497   match(Set dst (AbsVF src));
17498   ins_cost(INSN_COST * 3);
17499   format %{ "fabs  $dst,$src\t# vector (4S)" %}
17500   ins_encode %{
17501     __ fabs(as_FloatRegister($dst$$reg), __ T4S,
17502             as_FloatRegister($src$$reg));
17503   %}
17504   ins_pipe(vunop_fp128);
17505 %}
17506 
17507 instruct vabs2D(vecX dst, vecX src)
17508 %{
17509   predicate(n->as_Vector()->length() == 2);
17510   match(Set dst (AbsVD src));
17511   ins_cost(INSN_COST * 3);
17512   format %{ "fabs  $dst,$src\t# vector (2D)" %}
17513   ins_encode %{
17514     __ fabs(as_FloatRegister($dst$$reg), __ T2D,
17515             as_FloatRegister($src$$reg));
17516   %}
17517   ins_pipe(vunop_fp128);
17518 %}
17519 
17520 // --------------------------------- NEG --------------------------------------
17521 
17522 instruct vneg2F(vecD dst, vecD src)
17523 %{
17524   predicate(n->as_Vector()->length() == 2);
17525   match(Set dst (NegVF src));
17526   ins_cost(INSN_COST * 3);
17527   format %{ "fneg  $dst,$src\t# vector (2S)" %}
17528   ins_encode %{
17529     __ fneg(as_FloatRegister($dst$$reg), __ T2S,
17530             as_FloatRegister($src$$reg));
17531   %}
17532   ins_pipe(vunop_fp64);
17533 %}
17534 
17535 instruct vneg4F(vecX dst, vecX src)
17536 %{
17537   predicate(n->as_Vector()->length() == 4);
17538   match(Set dst (NegVF src));
17539   ins_cost(INSN_COST * 3);
17540   format %{ "fneg  $dst,$src\t# vector (4S)" %}
17541   ins_encode %{
17542     __ fneg(as_FloatRegister($dst$$reg), __ T4S,
17543             as_FloatRegister($src$$reg));
17544   %}
17545   ins_pipe(vunop_fp128);
17546 %}
17547 
17548 instruct vneg2D(vecX dst, vecX src)
17549 %{
17550   predicate(n->as_Vector()->length() == 2);
17551   match(Set dst (NegVD src));
17552   ins_cost(INSN_COST * 3);
17553   format %{ "fneg  $dst,$src\t# vector (2D)" %}
17554   ins_encode %{
17555     __ fneg(as_FloatRegister($dst$$reg), __ T2D,
17556             as_FloatRegister($src$$reg));
17557   %}
17558   ins_pipe(vunop_fp128);
17559 %}
17560 
17561 // --------------------------------- AND --------------------------------------
17562 
17563 instruct vand8B(vecD dst, vecD src1, vecD src2)
17564 %{
17565   predicate(n->as_Vector()->length_in_bytes() == 4 ||
17566             n->as_Vector()->length_in_bytes() == 8);
17567   match(Set dst (AndV src1 src2));
17568   ins_cost(INSN_COST);
17569   format %{ "and  $dst,$src1,$src2\t# vector (8B)" %}
17570   ins_encode %{
17571     __ andr(as_FloatRegister($dst$$reg), __ T8B,
17572             as_FloatRegister($src1$$reg),
17573             as_FloatRegister($src2$$reg));
17574   %}
17575   ins_pipe(vlogical64);
17576 %}
17577 
17578 instruct vand16B(vecX dst, vecX src1, vecX src2)
17579 %{
17580   predicate(n->as_Vector()->length_in_bytes() == 16);
17581   match(Set dst (AndV src1 src2));
17582   ins_cost(INSN_COST);
17583   format %{ "and  $dst,$src1,$src2\t# vector (16B)" %}
17584   ins_encode %{
17585     __ andr(as_FloatRegister($dst$$reg), __ T16B,
17586             as_FloatRegister($src1$$reg),
17587             as_FloatRegister($src2$$reg));
17588   %}
17589   ins_pipe(vlogical128);
17590 %}
17591 
17592 // --------------------------------- OR ---------------------------------------
17593 
17594 instruct vor8B(vecD dst, vecD src1, vecD src2)
17595 %{
17596   predicate(n->as_Vector()->length_in_bytes() == 4 ||
17597             n->as_Vector()->length_in_bytes() == 8);
17598   match(Set dst (OrV src1 src2));
17599   ins_cost(INSN_COST);
17600   format %{ "and  $dst,$src1,$src2\t# vector (8B)" %}
17601   ins_encode %{
17602     __ orr(as_FloatRegister($dst$$reg), __ T8B,
17603             as_FloatRegister($src1$$reg),
17604             as_FloatRegister($src2$$reg));
17605   %}
17606   ins_pipe(vlogical64);
17607 %}
17608 
17609 instruct vor16B(vecX dst, vecX src1, vecX src2)
17610 %{
17611   predicate(n->as_Vector()->length_in_bytes() == 16);
17612   match(Set dst (OrV src1 src2));
17613   ins_cost(INSN_COST);
17614   format %{ "orr  $dst,$src1,$src2\t# vector (16B)" %}
17615   ins_encode %{
17616     __ orr(as_FloatRegister($dst$$reg), __ T16B,
17617             as_FloatRegister($src1$$reg),
17618             as_FloatRegister($src2$$reg));
17619   %}
17620   ins_pipe(vlogical128);
17621 %}
17622 
17623 // --------------------------------- XOR --------------------------------------
17624 
17625 instruct vxor8B(vecD dst, vecD src1, vecD src2)
17626 %{
17627   predicate(n->as_Vector()->length_in_bytes() == 4 ||
17628             n->as_Vector()->length_in_bytes() == 8);
17629   match(Set dst (XorV src1 src2));
17630   ins_cost(INSN_COST);
17631   format %{ "xor  $dst,$src1,$src2\t# vector (8B)" %}
17632   ins_encode %{
17633     __ eor(as_FloatRegister($dst$$reg), __ T8B,
17634             as_FloatRegister($src1$$reg),
17635             as_FloatRegister($src2$$reg));
17636   %}
17637   ins_pipe(vlogical64);
17638 %}
17639 
17640 instruct vxor16B(vecX dst, vecX src1, vecX src2)
17641 %{
17642   predicate(n->as_Vector()->length_in_bytes() == 16);
17643   match(Set dst (XorV src1 src2));
17644   ins_cost(INSN_COST);
17645   format %{ "xor  $dst,$src1,$src2\t# vector (16B)" %}
17646   ins_encode %{
17647     __ eor(as_FloatRegister($dst$$reg), __ T16B,
17648             as_FloatRegister($src1$$reg),
17649             as_FloatRegister($src2$$reg));
17650   %}
17651   ins_pipe(vlogical128);
17652 %}
17653 
17654 // ------------------------------ Shift ---------------------------------------
17655 instruct vshiftcnt8B(vecD dst, iRegIorL2I cnt) %{
17656   predicate(n->as_Vector()->length_in_bytes() == 8);
17657   match(Set dst (LShiftCntV cnt));
17658   match(Set dst (RShiftCntV cnt));
17659   format %{ "dup  $dst, $cnt\t# shift count vector (8B)" %}
17660   ins_encode %{
17661     __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($cnt$$reg));
17662   %}
17663   ins_pipe(vdup_reg_reg64);
17664 %}
17665 
17666 instruct vshiftcnt16B(vecX dst, iRegIorL2I cnt) %{
17667   predicate(n->as_Vector()->length_in_bytes() == 16);
17668   match(Set dst (LShiftCntV cnt));
17669   match(Set dst (RShiftCntV cnt));
17670   format %{ "dup  $dst, $cnt\t# shift count vector (16B)" %}
17671   ins_encode %{
17672     __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($cnt$$reg));
17673   %}
17674   ins_pipe(vdup_reg_reg128);
17675 %}
17676 
17677 instruct vsll8B(vecD dst, vecD src, vecD shift) %{
17678   predicate(n->as_Vector()->length() == 4 ||
17679             n->as_Vector()->length() == 8);
17680   match(Set dst (LShiftVB src shift));
17681   ins_cost(INSN_COST);
17682   format %{ "sshl  $dst,$src,$shift\t# vector (8B)" %}
17683   ins_encode %{
17684     __ sshl(as_FloatRegister($dst$$reg), __ T8B,
17685             as_FloatRegister($src$$reg),
17686             as_FloatRegister($shift$$reg));
17687   %}
17688   ins_pipe(vshift64);
17689 %}
17690 
17691 instruct vsll16B(vecX dst, vecX src, vecX shift) %{
17692   predicate(n->as_Vector()->length() == 16);
17693   match(Set dst (LShiftVB src shift));
17694   ins_cost(INSN_COST);
17695   format %{ "sshl  $dst,$src,$shift\t# vector (16B)" %}
17696   ins_encode %{
17697     __ sshl(as_FloatRegister($dst$$reg), __ T16B,
17698             as_FloatRegister($src$$reg),
17699             as_FloatRegister($shift$$reg));
17700   %}
17701   ins_pipe(vshift128);
17702 %}
17703 
17704 // Right shifts with vector shift count on aarch64 SIMD are implemented
17705 // as left shift by negative shift count.
17706 // There are two cases for vector shift count.
17707 //
17708 // Case 1: The vector shift count is from replication.
17709 //        |            |
17710 //    LoadVector  RShiftCntV
17711 //        |       /
17712 //     RShiftVI
17713 // Note: In inner loop, multiple neg instructions are used, which can be
17714 // moved to outer loop and merge into one neg instruction.
17715 //
17716 // Case 2: The vector shift count is from loading.
17717 // This case isn't supported by middle-end now. But it's supported by
17718 // panama/vectorIntrinsics(JEP 338: Vector API).
17719 //        |            |
17720 //    LoadVector  LoadVector
17721 //        |       /
17722 //     RShiftVI
17723 //
17724 
17725 instruct vsra8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
17726   predicate(n->as_Vector()->length() == 4 ||
17727             n->as_Vector()->length() == 8);
17728   match(Set dst (RShiftVB src shift));
17729   ins_cost(INSN_COST);
17730   effect(TEMP tmp);
17731   format %{ "negr  $tmp,$shift\t"
17732             "sshl  $dst,$src,$tmp\t# vector (8B)" %}
17733   ins_encode %{
17734     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17735             as_FloatRegister($shift$$reg));
17736     __ sshl(as_FloatRegister($dst$$reg), __ T8B,
17737             as_FloatRegister($src$$reg),
17738             as_FloatRegister($tmp$$reg));
17739   %}
17740   ins_pipe(vshift64);
17741 %}
17742 
17743 instruct vsra16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
17744   predicate(n->as_Vector()->length() == 16);
17745   match(Set dst (RShiftVB src shift));
17746   ins_cost(INSN_COST);
17747   effect(TEMP tmp);
17748   format %{ "negr  $tmp,$shift\t"
17749             "sshl  $dst,$src,$tmp\t# vector (16B)" %}
17750   ins_encode %{
17751     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17752             as_FloatRegister($shift$$reg));
17753     __ sshl(as_FloatRegister($dst$$reg), __ T16B,
17754             as_FloatRegister($src$$reg),
17755             as_FloatRegister($tmp$$reg));
17756   %}
17757   ins_pipe(vshift128);
17758 %}
17759 
17760 instruct vsrl8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
17761   predicate(n->as_Vector()->length() == 4 ||
17762             n->as_Vector()->length() == 8);
17763   match(Set dst (URShiftVB src shift));
17764   ins_cost(INSN_COST);
17765   effect(TEMP tmp);
17766   format %{ "negr  $tmp,$shift\t"
17767             "ushl  $dst,$src,$tmp\t# vector (8B)" %}
17768   ins_encode %{
17769     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17770             as_FloatRegister($shift$$reg));
17771     __ ushl(as_FloatRegister($dst$$reg), __ T8B,
17772             as_FloatRegister($src$$reg),
17773             as_FloatRegister($tmp$$reg));
17774   %}
17775   ins_pipe(vshift64);
17776 %}
17777 
17778 instruct vsrl16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
17779   predicate(n->as_Vector()->length() == 16);
17780   match(Set dst (URShiftVB src shift));
17781   ins_cost(INSN_COST);
17782   effect(TEMP tmp);
17783   format %{ "negr  $tmp,$shift\t"
17784             "ushl  $dst,$src,$tmp\t# vector (16B)" %}
17785   ins_encode %{
17786     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17787             as_FloatRegister($shift$$reg));
17788     __ ushl(as_FloatRegister($dst$$reg), __ T16B,
17789             as_FloatRegister($src$$reg),
17790             as_FloatRegister($tmp$$reg));
17791   %}
17792   ins_pipe(vshift128);
17793 %}
17794 
17795 instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
17796   predicate(n->as_Vector()->length() == 4 ||
17797             n->as_Vector()->length() == 8);
17798   match(Set dst (LShiftVB src (LShiftCntV shift)));
17799   ins_cost(INSN_COST);
17800   format %{ "shl    $dst, $src, $shift\t# vector (8B)" %}
17801   ins_encode %{
17802     int sh = (int)$shift$$constant;
17803     if (sh >= 8) {
17804       __ eor(as_FloatRegister($dst$$reg), __ T8B,
17805              as_FloatRegister($src$$reg),
17806              as_FloatRegister($src$$reg));
17807     } else {
17808       __ shl(as_FloatRegister($dst$$reg), __ T8B,
17809              as_FloatRegister($src$$reg), sh);
17810     }
17811   %}
17812   ins_pipe(vshift64_imm);
17813 %}
17814 
17815 instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
17816   predicate(n->as_Vector()->length() == 16);
17817   match(Set dst (LShiftVB src (LShiftCntV shift)));
17818   ins_cost(INSN_COST);
17819   format %{ "shl    $dst, $src, $shift\t# vector (16B)" %}
17820   ins_encode %{
17821     int sh = (int)$shift$$constant;
17822     if (sh >= 8) {
17823       __ eor(as_FloatRegister($dst$$reg), __ T16B,
17824              as_FloatRegister($src$$reg),
17825              as_FloatRegister($src$$reg));
17826     } else {
17827       __ shl(as_FloatRegister($dst$$reg), __ T16B,
17828              as_FloatRegister($src$$reg), sh);
17829     }
17830   %}
17831   ins_pipe(vshift128_imm);
17832 %}
17833 
17834 instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{
17835   predicate(n->as_Vector()->length() == 4 ||
17836             n->as_Vector()->length() == 8);
17837   match(Set dst (RShiftVB src (RShiftCntV shift)));
17838   ins_cost(INSN_COST);
17839   format %{ "sshr    $dst, $src, $shift\t# vector (8B)" %}
17840   ins_encode %{
17841     int sh = (int)$shift$$constant;
17842     if (sh >= 8) sh = 7;
17843     __ sshr(as_FloatRegister($dst$$reg), __ T8B,
17844            as_FloatRegister($src$$reg), sh);
17845   %}
17846   ins_pipe(vshift64_imm);
17847 %}
17848 
17849 instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
17850   predicate(n->as_Vector()->length() == 16);
17851   match(Set dst (RShiftVB src (RShiftCntV shift)));
17852   ins_cost(INSN_COST);
17853   format %{ "sshr    $dst, $src, $shift\t# vector (16B)" %}
17854   ins_encode %{
17855     int sh = (int)$shift$$constant;
17856     if (sh >= 8) sh = 7;
17857     __ sshr(as_FloatRegister($dst$$reg), __ T16B,
17858            as_FloatRegister($src$$reg), sh);
17859   %}
17860   ins_pipe(vshift128_imm);
17861 %}
17862 
17863 instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{
17864   predicate(n->as_Vector()->length() == 4 ||
17865             n->as_Vector()->length() == 8);
17866   match(Set dst (URShiftVB src (RShiftCntV shift)));
17867   ins_cost(INSN_COST);
17868   format %{ "ushr    $dst, $src, $shift\t# vector (8B)" %}
17869   ins_encode %{
17870     int sh = (int)$shift$$constant;
17871     if (sh >= 8) {
17872       __ eor(as_FloatRegister($dst$$reg), __ T8B,
17873              as_FloatRegister($src$$reg),
17874              as_FloatRegister($src$$reg));
17875     } else {
17876       __ ushr(as_FloatRegister($dst$$reg), __ T8B,
17877              as_FloatRegister($src$$reg), sh);
17878     }
17879   %}
17880   ins_pipe(vshift64_imm);
17881 %}
17882 
17883 instruct vsrl16B_imm(vecX dst, vecX src, immI shift) %{
17884   predicate(n->as_Vector()->length() == 16);
17885   match(Set dst (URShiftVB src (RShiftCntV shift)));
17886   ins_cost(INSN_COST);
17887   format %{ "ushr    $dst, $src, $shift\t# vector (16B)" %}
17888   ins_encode %{
17889     int sh = (int)$shift$$constant;
17890     if (sh >= 8) {
17891       __ eor(as_FloatRegister($dst$$reg), __ T16B,
17892              as_FloatRegister($src$$reg),
17893              as_FloatRegister($src$$reg));
17894     } else {
17895       __ ushr(as_FloatRegister($dst$$reg), __ T16B,
17896              as_FloatRegister($src$$reg), sh);
17897     }
17898   %}
17899   ins_pipe(vshift128_imm);
17900 %}
17901 
17902 instruct vsll4S(vecD dst, vecD src, vecD shift) %{
17903   predicate(n->as_Vector()->length() == 2 ||
17904             n->as_Vector()->length() == 4);
17905   match(Set dst (LShiftVS src shift));
17906   ins_cost(INSN_COST);
17907   format %{ "sshl  $dst,$src,$shift\t# vector (4H)" %}
17908   ins_encode %{
17909     __ sshl(as_FloatRegister($dst$$reg), __ T4H,
17910             as_FloatRegister($src$$reg),
17911             as_FloatRegister($shift$$reg));
17912   %}
17913   ins_pipe(vshift64);
17914 %}
17915 
17916 instruct vsll8S(vecX dst, vecX src, vecX shift) %{
17917   predicate(n->as_Vector()->length() == 8);
17918   match(Set dst (LShiftVS src shift));
17919   ins_cost(INSN_COST);
17920   format %{ "sshl  $dst,$src,$shift\t# vector (8H)" %}
17921   ins_encode %{
17922     __ sshl(as_FloatRegister($dst$$reg), __ T8H,
17923             as_FloatRegister($src$$reg),
17924             as_FloatRegister($shift$$reg));
17925   %}
17926   ins_pipe(vshift128);
17927 %}
17928 
17929 instruct vsra4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
17930   predicate(n->as_Vector()->length() == 2 ||
17931             n->as_Vector()->length() == 4);
17932   match(Set dst (RShiftVS src shift));
17933   ins_cost(INSN_COST);
17934   effect(TEMP tmp);
17935   format %{ "negr  $tmp,$shift\t"
17936             "sshl  $dst,$src,$tmp\t# vector (4H)" %}
17937   ins_encode %{
17938     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17939             as_FloatRegister($shift$$reg));
17940     __ sshl(as_FloatRegister($dst$$reg), __ T4H,
17941             as_FloatRegister($src$$reg),
17942             as_FloatRegister($tmp$$reg));
17943   %}
17944   ins_pipe(vshift64);
17945 %}
17946 
17947 instruct vsra8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
17948   predicate(n->as_Vector()->length() == 8);
17949   match(Set dst (RShiftVS src shift));
17950   ins_cost(INSN_COST);
17951   effect(TEMP tmp);
17952   format %{ "negr  $tmp,$shift\t"
17953             "sshl  $dst,$src,$tmp\t# vector (8H)" %}
17954   ins_encode %{
17955     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17956             as_FloatRegister($shift$$reg));
17957     __ sshl(as_FloatRegister($dst$$reg), __ T8H,
17958             as_FloatRegister($src$$reg),
17959             as_FloatRegister($tmp$$reg));
17960   %}
17961   ins_pipe(vshift128);
17962 %}
17963 
17964 instruct vsrl4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
17965   predicate(n->as_Vector()->length() == 2 ||
17966             n->as_Vector()->length() == 4);
17967   match(Set dst (URShiftVS src shift));
17968   ins_cost(INSN_COST);
17969   effect(TEMP tmp);
17970   format %{ "negr  $tmp,$shift\t"
17971             "ushl  $dst,$src,$tmp\t# vector (4H)" %}
17972   ins_encode %{
17973     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17974             as_FloatRegister($shift$$reg));
17975     __ ushl(as_FloatRegister($dst$$reg), __ T4H,
17976             as_FloatRegister($src$$reg),
17977             as_FloatRegister($tmp$$reg));
17978   %}
17979   ins_pipe(vshift64);
17980 %}
17981 
17982 instruct vsrl8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
17983   predicate(n->as_Vector()->length() == 8);
17984   match(Set dst (URShiftVS src shift));
17985   ins_cost(INSN_COST);
17986   effect(TEMP tmp);
17987   format %{ "negr  $tmp,$shift\t"
17988             "ushl  $dst,$src,$tmp\t# vector (8H)" %}
17989   ins_encode %{
17990     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17991             as_FloatRegister($shift$$reg));
17992     __ ushl(as_FloatRegister($dst$$reg), __ T8H,
17993             as_FloatRegister($src$$reg),
17994             as_FloatRegister($tmp$$reg));
17995   %}
17996   ins_pipe(vshift128);
17997 %}
17998 
17999 instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
18000   predicate(n->as_Vector()->length() == 2 ||
18001             n->as_Vector()->length() == 4);
18002   match(Set dst (LShiftVS src (LShiftCntV shift)));
18003   ins_cost(INSN_COST);
18004   format %{ "shl    $dst, $src, $shift\t# vector (4H)" %}
18005   ins_encode %{
18006     int sh = (int)$shift$$constant;
18007     if (sh >= 16) {
18008       __ eor(as_FloatRegister($dst$$reg), __ T8B,
18009              as_FloatRegister($src$$reg),
18010              as_FloatRegister($src$$reg));
18011     } else {
18012       __ shl(as_FloatRegister($dst$$reg), __ T4H,
18013              as_FloatRegister($src$$reg), sh);
18014     }
18015   %}
18016   ins_pipe(vshift64_imm);
18017 %}
18018 
18019 instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
18020   predicate(n->as_Vector()->length() == 8);
18021   match(Set dst (LShiftVS src (LShiftCntV shift)));
18022   ins_cost(INSN_COST);
18023   format %{ "shl    $dst, $src, $shift\t# vector (8H)" %}
18024   ins_encode %{
18025     int sh = (int)$shift$$constant;
18026     if (sh >= 16) {
18027       __ eor(as_FloatRegister($dst$$reg), __ T16B,
18028              as_FloatRegister($src$$reg),
18029              as_FloatRegister($src$$reg));
18030     } else {
18031       __ shl(as_FloatRegister($dst$$reg), __ T8H,
18032              as_FloatRegister($src$$reg), sh);
18033     }
18034   %}
18035   ins_pipe(vshift128_imm);
18036 %}
18037 
18038 instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{
18039   predicate(n->as_Vector()->length() == 2 ||
18040             n->as_Vector()->length() == 4);
18041   match(Set dst (RShiftVS src (RShiftCntV shift)));
18042   ins_cost(INSN_COST);
18043   format %{ "sshr    $dst, $src, $shift\t# vector (4H)" %}
18044   ins_encode %{
18045     int sh = (int)$shift$$constant;
18046     if (sh >= 16) sh = 15;
18047     __ sshr(as_FloatRegister($dst$$reg), __ T4H,
18048            as_FloatRegister($src$$reg), sh);
18049   %}
18050   ins_pipe(vshift64_imm);
18051 %}
18052 
18053 instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
18054   predicate(n->as_Vector()->length() == 8);
18055   match(Set dst (RShiftVS src (RShiftCntV shift)));
18056   ins_cost(INSN_COST);
18057   format %{ "sshr    $dst, $src, $shift\t# vector (8H)" %}
18058   ins_encode %{
18059     int sh = (int)$shift$$constant;
18060     if (sh >= 16) sh = 15;
18061     __ sshr(as_FloatRegister($dst$$reg), __ T8H,
18062            as_FloatRegister($src$$reg), sh);
18063   %}
18064   ins_pipe(vshift128_imm);
18065 %}
18066 
18067 instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{
18068   predicate(n->as_Vector()->length() == 2 ||
18069             n->as_Vector()->length() == 4);
18070   match(Set dst (URShiftVS src (RShiftCntV shift)));
18071   ins_cost(INSN_COST);
18072   format %{ "ushr    $dst, $src, $shift\t# vector (4H)" %}
18073   ins_encode %{
18074     int sh = (int)$shift$$constant;
18075     if (sh >= 16) {
18076       __ eor(as_FloatRegister($dst$$reg), __ T8B,
18077              as_FloatRegister($src$$reg),
18078              as_FloatRegister($src$$reg));
18079     } else {
18080       __ ushr(as_FloatRegister($dst$$reg), __ T4H,
18081              as_FloatRegister($src$$reg), sh);
18082     }
18083   %}
18084   ins_pipe(vshift64_imm);
18085 %}
18086 
18087 instruct vsrl8S_imm(vecX dst, vecX src, immI shift) %{
18088   predicate(n->as_Vector()->length() == 8);
18089   match(Set dst (URShiftVS src (RShiftCntV shift)));
18090   ins_cost(INSN_COST);
18091   format %{ "ushr    $dst, $src, $shift\t# vector (8H)" %}
18092   ins_encode %{
18093     int sh = (int)$shift$$constant;
18094     if (sh >= 16) {
18095       __ eor(as_FloatRegister($dst$$reg), __ T16B,
18096              as_FloatRegister($src$$reg),
18097              as_FloatRegister($src$$reg));
18098     } else {
18099       __ ushr(as_FloatRegister($dst$$reg), __ T8H,
18100              as_FloatRegister($src$$reg), sh);
18101     }
18102   %}
18103   ins_pipe(vshift128_imm);
18104 %}
18105 
18106 instruct vsll2I(vecD dst, vecD src, vecD shift) %{
18107   predicate(n->as_Vector()->length() == 2);
18108   match(Set dst (LShiftVI src shift));
18109   ins_cost(INSN_COST);
18110   format %{ "sshl  $dst,$src,$shift\t# vector (2S)" %}
18111   ins_encode %{
18112     __ sshl(as_FloatRegister($dst$$reg), __ T2S,
18113             as_FloatRegister($src$$reg),
18114             as_FloatRegister($shift$$reg));
18115   %}
18116   ins_pipe(vshift64);
18117 %}
18118 
18119 instruct vsll4I(vecX dst, vecX src, vecX shift) %{
18120   predicate(n->as_Vector()->length() == 4);
18121   match(Set dst (LShiftVI src shift));
18122   ins_cost(INSN_COST);
18123   format %{ "sshl  $dst,$src,$shift\t# vector (4S)" %}
18124   ins_encode %{
18125     __ sshl(as_FloatRegister($dst$$reg), __ T4S,
18126             as_FloatRegister($src$$reg),
18127             as_FloatRegister($shift$$reg));
18128   %}
18129   ins_pipe(vshift128);
18130 %}
18131 
18132 instruct vsra2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
18133   predicate(n->as_Vector()->length() == 2);
18134   match(Set dst (RShiftVI src shift));
18135   ins_cost(INSN_COST);
18136   effect(TEMP tmp);
18137   format %{ "negr  $tmp,$shift\t"
18138             "sshl  $dst,$src,$tmp\t# vector (2S)" %}
18139   ins_encode %{
18140     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
18141             as_FloatRegister($shift$$reg));
18142     __ sshl(as_FloatRegister($dst$$reg), __ T2S,
18143             as_FloatRegister($src$$reg),
18144             as_FloatRegister($tmp$$reg));
18145   %}
18146   ins_pipe(vshift64);
18147 %}
18148 
18149 instruct vsra4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
18150   predicate(n->as_Vector()->length() == 4);
18151   match(Set dst (RShiftVI src shift));
18152   ins_cost(INSN_COST);
18153   effect(TEMP tmp);
18154   format %{ "negr  $tmp,$shift\t"
18155             "sshl  $dst,$src,$tmp\t# vector (4S)" %}
18156   ins_encode %{
18157     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
18158             as_FloatRegister($shift$$reg));
18159     __ sshl(as_FloatRegister($dst$$reg), __ T4S,
18160             as_FloatRegister($src$$reg),
18161             as_FloatRegister($tmp$$reg));
18162   %}
18163   ins_pipe(vshift128);
18164 %}
18165 
18166 instruct vsrl2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
18167   predicate(n->as_Vector()->length() == 2);
18168   match(Set dst (URShiftVI src shift));
18169   ins_cost(INSN_COST);
18170   effect(TEMP tmp);
18171   format %{ "negr  $tmp,$shift\t"
18172             "ushl  $dst,$src,$tmp\t# vector (2S)" %}
18173   ins_encode %{
18174     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
18175             as_FloatRegister($shift$$reg));
18176     __ ushl(as_FloatRegister($dst$$reg), __ T2S,
18177             as_FloatRegister($src$$reg),
18178             as_FloatRegister($tmp$$reg));
18179   %}
18180   ins_pipe(vshift64);
18181 %}
18182 
18183 instruct vsrl4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
18184   predicate(n->as_Vector()->length() == 4);
18185   match(Set dst (URShiftVI src shift));
18186   ins_cost(INSN_COST);
18187   effect(TEMP tmp);
18188   format %{ "negr  $tmp,$shift\t"
18189             "ushl  $dst,$src,$tmp\t# vector (4S)" %}
18190   ins_encode %{
18191     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
18192             as_FloatRegister($shift$$reg));
18193     __ ushl(as_FloatRegister($dst$$reg), __ T4S,
18194             as_FloatRegister($src$$reg),
18195             as_FloatRegister($tmp$$reg));
18196   %}
18197   ins_pipe(vshift128);
18198 %}
18199 
18200 instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
18201   predicate(n->as_Vector()->length() == 2);
18202   match(Set dst (LShiftVI src (LShiftCntV shift)));
18203   ins_cost(INSN_COST);
18204   format %{ "shl    $dst, $src, $shift\t# vector (2S)" %}
18205   ins_encode %{
18206     __ shl(as_FloatRegister($dst$$reg), __ T2S,
18207            as_FloatRegister($src$$reg),
18208            (int)$shift$$constant);
18209   %}
18210   ins_pipe(vshift64_imm);
18211 %}
18212 
18213 instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{
18214   predicate(n->as_Vector()->length() == 4);
18215   match(Set dst (LShiftVI src (LShiftCntV shift)));
18216   ins_cost(INSN_COST);
18217   format %{ "shl    $dst, $src, $shift\t# vector (4S)" %}
18218   ins_encode %{
18219     __ shl(as_FloatRegister($dst$$reg), __ T4S,
18220            as_FloatRegister($src$$reg),
18221            (int)$shift$$constant);
18222   %}
18223   ins_pipe(vshift128_imm);
18224 %}
18225 
18226 instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{
18227   predicate(n->as_Vector()->length() == 2);
18228   match(Set dst (RShiftVI src (RShiftCntV shift)));
18229   ins_cost(INSN_COST);
18230   format %{ "sshr    $dst, $src, $shift\t# vector (2S)" %}
18231   ins_encode %{
18232     __ sshr(as_FloatRegister($dst$$reg), __ T2S,
18233             as_FloatRegister($src$$reg),
18234             (int)$shift$$constant);
18235   %}
18236   ins_pipe(vshift64_imm);
18237 %}
18238 
18239 instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{
18240   predicate(n->as_Vector()->length() == 4);
18241   match(Set dst (RShiftVI src (RShiftCntV shift)));
18242   ins_cost(INSN_COST);
18243   format %{ "sshr    $dst, $src, $shift\t# vector (4S)" %}
18244   ins_encode %{
18245     __ sshr(as_FloatRegister($dst$$reg), __ T4S,
18246             as_FloatRegister($src$$reg),
18247             (int)$shift$$constant);
18248   %}
18249   ins_pipe(vshift128_imm);
18250 %}
18251 
18252 instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{
18253   predicate(n->as_Vector()->length() == 2);
18254   match(Set dst (URShiftVI src (RShiftCntV shift)));
18255   ins_cost(INSN_COST);
18256   format %{ "ushr    $dst, $src, $shift\t# vector (2S)" %}
18257   ins_encode %{
18258     __ ushr(as_FloatRegister($dst$$reg), __ T2S,
18259             as_FloatRegister($src$$reg),
18260             (int)$shift$$constant);
18261   %}
18262   ins_pipe(vshift64_imm);
18263 %}
18264 
18265 instruct vsrl4I_imm(vecX dst, vecX src, immI shift) %{
18266   predicate(n->as_Vector()->length() == 4);
18267   match(Set dst (URShiftVI src (RShiftCntV shift)));
18268   ins_cost(INSN_COST);
18269   format %{ "ushr    $dst, $src, $shift\t# vector (4S)" %}
18270   ins_encode %{
18271     __ ushr(as_FloatRegister($dst$$reg), __ T4S,
18272             as_FloatRegister($src$$reg),
18273             (int)$shift$$constant);
18274   %}
18275   ins_pipe(vshift128_imm);
18276 %}
18277 
18278 instruct vsll2L(vecX dst, vecX src, vecX shift) %{
18279   predicate(n->as_Vector()->length() == 2);
18280   match(Set dst (LShiftVL src shift));
18281   ins_cost(INSN_COST);
18282   format %{ "sshl  $dst,$src,$shift\t# vector (2D)" %}
18283   ins_encode %{
18284     __ sshl(as_FloatRegister($dst$$reg), __ T2D,
18285             as_FloatRegister($src$$reg),
18286             as_FloatRegister($shift$$reg));
18287   %}
18288   ins_pipe(vshift128);
18289 %}
18290 
18291 instruct vsra2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
18292   predicate(n->as_Vector()->length() == 2);
18293   match(Set dst (RShiftVL src shift));
18294   ins_cost(INSN_COST);
18295   effect(TEMP tmp);
18296   format %{ "negr  $tmp,$shift\t"
18297             "sshl  $dst,$src,$tmp\t# vector (2D)" %}
18298   ins_encode %{
18299     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
18300             as_FloatRegister($shift$$reg));
18301     __ sshl(as_FloatRegister($dst$$reg), __ T2D,
18302             as_FloatRegister($src$$reg),
18303             as_FloatRegister($tmp$$reg));
18304   %}
18305   ins_pipe(vshift128);
18306 %}
18307 
18308 instruct vsrl2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
18309   predicate(n->as_Vector()->length() == 2);
18310   match(Set dst (URShiftVL src shift));
18311   ins_cost(INSN_COST);
18312   effect(TEMP tmp);
18313   format %{ "negr  $tmp,$shift\t"
18314             "ushl  $dst,$src,$tmp\t# vector (2D)" %}
18315   ins_encode %{
18316     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
18317             as_FloatRegister($shift$$reg));
18318     __ ushl(as_FloatRegister($dst$$reg), __ T2D,
18319             as_FloatRegister($src$$reg),
18320             as_FloatRegister($tmp$$reg));
18321   %}
18322   ins_pipe(vshift128);
18323 %}
18324 
18325 instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
18326   predicate(n->as_Vector()->length() == 2);
18327   match(Set dst (LShiftVL src (LShiftCntV shift)));
18328   ins_cost(INSN_COST);
18329   format %{ "shl    $dst, $src, $shift\t# vector (2D)" %}
18330   ins_encode %{
18331     __ shl(as_FloatRegister($dst$$reg), __ T2D,
18332            as_FloatRegister($src$$reg),
18333            (int)$shift$$constant);
18334   %}
18335   ins_pipe(vshift128_imm);
18336 %}
18337 
18338 instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{
18339   predicate(n->as_Vector()->length() == 2);
18340   match(Set dst (RShiftVL src (RShiftCntV shift)));
18341   ins_cost(INSN_COST);
18342   format %{ "sshr    $dst, $src, $shift\t# vector (2D)" %}
18343   ins_encode %{
18344     __ sshr(as_FloatRegister($dst$$reg), __ T2D,
18345             as_FloatRegister($src$$reg),
18346             (int)$shift$$constant);
18347   %}
18348   ins_pipe(vshift128_imm);
18349 %}
18350 
18351 instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{
18352   predicate(n->as_Vector()->length() == 2);
18353   match(Set dst (URShiftVL src (RShiftCntV shift)));
18354   ins_cost(INSN_COST);
18355   format %{ "ushr    $dst, $src, $shift\t# vector (2D)" %}
18356   ins_encode %{
18357     __ ushr(as_FloatRegister($dst$$reg), __ T2D,
18358             as_FloatRegister($src$$reg),
18359             (int)$shift$$constant);
18360   %}
18361   ins_pipe(vshift128_imm);
18362 %}
18363 
18364 instruct vmax2F(vecD dst, vecD src1, vecD src2)
18365 %{
18366   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18367   match(Set dst (MaxV src1 src2));
18368   ins_cost(INSN_COST);
18369   format %{ "fmax  $dst,$src1,$src2\t# vector (2F)" %}
18370   ins_encode %{
18371     __ fmax(as_FloatRegister($dst$$reg), __ T2S,
18372             as_FloatRegister($src1$$reg),
18373             as_FloatRegister($src2$$reg));
18374   %}
18375   ins_pipe(vdop_fp64);
18376 %}
18377 
18378 instruct vmax4F(vecX dst, vecX src1, vecX src2)
18379 %{
18380   predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18381   match(Set dst (MaxV src1 src2));
18382   ins_cost(INSN_COST);
18383   format %{ "fmax  $dst,$src1,$src2\t# vector (4S)" %}
18384   ins_encode %{
18385     __ fmax(as_FloatRegister($dst$$reg), __ T4S,
18386             as_FloatRegister($src1$$reg),
18387             as_FloatRegister($src2$$reg));
18388   %}
18389   ins_pipe(vdop_fp128);
18390 %}
18391 
18392 instruct vmax2D(vecX dst, vecX src1, vecX src2)
18393 %{
18394   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
18395   match(Set dst (MaxV src1 src2));
18396   ins_cost(INSN_COST);
18397   format %{ "fmax  $dst,$src1,$src2\t# vector (2D)" %}
18398   ins_encode %{
18399     __ fmax(as_FloatRegister($dst$$reg), __ T2D,
18400             as_FloatRegister($src1$$reg),
18401             as_FloatRegister($src2$$reg));
18402   %}
18403   ins_pipe(vdop_fp128);
18404 %}
18405 
18406 instruct vmin2F(vecD dst, vecD src1, vecD src2)
18407 %{
18408   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18409   match(Set dst (MinV src1 src2));
18410   ins_cost(INSN_COST);
18411   format %{ "fmin  $dst,$src1,$src2\t# vector (2F)" %}
18412   ins_encode %{
18413     __ fmin(as_FloatRegister($dst$$reg), __ T2S,
18414             as_FloatRegister($src1$$reg),
18415             as_FloatRegister($src2$$reg));
18416   %}
18417   ins_pipe(vdop_fp64);
18418 %}
18419 
18420 instruct vmin4F(vecX dst, vecX src1, vecX src2)
18421 %{
18422   predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18423   match(Set dst (MinV src1 src2));
18424   ins_cost(INSN_COST);
18425   format %{ "fmin  $dst,$src1,$src2\t# vector (4S)" %}
18426   ins_encode %{
18427     __ fmin(as_FloatRegister($dst$$reg), __ T4S,
18428             as_FloatRegister($src1$$reg),
18429             as_FloatRegister($src2$$reg));
18430   %}
18431   ins_pipe(vdop_fp128);
18432 %}
18433 
18434 instruct vmin2D(vecX dst, vecX src1, vecX src2)
18435 %{
18436   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
18437   match(Set dst (MinV src1 src2));
18438   ins_cost(INSN_COST);
18439   format %{ "fmin  $dst,$src1,$src2\t# vector (2D)" %}
18440   ins_encode %{
18441     __ fmin(as_FloatRegister($dst$$reg), __ T2D,
18442             as_FloatRegister($src1$$reg),
18443             as_FloatRegister($src2$$reg));
18444   %}
18445   ins_pipe(vdop_fp128);
18446 %}
18447 
18448 instruct vround2D_reg(vecX dst, vecX src, immI rmode) %{
18449   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
18450   match(Set dst (RoundDoubleModeV src rmode));
18451   format %{ "frint  $dst, $src, $rmode" %}
18452   ins_encode %{
18453     switch ($rmode$$constant) {
18454       case RoundDoubleModeNode::rmode_rint:
18455         __ frintn(as_FloatRegister($dst$$reg), __ T2D,
18456                   as_FloatRegister($src$$reg));
18457         break;
18458       case RoundDoubleModeNode::rmode_floor:
18459         __ frintm(as_FloatRegister($dst$$reg), __ T2D,
18460                   as_FloatRegister($src$$reg));
18461         break;
18462       case RoundDoubleModeNode::rmode_ceil:
18463         __ frintp(as_FloatRegister($dst$$reg), __ T2D,
18464                   as_FloatRegister($src$$reg));
18465         break;
18466     }
18467   %}
18468   ins_pipe(vdop_fp128);
18469 %}
18470 
18471 instruct vpopcount4I(vecX dst, vecX src) %{
18472   predicate(UsePopCountInstruction && n->as_Vector()->length() == 4);
18473   match(Set dst (PopCountVI src));
18474   format %{
18475     "cnt     $dst, $src\t# vector (16B)\n\t"
18476     "uaddlp  $dst, $dst\t# vector (16B)\n\t"
18477     "uaddlp  $dst, $dst\t# vector (8H)"
18478   %}
18479   ins_encode %{
18480      __ cnt(as_FloatRegister($dst$$reg), __ T16B,
18481             as_FloatRegister($src$$reg));
18482      __ uaddlp(as_FloatRegister($dst$$reg), __ T16B,
18483                as_FloatRegister($dst$$reg));
18484      __ uaddlp(as_FloatRegister($dst$$reg), __ T8H,
18485                as_FloatRegister($dst$$reg));
18486   %}
18487   ins_pipe(pipe_class_default);
18488 %}
18489 
18490 instruct vpopcount2I(vecD dst, vecD src) %{
18491   predicate(UsePopCountInstruction && n->as_Vector()->length() == 2);
18492   match(Set dst (PopCountVI src));
18493   format %{
18494     "cnt     $dst, $src\t# vector (8B)\n\t"
18495     "uaddlp  $dst, $dst\t# vector (8B)\n\t"
18496     "uaddlp  $dst, $dst\t# vector (4H)"
18497   %}
18498   ins_encode %{
18499      __ cnt(as_FloatRegister($dst$$reg), __ T8B,
18500             as_FloatRegister($src$$reg));
18501      __ uaddlp(as_FloatRegister($dst$$reg), __ T8B,
18502                as_FloatRegister($dst$$reg));
18503      __ uaddlp(as_FloatRegister($dst$$reg), __ T4H,
18504                as_FloatRegister($dst$$reg));
18505   %}
18506   ins_pipe(pipe_class_default);
18507 %}
18508 
18509 //----------PEEPHOLE RULES-----------------------------------------------------
18510 // These must follow all instruction definitions as they use the names
18511 // defined in the instructions definitions.
18512 //
18513 // peepmatch ( root_instr_name [preceding_instruction]* );
18514 //
18515 // peepconstraint %{
18516 // (instruction_number.operand_name relational_op instruction_number.operand_name
18517 //  [, ...] );
18518 // // instruction numbers are zero-based using left to right order in peepmatch
18519 //
18520 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
18521 // // provide an instruction_number.operand_name for each operand that appears
18522 // // in the replacement instruction's match rule
18523 //
18524 // ---------VM FLAGS---------------------------------------------------------
18525 //
18526 // All peephole optimizations can be turned off using -XX:-OptoPeephole
18527 //
18528 // Each peephole rule is given an identifying number starting with zero and
18529 // increasing by one in the order seen by the parser.  An individual peephole
18530 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
18531 // on the command-line.
18532 //
18533 // ---------CURRENT LIMITATIONS----------------------------------------------
18534 //
18535 // Only match adjacent instructions in same basic block
18536 // Only equality constraints
18537 // Only constraints between operands, not (0.dest_reg == RAX_enc)
18538 // Only one replacement instruction
18539 //
18540 // ---------EXAMPLE----------------------------------------------------------
18541 //
18542 // // pertinent parts of existing instructions in architecture description
18543 // instruct movI(iRegINoSp dst, iRegI src)
18544 // %{
18545 //   match(Set dst (CopyI src));
18546 // %}
18547 //
18548 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
18549 // %{
18550 //   match(Set dst (AddI dst src));
18551 //   effect(KILL cr);
18552 // %}
18553 //
18554 // // Change (inc mov) to lea
18555 // peephole %{
18556 //   // increment preceeded by register-register move
18557 //   peepmatch ( incI_iReg movI );
18558 //   // require that the destination register of the increment
18559 //   // match the destination register of the move
18560 //   peepconstraint ( 0.dst == 1.dst );
18561 //   // construct a replacement instruction that sets
18562 //   // the destination to ( move's source register + one )
18563 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
18564 // %}
18565 //
18566 
18567 // Implementation no longer uses movX instructions since
18568 // machine-independent system no longer uses CopyX nodes.
18569 //
18570 // peephole
18571 // %{
18572 //   peepmatch (incI_iReg movI);
18573 //   peepconstraint (0.dst == 1.dst);
18574 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
18575 // %}
18576 
18577 // peephole
18578 // %{
18579 //   peepmatch (decI_iReg movI);
18580 //   peepconstraint (0.dst == 1.dst);
18581 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
18582 // %}
18583 
18584 // peephole
18585 // %{
18586 //   peepmatch (addI_iReg_imm movI);
18587 //   peepconstraint (0.dst == 1.dst);
18588 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
18589 // %}
18590 
18591 // peephole
18592 // %{
18593 //   peepmatch (incL_iReg movL);
18594 //   peepconstraint (0.dst == 1.dst);
18595 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
18596 // %}
18597 
18598 // peephole
18599 // %{
18600 //   peepmatch (decL_iReg movL);
18601 //   peepconstraint (0.dst == 1.dst);
18602 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
18603 // %}
18604 
18605 // peephole
18606 // %{
18607 //   peepmatch (addL_iReg_imm movL);
18608 //   peepconstraint (0.dst == 1.dst);
18609 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
18610 // %}
18611 
18612 // peephole
18613 // %{
18614 //   peepmatch (addP_iReg_imm movP);
18615 //   peepconstraint (0.dst == 1.dst);
18616 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
18617 // %}
18618 
18619 // // Change load of spilled value to only a spill
18620 // instruct storeI(memory mem, iRegI src)
18621 // %{
18622 //   match(Set mem (StoreI mem src));
18623 // %}
18624 //
18625 // instruct loadI(iRegINoSp dst, memory mem)
18626 // %{
18627 //   match(Set dst (LoadI mem));
18628 // %}
18629 //
18630 
18631 //----------SMARTSPILL RULES---------------------------------------------------
18632 // These must follow all instruction definitions as they use the names
18633 // defined in the instructions definitions.
18634 
18635 // Local Variables:
18636 // mode: c++
18637 // End: