1 //
   2 // Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 // AArch64 Architecture Description File
  27 
  28 //----------REGISTER DEFINITION BLOCK------------------------------------------
  29 // This information is used by the matcher and the register allocator to
  30 // describe individual registers and classes of registers within the target
  31 // archtecture.
  32 
  33 register %{
  34 //----------Architecture Description Register Definitions----------------------
  35 // General Registers
  36 // "reg_def"  name ( register save type, C convention save type,
  37 //                   ideal register type, encoding );
  38 // Register Save Types:
  39 //
  40 // NS  = No-Save:       The register allocator assumes that these registers
  41 //                      can be used without saving upon entry to the method, &
  42 //                      that they do not need to be saved at call sites.
  43 //
  44 // SOC = Save-On-Call:  The register allocator assumes that these registers
  45 //                      can be used without saving upon entry to the method,
  46 //                      but that they must be saved at call sites.
  47 //
  48 // SOE = Save-On-Entry: The register allocator assumes that these registers
  49 //                      must be saved before using them upon entry to the
  50 //                      method, but they do not need to be saved at call
  51 //                      sites.
  52 //
  53 // AS  = Always-Save:   The register allocator assumes that these registers
  54 //                      must be saved before using them upon entry to the
  55 //                      method, & that they must be saved at call sites.
  56 //
  57 // Ideal Register Type is used to determine how to save & restore a
  58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  60 //
  61 // The encoding number is the actual bit-pattern placed into the opcodes.
  62 
  63 // We must define the 64 bit int registers in two 32 bit halves, the
  64 // real lower register and a virtual upper half register. upper halves
  65 // are used by the register allocator but are not actually supplied as
  66 // operands to memory ops.
  67 //
  68 // follow the C1 compiler in making registers
  69 //
  70 //   r0-r7,r10-r26 volatile (caller save)
  71 //   r27-r32 system (no save, no allocate)
  72 //   r8-r9 invisible to the allocator (so we can use them as scratch regs)
  73 //
  74 // as regards Java usage. we don't use any callee save registers
  75 // because this makes it difficult to de-optimise a frame (see comment
  76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
  77 //
  78 
  79 // General Registers
  80 
  81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
  82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
  83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
  84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
  85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
  86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
  87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
  88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
  89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
  90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
  91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
  92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
  93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
  94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
  95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
  96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
  97 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  98 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  99 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
 100 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 101 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
 102 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
 103 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
 104 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
 105 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
 106 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
 107 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
 108 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
 109 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
 110 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
 111 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
 112 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
 113 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()        );
 114 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
 115 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
 116 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
 117 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
 118 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
 119 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
 120 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
 121 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
 122 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
 123 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
 124 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
 125 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
 126 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
 127 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
 128 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
 129 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
 130 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
 131 reg_def R27     (  NS, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
 132 reg_def R27_H   (  NS, SOE, Op_RegI, 27, r27->as_VMReg()->next());
 133 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
 134 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
 135 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
 136 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
 137 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
 138 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
 139 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
 140 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
 141 
 142 // ----------------------------
 143 // Float/Double Registers
 144 // ----------------------------
 145 
 146 // Double Registers
 147 
 148 // The rules of ADL require that double registers be defined in pairs.
 149 // Each pair must be two 32-bit values, but not necessarily a pair of
 150 // single float registers. In each pair, ADLC-assigned register numbers
 151 // must be adjacent, with the lower number even. Finally, when the
 152 // CPU stores such a register pair to memory, the word associated with
 153 // the lower ADLC-assigned number must be stored to the lower address.
 154 
 155 // AArch64 has 32 floating-point registers. Each can store a vector of
 156 // single or double precision floating-point values up to 8 * 32
 157 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
 158 // use the first float or double element of the vector.
 159 
 160 // for Java use float registers v0-v15 are always save on call whereas
 161 // the platform ABI treats v8-v15 as callee save). float registers
 162 // v16-v31 are SOC as per the platform spec
 163 
 164   reg_def V0   ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()          );
 165   reg_def V0_H ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next()  );
 166   reg_def V0_J ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next(2) );
 167   reg_def V0_K ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next(3) );
 168 
 169   reg_def V1   ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()          );
 170   reg_def V1_H ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next()  );
 171   reg_def V1_J ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next(2) );
 172   reg_def V1_K ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next(3) );
 173 
 174   reg_def V2   ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()          );
 175   reg_def V2_H ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next()  );
 176   reg_def V2_J ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next(2) );
 177   reg_def V2_K ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next(3) );
 178 
 179   reg_def V3   ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()          );
 180   reg_def V3_H ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next()  );
 181   reg_def V3_J ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next(2) );
 182   reg_def V3_K ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next(3) );
 183 
 184   reg_def V4   ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()          );
 185   reg_def V4_H ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next()  );
 186   reg_def V4_J ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next(2) );
 187   reg_def V4_K ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next(3) );
 188 
 189   reg_def V5   ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()          );
 190   reg_def V5_H ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next()  );
 191   reg_def V5_J ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next(2) );
 192   reg_def V5_K ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next(3) );
 193 
 194   reg_def V6   ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()          );
 195   reg_def V6_H ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next()  );
 196   reg_def V6_J ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next(2) );
 197   reg_def V6_K ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next(3) );
 198 
 199   reg_def V7   ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()          );
 200   reg_def V7_H ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next()  );
 201   reg_def V7_J ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next(2) );
 202   reg_def V7_K ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next(3) );
 203 
 204   reg_def V8   ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()          );
 205   reg_def V8_H ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next()  );
 206   reg_def V8_J ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next(2) );
 207   reg_def V8_K ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next(3) );
 208 
 209   reg_def V9   ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()          );
 210   reg_def V9_H ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next()  );
 211   reg_def V9_J ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next(2) );
 212   reg_def V9_K ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next(3) );
 213 
 214   reg_def V10  ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()         );
 215   reg_def V10_H( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next() );
 216   reg_def V10_J( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2));
 217   reg_def V10_K( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3));
 218 
 219   reg_def V11  ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()         );
 220   reg_def V11_H( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next() );
 221   reg_def V11_J( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2));
 222   reg_def V11_K( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3));
 223 
 224   reg_def V12  ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()         );
 225   reg_def V12_H( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next() );
 226   reg_def V12_J( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2));
 227   reg_def V12_K( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3));
 228 
 229   reg_def V13  ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()         );
 230   reg_def V13_H( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next() );
 231   reg_def V13_J( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2));
 232   reg_def V13_K( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3));
 233 
 234   reg_def V14  ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()         );
 235   reg_def V14_H( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next() );
 236   reg_def V14_J( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2));
 237   reg_def V14_K( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3));
 238 
 239   reg_def V15  ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()         );
 240   reg_def V15_H( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next() );
 241   reg_def V15_J( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2));
 242   reg_def V15_K( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3));
 243 
 244   reg_def V16  ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()         );
 245   reg_def V16_H( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
 246   reg_def V16_J( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2));
 247   reg_def V16_K( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3));
 248 
 249   reg_def V17  ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()         );
 250   reg_def V17_H( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
 251   reg_def V17_J( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2));
 252   reg_def V17_K( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3));
 253 
 254   reg_def V18  ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()         );
 255   reg_def V18_H( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
 256   reg_def V18_J( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2));
 257   reg_def V18_K( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3));
 258 
 259   reg_def V19  ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()         );
 260   reg_def V19_H( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
 261   reg_def V19_J( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2));
 262   reg_def V19_K( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3));
 263 
 264   reg_def V20  ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()         );
 265   reg_def V20_H( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
 266   reg_def V20_J( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2));
 267   reg_def V20_K( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3));
 268 
 269   reg_def V21  ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()         );
 270   reg_def V21_H( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
 271   reg_def V21_J( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2));
 272   reg_def V21_K( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3));
 273 
 274   reg_def V22  ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()         );
 275   reg_def V22_H( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
 276   reg_def V22_J( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2));
 277   reg_def V22_K( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3));
 278 
 279   reg_def V23  ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()         );
 280   reg_def V23_H( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
 281   reg_def V23_J( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2));
 282   reg_def V23_K( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3));
 283 
 284   reg_def V24  ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()         );
 285   reg_def V24_H( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
 286   reg_def V24_J( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2));
 287   reg_def V24_K( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3));
 288 
 289   reg_def V25  ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()         );
 290   reg_def V25_H( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
 291   reg_def V25_J( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2));
 292   reg_def V25_K( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3));
 293 
 294   reg_def V26  ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()         );
 295   reg_def V26_H( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
 296   reg_def V26_J( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2));
 297   reg_def V26_K( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3));
 298 
 299   reg_def V27  ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()         );
 300   reg_def V27_H( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
 301   reg_def V27_J( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2));
 302   reg_def V27_K( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3));
 303 
 304   reg_def V28  ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()         );
 305   reg_def V28_H( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
 306   reg_def V28_J( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2));
 307   reg_def V28_K( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3));
 308 
 309   reg_def V29  ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()         );
 310   reg_def V29_H( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
 311   reg_def V29_J( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2));
 312   reg_def V29_K( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3));
 313 
 314   reg_def V30  ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()         );
 315   reg_def V30_H( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
 316   reg_def V30_J( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2));
 317   reg_def V30_K( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3));
 318 
 319   reg_def V31  ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()         );
 320   reg_def V31_H( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
 321   reg_def V31_J( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2));
 322   reg_def V31_K( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3));
 323 
 324 // ----------------------------
 325 // Special Registers
 326 // ----------------------------
 327 
 328 // the AArch64 CSPR status flag register is not directly acessible as
 329 // instruction operand. the FPSR status flag register is a system
 330 // register which can be written/read using MSR/MRS but again does not
 331 // appear as an operand (a code identifying the FSPR occurs as an
 332 // immediate value in the instruction).
 333 
 334 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
 335 
 336 
 337 // Specify priority of register selection within phases of register
 338 // allocation.  Highest priority is first.  A useful heuristic is to
 339 // give registers a low priority when they are required by machine
 340 // instructions, like EAX and EDX on I486, and choose no-save registers
 341 // before save-on-call, & save-on-call before save-on-entry.  Registers
 342 // which participate in fixed calling sequences should come last.
 343 // Registers which are used as pairs must fall on an even boundary.
 344 
 345 alloc_class chunk0(
 346     // volatiles
 347     R10, R10_H,
 348     R11, R11_H,
 349     R12, R12_H,
 350     R13, R13_H,
 351     R14, R14_H,
 352     R15, R15_H,
 353     R16, R16_H,
 354     R17, R17_H,
 355     R18, R18_H,
 356 
 357     // arg registers
 358     R0, R0_H,
 359     R1, R1_H,
 360     R2, R2_H,
 361     R3, R3_H,
 362     R4, R4_H,
 363     R5, R5_H,
 364     R6, R6_H,
 365     R7, R7_H,
 366 
 367     // non-volatiles
 368     R19, R19_H,
 369     R20, R20_H,
 370     R21, R21_H,
 371     R22, R22_H,
 372     R23, R23_H,
 373     R24, R24_H,
 374     R25, R25_H,
 375     R26, R26_H,
 376 
 377     // non-allocatable registers
 378 
 379     R27, R27_H, // heapbase
 380     R28, R28_H, // thread
 381     R29, R29_H, // fp
 382     R30, R30_H, // lr
 383     R31, R31_H, // sp
 384 );
 385 
 386 alloc_class chunk1(
 387 
 388     // no save
 389     V16, V16_H, V16_J, V16_K,
 390     V17, V17_H, V17_J, V17_K,
 391     V18, V18_H, V18_J, V18_K,
 392     V19, V19_H, V19_J, V19_K,
 393     V20, V20_H, V20_J, V20_K,
 394     V21, V21_H, V21_J, V21_K,
 395     V22, V22_H, V22_J, V22_K,
 396     V23, V23_H, V23_J, V23_K,
 397     V24, V24_H, V24_J, V24_K,
 398     V25, V25_H, V25_J, V25_K,
 399     V26, V26_H, V26_J, V26_K,
 400     V27, V27_H, V27_J, V27_K,
 401     V28, V28_H, V28_J, V28_K,
 402     V29, V29_H, V29_J, V29_K,
 403     V30, V30_H, V30_J, V30_K,
 404     V31, V31_H, V31_J, V31_K,
 405 
 406     // arg registers
 407     V0, V0_H, V0_J, V0_K,
 408     V1, V1_H, V1_J, V1_K,
 409     V2, V2_H, V2_J, V2_K,
 410     V3, V3_H, V3_J, V3_K,
 411     V4, V4_H, V4_J, V4_K,
 412     V5, V5_H, V5_J, V5_K,
 413     V6, V6_H, V6_J, V6_K,
 414     V7, V7_H, V7_J, V7_K,
 415 
 416     // non-volatiles
 417     V8, V8_H, V8_J, V8_K,
 418     V9, V9_H, V9_J, V9_K,
 419     V10, V10_H, V10_J, V10_K,
 420     V11, V11_H, V11_J, V11_K,
 421     V12, V12_H, V12_J, V12_K,
 422     V13, V13_H, V13_J, V13_K,
 423     V14, V14_H, V14_J, V14_K,
 424     V15, V15_H, V15_J, V15_K,
 425 );
 426 
 427 alloc_class chunk2(RFLAGS);
 428 
 429 //----------Architecture Description Register Classes--------------------------
 430 // Several register classes are automatically defined based upon information in
 431 // this architecture description.
 432 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 433 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 434 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 435 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 436 //
 437 
 438 // Class for all 32 bit integer registers -- excludes SP which will
 439 // never be used as an integer register
 440 reg_class any_reg32(
 441     R0,
 442     R1,
 443     R2,
 444     R3,
 445     R4,
 446     R5,
 447     R6,
 448     R7,
 449     R10,
 450     R11,
 451     R12,
 452     R13,
 453     R14,
 454     R15,
 455     R16,
 456     R17,
 457     R18,
 458     R19,
 459     R20,
 460     R21,
 461     R22,
 462     R23,
 463     R24,
 464     R25,
 465     R26,
 466     R27,
 467     R28,
 468     R29,
 469     R30
 470 );
 471 
 472 // Singleton class for R0 int register
 473 reg_class int_r0_reg(R0);
 474 
 475 // Singleton class for R2 int register
 476 reg_class int_r2_reg(R2);
 477 
 478 // Singleton class for R3 int register
 479 reg_class int_r3_reg(R3);
 480 
 481 // Singleton class for R4 int register
 482 reg_class int_r4_reg(R4);
 483 
 484 // Class for all long integer registers (including RSP)
 485 reg_class any_reg(
 486     R0, R0_H,
 487     R1, R1_H,
 488     R2, R2_H,
 489     R3, R3_H,
 490     R4, R4_H,
 491     R5, R5_H,
 492     R6, R6_H,
 493     R7, R7_H,
 494     R10, R10_H,
 495     R11, R11_H,
 496     R12, R12_H,
 497     R13, R13_H,
 498     R14, R14_H,
 499     R15, R15_H,
 500     R16, R16_H,
 501     R17, R17_H,
 502     R18, R18_H,
 503     R19, R19_H,
 504     R20, R20_H,
 505     R21, R21_H,
 506     R22, R22_H,
 507     R23, R23_H,
 508     R24, R24_H,
 509     R25, R25_H,
 510     R26, R26_H,
 511     R27, R27_H,
 512     R28, R28_H,
 513     R29, R29_H,
 514     R30, R30_H,
 515     R31, R31_H
 516 );
 517 
 518 // Class for all non-special integer registers
 519 reg_class no_special_reg32_no_fp(
 520     R0,
 521     R1,
 522     R2,
 523     R3,
 524     R4,
 525     R5,
 526     R6,
 527     R7,
 528     R10,
 529     R11,
 530     R12,                        // rmethod
 531     R13,
 532     R14,
 533     R15,
 534     R16,
 535     R17,
 536     R18,
 537     R19,
 538     R20,
 539     R21,
 540     R22,
 541     R23,
 542     R24,
 543     R25,
 544     R26
 545  /* R27, */                     // heapbase
 546  /* R28, */                     // thread
 547  /* R29, */                     // fp
 548  /* R30, */                     // lr
 549  /* R31 */                      // sp
 550 );
 551 
 552 reg_class no_special_reg32_with_fp(
 553     R0,
 554     R1,
 555     R2,
 556     R3,
 557     R4,
 558     R5,
 559     R6,
 560     R7,
 561     R10,
 562     R11,
 563     R12,                        // rmethod
 564     R13,
 565     R14,
 566     R15,
 567     R16,
 568     R17,
 569     R18,
 570     R19,
 571     R20,
 572     R21,
 573     R22,
 574     R23,
 575     R24,
 576     R25,
 577     R26
 578  /* R27, */                     // heapbase
 579  /* R28, */                     // thread
 580     R29,                        // fp
 581  /* R30, */                     // lr
 582  /* R31 */                      // sp
 583 );
 584 
 585 reg_class_dynamic no_special_reg32(no_special_reg32_no_fp, no_special_reg32_with_fp, %{ PreserveFramePointer %});
 586 
 587 // Class for all non-special long integer registers
 588 reg_class no_special_reg_no_fp(
 589     R0, R0_H,
 590     R1, R1_H,
 591     R2, R2_H,
 592     R3, R3_H,
 593     R4, R4_H,
 594     R5, R5_H,
 595     R6, R6_H,
 596     R7, R7_H,
 597     R10, R10_H,
 598     R11, R11_H,
 599     R12, R12_H,                 // rmethod
 600     R13, R13_H,
 601     R14, R14_H,
 602     R15, R15_H,
 603     R16, R16_H,
 604     R17, R17_H,
 605     R18, R18_H,
 606     R19, R19_H,
 607     R20, R20_H,
 608     R21, R21_H,
 609     R22, R22_H,
 610     R23, R23_H,
 611     R24, R24_H,
 612     R25, R25_H,
 613     R26, R26_H,
 614  /* R27, R27_H, */              // heapbase
 615  /* R28, R28_H, */              // thread
 616  /* R29, R29_H, */              // fp
 617  /* R30, R30_H, */              // lr
 618  /* R31, R31_H */               // sp
 619 );
 620 
 621 reg_class no_special_reg_with_fp(
 622     R0, R0_H,
 623     R1, R1_H,
 624     R2, R2_H,
 625     R3, R3_H,
 626     R4, R4_H,
 627     R5, R5_H,
 628     R6, R6_H,
 629     R7, R7_H,
 630     R10, R10_H,
 631     R11, R11_H,
 632     R12, R12_H,                 // rmethod
 633     R13, R13_H,
 634     R14, R14_H,
 635     R15, R15_H,
 636     R16, R16_H,
 637     R17, R17_H,
 638     R18, R18_H,
 639     R19, R19_H,
 640     R20, R20_H,
 641     R21, R21_H,
 642     R22, R22_H,
 643     R23, R23_H,
 644     R24, R24_H,
 645     R25, R25_H,
 646     R26, R26_H,
 647  /* R27, R27_H, */              // heapbase
 648  /* R28, R28_H, */              // thread
 649     R29, R29_H,                 // fp
 650  /* R30, R30_H, */              // lr
 651  /* R31, R31_H */               // sp
 652 );
 653 
 654 reg_class_dynamic no_special_reg(no_special_reg_no_fp, no_special_reg_with_fp, %{ PreserveFramePointer %});
 655 
 656 // Class for 64 bit register r0
 657 reg_class r0_reg(
 658     R0, R0_H
 659 );
 660 
 661 // Class for 64 bit register r1
 662 reg_class r1_reg(
 663     R1, R1_H
 664 );
 665 
 666 // Class for 64 bit register r2
 667 reg_class r2_reg(
 668     R2, R2_H
 669 );
 670 
 671 // Class for 64 bit register r3
 672 reg_class r3_reg(
 673     R3, R3_H
 674 );
 675 
 676 // Class for 64 bit register r4
 677 reg_class r4_reg(
 678     R4, R4_H
 679 );
 680 
 681 // Class for 64 bit register r5
 682 reg_class r5_reg(
 683     R5, R5_H
 684 );
 685 
 686 // Class for 64 bit register r10
 687 reg_class r10_reg(
 688     R10, R10_H
 689 );
 690 
 691 // Class for 64 bit register r11
 692 reg_class r11_reg(
 693     R11, R11_H
 694 );
 695 
 696 // Class for method register
 697 reg_class method_reg(
 698     R12, R12_H
 699 );
 700 
 701 // Class for heapbase register
 702 reg_class heapbase_reg(
 703     R27, R27_H
 704 );
 705 
 706 // Class for thread register
 707 reg_class thread_reg(
 708     R28, R28_H
 709 );
 710 
 711 // Class for frame pointer register
 712 reg_class fp_reg(
 713     R29, R29_H
 714 );
 715 
 716 // Class for link register
 717 reg_class lr_reg(
 718     R30, R30_H
 719 );
 720 
 721 // Class for long sp register
 722 reg_class sp_reg(
 723   R31, R31_H
 724 );
 725 
 726 // Class for all pointer registers
 727 reg_class ptr_reg(
 728     R0, R0_H,
 729     R1, R1_H,
 730     R2, R2_H,
 731     R3, R3_H,
 732     R4, R4_H,
 733     R5, R5_H,
 734     R6, R6_H,
 735     R7, R7_H,
 736     R10, R10_H,
 737     R11, R11_H,
 738     R12, R12_H,
 739     R13, R13_H,
 740     R14, R14_H,
 741     R15, R15_H,
 742     R16, R16_H,
 743     R17, R17_H,
 744     R18, R18_H,
 745     R19, R19_H,
 746     R20, R20_H,
 747     R21, R21_H,
 748     R22, R22_H,
 749     R23, R23_H,
 750     R24, R24_H,
 751     R25, R25_H,
 752     R26, R26_H,
 753     R27, R27_H,
 754     R28, R28_H,
 755     R29, R29_H,
 756     R30, R30_H,
 757     R31, R31_H
 758 );
 759 
 760 // Class for all non_special pointer registers
 761 reg_class no_special_ptr_reg(
 762     R0, R0_H,
 763     R1, R1_H,
 764     R2, R2_H,
 765     R3, R3_H,
 766     R4, R4_H,
 767     R5, R5_H,
 768     R6, R6_H,
 769     R7, R7_H,
 770     R10, R10_H,
 771     R11, R11_H,
 772     R12, R12_H,
 773     R13, R13_H,
 774     R14, R14_H,
 775     R15, R15_H,
 776     R16, R16_H,
 777     R17, R17_H,
 778     R18, R18_H,
 779     R19, R19_H,
 780     R20, R20_H,
 781     R21, R21_H,
 782     R22, R22_H,
 783     R23, R23_H,
 784     R24, R24_H,
 785     R25, R25_H,
 786     R26, R26_H,
 787  /* R27, R27_H, */              // heapbase
 788  /* R28, R28_H, */              // thread
 789  /* R29, R29_H, */              // fp
 790  /* R30, R30_H, */              // lr
 791  /* R31, R31_H */               // sp
 792 );
 793 
 794 // Class for all float registers
 795 reg_class float_reg(
 796     V0,
 797     V1,
 798     V2,
 799     V3,
 800     V4,
 801     V5,
 802     V6,
 803     V7,
 804     V8,
 805     V9,
 806     V10,
 807     V11,
 808     V12,
 809     V13,
 810     V14,
 811     V15,
 812     V16,
 813     V17,
 814     V18,
 815     V19,
 816     V20,
 817     V21,
 818     V22,
 819     V23,
 820     V24,
 821     V25,
 822     V26,
 823     V27,
 824     V28,
 825     V29,
 826     V30,
 827     V31
 828 );
 829 
 830 // Double precision float registers have virtual `high halves' that
 831 // are needed by the allocator.
 832 // Class for all double registers
 833 reg_class double_reg(
 834     V0, V0_H,
 835     V1, V1_H,
 836     V2, V2_H,
 837     V3, V3_H,
 838     V4, V4_H,
 839     V5, V5_H,
 840     V6, V6_H,
 841     V7, V7_H,
 842     V8, V8_H,
 843     V9, V9_H,
 844     V10, V10_H,
 845     V11, V11_H,
 846     V12, V12_H,
 847     V13, V13_H,
 848     V14, V14_H,
 849     V15, V15_H,
 850     V16, V16_H,
 851     V17, V17_H,
 852     V18, V18_H,
 853     V19, V19_H,
 854     V20, V20_H,
 855     V21, V21_H,
 856     V22, V22_H,
 857     V23, V23_H,
 858     V24, V24_H,
 859     V25, V25_H,
 860     V26, V26_H,
 861     V27, V27_H,
 862     V28, V28_H,
 863     V29, V29_H,
 864     V30, V30_H,
 865     V31, V31_H
 866 );
 867 
 868 // Class for all 64bit vector registers
 869 reg_class vectord_reg(
 870     V0, V0_H,
 871     V1, V1_H,
 872     V2, V2_H,
 873     V3, V3_H,
 874     V4, V4_H,
 875     V5, V5_H,
 876     V6, V6_H,
 877     V7, V7_H,
 878     V8, V8_H,
 879     V9, V9_H,
 880     V10, V10_H,
 881     V11, V11_H,
 882     V12, V12_H,
 883     V13, V13_H,
 884     V14, V14_H,
 885     V15, V15_H,
 886     V16, V16_H,
 887     V17, V17_H,
 888     V18, V18_H,
 889     V19, V19_H,
 890     V20, V20_H,
 891     V21, V21_H,
 892     V22, V22_H,
 893     V23, V23_H,
 894     V24, V24_H,
 895     V25, V25_H,
 896     V26, V26_H,
 897     V27, V27_H,
 898     V28, V28_H,
 899     V29, V29_H,
 900     V30, V30_H,
 901     V31, V31_H
 902 );
 903 
 904 // Class for all 128bit vector registers
 905 reg_class vectorx_reg(
 906     V0, V0_H, V0_J, V0_K,
 907     V1, V1_H, V1_J, V1_K,
 908     V2, V2_H, V2_J, V2_K,
 909     V3, V3_H, V3_J, V3_K,
 910     V4, V4_H, V4_J, V4_K,
 911     V5, V5_H, V5_J, V5_K,
 912     V6, V6_H, V6_J, V6_K,
 913     V7, V7_H, V7_J, V7_K,
 914     V8, V8_H, V8_J, V8_K,
 915     V9, V9_H, V9_J, V9_K,
 916     V10, V10_H, V10_J, V10_K,
 917     V11, V11_H, V11_J, V11_K,
 918     V12, V12_H, V12_J, V12_K,
 919     V13, V13_H, V13_J, V13_K,
 920     V14, V14_H, V14_J, V14_K,
 921     V15, V15_H, V15_J, V15_K,
 922     V16, V16_H, V16_J, V16_K,
 923     V17, V17_H, V17_J, V17_K,
 924     V18, V18_H, V18_J, V18_K,
 925     V19, V19_H, V19_J, V19_K,
 926     V20, V20_H, V20_J, V20_K,
 927     V21, V21_H, V21_J, V21_K,
 928     V22, V22_H, V22_J, V22_K,
 929     V23, V23_H, V23_J, V23_K,
 930     V24, V24_H, V24_J, V24_K,
 931     V25, V25_H, V25_J, V25_K,
 932     V26, V26_H, V26_J, V26_K,
 933     V27, V27_H, V27_J, V27_K,
 934     V28, V28_H, V28_J, V28_K,
 935     V29, V29_H, V29_J, V29_K,
 936     V30, V30_H, V30_J, V30_K,
 937     V31, V31_H, V31_J, V31_K
 938 );
 939 
 940 // Class for 128 bit register v0
 941 reg_class v0_reg(
 942     V0, V0_H
 943 );
 944 
 945 // Class for 128 bit register v1
 946 reg_class v1_reg(
 947     V1, V1_H
 948 );
 949 
 950 // Class for 128 bit register v2
 951 reg_class v2_reg(
 952     V2, V2_H
 953 );
 954 
 955 // Class for 128 bit register v3
 956 reg_class v3_reg(
 957     V3, V3_H
 958 );
 959 
 960 // Singleton class for condition codes
 961 reg_class int_flags(RFLAGS);
 962 
 963 %}
 964 
 965 //----------DEFINITION BLOCK---------------------------------------------------
 966 // Define name --> value mappings to inform the ADLC of an integer valued name
 967 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 968 // Format:
 969 //        int_def  <name>         ( <int_value>, <expression>);
 970 // Generated Code in ad_<arch>.hpp
 971 //        #define  <name>   (<expression>)
 972 //        // value == <int_value>
 973 // Generated code in ad_<arch>.cpp adlc_verification()
 974 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 975 //
 976 
 977 // we follow the ppc-aix port in using a simple cost model which ranks
 978 // register operations as cheap, memory ops as more expensive and
 979 // branches as most expensive. the first two have a low as well as a
 980 // normal cost. huge cost appears to be a way of saying don't do
 981 // something
 982 
 983 definitions %{
 984   // The default cost (of a register move instruction).
 985   int_def INSN_COST            (    100,     100);
 986   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 987   int_def CALL_COST            (    200,     2 * INSN_COST);
 988   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 989 %}
 990 
 991 
 992 //----------SOURCE BLOCK-------------------------------------------------------
 993 // This is a block of C++ code which provides values, functions, and
 994 // definitions necessary in the rest of the architecture description
 995 
 996 source_hpp %{
 997 
 998 #include "asm/macroAssembler.hpp"
 999 #include "gc/shared/cardTable.hpp"
1000 #include "gc/shared/cardTableBarrierSet.hpp"
1001 #include "gc/shared/collectedHeap.hpp"
1002 #include "opto/addnode.hpp"
1003 
1004 class CallStubImpl {
1005 
1006   //--------------------------------------------------------------
1007   //---<  Used for optimization in Compile::shorten_branches  >---
1008   //--------------------------------------------------------------
1009 
1010  public:
1011   // Size of call trampoline stub.
1012   static uint size_call_trampoline() {
1013     return 0; // no call trampolines on this platform
1014   }
1015 
1016   // number of relocations needed by a call trampoline stub
1017   static uint reloc_call_trampoline() {
1018     return 0; // no call trampolines on this platform
1019   }
1020 };
1021 
1022 class HandlerImpl {
1023 
1024  public:
1025 
1026   static int emit_exception_handler(CodeBuffer &cbuf);
1027   static int emit_deopt_handler(CodeBuffer& cbuf);
1028 
1029   static uint size_exception_handler() {
1030     return MacroAssembler::far_branch_size();
1031   }
1032 
1033   static uint size_deopt_handler() {
1034     // count one adr and one far branch instruction
1035     return 4 * NativeInstruction::instruction_size;
1036   }
1037 };
1038 
1039   // graph traversal helpers
1040 
1041   MemBarNode *parent_membar(const Node *n);
1042   MemBarNode *child_membar(const MemBarNode *n);
1043   bool leading_membar(const MemBarNode *barrier);
1044 
1045   bool is_card_mark_membar(const MemBarNode *barrier);
1046   bool is_CAS(int opcode);
1047 
1048   MemBarNode *leading_to_normal(MemBarNode *leading);
1049   MemBarNode *normal_to_leading(const MemBarNode *barrier);
1050   MemBarNode *card_mark_to_trailing(const MemBarNode *barrier);
1051   MemBarNode *trailing_to_card_mark(const MemBarNode *trailing);
1052   MemBarNode *trailing_to_leading(const MemBarNode *trailing);
1053 
1054   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1055 
1056   bool unnecessary_acquire(const Node *barrier);
1057   bool needs_acquiring_load(const Node *load);
1058 
1059   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1060 
1061   bool unnecessary_release(const Node *barrier);
1062   bool unnecessary_volatile(const Node *barrier);
1063   bool needs_releasing_store(const Node *store);
1064 
1065   // predicate controlling translation of CompareAndSwapX
1066   bool needs_acquiring_load_exclusive(const Node *load);
1067 
1068   // predicate controlling translation of StoreCM
1069   bool unnecessary_storestore(const Node *storecm);
1070 
1071   // predicate controlling addressing modes
1072   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1073 %}
1074 
1075 source %{
1076 
1077   // Optimizaton of volatile gets and puts
1078   // -------------------------------------
1079   //
1080   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1081   // use to implement volatile reads and writes. For a volatile read
1082   // we simply need
1083   //
1084   //   ldar<x>
1085   //
1086   // and for a volatile write we need
1087   //
1088   //   stlr<x>
1089   //
1090   // Alternatively, we can implement them by pairing a normal
1091   // load/store with a memory barrier. For a volatile read we need
1092   //
1093   //   ldr<x>
1094   //   dmb ishld
1095   //
1096   // for a volatile write
1097   //
1098   //   dmb ish
1099   //   str<x>
1100   //   dmb ish
1101   //
1102   // We can also use ldaxr and stlxr to implement compare and swap CAS
1103   // sequences. These are normally translated to an instruction
1104   // sequence like the following
1105   //
1106   //   dmb      ish
1107   // retry:
1108   //   ldxr<x>   rval raddr
1109   //   cmp       rval rold
1110   //   b.ne done
1111   //   stlxr<x>  rval, rnew, rold
1112   //   cbnz      rval retry
1113   // done:
1114   //   cset      r0, eq
1115   //   dmb ishld
1116   //
1117   // Note that the exclusive store is already using an stlxr
1118   // instruction. That is required to ensure visibility to other
1119   // threads of the exclusive write (assuming it succeeds) before that
1120   // of any subsequent writes.
1121   //
1122   // The following instruction sequence is an improvement on the above
1123   //
1124   // retry:
1125   //   ldaxr<x>  rval raddr
1126   //   cmp       rval rold
1127   //   b.ne done
1128   //   stlxr<x>  rval, rnew, rold
1129   //   cbnz      rval retry
1130   // done:
1131   //   cset      r0, eq
1132   //
1133   // We don't need the leading dmb ish since the stlxr guarantees
1134   // visibility of prior writes in the case that the swap is
1135   // successful. Crucially we don't have to worry about the case where
1136   // the swap is not successful since no valid program should be
1137   // relying on visibility of prior changes by the attempting thread
1138   // in the case where the CAS fails.
1139   //
1140   // Similarly, we don't need the trailing dmb ishld if we substitute
1141   // an ldaxr instruction since that will provide all the guarantees we
1142   // require regarding observation of changes made by other threads
1143   // before any change to the CAS address observed by the load.
1144   //
1145   // In order to generate the desired instruction sequence we need to
1146   // be able to identify specific 'signature' ideal graph node
1147   // sequences which i) occur as a translation of a volatile reads or
1148   // writes or CAS operations and ii) do not occur through any other
1149   // translation or graph transformation. We can then provide
1150   // alternative aldc matching rules which translate these node
1151   // sequences to the desired machine code sequences. Selection of the
1152   // alternative rules can be implemented by predicates which identify
1153   // the relevant node sequences.
1154   //
1155   // The ideal graph generator translates a volatile read to the node
1156   // sequence
1157   //
1158   //   LoadX[mo_acquire]
1159   //   MemBarAcquire
1160   //
1161   // As a special case when using the compressed oops optimization we
1162   // may also see this variant
1163   //
1164   //   LoadN[mo_acquire]
1165   //   DecodeN
1166   //   MemBarAcquire
1167   //
1168   // A volatile write is translated to the node sequence
1169   //
1170   //   MemBarRelease
1171   //   StoreX[mo_release] {CardMark}-optional
1172   //   MemBarVolatile
1173   //
1174   // n.b. the above node patterns are generated with a strict
1175   // 'signature' configuration of input and output dependencies (see
1176   // the predicates below for exact details). The card mark may be as
1177   // simple as a few extra nodes or, in a few GC configurations, may
1178   // include more complex control flow between the leading and
1179   // trailing memory barriers. However, whatever the card mark
1180   // configuration these signatures are unique to translated volatile
1181   // reads/stores -- they will not appear as a result of any other
1182   // bytecode translation or inlining nor as a consequence of
1183   // optimizing transforms.
1184   //
1185   // We also want to catch inlined unsafe volatile gets and puts and
1186   // be able to implement them using either ldar<x>/stlr<x> or some
1187   // combination of ldr<x>/stlr<x> and dmb instructions.
1188   //
1189   // Inlined unsafe volatiles puts manifest as a minor variant of the
1190   // normal volatile put node sequence containing an extra cpuorder
1191   // membar
1192   //
1193   //   MemBarRelease
1194   //   MemBarCPUOrder
1195   //   StoreX[mo_release] {CardMark}-optional
1196   //   MemBarVolatile
1197   //
1198   // n.b. as an aside, the cpuorder membar is not itself subject to
1199   // matching and translation by adlc rules.  However, the rule
1200   // predicates need to detect its presence in order to correctly
1201   // select the desired adlc rules.
1202   //
1203   // Inlined unsafe volatile gets manifest as a somewhat different
1204   // node sequence to a normal volatile get
1205   //
1206   //   MemBarCPUOrder
1207   //        ||       \\
1208   //   MemBarAcquire LoadX[mo_acquire]
1209   //        ||
1210   //   MemBarCPUOrder
1211   //
1212   // In this case the acquire membar does not directly depend on the
1213   // load. However, we can be sure that the load is generated from an
1214   // inlined unsafe volatile get if we see it dependent on this unique
1215   // sequence of membar nodes. Similarly, given an acquire membar we
1216   // can know that it was added because of an inlined unsafe volatile
1217   // get if it is fed and feeds a cpuorder membar and if its feed
1218   // membar also feeds an acquiring load.
1219   //
1220   // Finally an inlined (Unsafe) CAS operation is translated to the
1221   // following ideal graph
1222   //
1223   //   MemBarRelease
1224   //   MemBarCPUOrder
1225   //   CompareAndSwapX {CardMark}-optional
1226   //   MemBarCPUOrder
1227   //   MemBarAcquire
1228   //
1229   // So, where we can identify these volatile read and write
1230   // signatures we can choose to plant either of the above two code
1231   // sequences. For a volatile read we can simply plant a normal
1232   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1233   // also choose to inhibit translation of the MemBarAcquire and
1234   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1235   //
1236   // When we recognise a volatile store signature we can choose to
1237   // plant at a dmb ish as a translation for the MemBarRelease, a
1238   // normal str<x> and then a dmb ish for the MemBarVolatile.
1239   // Alternatively, we can inhibit translation of the MemBarRelease
1240   // and MemBarVolatile and instead plant a simple stlr<x>
1241   // instruction.
1242   //
1243   // when we recognise a CAS signature we can choose to plant a dmb
1244   // ish as a translation for the MemBarRelease, the conventional
1245   // macro-instruction sequence for the CompareAndSwap node (which
1246   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1247   // Alternatively, we can elide generation of the dmb instructions
1248   // and plant the alternative CompareAndSwap macro-instruction
1249   // sequence (which uses ldaxr<x>).
1250   //
1251   // Of course, the above only applies when we see these signature
1252   // configurations. We still want to plant dmb instructions in any
1253   // other cases where we may see a MemBarAcquire, MemBarRelease or
1254   // MemBarVolatile. For example, at the end of a constructor which
1255   // writes final/volatile fields we will see a MemBarRelease
1256   // instruction and this needs a 'dmb ish' lest we risk the
1257   // constructed object being visible without making the
1258   // final/volatile field writes visible.
1259   //
1260   // n.b. the translation rules below which rely on detection of the
1261   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1262   // If we see anything other than the signature configurations we
1263   // always just translate the loads and stores to ldr<x> and str<x>
1264   // and translate acquire, release and volatile membars to the
1265   // relevant dmb instructions.
1266   //
1267 
1268   // graph traversal helpers used for volatile put/get and CAS
1269   // optimization
1270 
1271   // 1) general purpose helpers
1272 
1273   // if node n is linked to a parent MemBarNode by an intervening
1274   // Control and Memory ProjNode return the MemBarNode otherwise return
1275   // NULL.
1276   //
1277   // n may only be a Load or a MemBar.
1278 
1279   MemBarNode *parent_membar(const Node *n)
1280   {
1281     Node *ctl = NULL;
1282     Node *mem = NULL;
1283     Node *membar = NULL;
1284 
1285     if (n->is_Load()) {
1286       ctl = n->lookup(LoadNode::Control);
1287       mem = n->lookup(LoadNode::Memory);
1288     } else if (n->is_MemBar()) {
1289       ctl = n->lookup(TypeFunc::Control);
1290       mem = n->lookup(TypeFunc::Memory);
1291     } else {
1292         return NULL;
1293     }
1294 
1295     if (!ctl || !mem || !ctl->is_Proj() || !mem->is_Proj()) {
1296       return NULL;
1297     }
1298 
1299     membar = ctl->lookup(0);
1300 
1301     if (!membar || !membar->is_MemBar()) {
1302       return NULL;
1303     }
1304 
1305     if (mem->lookup(0) != membar) {
1306       return NULL;
1307     }
1308 
1309     return membar->as_MemBar();
1310   }
1311 
1312   // if n is linked to a child MemBarNode by intervening Control and
1313   // Memory ProjNodes return the MemBarNode otherwise return NULL.
1314 
1315   MemBarNode *child_membar(const MemBarNode *n)
1316   {
1317     ProjNode *ctl = n->proj_out(TypeFunc::Control);
1318     ProjNode *mem = n->proj_out(TypeFunc::Memory);
1319 
1320     // MemBar needs to have both a Ctl and Mem projection
1321     if (! ctl || ! mem)
1322       return NULL;
1323 
1324     MemBarNode *child = NULL;
1325     Node *x;
1326 
1327     for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {
1328       x = ctl->fast_out(i);
1329       // if we see a membar we keep hold of it. we may also see a new
1330       // arena copy of the original but it will appear later
1331       if (x->is_MemBar()) {
1332           child = x->as_MemBar();
1333           break;
1334       }
1335     }
1336 
1337     if (child == NULL) {
1338       return NULL;
1339     }
1340 
1341     for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
1342       x = mem->fast_out(i);
1343       // if we see a membar we keep hold of it. we may also see a new
1344       // arena copy of the original but it will appear later
1345       if (x == child) {
1346         return child;
1347       }
1348     }
1349     return NULL;
1350   }
1351 
1352   // helper predicate use to filter candidates for a leading memory
1353   // barrier
1354   //
1355   // returns true if barrier is a MemBarRelease or a MemBarCPUOrder
1356   // whose Ctl and Mem feeds come from a MemBarRelease otherwise false
1357 
1358   bool leading_membar(const MemBarNode *barrier)
1359   {
1360     int opcode = barrier->Opcode();
1361     // if this is a release membar we are ok
1362     if (opcode == Op_MemBarRelease) {
1363       return true;
1364     }
1365     // if its a cpuorder membar . . .
1366     if (opcode != Op_MemBarCPUOrder) {
1367       return false;
1368     }
1369     // then the parent has to be a release membar
1370     MemBarNode *parent = parent_membar(barrier);
1371     if (!parent) {
1372       return false;
1373     }
1374     opcode = parent->Opcode();
1375     return opcode == Op_MemBarRelease;
1376   }
1377 
1378   // 2) card mark detection helper
1379 
1380   // helper predicate which can be used to detect a volatile membar
1381   // introduced as part of a conditional card mark sequence either by
1382   // G1 or by CMS when UseCondCardMark is true.
1383   //
1384   // membar can be definitively determined to be part of a card mark
1385   // sequence if and only if all the following hold
1386   //
1387   // i) it is a MemBarVolatile
1388   //
1389   // ii) either UseG1GC or (UseConcMarkSweepGC && UseCondCardMark) is
1390   // true
1391   //
1392   // iii) the node's Mem projection feeds a StoreCM node.
1393 
1394   bool is_card_mark_membar(const MemBarNode *barrier)
1395   {
1396     if (!UseG1GC && !(UseConcMarkSweepGC && UseCondCardMark)) {
1397       return false;
1398     }
1399 
1400     if (barrier->Opcode() != Op_MemBarVolatile) {
1401       return false;
1402     }
1403 
1404     ProjNode *mem = barrier->proj_out(TypeFunc::Memory);
1405 
1406     for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax ; i++) {
1407       Node *y = mem->fast_out(i);
1408       if (y->Opcode() == Op_StoreCM) {
1409         return true;
1410       }
1411     }
1412 
1413     return false;
1414   }
1415 
1416 
1417   // 3) helper predicates to traverse volatile put or CAS graphs which
1418   // may contain GC barrier subgraphs
1419 
1420   // Preamble
1421   // --------
1422   //
1423   // for volatile writes we can omit generating barriers and employ a
1424   // releasing store when we see a node sequence sequence with a
1425   // leading MemBarRelease and a trailing MemBarVolatile as follows
1426   //
1427   //   MemBarRelease
1428   //  {      ||      } -- optional
1429   //  {MemBarCPUOrder}
1430   //         ||     \\
1431   //         ||     StoreX[mo_release]
1432   //         | \     /
1433   //         | MergeMem
1434   //         | /
1435   //   MemBarVolatile
1436   //
1437   // where
1438   //  || and \\ represent Ctl and Mem feeds via Proj nodes
1439   //  | \ and / indicate further routing of the Ctl and Mem feeds
1440   //
1441   // this is the graph we see for non-object stores. however, for a
1442   // volatile Object store (StoreN/P) we may see other nodes below the
1443   // leading membar because of the need for a GC pre- or post-write
1444   // barrier.
1445   //
1446   // with most GC configurations we with see this simple variant which
1447   // includes a post-write barrier card mark.
1448   //
1449   //   MemBarRelease______________________________
1450   //         ||    \\               Ctl \        \\
1451   //         ||    StoreN/P[mo_release] CastP2X  StoreB/CM
1452   //         | \     /                       . . .  /
1453   //         | MergeMem
1454   //         | /
1455   //         ||      /
1456   //   MemBarVolatile
1457   //
1458   // i.e. the leading membar feeds Ctl to a CastP2X (which converts
1459   // the object address to an int used to compute the card offset) and
1460   // Ctl+Mem to a StoreB node (which does the actual card mark).
1461   //
1462   // n.b. a StoreCM node will only appear in this configuration when
1463   // using CMS. StoreCM differs from a normal card mark write (StoreB)
1464   // because it implies a requirement to order visibility of the card
1465   // mark (StoreCM) relative to the object put (StoreP/N) using a
1466   // StoreStore memory barrier (arguably this ought to be represented
1467   // explicitly in the ideal graph but that is not how it works). This
1468   // ordering is required for both non-volatile and volatile
1469   // puts. Normally that means we need to translate a StoreCM using
1470   // the sequence
1471   //
1472   //   dmb ishst
1473   //   stlrb
1474   //
1475   // However, in the case of a volatile put if we can recognise this
1476   // configuration and plant an stlr for the object write then we can
1477   // omit the dmb and just plant an strb since visibility of the stlr
1478   // is ordered before visibility of subsequent stores. StoreCM nodes
1479   // also arise when using G1 or using CMS with conditional card
1480   // marking. In these cases (as we shall see) we don't need to insert
1481   // the dmb when translating StoreCM because there is already an
1482   // intervening StoreLoad barrier between it and the StoreP/N.
1483   //
1484   // It is also possible to perform the card mark conditionally on it
1485   // currently being unmarked in which case the volatile put graph
1486   // will look slightly different
1487   //
1488   //   MemBarRelease____________________________________________
1489   //         ||    \\               Ctl \     Ctl \     \\  Mem \
1490   //         ||    StoreN/P[mo_release] CastP2X   If   LoadB     |
1491   //         | \     /                              \            |
1492   //         | MergeMem                            . . .      StoreB
1493   //         | /                                                /
1494   //         ||     /
1495   //   MemBarVolatile
1496   //
1497   // It is worth noting at this stage that both the above
1498   // configurations can be uniquely identified by checking that the
1499   // memory flow includes the following subgraph:
1500   //
1501   //   MemBarRelease
1502   //  {MemBarCPUOrder}
1503   //          |  \      . . .
1504   //          |  StoreX[mo_release]  . . .
1505   //          |   /
1506   //         MergeMem
1507   //          |
1508   //   MemBarVolatile
1509   //
1510   // This is referred to as a *normal* subgraph. It can easily be
1511   // detected starting from any candidate MemBarRelease,
1512   // StoreX[mo_release] or MemBarVolatile.
1513   //
1514   // A simple variation on this normal case occurs for an unsafe CAS
1515   // operation. The basic graph for a non-object CAS is
1516   //
1517   //   MemBarRelease
1518   //         ||
1519   //   MemBarCPUOrder
1520   //         ||     \\   . . .
1521   //         ||     CompareAndSwapX
1522   //         ||       |
1523   //         ||     SCMemProj
1524   //         | \     /
1525   //         | MergeMem
1526   //         | /
1527   //   MemBarCPUOrder
1528   //         ||
1529   //   MemBarAcquire
1530   //
1531   // The same basic variations on this arrangement (mutatis mutandis)
1532   // occur when a card mark is introduced. i.e. we se the same basic
1533   // shape but the StoreP/N is replaced with CompareAndSawpP/N and the
1534   // tail of the graph is a pair comprising a MemBarCPUOrder +
1535   // MemBarAcquire.
1536   //
1537   // So, in the case of a CAS the normal graph has the variant form
1538   //
1539   //   MemBarRelease
1540   //   MemBarCPUOrder
1541   //          |   \      . . .
1542   //          |  CompareAndSwapX  . . .
1543   //          |    |
1544   //          |   SCMemProj
1545   //          |   /  . . .
1546   //         MergeMem
1547   //          |
1548   //   MemBarCPUOrder
1549   //   MemBarAcquire
1550   //
1551   // This graph can also easily be detected starting from any
1552   // candidate MemBarRelease, CompareAndSwapX or MemBarAcquire.
1553   //
1554   // the code below uses two helper predicates, leading_to_normal and
1555   // normal_to_leading to identify these normal graphs, one validating
1556   // the layout starting from the top membar and searching down and
1557   // the other validating the layout starting from the lower membar
1558   // and searching up.
1559   //
1560   // There are two special case GC configurations when a normal graph
1561   // may not be generated: when using G1 (which always employs a
1562   // conditional card mark); and when using CMS with conditional card
1563   // marking configured. These GCs are both concurrent rather than
1564   // stop-the world GCs. So they introduce extra Ctl+Mem flow into the
1565   // graph between the leading and trailing membar nodes, in
1566   // particular enforcing stronger memory serialisation beween the
1567   // object put and the corresponding conditional card mark. CMS
1568   // employs a post-write GC barrier while G1 employs both a pre- and
1569   // post-write GC barrier. Of course the extra nodes may be absent --
1570   // they are only inserted for object puts. This significantly
1571   // complicates the task of identifying whether a MemBarRelease,
1572   // StoreX[mo_release] or MemBarVolatile forms part of a volatile put
1573   // when using these GC configurations (see below). It adds similar
1574   // complexity to the task of identifying whether a MemBarRelease,
1575   // CompareAndSwapX or MemBarAcquire forms part of a CAS.
1576   //
1577   // In both cases the post-write subtree includes an auxiliary
1578   // MemBarVolatile (StoreLoad barrier) separating the object put and
1579   // the read of the corresponding card. This poses two additional
1580   // problems.
1581   //
1582   // Firstly, a card mark MemBarVolatile needs to be distinguished
1583   // from a normal trailing MemBarVolatile. Resolving this first
1584   // problem is straightforward: a card mark MemBarVolatile always
1585   // projects a Mem feed to a StoreCM node and that is a unique marker
1586   //
1587   //      MemBarVolatile (card mark)
1588   //       C |    \     . . .
1589   //         |   StoreCM   . . .
1590   //       . . .
1591   //
1592   // The second problem is how the code generator is to translate the
1593   // card mark barrier? It always needs to be translated to a "dmb
1594   // ish" instruction whether or not it occurs as part of a volatile
1595   // put. A StoreLoad barrier is needed after the object put to ensure
1596   // i) visibility to GC threads of the object put and ii) visibility
1597   // to the mutator thread of any card clearing write by a GC
1598   // thread. Clearly a normal store (str) will not guarantee this
1599   // ordering but neither will a releasing store (stlr). The latter
1600   // guarantees that the object put is visible but does not guarantee
1601   // that writes by other threads have also been observed.
1602   //
1603   // So, returning to the task of translating the object put and the
1604   // leading/trailing membar nodes: what do the non-normal node graph
1605   // look like for these 2 special cases? and how can we determine the
1606   // status of a MemBarRelease, StoreX[mo_release] or MemBarVolatile
1607   // in both normal and non-normal cases?
1608   //
1609   // A CMS GC post-barrier wraps its card write (StoreCM) inside an If
1610   // which selects conditonal execution based on the value loaded
1611   // (LoadB) from the card. Ctl and Mem are fed to the If via an
1612   // intervening StoreLoad barrier (MemBarVolatile).
1613   //
1614   // So, with CMS we may see a node graph for a volatile object store
1615   // which looks like this
1616   //
1617   //   MemBarRelease
1618   //   MemBarCPUOrder_(leading)__________________
1619   //     C |    M \       \\                   C \
1620   //       |       \    StoreN/P[mo_release]  CastP2X
1621   //       |    Bot \    /
1622   //       |       MergeMem
1623   //       |         /
1624   //      MemBarVolatile (card mark)
1625   //     C |  ||    M |
1626   //       | LoadB    |
1627   //       |   |      |
1628   //       | Cmp      |\
1629   //       | /        | \
1630   //       If         |  \
1631   //       | \        |   \
1632   // IfFalse  IfTrue  |    \
1633   //       \     / \  |     \
1634   //        \   / StoreCM    |
1635   //         \ /      |      |
1636   //        Region   . . .   |
1637   //          | \           /
1638   //          |  . . .  \  / Bot
1639   //          |       MergeMem
1640   //          |          |
1641   //        MemBarVolatile (trailing)
1642   //
1643   // The first MergeMem merges the AliasIdxBot Mem slice from the
1644   // leading membar and the oopptr Mem slice from the Store into the
1645   // card mark membar. The trailing MergeMem merges the AliasIdxBot
1646   // Mem slice from the card mark membar and the AliasIdxRaw slice
1647   // from the StoreCM into the trailing membar (n.b. the latter
1648   // proceeds via a Phi associated with the If region).
1649   //
1650   // The graph for a CAS varies slightly, the obvious difference being
1651   // that the StoreN/P node is replaced by a CompareAndSwapP/N node
1652   // and the trailing MemBarVolatile by a MemBarCPUOrder +
1653   // MemBarAcquire pair. The other important difference is that the
1654   // CompareAndSwap node's SCMemProj is not merged into the card mark
1655   // membar - it still feeds the trailing MergeMem. This also means
1656   // that the card mark membar receives its Mem feed directly from the
1657   // leading membar rather than via a MergeMem.
1658   //
1659   //   MemBarRelease
1660   //   MemBarCPUOrder__(leading)_________________________
1661   //       ||                       \\                 C \
1662   //   MemBarVolatile (card mark)  CompareAndSwapN/P  CastP2X
1663   //     C |  ||    M |              |
1664   //       | LoadB    |       ______/|
1665   //       |   |      |      /       |
1666   //       | Cmp      |     /      SCMemProj
1667   //       | /        |    /         |
1668   //       If         |   /         /
1669   //       | \        |  /         /
1670   // IfFalse  IfTrue  | /         /
1671   //       \     / \  |/ prec    /
1672   //        \   / StoreCM       /
1673   //         \ /      |        /
1674   //        Region   . . .    /
1675   //          | \            /
1676   //          |  . . .  \   / Bot
1677   //          |       MergeMem
1678   //          |          |
1679   //        MemBarCPUOrder
1680   //        MemBarAcquire (trailing)
1681   //
1682   // This has a slightly different memory subgraph to the one seen
1683   // previously but the core of it is the same as for the CAS normal
1684   // sungraph
1685   //
1686   //   MemBarRelease
1687   //   MemBarCPUOrder____
1688   //      ||             \      . . .
1689   //   MemBarVolatile  CompareAndSwapX  . . .
1690   //      |  \            |
1691   //        . . .   SCMemProj
1692   //          |     /  . . .
1693   //         MergeMem
1694   //          |
1695   //   MemBarCPUOrder
1696   //   MemBarAcquire
1697   //
1698   //
1699   // G1 is quite a lot more complicated. The nodes inserted on behalf
1700   // of G1 may comprise: a pre-write graph which adds the old value to
1701   // the SATB queue; the releasing store itself; and, finally, a
1702   // post-write graph which performs a card mark.
1703   //
1704   // The pre-write graph may be omitted, but only when the put is
1705   // writing to a newly allocated (young gen) object and then only if
1706   // there is a direct memory chain to the Initialize node for the
1707   // object allocation. This will not happen for a volatile put since
1708   // any memory chain passes through the leading membar.
1709   //
1710   // The pre-write graph includes a series of 3 If tests. The outermost
1711   // If tests whether SATB is enabled (no else case). The next If tests
1712   // whether the old value is non-NULL (no else case). The third tests
1713   // whether the SATB queue index is > 0, if so updating the queue. The
1714   // else case for this third If calls out to the runtime to allocate a
1715   // new queue buffer.
1716   //
1717   // So with G1 the pre-write and releasing store subgraph looks like
1718   // this (the nested Ifs are omitted).
1719   //
1720   //  MemBarRelease (leading)____________
1721   //     C |  ||  M \   M \    M \  M \ . . .
1722   //       | LoadB   \  LoadL  LoadN   \
1723   //       | /        \                 \
1724   //       If         |\                 \
1725   //       | \        | \                 \
1726   //  IfFalse  IfTrue |  \                 \
1727   //       |     |    |   \                 |
1728   //       |     If   |   /\                |
1729   //       |     |          \               |
1730   //       |                 \              |
1731   //       |    . . .         \             |
1732   //       | /       | /       |            |
1733   //      Region  Phi[M]       |            |
1734   //       | \       |         |            |
1735   //       |  \_____ | ___     |            |
1736   //     C | C \     |   C \ M |            |
1737   //       | CastP2X | StoreN/P[mo_release] |
1738   //       |         |         |            |
1739   //     C |       M |       M |          M |
1740   //        \        |         |           /
1741   //                  . . .
1742   //          (post write subtree elided)
1743   //                    . . .
1744   //             C \         M /
1745   //         MemBarVolatile (trailing)
1746   //
1747   // n.b. the LoadB in this subgraph is not the card read -- it's a
1748   // read of the SATB queue active flag.
1749   //
1750   // Once again the CAS graph is a minor variant on the above with the
1751   // expected substitutions of CompareAndSawpX for StoreN/P and
1752   // MemBarCPUOrder + MemBarAcquire for trailing MemBarVolatile.
1753   //
1754   // The G1 post-write subtree is also optional, this time when the
1755   // new value being written is either null or can be identified as a
1756   // newly allocated (young gen) object with no intervening control
1757   // flow. The latter cannot happen but the former may, in which case
1758   // the card mark membar is omitted and the memory feeds form the
1759   // leading membar and the SToreN/P are merged direct into the
1760   // trailing membar as per the normal subgraph. So, the only special
1761   // case which arises is when the post-write subgraph is generated.
1762   //
1763   // The kernel of the post-write G1 subgraph is the card mark itself
1764   // which includes a card mark memory barrier (MemBarVolatile), a
1765   // card test (LoadB), and a conditional update (If feeding a
1766   // StoreCM). These nodes are surrounded by a series of nested Ifs
1767   // which try to avoid doing the card mark. The top level If skips if
1768   // the object reference does not cross regions (i.e. it tests if
1769   // (adr ^ val) >> log2(regsize) != 0) -- intra-region references
1770   // need not be recorded. The next If, which skips on a NULL value,
1771   // may be absent (it is not generated if the type of value is >=
1772   // OopPtr::NotNull). The 3rd If skips writes to young regions (by
1773   // checking if card_val != young).  n.b. although this test requires
1774   // a pre-read of the card it can safely be done before the StoreLoad
1775   // barrier. However that does not bypass the need to reread the card
1776   // after the barrier.
1777   //
1778   //                (pre-write subtree elided)
1779   //        . . .                  . . .    . . .  . . .
1780   //        C |                    M |     M |    M |
1781   //       Region                  Phi[M] StoreN    |
1782   //          |                     / \      |      |
1783   //         / \_______            /   \     |      |
1784   //      C / C \      . . .            \    |      |
1785   //       If   CastP2X . . .            |   |      |
1786   //       / \                           |   |      |
1787   //      /   \                          |   |      |
1788   // IfFalse IfTrue                      |   |      |
1789   //   |       |                         |   |     /|
1790   //   |       If                        |   |    / |
1791   //   |      / \                        |   |   /  |
1792   //   |     /   \                        \  |  /   |
1793   //   | IfFalse IfTrue                   MergeMem  |
1794   //   |  . . .    / \                       /      |
1795   //   |          /   \                     /       |
1796   //   |     IfFalse IfTrue                /        |
1797   //   |      . . .    |                  /         |
1798   //   |               If                /          |
1799   //   |               / \              /           |
1800   //   |              /   \            /            |
1801   //   |         IfFalse IfTrue       /             |
1802   //   |           . . .   |         /              |
1803   //   |                    \       /               |
1804   //   |                     \     /                |
1805   //   |             MemBarVolatile__(card mark)    |
1806   //   |                ||   C |  M \  M \          |
1807   //   |               LoadB   If    |    |         |
1808   //   |                      / \    |    |         |
1809   //   |                     . . .   |    |         |
1810   //   |                          \  |    |        /
1811   //   |                        StoreCM   |       /
1812   //   |                          . . .   |      /
1813   //   |                        _________/      /
1814   //   |                       /  _____________/
1815   //   |   . . .       . . .  |  /            /
1816   //   |    |                 | /   _________/
1817   //   |    |               Phi[M] /        /
1818   //   |    |                 |   /        /
1819   //   |    |                 |  /        /
1820   //   |  Region  . . .     Phi[M]  _____/
1821   //   |    /                 |    /
1822   //   |                      |   /
1823   //   | . . .   . . .        |  /
1824   //   | /                    | /
1825   // Region           |  |  Phi[M]
1826   //   |              |  |  / Bot
1827   //    \            MergeMem
1828   //     \            /
1829   //     MemBarVolatile
1830   //
1831   // As with CMS the initial MergeMem merges the AliasIdxBot Mem slice
1832   // from the leading membar and the oopptr Mem slice from the Store
1833   // into the card mark membar i.e. the memory flow to the card mark
1834   // membar still looks like a normal graph.
1835   //
1836   // The trailing MergeMem merges an AliasIdxBot Mem slice with other
1837   // Mem slices (from the StoreCM and other card mark queue stores).
1838   // However in this case the AliasIdxBot Mem slice does not come
1839   // direct from the card mark membar. It is merged through a series
1840   // of Phi nodes. These are needed to merge the AliasIdxBot Mem flow
1841   // from the leading membar with the Mem feed from the card mark
1842   // membar. Each Phi corresponds to one of the Ifs which may skip
1843   // around the card mark membar. So when the If implementing the NULL
1844   // value check has been elided the total number of Phis is 2
1845   // otherwise it is 3.
1846   //
1847   // The CAS graph when using G1GC also includes a pre-write subgraph
1848   // and an optional post-write subgraph. Teh sam evarioations are
1849   // introduced as for CMS with conditional card marking i.e. the
1850   // StoreP/N is swapped for a CompareAndSwapP/N, the tariling
1851   // MemBarVolatile for a MemBarCPUOrder + MemBarAcquire pair and the
1852   // Mem feed from the CompareAndSwapP/N includes a precedence
1853   // dependency feed to the StoreCM and a feed via an SCMemProj to the
1854   // trailing membar. So, as before the configuration includes the
1855   // normal CAS graph as a subgraph of the memory flow.
1856   //
1857   // So, the upshot is that in all cases the volatile put graph will
1858   // include a *normal* memory subgraph betwen the leading membar and
1859   // its child membar, either a volatile put graph (including a
1860   // releasing StoreX) or a CAS graph (including a CompareAndSwapX).
1861   // When that child is not a card mark membar then it marks the end
1862   // of the volatile put or CAS subgraph. If the child is a card mark
1863   // membar then the normal subgraph will form part of a volatile put
1864   // subgraph if and only if the child feeds an AliasIdxBot Mem feed
1865   // to a trailing barrier via a MergeMem. That feed is either direct
1866   // (for CMS) or via 2 or 3 Phi nodes merging the leading barrier
1867   // memory flow (for G1).
1868   //
1869   // The predicates controlling generation of instructions for store
1870   // and barrier nodes employ a few simple helper functions (described
1871   // below) which identify the presence or absence of all these
1872   // subgraph configurations and provide a means of traversing from
1873   // one node in the subgraph to another.
1874 
1875   // is_CAS(int opcode)
1876   //
1877   // return true if opcode is one of the possible CompareAndSwapX
1878   // values otherwise false.
1879 
1880   bool is_CAS(int opcode)
1881   {
1882     switch(opcode) {
1883       // We handle these
1884     case Op_CompareAndSwapI:
1885     case Op_CompareAndSwapL:
1886     case Op_CompareAndSwapP:
1887     case Op_CompareAndSwapN:
1888  // case Op_CompareAndSwapB:
1889  // case Op_CompareAndSwapS:
1890       return true;
1891       // These are TBD
1892     case Op_WeakCompareAndSwapB:
1893     case Op_WeakCompareAndSwapS:
1894     case Op_WeakCompareAndSwapI:
1895     case Op_WeakCompareAndSwapL:
1896     case Op_WeakCompareAndSwapP:
1897     case Op_WeakCompareAndSwapN:
1898     case Op_CompareAndExchangeB:
1899     case Op_CompareAndExchangeS:
1900     case Op_CompareAndExchangeI:
1901     case Op_CompareAndExchangeL:
1902     case Op_CompareAndExchangeP:
1903     case Op_CompareAndExchangeN:
1904       return false;
1905     default:
1906       return false;
1907     }
1908   }
1909 
1910 
1911   // leading_to_normal
1912   //
1913   //graph traversal helper which detects the normal case Mem feed from
1914   // a release membar (or, optionally, its cpuorder child) to a
1915   // dependent volatile membar i.e. it ensures that one or other of
1916   // the following Mem flow subgraph is present.
1917   //
1918   //   MemBarRelease
1919   //   MemBarCPUOrder {leading}
1920   //          |  \      . . .
1921   //          |  StoreN/P[mo_release]  . . .
1922   //          |   /
1923   //         MergeMem
1924   //          |
1925   //   MemBarVolatile {trailing or card mark}
1926   //
1927   //   MemBarRelease
1928   //   MemBarCPUOrder {leading}
1929   //      |       \      . . .
1930   //      |     CompareAndSwapX  . . .
1931   //               |
1932   //     . . .    SCMemProj
1933   //           \   |
1934   //      |    MergeMem
1935   //      |       /
1936   //    MemBarCPUOrder
1937   //    MemBarAcquire {trailing}
1938   //
1939   // if the correct configuration is present returns the trailing
1940   // membar otherwise NULL.
1941   //
1942   // the input membar is expected to be either a cpuorder membar or a
1943   // release membar. in the latter case it should not have a cpu membar
1944   // child.
1945   //
1946   // the returned value may be a card mark or trailing membar
1947   //
1948 
1949   MemBarNode *leading_to_normal(MemBarNode *leading)
1950   {
1951     assert((leading->Opcode() == Op_MemBarRelease ||
1952             leading->Opcode() == Op_MemBarCPUOrder),
1953            "expecting a volatile or cpuroder membar!");
1954 
1955     // check the mem flow
1956     ProjNode *mem = leading->proj_out(TypeFunc::Memory);
1957 
1958     if (!mem) {
1959       return NULL;
1960     }
1961 
1962     Node *x = NULL;
1963     StoreNode * st = NULL;
1964     LoadStoreNode *cas = NULL;
1965     MergeMemNode *mm = NULL;
1966 
1967     for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
1968       x = mem->fast_out(i);
1969       if (x->is_MergeMem()) {
1970         if (mm != NULL) {
1971           return NULL;
1972         }
1973         // two merge mems is one too many
1974         mm = x->as_MergeMem();
1975       } else if (x->is_Store() && x->as_Store()->is_release() && x->Opcode() != Op_StoreCM) {
1976         // two releasing stores/CAS nodes is one too many
1977         if (st != NULL || cas != NULL) {
1978           return NULL;
1979         }
1980         st = x->as_Store();
1981       } else if (is_CAS(x->Opcode())) {
1982         if (st != NULL || cas != NULL) {
1983           return NULL;
1984         }
1985         cas = x->as_LoadStore();
1986       }
1987     }
1988 
1989     // must have a store or a cas
1990     if (!st && !cas) {
1991       return NULL;
1992     }
1993 
1994     // must have a merge if we also have st
1995     if (st && !mm) {
1996       return NULL;
1997     }
1998 
1999     Node *y = NULL;
2000     if (cas) {
2001       // look for an SCMemProj
2002       for (DUIterator_Fast imax, i = cas->fast_outs(imax); i < imax; i++) {
2003         x = cas->fast_out(i);
2004         if (x->is_Proj()) {
2005           y = x;
2006           break;
2007         }
2008       }
2009       if (y == NULL) {
2010         return NULL;
2011       }
2012       // the proj must feed a MergeMem
2013       for (DUIterator_Fast imax, i = y->fast_outs(imax); i < imax; i++) {
2014         x = y->fast_out(i);
2015         if (x->is_MergeMem()) {
2016           mm = x->as_MergeMem();
2017           break;
2018         }
2019       }
2020       if (mm == NULL)
2021         return NULL;
2022     } else {
2023       // ensure the store feeds the existing mergemem;
2024       for (DUIterator_Fast imax, i = st->fast_outs(imax); i < imax; i++) {
2025         if (st->fast_out(i) == mm) {
2026           y = st;
2027           break;
2028         }
2029       }
2030       if (y == NULL) {
2031         return NULL;
2032       }
2033     }
2034 
2035     MemBarNode *mbar = NULL;
2036     // ensure the merge feeds to the expected type of membar
2037     for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {
2038       x = mm->fast_out(i);
2039       if (x->is_MemBar()) {
2040         int opcode = x->Opcode();
2041         if (opcode == Op_MemBarVolatile && st) {
2042           mbar = x->as_MemBar();
2043         } else if (cas && opcode == Op_MemBarCPUOrder) {
2044           MemBarNode *y =  x->as_MemBar();
2045           y = child_membar(y);
2046           if (y != NULL && y->Opcode() == Op_MemBarAcquire) {
2047             mbar = y;
2048           }
2049         }
2050         break;
2051       }
2052     }
2053 
2054     return mbar;
2055   }
2056 
2057   // normal_to_leading
2058   //
2059   // graph traversal helper which detects the normal case Mem feed
2060   // from either a card mark or a trailing membar to a preceding
2061   // release membar (optionally its cpuorder child) i.e. it ensures
2062   // that one or other of the following Mem flow subgraphs is present.
2063   //
2064   //   MemBarRelease
2065   //   MemBarCPUOrder {leading}
2066   //          |  \      . . .
2067   //          |  StoreN/P[mo_release]  . . .
2068   //          |   /
2069   //         MergeMem
2070   //          |
2071   //   MemBarVolatile {card mark or trailing}
2072   //
2073   //   MemBarRelease
2074   //   MemBarCPUOrder {leading}
2075   //      |       \      . . .
2076   //      |     CompareAndSwapX  . . .
2077   //               |
2078   //     . . .    SCMemProj
2079   //           \   |
2080   //      |    MergeMem
2081   //      |        /
2082   //    MemBarCPUOrder
2083   //    MemBarAcquire {trailing}
2084   //
2085   // this predicate checks for the same flow as the previous predicate
2086   // but starting from the bottom rather than the top.
2087   //
2088   // if the configuration is present returns the cpuorder member for
2089   // preference or when absent the release membar otherwise NULL.
2090   //
2091   // n.b. the input membar is expected to be a MemBarVolatile but
2092   // need not be a card mark membar.
2093 
2094   MemBarNode *normal_to_leading(const MemBarNode *barrier)
2095   {
2096     // input must be a volatile membar
2097     assert((barrier->Opcode() == Op_MemBarVolatile ||
2098             barrier->Opcode() == Op_MemBarAcquire),
2099            "expecting a volatile or an acquire membar");
2100     Node *x;
2101     bool is_cas = barrier->Opcode() == Op_MemBarAcquire;
2102 
2103     // if we have an acquire membar then it must be fed via a CPUOrder
2104     // membar
2105 
2106     if (is_cas) {
2107       // skip to parent barrier which must be a cpuorder
2108       x = parent_membar(barrier);
2109       if (x->Opcode() != Op_MemBarCPUOrder)
2110         return NULL;
2111     } else {
2112       // start from the supplied barrier
2113       x = (Node *)barrier;
2114     }
2115 
2116     // the Mem feed to the membar should be a merge
2117     x = x ->in(TypeFunc::Memory);
2118     if (!x->is_MergeMem())
2119       return NULL;
2120 
2121     MergeMemNode *mm = x->as_MergeMem();
2122 
2123     if (is_cas) {
2124       // the merge should be fed from the CAS via an SCMemProj node
2125       x = NULL;
2126       for (uint idx = 1; idx < mm->req(); idx++) {
2127         if (mm->in(idx)->Opcode() == Op_SCMemProj) {
2128           x = mm->in(idx);
2129           break;
2130         }
2131       }
2132       if (x == NULL) {
2133         return NULL;
2134       }
2135       // check for a CAS feeding this proj
2136       x = x->in(0);
2137       int opcode = x->Opcode();
2138       if (!is_CAS(opcode)) {
2139         return NULL;
2140       }
2141       // the CAS should get its mem feed from the leading membar
2142       x = x->in(MemNode::Memory);
2143     } else {
2144       // the merge should get its Bottom mem feed from the leading membar
2145       x = mm->in(Compile::AliasIdxBot);
2146     }
2147 
2148     // ensure this is a non control projection
2149     if (!x->is_Proj() || x->is_CFG()) {
2150       return NULL;
2151     }
2152     // if it is fed by a membar that's the one we want
2153     x = x->in(0);
2154 
2155     if (!x->is_MemBar()) {
2156       return NULL;
2157     }
2158 
2159     MemBarNode *leading = x->as_MemBar();
2160     // reject invalid candidates
2161     if (!leading_membar(leading)) {
2162       return NULL;
2163     }
2164 
2165     // ok, we have a leading membar, now for the sanity clauses
2166 
2167     // the leading membar must feed Mem to a releasing store or CAS
2168     ProjNode *mem = leading->proj_out(TypeFunc::Memory);
2169     StoreNode *st = NULL;
2170     LoadStoreNode *cas = NULL;
2171     for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
2172       x = mem->fast_out(i);
2173       if (x->is_Store() && x->as_Store()->is_release() && x->Opcode() != Op_StoreCM) {
2174         // two stores or CASes is one too many
2175         if (st != NULL || cas != NULL) {
2176           return NULL;
2177         }
2178         st = x->as_Store();
2179       } else if (is_CAS(x->Opcode())) {
2180         if (st != NULL || cas != NULL) {
2181           return NULL;
2182         }
2183         cas = x->as_LoadStore();
2184       }
2185     }
2186 
2187     // we should not have both a store and a cas
2188     if (st == NULL & cas == NULL) {
2189       return NULL;
2190     }
2191 
2192     if (st == NULL) {
2193       // nothing more to check
2194       return leading;
2195     } else {
2196       // we should not have a store if we started from an acquire
2197       if (is_cas) {
2198         return NULL;
2199       }
2200 
2201       // the store should feed the merge we used to get here
2202       for (DUIterator_Fast imax, i = st->fast_outs(imax); i < imax; i++) {
2203         if (st->fast_out(i) == mm) {
2204           return leading;
2205         }
2206       }
2207     }
2208 
2209     return NULL;
2210   }
2211 
2212   // card_mark_to_trailing
2213   //
2214   // graph traversal helper which detects extra, non-normal Mem feed
2215   // from a card mark volatile membar to a trailing membar i.e. it
2216   // ensures that one of the following three GC post-write Mem flow
2217   // subgraphs is present.
2218   //
2219   // 1)
2220   //     . . .
2221   //       |
2222   //   MemBarVolatile (card mark)
2223   //      |          |
2224   //      |        StoreCM
2225   //      |          |
2226   //      |        . . .
2227   //  Bot |  /
2228   //   MergeMem
2229   //      |
2230   //      |
2231   //    MemBarVolatile {trailing}
2232   //
2233   // 2)
2234   //   MemBarRelease/CPUOrder (leading)
2235   //    |
2236   //    |
2237   //    |\       . . .
2238   //    | \        |
2239   //    |  \  MemBarVolatile (card mark)
2240   //    |   \   |     |
2241   //     \   \  |   StoreCM    . . .
2242   //      \   \ |
2243   //       \  Phi
2244   //        \ /
2245   //        Phi  . . .
2246   //     Bot |   /
2247   //       MergeMem
2248   //         |
2249   //    MemBarVolatile {trailing}
2250   //
2251   //
2252   // 3)
2253   //   MemBarRelease/CPUOrder (leading)
2254   //    |
2255   //    |\
2256   //    | \
2257   //    |  \      . . .
2258   //    |   \       |
2259   //    |\   \  MemBarVolatile (card mark)
2260   //    | \   \   |     |
2261   //    |  \   \  |   StoreCM    . . .
2262   //    |   \   \ |
2263   //     \   \  Phi
2264   //      \   \ /
2265   //       \  Phi
2266   //        \ /
2267   //        Phi  . . .
2268   //     Bot |   /
2269   //       MergeMem
2270   //         |
2271   //         |
2272   //    MemBarVolatile {trailing}
2273   //
2274   // configuration 1 is only valid if UseConcMarkSweepGC &&
2275   // UseCondCardMark
2276   //
2277   // configurations 2 and 3 are only valid if UseG1GC.
2278   //
2279   // if a valid configuration is present returns the trailing membar
2280   // otherwise NULL.
2281   //
2282   // n.b. the supplied membar is expected to be a card mark
2283   // MemBarVolatile i.e. the caller must ensure the input node has the
2284   // correct operand and feeds Mem to a StoreCM node
2285 
2286   MemBarNode *card_mark_to_trailing(const MemBarNode *barrier)
2287   {
2288     // input must be a card mark volatile membar
2289     assert(is_card_mark_membar(barrier), "expecting a card mark membar");
2290 
2291     Node *feed = barrier->proj_out(TypeFunc::Memory);
2292     Node *x;
2293     MergeMemNode *mm = NULL;
2294 
2295     const int MAX_PHIS = 3;     // max phis we will search through
2296     int phicount = 0;           // current search count
2297 
2298     bool retry_feed = true;
2299     while (retry_feed) {
2300       // see if we have a direct MergeMem feed
2301       for (DUIterator_Fast imax, i = feed->fast_outs(imax); i < imax; i++) {
2302         x = feed->fast_out(i);
2303         // the correct Phi will be merging a Bot memory slice
2304         if (x->is_MergeMem()) {
2305           mm = x->as_MergeMem();
2306           break;
2307         }
2308       }
2309       if (mm) {
2310         retry_feed = false;
2311       } else if (UseG1GC & phicount++ < MAX_PHIS) {
2312         // the barrier may feed indirectly via one or two Phi nodes
2313         PhiNode *phi = NULL;
2314         for (DUIterator_Fast imax, i = feed->fast_outs(imax); i < imax; i++) {
2315           x = feed->fast_out(i);
2316           // the correct Phi will be merging a Bot memory slice
2317           if (x->is_Phi() && x->adr_type() == TypePtr::BOTTOM) {
2318             phi = x->as_Phi();
2319             break;
2320           }
2321         }
2322         if (!phi) {
2323           return NULL;
2324         }
2325         // look for another merge below this phi
2326         feed = phi;
2327       } else {
2328         // couldn't find a merge
2329         return NULL;
2330       }
2331     }
2332 
2333     // sanity check this feed turns up as the expected slice
2334     assert(mm->as_MergeMem()->in(Compile::AliasIdxBot) == feed, "expecting membar to feed AliasIdxBot slice to Merge");
2335 
2336     MemBarNode *trailing = NULL;
2337     // be sure we have a trailing membar the merge
2338     for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {
2339       x = mm->fast_out(i);
2340       if (x->is_MemBar() && x->Opcode() == Op_MemBarVolatile) {
2341         trailing = x->as_MemBar();
2342         break;
2343       }
2344     }
2345 
2346     return trailing;
2347   }
2348 
2349   // trailing_to_card_mark
2350   //
2351   // graph traversal helper which detects extra, non-normal Mem feed
2352   // from a trailing volatile membar to a preceding card mark volatile
2353   // membar i.e. it identifies whether one of the three possible extra
2354   // GC post-write Mem flow subgraphs is present
2355   //
2356   // this predicate checks for the same flow as the previous predicate
2357   // but starting from the bottom rather than the top.
2358   //
2359   // if the configuration is present returns the card mark membar
2360   // otherwise NULL
2361   //
2362   // n.b. the supplied membar is expected to be a trailing
2363   // MemBarVolatile i.e. the caller must ensure the input node has the
2364   // correct opcode
2365 
2366   MemBarNode *trailing_to_card_mark(const MemBarNode *trailing)
2367   {
2368     assert(trailing->Opcode() == Op_MemBarVolatile,
2369            "expecting a volatile membar");
2370     assert(!is_card_mark_membar(trailing),
2371            "not expecting a card mark membar");
2372 
2373     // the Mem feed to the membar should be a merge
2374     Node *x = trailing->in(TypeFunc::Memory);
2375     if (!x->is_MergeMem()) {
2376       return NULL;
2377     }
2378 
2379     MergeMemNode *mm = x->as_MergeMem();
2380 
2381     x = mm->in(Compile::AliasIdxBot);
2382     // with G1 we may possibly see a Phi or two before we see a Memory
2383     // Proj from the card mark membar
2384 
2385     const int MAX_PHIS = 3;     // max phis we will search through
2386     int phicount = 0;           // current search count
2387 
2388     bool retry_feed = !x->is_Proj();
2389 
2390     while (retry_feed) {
2391       if (UseG1GC && x->is_Phi() && phicount++ < MAX_PHIS) {
2392         PhiNode *phi = x->as_Phi();
2393         ProjNode *proj = NULL;
2394         PhiNode *nextphi = NULL;
2395         bool found_leading = false;
2396         for (uint i = 1; i < phi->req(); i++) {
2397           x = phi->in(i);
2398           if (x->is_Phi()) {
2399             nextphi = x->as_Phi();
2400           } else if (x->is_Proj()) {
2401             int opcode = x->in(0)->Opcode();
2402             if (opcode == Op_MemBarVolatile) {
2403               proj = x->as_Proj();
2404             } else if (opcode == Op_MemBarRelease ||
2405                        opcode == Op_MemBarCPUOrder) {
2406               // probably a leading membar
2407               found_leading = true;
2408             }
2409           }
2410         }
2411         // if we found a correct looking proj then retry from there
2412         // otherwise we must see a leading and a phi or this the
2413         // wrong config
2414         if (proj != NULL) {
2415           x = proj;
2416           retry_feed = false;
2417         } else if (found_leading && nextphi != NULL) {
2418           // retry from this phi to check phi2
2419           x = nextphi;
2420         } else {
2421           // not what we were looking for
2422           return NULL;
2423         }
2424       } else {
2425         return NULL;
2426       }
2427     }
2428     // the proj has to come from the card mark membar
2429     x = x->in(0);
2430     if (!x->is_MemBar()) {
2431       return NULL;
2432     }
2433 
2434     MemBarNode *card_mark_membar = x->as_MemBar();
2435 
2436     if (!is_card_mark_membar(card_mark_membar)) {
2437       return NULL;
2438     }
2439 
2440     return card_mark_membar;
2441   }
2442 
2443   // trailing_to_leading
2444   //
2445   // graph traversal helper which checks the Mem flow up the graph
2446   // from a (non-card mark) trailing membar attempting to locate and
2447   // return an associated leading membar. it first looks for a
2448   // subgraph in the normal configuration (relying on helper
2449   // normal_to_leading). failing that it then looks for one of the
2450   // possible post-write card mark subgraphs linking the trailing node
2451   // to a the card mark membar (relying on helper
2452   // trailing_to_card_mark), and then checks that the card mark membar
2453   // is fed by a leading membar (once again relying on auxiliary
2454   // predicate normal_to_leading).
2455   //
2456   // if the configuration is valid returns the cpuorder member for
2457   // preference or when absent the release membar otherwise NULL.
2458   //
2459   // n.b. the input membar is expected to be either a volatile or
2460   // acquire membar but in the former case must *not* be a card mark
2461   // membar.
2462 
2463   MemBarNode *trailing_to_leading(const MemBarNode *trailing)
2464   {
2465     assert((trailing->Opcode() == Op_MemBarAcquire ||
2466             trailing->Opcode() == Op_MemBarVolatile),
2467            "expecting an acquire or volatile membar");
2468     assert((trailing->Opcode() != Op_MemBarVolatile ||
2469             !is_card_mark_membar(trailing)),
2470            "not expecting a card mark membar");
2471 
2472     MemBarNode *leading = normal_to_leading(trailing);
2473 
2474     if (leading) {
2475       return leading;
2476     }
2477 
2478     // nothing more to do if this is an acquire
2479     if (trailing->Opcode() == Op_MemBarAcquire) {
2480       return NULL;
2481     }
2482 
2483     MemBarNode *card_mark_membar = trailing_to_card_mark(trailing);
2484 
2485     if (!card_mark_membar) {
2486       return NULL;
2487     }
2488 
2489     return normal_to_leading(card_mark_membar);
2490   }
2491 
2492   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
2493 
2494 bool unnecessary_acquire(const Node *barrier)
2495 {
2496   assert(barrier->is_MemBar(), "expecting a membar");
2497 
2498   if (UseBarriersForVolatile) {
2499     // we need to plant a dmb
2500     return false;
2501   }
2502 
2503   // a volatile read derived from bytecode (or also from an inlined
2504   // SHA field read via LibraryCallKit::load_field_from_object)
2505   // manifests as a LoadX[mo_acquire] followed by an acquire membar
2506   // with a bogus read dependency on it's preceding load. so in those
2507   // cases we will find the load node at the PARMS offset of the
2508   // acquire membar.  n.b. there may be an intervening DecodeN node.
2509   //
2510   // a volatile load derived from an inlined unsafe field access
2511   // manifests as a cpuorder membar with Ctl and Mem projections
2512   // feeding both an acquire membar and a LoadX[mo_acquire]. The
2513   // acquire then feeds another cpuorder membar via Ctl and Mem
2514   // projections. The load has no output dependency on these trailing
2515   // membars because subsequent nodes inserted into the graph take
2516   // their control feed from the final membar cpuorder meaning they
2517   // are all ordered after the load.
2518 
2519   Node *x = barrier->lookup(TypeFunc::Parms);
2520   if (x) {
2521     // we are starting from an acquire and it has a fake dependency
2522     //
2523     // need to check for
2524     //
2525     //   LoadX[mo_acquire]
2526     //   {  |1   }
2527     //   {DecodeN}
2528     //      |Parms
2529     //   MemBarAcquire*
2530     //
2531     // where * tags node we were passed
2532     // and |k means input k
2533     if (x->is_DecodeNarrowPtr()) {
2534       x = x->in(1);
2535     }
2536 
2537     return (x->is_Load() && x->as_Load()->is_acquire());
2538   }
2539 
2540   // now check for an unsafe volatile get
2541 
2542   // need to check for
2543   //
2544   //   MemBarCPUOrder
2545   //        ||       \\
2546   //   MemBarAcquire* LoadX[mo_acquire]
2547   //        ||
2548   //   MemBarCPUOrder
2549   //
2550   // where * tags node we were passed
2551   // and || or \\ are Ctl+Mem feeds via intermediate Proj Nodes
2552 
2553   // check for a parent MemBarCPUOrder
2554   ProjNode *ctl;
2555   ProjNode *mem;
2556   MemBarNode *parent = parent_membar(barrier);
2557   if (!parent || parent->Opcode() != Op_MemBarCPUOrder)
2558     return false;
2559   ctl = parent->proj_out(TypeFunc::Control);
2560   mem = parent->proj_out(TypeFunc::Memory);
2561   if (!ctl || !mem) {
2562     return false;
2563   }
2564   // ensure the proj nodes both feed a LoadX[mo_acquire]
2565   LoadNode *ld = NULL;
2566   for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {
2567     x = ctl->fast_out(i);
2568     // if we see a load we keep hold of it and stop searching
2569     if (x->is_Load()) {
2570       ld = x->as_Load();
2571       break;
2572     }
2573   }
2574   // it must be an acquiring load
2575   if (ld && ld->is_acquire()) {
2576 
2577     for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
2578       x = mem->fast_out(i);
2579       // if we see the same load we drop it and stop searching
2580       if (x == ld) {
2581         ld = NULL;
2582         break;
2583       }
2584     }
2585     // we must have dropped the load
2586     if (ld == NULL) {
2587       // check for a child cpuorder membar
2588       MemBarNode *child  = child_membar(barrier->as_MemBar());
2589       if (child && child->Opcode() == Op_MemBarCPUOrder)
2590         return true;
2591     }
2592   }
2593 
2594   // final option for unnecessary mebar is that it is a trailing node
2595   // belonging to a CAS
2596 
2597   MemBarNode *leading = trailing_to_leading(barrier->as_MemBar());
2598 
2599   return leading != NULL;
2600 }
2601 
2602 bool needs_acquiring_load(const Node *n)
2603 {
2604   assert(n->is_Load(), "expecting a load");
2605   if (UseBarriersForVolatile) {
2606     // we use a normal load and a dmb
2607     return false;
2608   }
2609 
2610   LoadNode *ld = n->as_Load();
2611 
2612   if (!ld->is_acquire()) {
2613     return false;
2614   }
2615 
2616   // check if this load is feeding an acquire membar
2617   //
2618   //   LoadX[mo_acquire]
2619   //   {  |1   }
2620   //   {DecodeN}
2621   //      |Parms
2622   //   MemBarAcquire*
2623   //
2624   // where * tags node we were passed
2625   // and |k means input k
2626 
2627   Node *start = ld;
2628   Node *mbacq = NULL;
2629 
2630   // if we hit a DecodeNarrowPtr we reset the start node and restart
2631   // the search through the outputs
2632  restart:
2633 
2634   for (DUIterator_Fast imax, i = start->fast_outs(imax); i < imax; i++) {
2635     Node *x = start->fast_out(i);
2636     if (x->is_MemBar() && x->Opcode() == Op_MemBarAcquire) {
2637       mbacq = x;
2638     } else if (!mbacq &&
2639                (x->is_DecodeNarrowPtr() ||
2640                 (x->is_Mach() && x->Opcode() == Op_DecodeN))) {
2641       start = x;
2642       goto restart;
2643     }
2644   }
2645 
2646   if (mbacq) {
2647     return true;
2648   }
2649 
2650   // now check for an unsafe volatile get
2651 
2652   // check if Ctl and Proj feed comes from a MemBarCPUOrder
2653   //
2654   //     MemBarCPUOrder
2655   //        ||       \\
2656   //   MemBarAcquire* LoadX[mo_acquire]
2657   //        ||
2658   //   MemBarCPUOrder
2659 
2660   MemBarNode *membar;
2661 
2662   membar = parent_membar(ld);
2663 
2664   if (!membar || !membar->Opcode() == Op_MemBarCPUOrder) {
2665     return false;
2666   }
2667 
2668   // ensure that there is a CPUOrder->Acquire->CPUOrder membar chain
2669 
2670   membar = child_membar(membar);
2671 
2672   if (!membar || !membar->Opcode() == Op_MemBarAcquire) {
2673     return false;
2674   }
2675 
2676   membar = child_membar(membar);
2677 
2678   if (!membar || !membar->Opcode() == Op_MemBarCPUOrder) {
2679     return false;
2680   }
2681 
2682   return true;
2683 }
2684 
2685 bool unnecessary_release(const Node *n)
2686 {
2687   assert((n->is_MemBar() &&
2688           n->Opcode() == Op_MemBarRelease),
2689          "expecting a release membar");
2690 
2691   if (UseBarriersForVolatile) {
2692     // we need to plant a dmb
2693     return false;
2694   }
2695 
2696   // if there is a dependent CPUOrder barrier then use that as the
2697   // leading
2698 
2699   MemBarNode *barrier = n->as_MemBar();
2700   // check for an intervening cpuorder membar
2701   MemBarNode *b = child_membar(barrier);
2702   if (b && b->Opcode() == Op_MemBarCPUOrder) {
2703     // ok, so start the check from the dependent cpuorder barrier
2704     barrier = b;
2705   }
2706 
2707   // must start with a normal feed
2708   MemBarNode *child_barrier = leading_to_normal(barrier);
2709 
2710   if (!child_barrier) {
2711     return false;
2712   }
2713 
2714   if (!is_card_mark_membar(child_barrier)) {
2715     // this is the trailing membar and we are done
2716     return true;
2717   }
2718 
2719   // must be sure this card mark feeds a trailing membar
2720   MemBarNode *trailing = card_mark_to_trailing(child_barrier);
2721   return (trailing != NULL);
2722 }
2723 
2724 bool unnecessary_volatile(const Node *n)
2725 {
2726   // assert n->is_MemBar();
2727   if (UseBarriersForVolatile) {
2728     // we need to plant a dmb
2729     return false;
2730   }
2731 
2732   MemBarNode *mbvol = n->as_MemBar();
2733 
2734   // first we check if this is part of a card mark. if so then we have
2735   // to generate a StoreLoad barrier
2736 
2737   if (is_card_mark_membar(mbvol)) {
2738       return false;
2739   }
2740 
2741   // ok, if it's not a card mark then we still need to check if it is
2742   // a trailing membar of a volatile put hgraph.
2743 
2744   return (trailing_to_leading(mbvol) != NULL);
2745 }
2746 
2747 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
2748 
2749 bool needs_releasing_store(const Node *n)
2750 {
2751   // assert n->is_Store();
2752   if (UseBarriersForVolatile) {
2753     // we use a normal store and dmb combination
2754     return false;
2755   }
2756 
2757   StoreNode *st = n->as_Store();
2758 
2759   // the store must be marked as releasing
2760   if (!st->is_release()) {
2761     return false;
2762   }
2763 
2764   // the store must be fed by a membar
2765 
2766   Node *x = st->lookup(StoreNode::Memory);
2767 
2768   if (! x || !x->is_Proj()) {
2769     return false;
2770   }
2771 
2772   ProjNode *proj = x->as_Proj();
2773 
2774   x = proj->lookup(0);
2775 
2776   if (!x || !x->is_MemBar()) {
2777     return false;
2778   }
2779 
2780   MemBarNode *barrier = x->as_MemBar();
2781 
2782   // if the barrier is a release membar or a cpuorder mmebar fed by a
2783   // release membar then we need to check whether that forms part of a
2784   // volatile put graph.
2785 
2786   // reject invalid candidates
2787   if (!leading_membar(barrier)) {
2788     return false;
2789   }
2790 
2791   // does this lead a normal subgraph?
2792   MemBarNode *mbvol = leading_to_normal(barrier);
2793 
2794   if (!mbvol) {
2795     return false;
2796   }
2797 
2798   // all done unless this is a card mark
2799   if (!is_card_mark_membar(mbvol)) {
2800     return true;
2801   }
2802 
2803   // we found a card mark -- just make sure we have a trailing barrier
2804 
2805   return (card_mark_to_trailing(mbvol) != NULL);
2806 }
2807 
2808 // predicate controlling translation of CAS
2809 //
2810 // returns true if CAS needs to use an acquiring load otherwise false
2811 
2812 bool needs_acquiring_load_exclusive(const Node *n)
2813 {
2814   assert(is_CAS(n->Opcode()), "expecting a compare and swap");
2815   if (UseBarriersForVolatile) {
2816     return false;
2817   }
2818 
2819   // CAS nodes only ought to turn up in inlined unsafe CAS operations
2820 #ifdef ASSERT
2821   LoadStoreNode *st = n->as_LoadStore();
2822 
2823   // the store must be fed by a membar
2824 
2825   Node *x = st->lookup(StoreNode::Memory);
2826 
2827   assert (x && x->is_Proj(), "CAS not fed by memory proj!");
2828 
2829   ProjNode *proj = x->as_Proj();
2830 
2831   x = proj->lookup(0);
2832 
2833   assert (x && x->is_MemBar(), "CAS not fed by membar!");
2834 
2835   MemBarNode *barrier = x->as_MemBar();
2836 
2837   // the barrier must be a cpuorder mmebar fed by a release membar
2838 
2839   assert(barrier->Opcode() == Op_MemBarCPUOrder,
2840          "CAS not fed by cpuorder membar!");
2841 
2842   MemBarNode *b = parent_membar(barrier);
2843   assert ((b != NULL && b->Opcode() == Op_MemBarRelease),
2844           "CAS not fed by cpuorder+release membar pair!");
2845 
2846   // does this lead a normal subgraph?
2847   MemBarNode *mbar = leading_to_normal(barrier);
2848 
2849   assert(mbar != NULL, "CAS not embedded in normal graph!");
2850 
2851   assert(mbar->Opcode() == Op_MemBarAcquire, "trailing membar should be an acquire");
2852 #endif // ASSERT
2853   // so we can just return true here
2854   return true;
2855 }
2856 
2857 // predicate controlling translation of StoreCM
2858 //
2859 // returns true if a StoreStore must precede the card write otherwise
2860 // false
2861 
2862 bool unnecessary_storestore(const Node *storecm)
2863 {
2864   assert(storecm->Opcode()  == Op_StoreCM, "expecting a StoreCM");
2865 
2866   // we only ever need to generate a dmb ishst between an object put
2867   // and the associated card mark when we are using CMS without
2868   // conditional card marking
2869 
2870   if (!UseConcMarkSweepGC || UseCondCardMark) {
2871     return true;
2872   }
2873 
2874   // if we are implementing volatile puts using barriers then the
2875   // object put as an str so we must insert the dmb ishst
2876 
2877   if (UseBarriersForVolatile) {
2878     return false;
2879   }
2880 
2881   // we can omit the dmb ishst if this StoreCM is part of a volatile
2882   // put because in thta case the put will be implemented by stlr
2883   //
2884   // we need to check for a normal subgraph feeding this StoreCM.
2885   // that means the StoreCM must be fed Memory from a leading membar,
2886   // either a MemBarRelease or its dependent MemBarCPUOrder, and the
2887   // leading membar must be part of a normal subgraph
2888 
2889   Node *x = storecm->in(StoreNode::Memory);
2890 
2891   if (!x->is_Proj()) {
2892     return false;
2893   }
2894 
2895   x = x->in(0);
2896 
2897   if (!x->is_MemBar()) {
2898     return false;
2899   }
2900 
2901   MemBarNode *leading = x->as_MemBar();
2902 
2903   // reject invalid candidates
2904   if (!leading_membar(leading)) {
2905     return false;
2906   }
2907 
2908   // we can omit the StoreStore if it is the head of a normal subgraph
2909   return (leading_to_normal(leading) != NULL);
2910 }
2911 
2912 
2913 #define __ _masm.
2914 
2915 // advance declarations for helper functions to convert register
2916 // indices to register objects
2917 
2918 // the ad file has to provide implementations of certain methods
2919 // expected by the generic code
2920 //
2921 // REQUIRED FUNCTIONALITY
2922 
2923 //=============================================================================
2924 
2925 // !!!!! Special hack to get all types of calls to specify the byte offset
2926 //       from the start of the call to the point where the return address
2927 //       will point.
2928 
2929 int MachCallStaticJavaNode::ret_addr_offset()
2930 {
2931   // call should be a simple bl
2932   int off = 4;
2933   return off;
2934 }
2935 
2936 int MachCallDynamicJavaNode::ret_addr_offset()
2937 {
2938   return 16; // movz, movk, movk, bl
2939 }
2940 
2941 int MachCallRuntimeNode::ret_addr_offset() {
2942   // for generated stubs the call will be
2943   //   far_call(addr)
2944   // for real runtime callouts it will be six instructions
2945   // see aarch64_enc_java_to_runtime
2946   //   adr(rscratch2, retaddr)
2947   //   lea(rscratch1, RuntimeAddress(addr)
2948   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
2949   //   blrt rscratch1
2950   CodeBlob *cb = CodeCache::find_blob(_entry_point);
2951   if (cb) {
2952     return MacroAssembler::far_branch_size();
2953   } else {
2954     return 6 * NativeInstruction::instruction_size;
2955   }
2956 }
2957 
2958 // Indicate if the safepoint node needs the polling page as an input
2959 
2960 // the shared code plants the oop data at the start of the generated
2961 // code for the safepoint node and that needs ot be at the load
2962 // instruction itself. so we cannot plant a mov of the safepoint poll
2963 // address followed by a load. setting this to true means the mov is
2964 // scheduled as a prior instruction. that's better for scheduling
2965 // anyway.
2966 
2967 bool SafePointNode::needs_polling_address_input()
2968 {
2969   return true;
2970 }
2971 
2972 //=============================================================================
2973 
2974 #ifndef PRODUCT
2975 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2976   st->print("BREAKPOINT");
2977 }
2978 #endif
2979 
2980 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2981   MacroAssembler _masm(&cbuf);
2982   __ brk(0);
2983 }
2984 
2985 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
2986   return MachNode::size(ra_);
2987 }
2988 
2989 //=============================================================================
2990 
2991 #ifndef PRODUCT
2992   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
2993     st->print("nop \t# %d bytes pad for loops and calls", _count);
2994   }
2995 #endif
2996 
2997   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
2998     MacroAssembler _masm(&cbuf);
2999     for (int i = 0; i < _count; i++) {
3000       __ nop();
3001     }
3002   }
3003 
3004   uint MachNopNode::size(PhaseRegAlloc*) const {
3005     return _count * NativeInstruction::instruction_size;
3006   }
3007 
3008 //=============================================================================
3009 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
3010 
3011 int Compile::ConstantTable::calculate_table_base_offset() const {
3012   return 0;  // absolute addressing, no offset
3013 }
3014 
3015 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
3016 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
3017   ShouldNotReachHere();
3018 }
3019 
3020 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
3021   // Empty encoding
3022 }
3023 
3024 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
3025   return 0;
3026 }
3027 
3028 #ifndef PRODUCT
3029 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
3030   st->print("-- \t// MachConstantBaseNode (empty encoding)");
3031 }
3032 #endif
3033 
3034 #ifndef PRODUCT
3035 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
3036   Compile* C = ra_->C;
3037 
3038   int framesize = C->frame_slots() << LogBytesPerInt;
3039 
3040   if (C->need_stack_bang(framesize))
3041     st->print("# stack bang size=%d\n\t", framesize);
3042 
3043   if (framesize < ((1 << 9) + 2 * wordSize)) {
3044     st->print("sub  sp, sp, #%d\n\t", framesize);
3045     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
3046     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
3047   } else {
3048     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
3049     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
3050     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
3051     st->print("sub  sp, sp, rscratch1");
3052   }
3053 }
3054 #endif
3055 
3056 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
3057   Compile* C = ra_->C;
3058   MacroAssembler _masm(&cbuf);
3059 
3060   // n.b. frame size includes space for return pc and rfp
3061   const long framesize = C->frame_size_in_bytes();
3062   assert(framesize%(2*wordSize) == 0, "must preserve 2*wordSize alignment");
3063 
3064   // insert a nop at the start of the prolog so we can patch in a
3065   // branch if we need to invalidate the method later
3066   __ nop();
3067 
3068   int bangsize = C->bang_size_in_bytes();
3069   if (C->need_stack_bang(bangsize) && UseStackBanging)
3070     __ generate_stack_overflow_check(bangsize);
3071 
3072   __ build_frame(framesize);
3073 
3074   if (NotifySimulator) {
3075     __ notify(Assembler::method_entry);
3076   }
3077 
3078   if (VerifyStackAtCalls) {
3079     Unimplemented();
3080   }
3081 
3082   C->set_frame_complete(cbuf.insts_size());
3083 
3084   if (C->has_mach_constant_base_node()) {
3085     // NOTE: We set the table base offset here because users might be
3086     // emitted before MachConstantBaseNode.
3087     Compile::ConstantTable& constant_table = C->constant_table();
3088     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
3089   }
3090 }
3091 
3092 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
3093 {
3094   return MachNode::size(ra_); // too many variables; just compute it
3095                               // the hard way
3096 }
3097 
3098 int MachPrologNode::reloc() const
3099 {
3100   return 0;
3101 }
3102 
3103 //=============================================================================
3104 
3105 #ifndef PRODUCT
3106 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
3107   Compile* C = ra_->C;
3108   int framesize = C->frame_slots() << LogBytesPerInt;
3109 
3110   st->print("# pop frame %d\n\t",framesize);
3111 
3112   if (framesize == 0) {
3113     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
3114   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
3115     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
3116     st->print("add  sp, sp, #%d\n\t", framesize);
3117   } else {
3118     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
3119     st->print("add  sp, sp, rscratch1\n\t");
3120     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
3121   }
3122 
3123   if (do_polling() && C->is_method_compilation()) {
3124     st->print("# touch polling page\n\t");
3125     st->print("mov  rscratch1, #0x%lx\n\t", p2i(os::get_polling_page()));
3126     st->print("ldr zr, [rscratch1]");
3127   }
3128 }
3129 #endif
3130 
3131 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
3132   Compile* C = ra_->C;
3133   MacroAssembler _masm(&cbuf);
3134   int framesize = C->frame_slots() << LogBytesPerInt;
3135 
3136   __ remove_frame(framesize);
3137 
3138   if (NotifySimulator) {
3139     __ notify(Assembler::method_reentry);
3140   }
3141 
3142   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
3143     __ reserved_stack_check();
3144   }
3145 
3146   if (do_polling() && C->is_method_compilation()) {
3147     __ read_polling_page(rscratch1, os::get_polling_page(), relocInfo::poll_return_type);
3148   }
3149 }
3150 
3151 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
3152   // Variable size. Determine dynamically.
3153   return MachNode::size(ra_);
3154 }
3155 
3156 int MachEpilogNode::reloc() const {
3157   // Return number of relocatable values contained in this instruction.
3158   return 1; // 1 for polling page.
3159 }
3160 
3161 const Pipeline * MachEpilogNode::pipeline() const {
3162   return MachNode::pipeline_class();
3163 }
3164 
3165 // This method seems to be obsolete. It is declared in machnode.hpp
3166 // and defined in all *.ad files, but it is never called. Should we
3167 // get rid of it?
3168 int MachEpilogNode::safepoint_offset() const {
3169   assert(do_polling(), "no return for this epilog node");
3170   return 4;
3171 }
3172 
3173 //=============================================================================
3174 
3175 // Figure out which register class each belongs in: rc_int, rc_float or
3176 // rc_stack.
3177 enum RC { rc_bad, rc_int, rc_float, rc_stack };
3178 
3179 static enum RC rc_class(OptoReg::Name reg) {
3180 
3181   if (reg == OptoReg::Bad) {
3182     return rc_bad;
3183   }
3184 
3185   // we have 30 int registers * 2 halves
3186   // (rscratch1 and rscratch2 are omitted)
3187 
3188   if (reg < 60) {
3189     return rc_int;
3190   }
3191 
3192   // we have 32 float register * 2 halves
3193   if (reg < 60 + 128) {
3194     return rc_float;
3195   }
3196 
3197   // Between float regs & stack is the flags regs.
3198   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
3199 
3200   return rc_stack;
3201 }
3202 
3203 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
3204   Compile* C = ra_->C;
3205 
3206   // Get registers to move.
3207   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
3208   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
3209   OptoReg::Name dst_hi = ra_->get_reg_second(this);
3210   OptoReg::Name dst_lo = ra_->get_reg_first(this);
3211 
3212   enum RC src_hi_rc = rc_class(src_hi);
3213   enum RC src_lo_rc = rc_class(src_lo);
3214   enum RC dst_hi_rc = rc_class(dst_hi);
3215   enum RC dst_lo_rc = rc_class(dst_lo);
3216 
3217   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
3218 
3219   if (src_hi != OptoReg::Bad) {
3220     assert((src_lo&1)==0 && src_lo+1==src_hi &&
3221            (dst_lo&1)==0 && dst_lo+1==dst_hi,
3222            "expected aligned-adjacent pairs");
3223   }
3224 
3225   if (src_lo == dst_lo && src_hi == dst_hi) {
3226     return 0;            // Self copy, no move.
3227   }
3228 
3229   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
3230               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
3231   int src_offset = ra_->reg2offset(src_lo);
3232   int dst_offset = ra_->reg2offset(dst_lo);
3233 
3234   if (bottom_type()->isa_vect() != NULL) {
3235     uint ireg = ideal_reg();
3236     assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
3237     if (cbuf) {
3238       MacroAssembler _masm(cbuf);
3239       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
3240       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
3241         // stack->stack
3242         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
3243         if (ireg == Op_VecD) {
3244           __ unspill(rscratch1, true, src_offset);
3245           __ spill(rscratch1, true, dst_offset);
3246         } else {
3247           __ spill_copy128(src_offset, dst_offset);
3248         }
3249       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
3250         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
3251                ireg == Op_VecD ? __ T8B : __ T16B,
3252                as_FloatRegister(Matcher::_regEncode[src_lo]));
3253       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
3254         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
3255                        ireg == Op_VecD ? __ D : __ Q,
3256                        ra_->reg2offset(dst_lo));
3257       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
3258         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
3259                        ireg == Op_VecD ? __ D : __ Q,
3260                        ra_->reg2offset(src_lo));
3261       } else {
3262         ShouldNotReachHere();
3263       }
3264     }
3265   } else if (cbuf) {
3266     MacroAssembler _masm(cbuf);
3267     switch (src_lo_rc) {
3268     case rc_int:
3269       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
3270         if (is64) {
3271             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
3272                    as_Register(Matcher::_regEncode[src_lo]));
3273         } else {
3274             MacroAssembler _masm(cbuf);
3275             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
3276                     as_Register(Matcher::_regEncode[src_lo]));
3277         }
3278       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
3279         if (is64) {
3280             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
3281                      as_Register(Matcher::_regEncode[src_lo]));
3282         } else {
3283             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
3284                      as_Register(Matcher::_regEncode[src_lo]));
3285         }
3286       } else {                    // gpr --> stack spill
3287         assert(dst_lo_rc == rc_stack, "spill to bad register class");
3288         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
3289       }
3290       break;
3291     case rc_float:
3292       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
3293         if (is64) {
3294             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
3295                      as_FloatRegister(Matcher::_regEncode[src_lo]));
3296         } else {
3297             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
3298                      as_FloatRegister(Matcher::_regEncode[src_lo]));
3299         }
3300       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
3301           if (cbuf) {
3302             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
3303                      as_FloatRegister(Matcher::_regEncode[src_lo]));
3304         } else {
3305             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
3306                      as_FloatRegister(Matcher::_regEncode[src_lo]));
3307         }
3308       } else {                    // fpr --> stack spill
3309         assert(dst_lo_rc == rc_stack, "spill to bad register class");
3310         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
3311                  is64 ? __ D : __ S, dst_offset);
3312       }
3313       break;
3314     case rc_stack:
3315       if (dst_lo_rc == rc_int) {  // stack --> gpr load
3316         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
3317       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
3318         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
3319                    is64 ? __ D : __ S, src_offset);
3320       } else {                    // stack --> stack copy
3321         assert(dst_lo_rc == rc_stack, "spill to bad register class");
3322         __ unspill(rscratch1, is64, src_offset);
3323         __ spill(rscratch1, is64, dst_offset);
3324       }
3325       break;
3326     default:
3327       assert(false, "bad rc_class for spill");
3328       ShouldNotReachHere();
3329     }
3330   }
3331 
3332   if (st) {
3333     st->print("spill ");
3334     if (src_lo_rc == rc_stack) {
3335       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
3336     } else {
3337       st->print("%s -> ", Matcher::regName[src_lo]);
3338     }
3339     if (dst_lo_rc == rc_stack) {
3340       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
3341     } else {
3342       st->print("%s", Matcher::regName[dst_lo]);
3343     }
3344     if (bottom_type()->isa_vect() != NULL) {
3345       st->print("\t# vector spill size = %d", ideal_reg()==Op_VecD ? 64:128);
3346     } else {
3347       st->print("\t# spill size = %d", is64 ? 64:32);
3348     }
3349   }
3350 
3351   return 0;
3352 
3353 }
3354 
3355 #ifndef PRODUCT
3356 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
3357   if (!ra_)
3358     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
3359   else
3360     implementation(NULL, ra_, false, st);
3361 }
3362 #endif
3363 
3364 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
3365   implementation(&cbuf, ra_, false, NULL);
3366 }
3367 
3368 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
3369   return MachNode::size(ra_);
3370 }
3371 
3372 //=============================================================================
3373 
3374 #ifndef PRODUCT
3375 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
3376   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
3377   int reg = ra_->get_reg_first(this);
3378   st->print("add %s, rsp, #%d]\t# box lock",
3379             Matcher::regName[reg], offset);
3380 }
3381 #endif
3382 
3383 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
3384   MacroAssembler _masm(&cbuf);
3385 
3386   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
3387   int reg    = ra_->get_encode(this);
3388 
3389   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
3390     __ add(as_Register(reg), sp, offset);
3391   } else {
3392     ShouldNotReachHere();
3393   }
3394 }
3395 
3396 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
3397   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
3398   return 4;
3399 }
3400 
3401 //=============================================================================
3402 
3403 #ifndef PRODUCT
3404 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
3405 {
3406   st->print_cr("# MachUEPNode");
3407   if (UseCompressedClassPointers) {
3408     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
3409     if (Universe::narrow_klass_shift() != 0) {
3410       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
3411     }
3412   } else {
3413    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
3414   }
3415   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
3416   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
3417 }
3418 #endif
3419 
3420 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
3421 {
3422   // This is the unverified entry point.
3423   MacroAssembler _masm(&cbuf);
3424 
3425   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
3426   Label skip;
3427   // TODO
3428   // can we avoid this skip and still use a reloc?
3429   __ br(Assembler::EQ, skip);
3430   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
3431   __ bind(skip);
3432 }
3433 
3434 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
3435 {
3436   return MachNode::size(ra_);
3437 }
3438 
3439 // REQUIRED EMIT CODE
3440 
3441 //=============================================================================
3442 
3443 // Emit exception handler code.
3444 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
3445 {
3446   // mov rscratch1 #exception_blob_entry_point
3447   // br rscratch1
3448   // Note that the code buffer's insts_mark is always relative to insts.
3449   // That's why we must use the macroassembler to generate a handler.
3450   MacroAssembler _masm(&cbuf);
3451   address base = __ start_a_stub(size_exception_handler());
3452   if (base == NULL) {
3453     ciEnv::current()->record_failure("CodeCache is full");
3454     return 0;  // CodeBuffer::expand failed
3455   }
3456   int offset = __ offset();
3457   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
3458   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
3459   __ end_a_stub();
3460   return offset;
3461 }
3462 
3463 // Emit deopt handler code.
3464 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
3465 {
3466   // Note that the code buffer's insts_mark is always relative to insts.
3467   // That's why we must use the macroassembler to generate a handler.
3468   MacroAssembler _masm(&cbuf);
3469   address base = __ start_a_stub(size_deopt_handler());
3470   if (base == NULL) {
3471     ciEnv::current()->record_failure("CodeCache is full");
3472     return 0;  // CodeBuffer::expand failed
3473   }
3474   int offset = __ offset();
3475 
3476   __ adr(lr, __ pc());
3477   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
3478 
3479   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
3480   __ end_a_stub();
3481   return offset;
3482 }
3483 
3484 // REQUIRED MATCHER CODE
3485 
3486 //=============================================================================
3487 
3488 const bool Matcher::match_rule_supported(int opcode) {
3489 
3490   switch (opcode) {
3491   default:
3492     break;
3493   }
3494 
3495   if (!has_match_rule(opcode)) {
3496     return false;
3497   }
3498 
3499   return true;  // Per default match rules are supported.
3500 }
3501 
3502 const bool Matcher::match_rule_supported_vector(int opcode, int vlen) {
3503 
3504   // TODO
3505   // identify extra cases that we might want to provide match rules for
3506   // e.g. Op_ vector nodes and other intrinsics while guarding with vlen
3507   bool ret_value = match_rule_supported(opcode);
3508   // Add rules here.
3509 
3510   return ret_value;  // Per default match rules are supported.
3511 }
3512 
3513 const bool Matcher::has_predicated_vectors(void) {
3514   return false;
3515 }
3516 
3517 const int Matcher::float_pressure(int default_pressure_threshold) {
3518   return default_pressure_threshold;
3519 }
3520 
3521 int Matcher::regnum_to_fpu_offset(int regnum)
3522 {
3523   Unimplemented();
3524   return 0;
3525 }
3526 
3527 // Is this branch offset short enough that a short branch can be used?
3528 //
3529 // NOTE: If the platform does not provide any short branch variants, then
3530 //       this method should return false for offset 0.
3531 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
3532   // The passed offset is relative to address of the branch.
3533 
3534   return (-32768 <= offset && offset < 32768);
3535 }
3536 
3537 const bool Matcher::isSimpleConstant64(jlong value) {
3538   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
3539   // Probably always true, even if a temp register is required.
3540   return true;
3541 }
3542 
3543 // true just means we have fast l2f conversion
3544 const bool Matcher::convL2FSupported(void) {
3545   return true;
3546 }
3547 
3548 // Vector width in bytes.
3549 const int Matcher::vector_width_in_bytes(BasicType bt) {
3550   int size = MIN2(16,(int)MaxVectorSize);
3551   // Minimum 2 values in vector
3552   if (size < 2*type2aelembytes(bt)) size = 0;
3553   // But never < 4
3554   if (size < 4) size = 0;
3555   return size;
3556 }
3557 
3558 // Limits on vector size (number of elements) loaded into vector.
3559 const int Matcher::max_vector_size(const BasicType bt) {
3560   return vector_width_in_bytes(bt)/type2aelembytes(bt);
3561 }
3562 const int Matcher::min_vector_size(const BasicType bt) {
3563 //  For the moment limit the vector size to 8 bytes
3564     int size = 8 / type2aelembytes(bt);
3565     if (size < 2) size = 2;
3566     return size;
3567 }
3568 
3569 // Vector ideal reg.
3570 const uint Matcher::vector_ideal_reg(int len) {
3571   switch(len) {
3572     case  8: return Op_VecD;
3573     case 16: return Op_VecX;
3574   }
3575   ShouldNotReachHere();
3576   return 0;
3577 }
3578 
3579 const uint Matcher::vector_shift_count_ideal_reg(int size) {
3580   return Op_VecX;
3581 }
3582 
3583 // AES support not yet implemented
3584 const bool Matcher::pass_original_key_for_aes() {
3585   return false;
3586 }
3587 
3588 // x86 supports misaligned vectors store/load.
3589 const bool Matcher::misaligned_vectors_ok() {
3590   return !AlignVector; // can be changed by flag
3591 }
3592 
3593 // false => size gets scaled to BytesPerLong, ok.
3594 const bool Matcher::init_array_count_is_in_bytes = false;
3595 
3596 // Use conditional move (CMOVL)
3597 const int Matcher::long_cmove_cost() {
3598   // long cmoves are no more expensive than int cmoves
3599   return 0;
3600 }
3601 
3602 const int Matcher::float_cmove_cost() {
3603   // float cmoves are no more expensive than int cmoves
3604   return 0;
3605 }
3606 
3607 // Does the CPU require late expand (see block.cpp for description of late expand)?
3608 const bool Matcher::require_postalloc_expand = false;
3609 
3610 // Do we need to mask the count passed to shift instructions or does
3611 // the cpu only look at the lower 5/6 bits anyway?
3612 const bool Matcher::need_masked_shift_count = false;
3613 
3614 // This affects two different things:
3615 //  - how Decode nodes are matched
3616 //  - how ImplicitNullCheck opportunities are recognized
3617 // If true, the matcher will try to remove all Decodes and match them
3618 // (as operands) into nodes. NullChecks are not prepared to deal with
3619 // Decodes by final_graph_reshaping().
3620 // If false, final_graph_reshaping() forces the decode behind the Cmp
3621 // for a NullCheck. The matcher matches the Decode node into a register.
3622 // Implicit_null_check optimization moves the Decode along with the
3623 // memory operation back up before the NullCheck.
3624 bool Matcher::narrow_oop_use_complex_address() {
3625   return Universe::narrow_oop_shift() == 0;
3626 }
3627 
3628 bool Matcher::narrow_klass_use_complex_address() {
3629 // TODO
3630 // decide whether we need to set this to true
3631   return false;
3632 }
3633 
3634 bool Matcher::const_oop_prefer_decode() {
3635   // Prefer ConN+DecodeN over ConP in simple compressed oops mode.
3636   return Universe::narrow_oop_base() == NULL;
3637 }
3638 
3639 bool Matcher::const_klass_prefer_decode() {
3640   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
3641   return Universe::narrow_klass_base() == NULL;
3642 }
3643 
3644 // Is it better to copy float constants, or load them directly from
3645 // memory?  Intel can load a float constant from a direct address,
3646 // requiring no extra registers.  Most RISCs will have to materialize
3647 // an address into a register first, so they would do better to copy
3648 // the constant from stack.
3649 const bool Matcher::rematerialize_float_constants = false;
3650 
3651 // If CPU can load and store mis-aligned doubles directly then no
3652 // fixup is needed.  Else we split the double into 2 integer pieces
3653 // and move it piece-by-piece.  Only happens when passing doubles into
3654 // C code as the Java calling convention forces doubles to be aligned.
3655 const bool Matcher::misaligned_doubles_ok = true;
3656 
3657 // No-op on amd64
3658 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
3659   Unimplemented();
3660 }
3661 
3662 // Advertise here if the CPU requires explicit rounding operations to
3663 // implement the UseStrictFP mode.
3664 const bool Matcher::strict_fp_requires_explicit_rounding = false;
3665 
3666 // Are floats converted to double when stored to stack during
3667 // deoptimization?
3668 bool Matcher::float_in_double() { return false; }
3669 
3670 // Do ints take an entire long register or just half?
3671 // The relevant question is how the int is callee-saved:
3672 // the whole long is written but de-opt'ing will have to extract
3673 // the relevant 32 bits.
3674 const bool Matcher::int_in_long = true;
3675 
3676 // Return whether or not this register is ever used as an argument.
3677 // This function is used on startup to build the trampoline stubs in
3678 // generateOptoStub.  Registers not mentioned will be killed by the VM
3679 // call in the trampoline, and arguments in those registers not be
3680 // available to the callee.
3681 bool Matcher::can_be_java_arg(int reg)
3682 {
3683   return
3684     reg ==  R0_num || reg == R0_H_num ||
3685     reg ==  R1_num || reg == R1_H_num ||
3686     reg ==  R2_num || reg == R2_H_num ||
3687     reg ==  R3_num || reg == R3_H_num ||
3688     reg ==  R4_num || reg == R4_H_num ||
3689     reg ==  R5_num || reg == R5_H_num ||
3690     reg ==  R6_num || reg == R6_H_num ||
3691     reg ==  R7_num || reg == R7_H_num ||
3692     reg ==  V0_num || reg == V0_H_num ||
3693     reg ==  V1_num || reg == V1_H_num ||
3694     reg ==  V2_num || reg == V2_H_num ||
3695     reg ==  V3_num || reg == V3_H_num ||
3696     reg ==  V4_num || reg == V4_H_num ||
3697     reg ==  V5_num || reg == V5_H_num ||
3698     reg ==  V6_num || reg == V6_H_num ||
3699     reg ==  V7_num || reg == V7_H_num;
3700 }
3701 
3702 bool Matcher::is_spillable_arg(int reg)
3703 {
3704   return can_be_java_arg(reg);
3705 }
3706 
3707 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
3708   return false;
3709 }
3710 
3711 RegMask Matcher::divI_proj_mask() {
3712   ShouldNotReachHere();
3713   return RegMask();
3714 }
3715 
3716 // Register for MODI projection of divmodI.
3717 RegMask Matcher::modI_proj_mask() {
3718   ShouldNotReachHere();
3719   return RegMask();
3720 }
3721 
3722 // Register for DIVL projection of divmodL.
3723 RegMask Matcher::divL_proj_mask() {
3724   ShouldNotReachHere();
3725   return RegMask();
3726 }
3727 
3728 // Register for MODL projection of divmodL.
3729 RegMask Matcher::modL_proj_mask() {
3730   ShouldNotReachHere();
3731   return RegMask();
3732 }
3733 
3734 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
3735   return FP_REG_mask();
3736 }
3737 
3738 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
3739   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
3740     Node* u = addp->fast_out(i);
3741     if (u->is_Mem()) {
3742       int opsize = u->as_Mem()->memory_size();
3743       assert(opsize > 0, "unexpected memory operand size");
3744       if (u->as_Mem()->memory_size() != (1<<shift)) {
3745         return false;
3746       }
3747     }
3748   }
3749   return true;
3750 }
3751 
3752 const bool Matcher::convi2l_type_required = false;
3753 
3754 // Should the Matcher clone shifts on addressing modes, expecting them
3755 // to be subsumed into complex addressing expressions or compute them
3756 // into registers?
3757 bool Matcher::clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
3758   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
3759     return true;
3760   }
3761 
3762   Node *off = m->in(AddPNode::Offset);
3763   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
3764       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
3765       // Are there other uses besides address expressions?
3766       !is_visited(off)) {
3767     address_visited.set(off->_idx); // Flag as address_visited
3768     mstack.push(off->in(2), Visit);
3769     Node *conv = off->in(1);
3770     if (conv->Opcode() == Op_ConvI2L &&
3771         // Are there other uses besides address expressions?
3772         !is_visited(conv)) {
3773       address_visited.set(conv->_idx); // Flag as address_visited
3774       mstack.push(conv->in(1), Pre_Visit);
3775     } else {
3776       mstack.push(conv, Pre_Visit);
3777     }
3778     address_visited.test_set(m->_idx); // Flag as address_visited
3779     mstack.push(m->in(AddPNode::Address), Pre_Visit);
3780     mstack.push(m->in(AddPNode::Base), Pre_Visit);
3781     return true;
3782   } else if (off->Opcode() == Op_ConvI2L &&
3783              // Are there other uses besides address expressions?
3784              !is_visited(off)) {
3785     address_visited.test_set(m->_idx); // Flag as address_visited
3786     address_visited.set(off->_idx); // Flag as address_visited
3787     mstack.push(off->in(1), Pre_Visit);
3788     mstack.push(m->in(AddPNode::Address), Pre_Visit);
3789     mstack.push(m->in(AddPNode::Base), Pre_Visit);
3790     return true;
3791   }
3792   return false;
3793 }
3794 
3795 // Transform:
3796 // (AddP base (AddP base address (LShiftL index con)) offset)
3797 // into:
3798 // (AddP base (AddP base offset) (LShiftL index con))
3799 // to take full advantage of ARM's addressing modes
3800 void Compile::reshape_address(AddPNode* addp) {
3801   Node *addr = addp->in(AddPNode::Address);
3802   if (addr->is_AddP() && addr->in(AddPNode::Base) == addp->in(AddPNode::Base)) {
3803     const AddPNode *addp2 = addr->as_AddP();
3804     if ((addp2->in(AddPNode::Offset)->Opcode() == Op_LShiftL &&
3805          addp2->in(AddPNode::Offset)->in(2)->is_Con() &&
3806          size_fits_all_mem_uses(addp, addp2->in(AddPNode::Offset)->in(2)->get_int())) ||
3807         addp2->in(AddPNode::Offset)->Opcode() == Op_ConvI2L) {
3808 
3809       // Any use that can't embed the address computation?
3810       for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
3811         Node* u = addp->fast_out(i);
3812         if (!u->is_Mem()) {
3813           return;
3814         }
3815         if (u->is_LoadVector() || u->is_StoreVector() || u->Opcode() == Op_StoreCM) {
3816           return;
3817         }
3818         if (addp2->in(AddPNode::Offset)->Opcode() != Op_ConvI2L) {
3819           int scale = 1 << addp2->in(AddPNode::Offset)->in(2)->get_int();
3820           if (VM_Version::expensive_load(u->as_Mem()->memory_size(), scale)) {
3821             return;
3822           }
3823         }
3824       }
3825 
3826       Node* off = addp->in(AddPNode::Offset);
3827       Node* addr2 = addp2->in(AddPNode::Address);
3828       Node* base = addp->in(AddPNode::Base);
3829 
3830       Node* new_addr = NULL;
3831       // Check whether the graph already has the new AddP we need
3832       // before we create one (no GVN available here).
3833       for (DUIterator_Fast imax, i = addr2->fast_outs(imax); i < imax; i++) {
3834         Node* u = addr2->fast_out(i);
3835         if (u->is_AddP() &&
3836             u->in(AddPNode::Base) == base &&
3837             u->in(AddPNode::Address) == addr2 &&
3838             u->in(AddPNode::Offset) == off) {
3839           new_addr = u;
3840           break;
3841         }
3842       }
3843 
3844       if (new_addr == NULL) {
3845         new_addr = new AddPNode(base, addr2, off);
3846       }
3847       Node* new_off = addp2->in(AddPNode::Offset);
3848       addp->set_req(AddPNode::Address, new_addr);
3849       if (addr->outcnt() == 0) {
3850         addr->disconnect_inputs(NULL, this);
3851       }
3852       addp->set_req(AddPNode::Offset, new_off);
3853       if (off->outcnt() == 0) {
3854         off->disconnect_inputs(NULL, this);
3855       }
3856     }
3857   }
3858 }
3859 
3860 // helper for encoding java_to_runtime calls on sim
3861 //
3862 // this is needed to compute the extra arguments required when
3863 // planting a call to the simulator blrt instruction. the TypeFunc
3864 // can be queried to identify the counts for integral, and floating
3865 // arguments and the return type
3866 
3867 static void getCallInfo(const TypeFunc *tf, int &gpcnt, int &fpcnt, int &rtype)
3868 {
3869   int gps = 0;
3870   int fps = 0;
3871   const TypeTuple *domain = tf->domain();
3872   int max = domain->cnt();
3873   for (int i = TypeFunc::Parms; i < max; i++) {
3874     const Type *t = domain->field_at(i);
3875     switch(t->basic_type()) {
3876     case T_FLOAT:
3877     case T_DOUBLE:
3878       fps++;
3879     default:
3880       gps++;
3881     }
3882   }
3883   gpcnt = gps;
3884   fpcnt = fps;
3885   BasicType rt = tf->return_type();
3886   switch (rt) {
3887   case T_VOID:
3888     rtype = MacroAssembler::ret_type_void;
3889     break;
3890   default:
3891     rtype = MacroAssembler::ret_type_integral;
3892     break;
3893   case T_FLOAT:
3894     rtype = MacroAssembler::ret_type_float;
3895     break;
3896   case T_DOUBLE:
3897     rtype = MacroAssembler::ret_type_double;
3898     break;
3899   }
3900 }
3901 
3902 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
3903   MacroAssembler _masm(&cbuf);                                          \
3904   {                                                                     \
3905     guarantee(INDEX == -1, "mode not permitted for volatile");          \
3906     guarantee(DISP == 0, "mode not permitted for volatile");            \
3907     guarantee(SCALE == 0, "mode not permitted for volatile");           \
3908     __ INSN(REG, as_Register(BASE));                                    \
3909   }
3910 
3911 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
3912 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
3913 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
3914                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
3915 
3916   // Used for all non-volatile memory accesses.  The use of
3917   // $mem->opcode() to discover whether this pattern uses sign-extended
3918   // offsets is something of a kludge.
3919   static void loadStore(MacroAssembler masm, mem_insn insn,
3920                          Register reg, int opcode,
3921                          Register base, int index, int size, int disp)
3922   {
3923     Address::extend scale;
3924 
3925     // Hooboy, this is fugly.  We need a way to communicate to the
3926     // encoder that the index needs to be sign extended, so we have to
3927     // enumerate all the cases.
3928     switch (opcode) {
3929     case INDINDEXSCALEDI2L:
3930     case INDINDEXSCALEDI2LN:
3931     case INDINDEXI2L:
3932     case INDINDEXI2LN:
3933       scale = Address::sxtw(size);
3934       break;
3935     default:
3936       scale = Address::lsl(size);
3937     }
3938 
3939     if (index == -1) {
3940       (masm.*insn)(reg, Address(base, disp));
3941     } else {
3942       assert(disp == 0, "unsupported address mode: disp = %d", disp);
3943       (masm.*insn)(reg, Address(base, as_Register(index), scale));
3944     }
3945   }
3946 
3947   static void loadStore(MacroAssembler masm, mem_float_insn insn,
3948                          FloatRegister reg, int opcode,
3949                          Register base, int index, int size, int disp)
3950   {
3951     Address::extend scale;
3952 
3953     switch (opcode) {
3954     case INDINDEXSCALEDI2L:
3955     case INDINDEXSCALEDI2LN:
3956       scale = Address::sxtw(size);
3957       break;
3958     default:
3959       scale = Address::lsl(size);
3960     }
3961 
3962      if (index == -1) {
3963       (masm.*insn)(reg, Address(base, disp));
3964     } else {
3965       assert(disp == 0, "unsupported address mode: disp = %d", disp);
3966       (masm.*insn)(reg, Address(base, as_Register(index), scale));
3967     }
3968   }
3969 
3970   static void loadStore(MacroAssembler masm, mem_vector_insn insn,
3971                          FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
3972                          int opcode, Register base, int index, int size, int disp)
3973   {
3974     if (index == -1) {
3975       (masm.*insn)(reg, T, Address(base, disp));
3976     } else {
3977       assert(disp == 0, "unsupported address mode");
3978       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
3979     }
3980   }
3981 
3982 %}
3983 
3984 
3985 
3986 //----------ENCODING BLOCK-----------------------------------------------------
3987 // This block specifies the encoding classes used by the compiler to
3988 // output byte streams.  Encoding classes are parameterized macros
3989 // used by Machine Instruction Nodes in order to generate the bit
3990 // encoding of the instruction.  Operands specify their base encoding
3991 // interface with the interface keyword.  There are currently
3992 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
3993 // COND_INTER.  REG_INTER causes an operand to generate a function
3994 // which returns its register number when queried.  CONST_INTER causes
3995 // an operand to generate a function which returns the value of the
3996 // constant when queried.  MEMORY_INTER causes an operand to generate
3997 // four functions which return the Base Register, the Index Register,
3998 // the Scale Value, and the Offset Value of the operand when queried.
3999 // COND_INTER causes an operand to generate six functions which return
4000 // the encoding code (ie - encoding bits for the instruction)
4001 // associated with each basic boolean condition for a conditional
4002 // instruction.
4003 //
4004 // Instructions specify two basic values for encoding.  Again, a
4005 // function is available to check if the constant displacement is an
4006 // oop. They use the ins_encode keyword to specify their encoding
4007 // classes (which must be a sequence of enc_class names, and their
4008 // parameters, specified in the encoding block), and they use the
4009 // opcode keyword to specify, in order, their primary, secondary, and
4010 // tertiary opcode.  Only the opcode sections which a particular
4011 // instruction needs for encoding need to be specified.
4012 encode %{
4013   // Build emit functions for each basic byte or larger field in the
4014   // intel encoding scheme (opcode, rm, sib, immediate), and call them
4015   // from C++ code in the enc_class source block.  Emit functions will
4016   // live in the main source block for now.  In future, we can
4017   // generalize this by adding a syntax that specifies the sizes of
4018   // fields in an order, so that the adlc can build the emit functions
4019   // automagically
4020 
4021   // catch all for unimplemented encodings
4022   enc_class enc_unimplemented %{
4023     MacroAssembler _masm(&cbuf);
4024     __ unimplemented("C2 catch all");
4025   %}
4026 
4027   // BEGIN Non-volatile memory access
4028 
4029   enc_class aarch64_enc_ldrsbw(iRegI dst, memory mem) %{
4030     Register dst_reg = as_Register($dst$$reg);
4031     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
4032                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4033   %}
4034 
4035   enc_class aarch64_enc_ldrsb(iRegI dst, memory mem) %{
4036     Register dst_reg = as_Register($dst$$reg);
4037     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
4038                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4039   %}
4040 
4041   enc_class aarch64_enc_ldrb(iRegI dst, memory mem) %{
4042     Register dst_reg = as_Register($dst$$reg);
4043     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
4044                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4045   %}
4046 
4047   enc_class aarch64_enc_ldrb(iRegL dst, memory mem) %{
4048     Register dst_reg = as_Register($dst$$reg);
4049     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
4050                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4051   %}
4052 
4053   enc_class aarch64_enc_ldrshw(iRegI dst, memory mem) %{
4054     Register dst_reg = as_Register($dst$$reg);
4055     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
4056                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4057   %}
4058 
4059   enc_class aarch64_enc_ldrsh(iRegI dst, memory mem) %{
4060     Register dst_reg = as_Register($dst$$reg);
4061     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
4062                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4063   %}
4064 
4065   enc_class aarch64_enc_ldrh(iRegI dst, memory mem) %{
4066     Register dst_reg = as_Register($dst$$reg);
4067     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
4068                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4069   %}
4070 
4071   enc_class aarch64_enc_ldrh(iRegL dst, memory mem) %{
4072     Register dst_reg = as_Register($dst$$reg);
4073     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
4074                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4075   %}
4076 
4077   enc_class aarch64_enc_ldrw(iRegI dst, memory mem) %{
4078     Register dst_reg = as_Register($dst$$reg);
4079     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
4080                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4081   %}
4082 
4083   enc_class aarch64_enc_ldrw(iRegL dst, memory mem) %{
4084     Register dst_reg = as_Register($dst$$reg);
4085     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
4086                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4087   %}
4088 
4089   enc_class aarch64_enc_ldrsw(iRegL dst, memory mem) %{
4090     Register dst_reg = as_Register($dst$$reg);
4091     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
4092                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4093   %}
4094 
4095   enc_class aarch64_enc_ldr(iRegL dst, memory mem) %{
4096     Register dst_reg = as_Register($dst$$reg);
4097     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
4098                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4099   %}
4100 
4101   enc_class aarch64_enc_ldrs(vRegF dst, memory mem) %{
4102     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
4103     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
4104                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4105   %}
4106 
4107   enc_class aarch64_enc_ldrd(vRegD dst, memory mem) %{
4108     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
4109     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
4110                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4111   %}
4112 
4113   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
4114     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
4115     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
4116        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4117   %}
4118 
4119   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
4120     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
4121     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
4122        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4123   %}
4124 
4125   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
4126     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
4127     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
4128        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4129   %}
4130 
4131   enc_class aarch64_enc_strb(iRegI src, memory mem) %{
4132     Register src_reg = as_Register($src$$reg);
4133     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
4134                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4135   %}
4136 
4137   enc_class aarch64_enc_strb0(memory mem) %{
4138     MacroAssembler _masm(&cbuf);
4139     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
4140                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4141   %}
4142 
4143   enc_class aarch64_enc_strb0_ordered(memory mem) %{
4144     MacroAssembler _masm(&cbuf);
4145     __ membar(Assembler::StoreStore);
4146     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
4147                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4148   %}
4149 
4150   enc_class aarch64_enc_strh(iRegI src, memory mem) %{
4151     Register src_reg = as_Register($src$$reg);
4152     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
4153                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4154   %}
4155 
4156   enc_class aarch64_enc_strh0(memory mem) %{
4157     MacroAssembler _masm(&cbuf);
4158     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
4159                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4160   %}
4161 
4162   enc_class aarch64_enc_strw(iRegI src, memory mem) %{
4163     Register src_reg = as_Register($src$$reg);
4164     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
4165                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4166   %}
4167 
4168   enc_class aarch64_enc_strw0(memory mem) %{
4169     MacroAssembler _masm(&cbuf);
4170     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
4171                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4172   %}
4173 
4174   enc_class aarch64_enc_str(iRegL src, memory mem) %{
4175     Register src_reg = as_Register($src$$reg);
4176     // we sometimes get asked to store the stack pointer into the
4177     // current thread -- we cannot do that directly on AArch64
4178     if (src_reg == r31_sp) {
4179       MacroAssembler _masm(&cbuf);
4180       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
4181       __ mov(rscratch2, sp);
4182       src_reg = rscratch2;
4183     }
4184     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
4185                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4186   %}
4187 
4188   enc_class aarch64_enc_str0(memory mem) %{
4189     MacroAssembler _masm(&cbuf);
4190     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
4191                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4192   %}
4193 
4194   enc_class aarch64_enc_strs(vRegF src, memory mem) %{
4195     FloatRegister src_reg = as_FloatRegister($src$$reg);
4196     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
4197                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4198   %}
4199 
4200   enc_class aarch64_enc_strd(vRegD src, memory mem) %{
4201     FloatRegister src_reg = as_FloatRegister($src$$reg);
4202     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
4203                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4204   %}
4205 
4206   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
4207     FloatRegister src_reg = as_FloatRegister($src$$reg);
4208     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
4209        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4210   %}
4211 
4212   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
4213     FloatRegister src_reg = as_FloatRegister($src$$reg);
4214     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
4215        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4216   %}
4217 
4218   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
4219     FloatRegister src_reg = as_FloatRegister($src$$reg);
4220     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
4221        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
4222   %}
4223 
4224   // END Non-volatile memory access
4225 
4226   // volatile loads and stores
4227 
4228   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
4229     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4230                  rscratch1, stlrb);
4231   %}
4232 
4233   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
4234     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4235                  rscratch1, stlrh);
4236   %}
4237 
4238   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
4239     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4240                  rscratch1, stlrw);
4241   %}
4242 
4243 
4244   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
4245     Register dst_reg = as_Register($dst$$reg);
4246     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4247              rscratch1, ldarb);
4248     __ sxtbw(dst_reg, dst_reg);
4249   %}
4250 
4251   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
4252     Register dst_reg = as_Register($dst$$reg);
4253     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4254              rscratch1, ldarb);
4255     __ sxtb(dst_reg, dst_reg);
4256   %}
4257 
4258   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
4259     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4260              rscratch1, ldarb);
4261   %}
4262 
4263   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
4264     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4265              rscratch1, ldarb);
4266   %}
4267 
4268   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
4269     Register dst_reg = as_Register($dst$$reg);
4270     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4271              rscratch1, ldarh);
4272     __ sxthw(dst_reg, dst_reg);
4273   %}
4274 
4275   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
4276     Register dst_reg = as_Register($dst$$reg);
4277     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4278              rscratch1, ldarh);
4279     __ sxth(dst_reg, dst_reg);
4280   %}
4281 
4282   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
4283     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4284              rscratch1, ldarh);
4285   %}
4286 
4287   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
4288     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4289              rscratch1, ldarh);
4290   %}
4291 
4292   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
4293     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4294              rscratch1, ldarw);
4295   %}
4296 
4297   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
4298     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4299              rscratch1, ldarw);
4300   %}
4301 
4302   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
4303     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4304              rscratch1, ldar);
4305   %}
4306 
4307   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
4308     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4309              rscratch1, ldarw);
4310     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
4311   %}
4312 
4313   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
4314     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4315              rscratch1, ldar);
4316     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
4317   %}
4318 
4319   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
4320     Register src_reg = as_Register($src$$reg);
4321     // we sometimes get asked to store the stack pointer into the
4322     // current thread -- we cannot do that directly on AArch64
4323     if (src_reg == r31_sp) {
4324         MacroAssembler _masm(&cbuf);
4325       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
4326       __ mov(rscratch2, sp);
4327       src_reg = rscratch2;
4328     }
4329     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4330                  rscratch1, stlr);
4331   %}
4332 
4333   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
4334     {
4335       MacroAssembler _masm(&cbuf);
4336       FloatRegister src_reg = as_FloatRegister($src$$reg);
4337       __ fmovs(rscratch2, src_reg);
4338     }
4339     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4340                  rscratch1, stlrw);
4341   %}
4342 
4343   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
4344     {
4345       MacroAssembler _masm(&cbuf);
4346       FloatRegister src_reg = as_FloatRegister($src$$reg);
4347       __ fmovd(rscratch2, src_reg);
4348     }
4349     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
4350                  rscratch1, stlr);
4351   %}
4352 
4353   // synchronized read/update encodings
4354 
4355   enc_class aarch64_enc_ldaxr(iRegL dst, memory mem) %{
4356     MacroAssembler _masm(&cbuf);
4357     Register dst_reg = as_Register($dst$$reg);
4358     Register base = as_Register($mem$$base);
4359     int index = $mem$$index;
4360     int scale = $mem$$scale;
4361     int disp = $mem$$disp;
4362     if (index == -1) {
4363        if (disp != 0) {
4364         __ lea(rscratch1, Address(base, disp));
4365         __ ldaxr(dst_reg, rscratch1);
4366       } else {
4367         // TODO
4368         // should we ever get anything other than this case?
4369         __ ldaxr(dst_reg, base);
4370       }
4371     } else {
4372       Register index_reg = as_Register(index);
4373       if (disp == 0) {
4374         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
4375         __ ldaxr(dst_reg, rscratch1);
4376       } else {
4377         __ lea(rscratch1, Address(base, disp));
4378         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
4379         __ ldaxr(dst_reg, rscratch1);
4380       }
4381     }
4382   %}
4383 
4384   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory mem) %{
4385     MacroAssembler _masm(&cbuf);
4386     Register src_reg = as_Register($src$$reg);
4387     Register base = as_Register($mem$$base);
4388     int index = $mem$$index;
4389     int scale = $mem$$scale;
4390     int disp = $mem$$disp;
4391     if (index == -1) {
4392        if (disp != 0) {
4393         __ lea(rscratch2, Address(base, disp));
4394         __ stlxr(rscratch1, src_reg, rscratch2);
4395       } else {
4396         // TODO
4397         // should we ever get anything other than this case?
4398         __ stlxr(rscratch1, src_reg, base);
4399       }
4400     } else {
4401       Register index_reg = as_Register(index);
4402       if (disp == 0) {
4403         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
4404         __ stlxr(rscratch1, src_reg, rscratch2);
4405       } else {
4406         __ lea(rscratch2, Address(base, disp));
4407         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
4408         __ stlxr(rscratch1, src_reg, rscratch2);
4409       }
4410     }
4411     __ cmpw(rscratch1, zr);
4412   %}
4413 
4414   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
4415     MacroAssembler _masm(&cbuf);
4416     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4417     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4418                Assembler::xword, /*acquire*/ false, /*release*/ true,
4419                /*weak*/ false, noreg);
4420   %}
4421 
4422   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4423     MacroAssembler _masm(&cbuf);
4424     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4425     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4426                Assembler::word, /*acquire*/ false, /*release*/ true,
4427                /*weak*/ false, noreg);
4428   %}
4429 
4430   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4431     MacroAssembler _masm(&cbuf);
4432     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4433     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4434                Assembler::halfword, /*acquire*/ false, /*release*/ true,
4435                /*weak*/ false, noreg);
4436   %}
4437 
4438   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4439     MacroAssembler _masm(&cbuf);
4440     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4441     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4442                Assembler::byte, /*acquire*/ false, /*release*/ true,
4443                /*weak*/ false, noreg);
4444   %}
4445 
4446 
4447   // The only difference between aarch64_enc_cmpxchg and
4448   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
4449   // CompareAndSwap sequence to serve as a barrier on acquiring a
4450   // lock.
4451   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
4452     MacroAssembler _masm(&cbuf);
4453     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4454     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4455                Assembler::xword, /*acquire*/ true, /*release*/ true,
4456                /*weak*/ false, noreg);
4457   %}
4458 
4459   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4460     MacroAssembler _masm(&cbuf);
4461     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4462     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4463                Assembler::word, /*acquire*/ true, /*release*/ true,
4464                /*weak*/ false, noreg);
4465   %}
4466 
4467 
4468   // auxiliary used for CompareAndSwapX to set result register
4469   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
4470     MacroAssembler _masm(&cbuf);
4471     Register res_reg = as_Register($res$$reg);
4472     __ cset(res_reg, Assembler::EQ);
4473   %}
4474 
4475   // prefetch encodings
4476 
4477   enc_class aarch64_enc_prefetchw(memory mem) %{
4478     MacroAssembler _masm(&cbuf);
4479     Register base = as_Register($mem$$base);
4480     int index = $mem$$index;
4481     int scale = $mem$$scale;
4482     int disp = $mem$$disp;
4483     if (index == -1) {
4484       __ prfm(Address(base, disp), PSTL1KEEP);
4485     } else {
4486       Register index_reg = as_Register(index);
4487       if (disp == 0) {
4488         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
4489       } else {
4490         __ lea(rscratch1, Address(base, disp));
4491         __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
4492       }
4493     }
4494   %}
4495 
4496   /// mov envcodings
4497 
4498   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
4499     MacroAssembler _masm(&cbuf);
4500     u_int32_t con = (u_int32_t)$src$$constant;
4501     Register dst_reg = as_Register($dst$$reg);
4502     if (con == 0) {
4503       __ movw(dst_reg, zr);
4504     } else {
4505       __ movw(dst_reg, con);
4506     }
4507   %}
4508 
4509   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
4510     MacroAssembler _masm(&cbuf);
4511     Register dst_reg = as_Register($dst$$reg);
4512     u_int64_t con = (u_int64_t)$src$$constant;
4513     if (con == 0) {
4514       __ mov(dst_reg, zr);
4515     } else {
4516       __ mov(dst_reg, con);
4517     }
4518   %}
4519 
4520   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
4521     MacroAssembler _masm(&cbuf);
4522     Register dst_reg = as_Register($dst$$reg);
4523     address con = (address)$src$$constant;
4524     if (con == NULL || con == (address)1) {
4525       ShouldNotReachHere();
4526     } else {
4527       relocInfo::relocType rtype = $src->constant_reloc();
4528       if (rtype == relocInfo::oop_type) {
4529         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
4530       } else if (rtype == relocInfo::metadata_type) {
4531         __ mov_metadata(dst_reg, (Metadata*)con);
4532       } else {
4533         assert(rtype == relocInfo::none, "unexpected reloc type");
4534         if (con < (address)(uintptr_t)os::vm_page_size()) {
4535           __ mov(dst_reg, con);
4536         } else {
4537           unsigned long offset;
4538           __ adrp(dst_reg, con, offset);
4539           __ add(dst_reg, dst_reg, offset);
4540         }
4541       }
4542     }
4543   %}
4544 
4545   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
4546     MacroAssembler _masm(&cbuf);
4547     Register dst_reg = as_Register($dst$$reg);
4548     __ mov(dst_reg, zr);
4549   %}
4550 
4551   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
4552     MacroAssembler _masm(&cbuf);
4553     Register dst_reg = as_Register($dst$$reg);
4554     __ mov(dst_reg, (u_int64_t)1);
4555   %}
4556 
4557   enc_class aarch64_enc_mov_poll_page(iRegP dst, immPollPage src) %{
4558     MacroAssembler _masm(&cbuf);
4559     address page = (address)$src$$constant;
4560     Register dst_reg = as_Register($dst$$reg);
4561     unsigned long off;
4562     __ adrp(dst_reg, Address(page, relocInfo::poll_type), off);
4563     assert(off == 0, "assumed offset == 0");
4564   %}
4565 
4566   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
4567     MacroAssembler _masm(&cbuf);
4568     __ load_byte_map_base($dst$$Register);
4569   %}
4570 
4571   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
4572     MacroAssembler _masm(&cbuf);
4573     Register dst_reg = as_Register($dst$$reg);
4574     address con = (address)$src$$constant;
4575     if (con == NULL) {
4576       ShouldNotReachHere();
4577     } else {
4578       relocInfo::relocType rtype = $src->constant_reloc();
4579       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
4580       __ set_narrow_oop(dst_reg, (jobject)con);
4581     }
4582   %}
4583 
4584   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
4585     MacroAssembler _masm(&cbuf);
4586     Register dst_reg = as_Register($dst$$reg);
4587     __ mov(dst_reg, zr);
4588   %}
4589 
4590   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
4591     MacroAssembler _masm(&cbuf);
4592     Register dst_reg = as_Register($dst$$reg);
4593     address con = (address)$src$$constant;
4594     if (con == NULL) {
4595       ShouldNotReachHere();
4596     } else {
4597       relocInfo::relocType rtype = $src->constant_reloc();
4598       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
4599       __ set_narrow_klass(dst_reg, (Klass *)con);
4600     }
4601   %}
4602 
4603   // arithmetic encodings
4604 
4605   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
4606     MacroAssembler _masm(&cbuf);
4607     Register dst_reg = as_Register($dst$$reg);
4608     Register src_reg = as_Register($src1$$reg);
4609     int32_t con = (int32_t)$src2$$constant;
4610     // add has primary == 0, subtract has primary == 1
4611     if ($primary) { con = -con; }
4612     if (con < 0) {
4613       __ subw(dst_reg, src_reg, -con);
4614     } else {
4615       __ addw(dst_reg, src_reg, con);
4616     }
4617   %}
4618 
4619   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
4620     MacroAssembler _masm(&cbuf);
4621     Register dst_reg = as_Register($dst$$reg);
4622     Register src_reg = as_Register($src1$$reg);
4623     int32_t con = (int32_t)$src2$$constant;
4624     // add has primary == 0, subtract has primary == 1
4625     if ($primary) { con = -con; }
4626     if (con < 0) {
4627       __ sub(dst_reg, src_reg, -con);
4628     } else {
4629       __ add(dst_reg, src_reg, con);
4630     }
4631   %}
4632 
4633   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
4634     MacroAssembler _masm(&cbuf);
4635    Register dst_reg = as_Register($dst$$reg);
4636    Register src1_reg = as_Register($src1$$reg);
4637    Register src2_reg = as_Register($src2$$reg);
4638     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
4639   %}
4640 
4641   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
4642     MacroAssembler _masm(&cbuf);
4643    Register dst_reg = as_Register($dst$$reg);
4644    Register src1_reg = as_Register($src1$$reg);
4645    Register src2_reg = as_Register($src2$$reg);
4646     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
4647   %}
4648 
4649   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
4650     MacroAssembler _masm(&cbuf);
4651    Register dst_reg = as_Register($dst$$reg);
4652    Register src1_reg = as_Register($src1$$reg);
4653    Register src2_reg = as_Register($src2$$reg);
4654     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
4655   %}
4656 
4657   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
4658     MacroAssembler _masm(&cbuf);
4659    Register dst_reg = as_Register($dst$$reg);
4660    Register src1_reg = as_Register($src1$$reg);
4661    Register src2_reg = as_Register($src2$$reg);
4662     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
4663   %}
4664 
4665   // compare instruction encodings
4666 
4667   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
4668     MacroAssembler _masm(&cbuf);
4669     Register reg1 = as_Register($src1$$reg);
4670     Register reg2 = as_Register($src2$$reg);
4671     __ cmpw(reg1, reg2);
4672   %}
4673 
4674   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
4675     MacroAssembler _masm(&cbuf);
4676     Register reg = as_Register($src1$$reg);
4677     int32_t val = $src2$$constant;
4678     if (val >= 0) {
4679       __ subsw(zr, reg, val);
4680     } else {
4681       __ addsw(zr, reg, -val);
4682     }
4683   %}
4684 
4685   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
4686     MacroAssembler _masm(&cbuf);
4687     Register reg1 = as_Register($src1$$reg);
4688     u_int32_t val = (u_int32_t)$src2$$constant;
4689     __ movw(rscratch1, val);
4690     __ cmpw(reg1, rscratch1);
4691   %}
4692 
4693   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
4694     MacroAssembler _masm(&cbuf);
4695     Register reg1 = as_Register($src1$$reg);
4696     Register reg2 = as_Register($src2$$reg);
4697     __ cmp(reg1, reg2);
4698   %}
4699 
4700   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
4701     MacroAssembler _masm(&cbuf);
4702     Register reg = as_Register($src1$$reg);
4703     int64_t val = $src2$$constant;
4704     if (val >= 0) {
4705       __ subs(zr, reg, val);
4706     } else if (val != -val) {
4707       __ adds(zr, reg, -val);
4708     } else {
4709     // aargh, Long.MIN_VALUE is a special case
4710       __ orr(rscratch1, zr, (u_int64_t)val);
4711       __ subs(zr, reg, rscratch1);
4712     }
4713   %}
4714 
4715   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
4716     MacroAssembler _masm(&cbuf);
4717     Register reg1 = as_Register($src1$$reg);
4718     u_int64_t val = (u_int64_t)$src2$$constant;
4719     __ mov(rscratch1, val);
4720     __ cmp(reg1, rscratch1);
4721   %}
4722 
4723   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
4724     MacroAssembler _masm(&cbuf);
4725     Register reg1 = as_Register($src1$$reg);
4726     Register reg2 = as_Register($src2$$reg);
4727     __ cmp(reg1, reg2);
4728   %}
4729 
4730   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
4731     MacroAssembler _masm(&cbuf);
4732     Register reg1 = as_Register($src1$$reg);
4733     Register reg2 = as_Register($src2$$reg);
4734     __ cmpw(reg1, reg2);
4735   %}
4736 
4737   enc_class aarch64_enc_testp(iRegP src) %{
4738     MacroAssembler _masm(&cbuf);
4739     Register reg = as_Register($src$$reg);
4740     __ cmp(reg, zr);
4741   %}
4742 
4743   enc_class aarch64_enc_testn(iRegN src) %{
4744     MacroAssembler _masm(&cbuf);
4745     Register reg = as_Register($src$$reg);
4746     __ cmpw(reg, zr);
4747   %}
4748 
4749   enc_class aarch64_enc_b(label lbl) %{
4750     MacroAssembler _masm(&cbuf);
4751     Label *L = $lbl$$label;
4752     __ b(*L);
4753   %}
4754 
4755   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
4756     MacroAssembler _masm(&cbuf);
4757     Label *L = $lbl$$label;
4758     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
4759   %}
4760 
4761   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
4762     MacroAssembler _masm(&cbuf);
4763     Label *L = $lbl$$label;
4764     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
4765   %}
4766 
4767   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
4768   %{
4769      Register sub_reg = as_Register($sub$$reg);
4770      Register super_reg = as_Register($super$$reg);
4771      Register temp_reg = as_Register($temp$$reg);
4772      Register result_reg = as_Register($result$$reg);
4773 
4774      Label miss;
4775      MacroAssembler _masm(&cbuf);
4776      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
4777                                      NULL, &miss,
4778                                      /*set_cond_codes:*/ true);
4779      if ($primary) {
4780        __ mov(result_reg, zr);
4781      }
4782      __ bind(miss);
4783   %}
4784 
4785   enc_class aarch64_enc_java_static_call(method meth) %{
4786     MacroAssembler _masm(&cbuf);
4787 
4788     address addr = (address)$meth$$method;
4789     address call;
4790     if (!_method) {
4791       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
4792       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
4793     } else {
4794       int method_index = resolved_method_index(cbuf);
4795       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
4796                                                   : static_call_Relocation::spec(method_index);
4797       call = __ trampoline_call(Address(addr, rspec), &cbuf);
4798 
4799       // Emit stub for static call
4800       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
4801       if (stub == NULL) {
4802         ciEnv::current()->record_failure("CodeCache is full");
4803         return;
4804       }
4805     }
4806     if (call == NULL) {
4807       ciEnv::current()->record_failure("CodeCache is full");
4808       return;
4809     }
4810   %}
4811 
4812   enc_class aarch64_enc_java_dynamic_call(method meth) %{
4813     MacroAssembler _masm(&cbuf);
4814     int method_index = resolved_method_index(cbuf);
4815     address call = __ ic_call((address)$meth$$method, method_index);
4816     if (call == NULL) {
4817       ciEnv::current()->record_failure("CodeCache is full");
4818       return;
4819     }
4820   %}
4821 
4822   enc_class aarch64_enc_call_epilog() %{
4823     MacroAssembler _masm(&cbuf);
4824     if (VerifyStackAtCalls) {
4825       // Check that stack depth is unchanged: find majik cookie on stack
4826       __ call_Unimplemented();
4827     }
4828   %}
4829 
4830   enc_class aarch64_enc_java_to_runtime(method meth) %{
4831     MacroAssembler _masm(&cbuf);
4832 
4833     // some calls to generated routines (arraycopy code) are scheduled
4834     // by C2 as runtime calls. if so we can call them using a br (they
4835     // will be in a reachable segment) otherwise we have to use a blrt
4836     // which loads the absolute address into a register.
4837     address entry = (address)$meth$$method;
4838     CodeBlob *cb = CodeCache::find_blob(entry);
4839     if (cb) {
4840       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
4841       if (call == NULL) {
4842         ciEnv::current()->record_failure("CodeCache is full");
4843         return;
4844       }
4845     } else {
4846       int gpcnt;
4847       int fpcnt;
4848       int rtype;
4849       getCallInfo(tf(), gpcnt, fpcnt, rtype);
4850       Label retaddr;
4851       __ adr(rscratch2, retaddr);
4852       __ lea(rscratch1, RuntimeAddress(entry));
4853       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
4854       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
4855       __ blrt(rscratch1, gpcnt, fpcnt, rtype);
4856       __ bind(retaddr);
4857       __ add(sp, sp, 2 * wordSize);
4858     }
4859   %}
4860 
4861   enc_class aarch64_enc_rethrow() %{
4862     MacroAssembler _masm(&cbuf);
4863     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
4864   %}
4865 
4866   enc_class aarch64_enc_ret() %{
4867     MacroAssembler _masm(&cbuf);
4868     __ ret(lr);
4869   %}
4870 
4871   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
4872     MacroAssembler _masm(&cbuf);
4873     Register target_reg = as_Register($jump_target$$reg);
4874     __ br(target_reg);
4875   %}
4876 
4877   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
4878     MacroAssembler _masm(&cbuf);
4879     Register target_reg = as_Register($jump_target$$reg);
4880     // exception oop should be in r0
4881     // ret addr has been popped into lr
4882     // callee expects it in r3
4883     __ mov(r3, lr);
4884     __ br(target_reg);
4885   %}
4886 
4887   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
4888     MacroAssembler _masm(&cbuf);
4889     Register oop = as_Register($object$$reg);
4890     Register box = as_Register($box$$reg);
4891     Register disp_hdr = as_Register($tmp$$reg);
4892     Register tmp = as_Register($tmp2$$reg);
4893     Label cont;
4894     Label object_has_monitor;
4895     Label cas_failed;
4896 
4897     assert_different_registers(oop, box, tmp, disp_hdr);
4898 
4899     // Load markOop from object into displaced_header.
4900     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
4901 
4902     // Always do locking in runtime.
4903     if (EmitSync & 0x01) {
4904       __ cmp(oop, zr);
4905       return;
4906     }
4907 
4908     if (UseBiasedLocking && !UseOptoBiasInlining) {
4909       __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
4910     }
4911 
4912     // Handle existing monitor
4913     if ((EmitSync & 0x02) == 0) {
4914       // we can use AArch64's bit test and branch here but
4915       // markoopDesc does not define a bit index just the bit value
4916       // so assert in case the bit pos changes
4917 #     define __monitor_value_log2 1
4918       assert(markOopDesc::monitor_value == (1 << __monitor_value_log2), "incorrect bit position");
4919       __ tbnz(disp_hdr, __monitor_value_log2, object_has_monitor);
4920 #     undef __monitor_value_log2
4921     }
4922 
4923     // Set displaced_header to be (markOop of object | UNLOCK_VALUE).
4924     __ orr(disp_hdr, disp_hdr, markOopDesc::unlocked_value);
4925 
4926     // Load Compare Value application register.
4927 
4928     // Initialize the box. (Must happen before we update the object mark!)
4929     __ str(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
4930 
4931     // Compare object markOop with mark and if equal exchange scratch1
4932     // with object markOop.
4933     if (UseLSE) {
4934       __ mov(tmp, disp_hdr);
4935       __ casal(Assembler::xword, tmp, box, oop);
4936       __ cmp(tmp, disp_hdr);
4937       __ br(Assembler::EQ, cont);
4938     } else {
4939       Label retry_load;
4940       if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
4941         __ prfm(Address(oop), PSTL1STRM);
4942       __ bind(retry_load);
4943       __ ldaxr(tmp, oop);
4944       __ cmp(tmp, disp_hdr);
4945       __ br(Assembler::NE, cas_failed);
4946       // use stlxr to ensure update is immediately visible
4947       __ stlxr(tmp, box, oop);
4948       __ cbzw(tmp, cont);
4949       __ b(retry_load);
4950     }
4951 
4952     // Formerly:
4953     // __ cmpxchgptr(/*oldv=*/disp_hdr,
4954     //               /*newv=*/box,
4955     //               /*addr=*/oop,
4956     //               /*tmp=*/tmp,
4957     //               cont,
4958     //               /*fail*/NULL);
4959 
4960     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
4961 
4962     // If the compare-and-exchange succeeded, then we found an unlocked
4963     // object, will have now locked it will continue at label cont
4964 
4965     __ bind(cas_failed);
4966     // We did not see an unlocked object so try the fast recursive case.
4967 
4968     // Check if the owner is self by comparing the value in the
4969     // markOop of object (disp_hdr) with the stack pointer.
4970     __ mov(rscratch1, sp);
4971     __ sub(disp_hdr, disp_hdr, rscratch1);
4972     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place));
4973     // If condition is true we are cont and hence we can store 0 as the
4974     // displaced header in the box, which indicates that it is a recursive lock.
4975     __ ands(tmp/*==0?*/, disp_hdr, tmp);
4976     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
4977 
4978     // Handle existing monitor.
4979     if ((EmitSync & 0x02) == 0) {
4980       __ b(cont);
4981 
4982       __ bind(object_has_monitor);
4983       // The object's monitor m is unlocked iff m->owner == NULL,
4984       // otherwise m->owner may contain a thread or a stack address.
4985       //
4986       // Try to CAS m->owner from NULL to current thread.
4987       __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
4988       __ mov(disp_hdr, zr);
4989 
4990       if (UseLSE) {
4991         __ mov(rscratch1, disp_hdr);
4992         __ casal(Assembler::xword, rscratch1, rthread, tmp);
4993         __ cmp(rscratch1, disp_hdr);
4994       } else {
4995         Label retry_load, fail;
4996         if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
4997           __ prfm(Address(tmp), PSTL1STRM);
4998         __ bind(retry_load);
4999         __ ldaxr(rscratch1, tmp);
5000         __ cmp(disp_hdr, rscratch1);
5001         __ br(Assembler::NE, fail);
5002         // use stlxr to ensure update is immediately visible
5003         __ stlxr(rscratch1, rthread, tmp);
5004         __ cbnzw(rscratch1, retry_load);
5005         __ bind(fail);
5006       }
5007 
5008       // Label next;
5009       // __ cmpxchgptr(/*oldv=*/disp_hdr,
5010       //               /*newv=*/rthread,
5011       //               /*addr=*/tmp,
5012       //               /*tmp=*/rscratch1,
5013       //               /*succeed*/next,
5014       //               /*fail*/NULL);
5015       // __ bind(next);
5016 
5017       // store a non-null value into the box.
5018       __ str(box, Address(box, BasicLock::displaced_header_offset_in_bytes()));
5019 
5020       // PPC port checks the following invariants
5021       // #ifdef ASSERT
5022       // bne(flag, cont);
5023       // We have acquired the monitor, check some invariants.
5024       // addw(/*monitor=*/tmp, tmp, -ObjectMonitor::owner_offset_in_bytes());
5025       // Invariant 1: _recursions should be 0.
5026       // assert(ObjectMonitor::recursions_size_in_bytes() == 8, "unexpected size");
5027       // assert_mem8_is_zero(ObjectMonitor::recursions_offset_in_bytes(), tmp,
5028       //                        "monitor->_recursions should be 0", -1);
5029       // Invariant 2: OwnerIsThread shouldn't be 0.
5030       // assert(ObjectMonitor::OwnerIsThread_size_in_bytes() == 4, "unexpected size");
5031       //assert_mem4_isnot_zero(ObjectMonitor::OwnerIsThread_offset_in_bytes(), tmp,
5032       //                           "monitor->OwnerIsThread shouldn't be 0", -1);
5033       // #endif
5034     }
5035 
5036     __ bind(cont);
5037     // flag == EQ indicates success
5038     // flag == NE indicates failure
5039 
5040   %}
5041 
5042   // TODO
5043   // reimplement this with custom cmpxchgptr code
5044   // which avoids some of the unnecessary branching
5045   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
5046     MacroAssembler _masm(&cbuf);
5047     Register oop = as_Register($object$$reg);
5048     Register box = as_Register($box$$reg);
5049     Register disp_hdr = as_Register($tmp$$reg);
5050     Register tmp = as_Register($tmp2$$reg);
5051     Label cont;
5052     Label object_has_monitor;
5053     Label cas_failed;
5054 
5055     assert_different_registers(oop, box, tmp, disp_hdr);
5056 
5057     // Always do locking in runtime.
5058     if (EmitSync & 0x01) {
5059       __ cmp(oop, zr); // Oop can't be 0 here => always false.
5060       return;
5061     }
5062 
5063     if (UseBiasedLocking && !UseOptoBiasInlining) {
5064       __ biased_locking_exit(oop, tmp, cont);
5065     }
5066 
5067     // Find the lock address and load the displaced header from the stack.
5068     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
5069 
5070     // If the displaced header is 0, we have a recursive unlock.
5071     __ cmp(disp_hdr, zr);
5072     __ br(Assembler::EQ, cont);
5073 
5074 
5075     // Handle existing monitor.
5076     if ((EmitSync & 0x02) == 0) {
5077       __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
5078       __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
5079     }
5080 
5081     // Check if it is still a light weight lock, this is is true if we
5082     // see the stack address of the basicLock in the markOop of the
5083     // object.
5084 
5085       if (UseLSE) {
5086         __ mov(tmp, box);
5087         __ casl(Assembler::xword, tmp, disp_hdr, oop);
5088         __ cmp(tmp, box);
5089       } else {
5090         Label retry_load;
5091         if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
5092           __ prfm(Address(oop), PSTL1STRM);
5093         __ bind(retry_load);
5094         __ ldxr(tmp, oop);
5095         __ cmp(box, tmp);
5096         __ br(Assembler::NE, cas_failed);
5097         // use stlxr to ensure update is immediately visible
5098         __ stlxr(tmp, disp_hdr, oop);
5099         __ cbzw(tmp, cont);
5100         __ b(retry_load);
5101       }
5102 
5103     // __ cmpxchgptr(/*compare_value=*/box,
5104     //               /*exchange_value=*/disp_hdr,
5105     //               /*where=*/oop,
5106     //               /*result=*/tmp,
5107     //               cont,
5108     //               /*cas_failed*/NULL);
5109     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
5110 
5111     __ bind(cas_failed);
5112 
5113     // Handle existing monitor.
5114     if ((EmitSync & 0x02) == 0) {
5115       __ b(cont);
5116 
5117       __ bind(object_has_monitor);
5118       __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor
5119       __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
5120       __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
5121       __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
5122       __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
5123       __ cmp(rscratch1, zr);
5124       __ br(Assembler::NE, cont);
5125 
5126       __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
5127       __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
5128       __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
5129       __ cmp(rscratch1, zr);
5130       __ cbnz(rscratch1, cont);
5131       // need a release store here
5132       __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
5133       __ stlr(rscratch1, tmp); // rscratch1 is zero
5134     }
5135 
5136     __ bind(cont);
5137     // flag == EQ indicates success
5138     // flag == NE indicates failure
5139   %}
5140 
5141 %}
5142 
5143 //----------FRAME--------------------------------------------------------------
5144 // Definition of frame structure and management information.
5145 //
5146 //  S T A C K   L A Y O U T    Allocators stack-slot number
5147 //                             |   (to get allocators register number
5148 //  G  Owned by    |        |  v    add OptoReg::stack0())
5149 //  r   CALLER     |        |
5150 //  o     |        +--------+      pad to even-align allocators stack-slot
5151 //  w     V        |  pad0  |        numbers; owned by CALLER
5152 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
5153 //  h     ^        |   in   |  5
5154 //        |        |  args  |  4   Holes in incoming args owned by SELF
5155 //  |     |        |        |  3
5156 //  |     |        +--------+
5157 //  V     |        | old out|      Empty on Intel, window on Sparc
5158 //        |    old |preserve|      Must be even aligned.
5159 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
5160 //        |        |   in   |  3   area for Intel ret address
5161 //     Owned by    |preserve|      Empty on Sparc.
5162 //       SELF      +--------+
5163 //        |        |  pad2  |  2   pad to align old SP
5164 //        |        +--------+  1
5165 //        |        | locks  |  0
5166 //        |        +--------+----> OptoReg::stack0(), even aligned
5167 //        |        |  pad1  | 11   pad to align new SP
5168 //        |        +--------+
5169 //        |        |        | 10
5170 //        |        | spills |  9   spills
5171 //        V        |        |  8   (pad0 slot for callee)
5172 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
5173 //        ^        |  out   |  7
5174 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
5175 //     Owned by    +--------+
5176 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
5177 //        |    new |preserve|      Must be even-aligned.
5178 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
5179 //        |        |        |
5180 //
5181 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
5182 //         known from SELF's arguments and the Java calling convention.
5183 //         Region 6-7 is determined per call site.
5184 // Note 2: If the calling convention leaves holes in the incoming argument
5185 //         area, those holes are owned by SELF.  Holes in the outgoing area
5186 //         are owned by the CALLEE.  Holes should not be nessecary in the
5187 //         incoming area, as the Java calling convention is completely under
5188 //         the control of the AD file.  Doubles can be sorted and packed to
5189 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
5190 //         varargs C calling conventions.
5191 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
5192 //         even aligned with pad0 as needed.
5193 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
5194 //           (the latter is true on Intel but is it false on AArch64?)
5195 //         region 6-11 is even aligned; it may be padded out more so that
5196 //         the region from SP to FP meets the minimum stack alignment.
5197 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
5198 //         alignment.  Region 11, pad1, may be dynamically extended so that
5199 //         SP meets the minimum alignment.
5200 
5201 frame %{
5202   // What direction does stack grow in (assumed to be same for C & Java)
5203   stack_direction(TOWARDS_LOW);
5204 
5205   // These three registers define part of the calling convention
5206   // between compiled code and the interpreter.
5207 
5208   // Inline Cache Register or methodOop for I2C.
5209   inline_cache_reg(R12);
5210 
5211   // Method Oop Register when calling interpreter.
5212   interpreter_method_oop_reg(R12);
5213 
5214   // Number of stack slots consumed by locking an object
5215   sync_stack_slots(2);
5216 
5217   // Compiled code's Frame Pointer
5218   frame_pointer(R31);
5219 
5220   // Interpreter stores its frame pointer in a register which is
5221   // stored to the stack by I2CAdaptors.
5222   // I2CAdaptors convert from interpreted java to compiled java.
5223   interpreter_frame_pointer(R29);
5224 
5225   // Stack alignment requirement
5226   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
5227 
5228   // Number of stack slots between incoming argument block and the start of
5229   // a new frame.  The PROLOG must add this many slots to the stack.  The
5230   // EPILOG must remove this many slots. aarch64 needs two slots for
5231   // return address and fp.
5232   // TODO think this is correct but check
5233   in_preserve_stack_slots(4);
5234 
5235   // Number of outgoing stack slots killed above the out_preserve_stack_slots
5236   // for calls to C.  Supports the var-args backing area for register parms.
5237   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
5238 
5239   // The after-PROLOG location of the return address.  Location of
5240   // return address specifies a type (REG or STACK) and a number
5241   // representing the register number (i.e. - use a register name) or
5242   // stack slot.
5243   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
5244   // Otherwise, it is above the locks and verification slot and alignment word
5245   // TODO this may well be correct but need to check why that - 2 is there
5246   // ppc port uses 0 but we definitely need to allow for fixed_slots
5247   // which folds in the space used for monitors
5248   return_addr(STACK - 2 +
5249               align_up((Compile::current()->in_preserve_stack_slots() +
5250                         Compile::current()->fixed_slots()),
5251                        stack_alignment_in_slots()));
5252 
5253   // Body of function which returns an integer array locating
5254   // arguments either in registers or in stack slots.  Passed an array
5255   // of ideal registers called "sig" and a "length" count.  Stack-slot
5256   // offsets are based on outgoing arguments, i.e. a CALLER setting up
5257   // arguments for a CALLEE.  Incoming stack arguments are
5258   // automatically biased by the preserve_stack_slots field above.
5259 
5260   calling_convention
5261   %{
5262     // No difference between ingoing/outgoing just pass false
5263     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
5264   %}
5265 
5266   c_calling_convention
5267   %{
5268     // This is obviously always outgoing
5269     (void) SharedRuntime::c_calling_convention(sig_bt, regs, NULL, length);
5270   %}
5271 
5272   // Location of compiled Java return values.  Same as C for now.
5273   return_value
5274   %{
5275     // TODO do we allow ideal_reg == Op_RegN???
5276     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
5277            "only return normal values");
5278 
5279     static const int lo[Op_RegL + 1] = { // enum name
5280       0,                                 // Op_Node
5281       0,                                 // Op_Set
5282       R0_num,                            // Op_RegN
5283       R0_num,                            // Op_RegI
5284       R0_num,                            // Op_RegP
5285       V0_num,                            // Op_RegF
5286       V0_num,                            // Op_RegD
5287       R0_num                             // Op_RegL
5288     };
5289 
5290     static const int hi[Op_RegL + 1] = { // enum name
5291       0,                                 // Op_Node
5292       0,                                 // Op_Set
5293       OptoReg::Bad,                       // Op_RegN
5294       OptoReg::Bad,                      // Op_RegI
5295       R0_H_num,                          // Op_RegP
5296       OptoReg::Bad,                      // Op_RegF
5297       V0_H_num,                          // Op_RegD
5298       R0_H_num                           // Op_RegL
5299     };
5300 
5301     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
5302   %}
5303 %}
5304 
5305 //----------ATTRIBUTES---------------------------------------------------------
5306 //----------Operand Attributes-------------------------------------------------
5307 op_attrib op_cost(1);        // Required cost attribute
5308 
5309 //----------Instruction Attributes---------------------------------------------
5310 ins_attrib ins_cost(INSN_COST); // Required cost attribute
5311 ins_attrib ins_size(32);        // Required size attribute (in bits)
5312 ins_attrib ins_short_branch(0); // Required flag: is this instruction
5313                                 // a non-matching short branch variant
5314                                 // of some long branch?
5315 ins_attrib ins_alignment(4);    // Required alignment attribute (must
5316                                 // be a power of 2) specifies the
5317                                 // alignment that some part of the
5318                                 // instruction (not necessarily the
5319                                 // start) requires.  If > 1, a
5320                                 // compute_padding() function must be
5321                                 // provided for the instruction
5322 
5323 //----------OPERANDS-----------------------------------------------------------
5324 // Operand definitions must precede instruction definitions for correct parsing
5325 // in the ADLC because operands constitute user defined types which are used in
5326 // instruction definitions.
5327 
5328 //----------Simple Operands----------------------------------------------------
5329 
5330 // Integer operands 32 bit
5331 // 32 bit immediate
5332 operand immI()
5333 %{
5334   match(ConI);
5335 
5336   op_cost(0);
5337   format %{ %}
5338   interface(CONST_INTER);
5339 %}
5340 
5341 // 32 bit zero
5342 operand immI0()
5343 %{
5344   predicate(n->get_int() == 0);
5345   match(ConI);
5346 
5347   op_cost(0);
5348   format %{ %}
5349   interface(CONST_INTER);
5350 %}
5351 
5352 // 32 bit unit increment
5353 operand immI_1()
5354 %{
5355   predicate(n->get_int() == 1);
5356   match(ConI);
5357 
5358   op_cost(0);
5359   format %{ %}
5360   interface(CONST_INTER);
5361 %}
5362 
5363 // 32 bit unit decrement
5364 operand immI_M1()
5365 %{
5366   predicate(n->get_int() == -1);
5367   match(ConI);
5368 
5369   op_cost(0);
5370   format %{ %}
5371   interface(CONST_INTER);
5372 %}
5373 
5374 // Shift values for add/sub extension shift
5375 operand immIExt()
5376 %{
5377   predicate(0 <= n->get_int() && (n->get_int() <= 4));
5378   match(ConI);
5379 
5380   op_cost(0);
5381   format %{ %}
5382   interface(CONST_INTER);
5383 %}
5384 
5385 operand immI_le_4()
5386 %{
5387   predicate(n->get_int() <= 4);
5388   match(ConI);
5389 
5390   op_cost(0);
5391   format %{ %}
5392   interface(CONST_INTER);
5393 %}
5394 
5395 operand immI_31()
5396 %{
5397   predicate(n->get_int() == 31);
5398   match(ConI);
5399 
5400   op_cost(0);
5401   format %{ %}
5402   interface(CONST_INTER);
5403 %}
5404 
5405 operand immI_8()
5406 %{
5407   predicate(n->get_int() == 8);
5408   match(ConI);
5409 
5410   op_cost(0);
5411   format %{ %}
5412   interface(CONST_INTER);
5413 %}
5414 
5415 operand immI_16()
5416 %{
5417   predicate(n->get_int() == 16);
5418   match(ConI);
5419 
5420   op_cost(0);
5421   format %{ %}
5422   interface(CONST_INTER);
5423 %}
5424 
5425 operand immI_24()
5426 %{
5427   predicate(n->get_int() == 24);
5428   match(ConI);
5429 
5430   op_cost(0);
5431   format %{ %}
5432   interface(CONST_INTER);
5433 %}
5434 
5435 operand immI_32()
5436 %{
5437   predicate(n->get_int() == 32);
5438   match(ConI);
5439 
5440   op_cost(0);
5441   format %{ %}
5442   interface(CONST_INTER);
5443 %}
5444 
5445 operand immI_48()
5446 %{
5447   predicate(n->get_int() == 48);
5448   match(ConI);
5449 
5450   op_cost(0);
5451   format %{ %}
5452   interface(CONST_INTER);
5453 %}
5454 
5455 operand immI_56()
5456 %{
5457   predicate(n->get_int() == 56);
5458   match(ConI);
5459 
5460   op_cost(0);
5461   format %{ %}
5462   interface(CONST_INTER);
5463 %}
5464 
5465 operand immI_63()
5466 %{
5467   predicate(n->get_int() == 63);
5468   match(ConI);
5469 
5470   op_cost(0);
5471   format %{ %}
5472   interface(CONST_INTER);
5473 %}
5474 
5475 operand immI_64()
5476 %{
5477   predicate(n->get_int() == 64);
5478   match(ConI);
5479 
5480   op_cost(0);
5481   format %{ %}
5482   interface(CONST_INTER);
5483 %}
5484 
5485 operand immI_255()
5486 %{
5487   predicate(n->get_int() == 255);
5488   match(ConI);
5489 
5490   op_cost(0);
5491   format %{ %}
5492   interface(CONST_INTER);
5493 %}
5494 
5495 operand immI_65535()
5496 %{
5497   predicate(n->get_int() == 65535);
5498   match(ConI);
5499 
5500   op_cost(0);
5501   format %{ %}
5502   interface(CONST_INTER);
5503 %}
5504 
5505 operand immL_255()
5506 %{
5507   predicate(n->get_long() == 255L);
5508   match(ConL);
5509 
5510   op_cost(0);
5511   format %{ %}
5512   interface(CONST_INTER);
5513 %}
5514 
5515 operand immL_65535()
5516 %{
5517   predicate(n->get_long() == 65535L);
5518   match(ConL);
5519 
5520   op_cost(0);
5521   format %{ %}
5522   interface(CONST_INTER);
5523 %}
5524 
5525 operand immL_4294967295()
5526 %{
5527   predicate(n->get_long() == 4294967295L);
5528   match(ConL);
5529 
5530   op_cost(0);
5531   format %{ %}
5532   interface(CONST_INTER);
5533 %}
5534 
5535 operand immL_bitmask()
5536 %{
5537   predicate(((n->get_long() & 0xc000000000000000l) == 0)
5538             && is_power_of_2(n->get_long() + 1));
5539   match(ConL);
5540 
5541   op_cost(0);
5542   format %{ %}
5543   interface(CONST_INTER);
5544 %}
5545 
5546 operand immI_bitmask()
5547 %{
5548   predicate(((n->get_int() & 0xc0000000) == 0)
5549             && is_power_of_2(n->get_int() + 1));
5550   match(ConI);
5551 
5552   op_cost(0);
5553   format %{ %}
5554   interface(CONST_INTER);
5555 %}
5556 
5557 // Scale values for scaled offset addressing modes (up to long but not quad)
5558 operand immIScale()
5559 %{
5560   predicate(0 <= n->get_int() && (n->get_int() <= 3));
5561   match(ConI);
5562 
5563   op_cost(0);
5564   format %{ %}
5565   interface(CONST_INTER);
5566 %}
5567 
5568 // 26 bit signed offset -- for pc-relative branches
5569 operand immI26()
5570 %{
5571   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
5572   match(ConI);
5573 
5574   op_cost(0);
5575   format %{ %}
5576   interface(CONST_INTER);
5577 %}
5578 
5579 // 19 bit signed offset -- for pc-relative loads
5580 operand immI19()
5581 %{
5582   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
5583   match(ConI);
5584 
5585   op_cost(0);
5586   format %{ %}
5587   interface(CONST_INTER);
5588 %}
5589 
5590 // 12 bit unsigned offset -- for base plus immediate loads
5591 operand immIU12()
5592 %{
5593   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
5594   match(ConI);
5595 
5596   op_cost(0);
5597   format %{ %}
5598   interface(CONST_INTER);
5599 %}
5600 
5601 operand immLU12()
5602 %{
5603   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
5604   match(ConL);
5605 
5606   op_cost(0);
5607   format %{ %}
5608   interface(CONST_INTER);
5609 %}
5610 
5611 // Offset for scaled or unscaled immediate loads and stores
5612 operand immIOffset()
5613 %{
5614   predicate(Address::offset_ok_for_immed(n->get_int()));
5615   match(ConI);
5616 
5617   op_cost(0);
5618   format %{ %}
5619   interface(CONST_INTER);
5620 %}
5621 
5622 operand immIOffset4()
5623 %{
5624   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
5625   match(ConI);
5626 
5627   op_cost(0);
5628   format %{ %}
5629   interface(CONST_INTER);
5630 %}
5631 
5632 operand immIOffset8()
5633 %{
5634   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
5635   match(ConI);
5636 
5637   op_cost(0);
5638   format %{ %}
5639   interface(CONST_INTER);
5640 %}
5641 
5642 operand immIOffset16()
5643 %{
5644   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
5645   match(ConI);
5646 
5647   op_cost(0);
5648   format %{ %}
5649   interface(CONST_INTER);
5650 %}
5651 
5652 operand immLoffset()
5653 %{
5654   predicate(Address::offset_ok_for_immed(n->get_long()));
5655   match(ConL);
5656 
5657   op_cost(0);
5658   format %{ %}
5659   interface(CONST_INTER);
5660 %}
5661 
5662 operand immLoffset4()
5663 %{
5664   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
5665   match(ConL);
5666 
5667   op_cost(0);
5668   format %{ %}
5669   interface(CONST_INTER);
5670 %}
5671 
5672 operand immLoffset8()
5673 %{
5674   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
5675   match(ConL);
5676 
5677   op_cost(0);
5678   format %{ %}
5679   interface(CONST_INTER);
5680 %}
5681 
5682 operand immLoffset16()
5683 %{
5684   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
5685   match(ConL);
5686 
5687   op_cost(0);
5688   format %{ %}
5689   interface(CONST_INTER);
5690 %}
5691 
5692 // 32 bit integer valid for add sub immediate
5693 operand immIAddSub()
5694 %{
5695   predicate(Assembler::operand_valid_for_add_sub_immediate((long)n->get_int()));
5696   match(ConI);
5697   op_cost(0);
5698   format %{ %}
5699   interface(CONST_INTER);
5700 %}
5701 
5702 // 32 bit unsigned integer valid for logical immediate
5703 // TODO -- check this is right when e.g the mask is 0x80000000
5704 operand immILog()
5705 %{
5706   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (unsigned long)n->get_int()));
5707   match(ConI);
5708 
5709   op_cost(0);
5710   format %{ %}
5711   interface(CONST_INTER);
5712 %}
5713 
5714 // Integer operands 64 bit
5715 // 64 bit immediate
5716 operand immL()
5717 %{
5718   match(ConL);
5719 
5720   op_cost(0);
5721   format %{ %}
5722   interface(CONST_INTER);
5723 %}
5724 
5725 // 64 bit zero
5726 operand immL0()
5727 %{
5728   predicate(n->get_long() == 0);
5729   match(ConL);
5730 
5731   op_cost(0);
5732   format %{ %}
5733   interface(CONST_INTER);
5734 %}
5735 
5736 // 64 bit unit increment
5737 operand immL_1()
5738 %{
5739   predicate(n->get_long() == 1);
5740   match(ConL);
5741 
5742   op_cost(0);
5743   format %{ %}
5744   interface(CONST_INTER);
5745 %}
5746 
5747 // 64 bit unit decrement
5748 operand immL_M1()
5749 %{
5750   predicate(n->get_long() == -1);
5751   match(ConL);
5752 
5753   op_cost(0);
5754   format %{ %}
5755   interface(CONST_INTER);
5756 %}
5757 
5758 // 32 bit offset of pc in thread anchor
5759 
5760 operand immL_pc_off()
5761 %{
5762   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
5763                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
5764   match(ConL);
5765 
5766   op_cost(0);
5767   format %{ %}
5768   interface(CONST_INTER);
5769 %}
5770 
5771 // 64 bit integer valid for add sub immediate
5772 operand immLAddSub()
5773 %{
5774   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
5775   match(ConL);
5776   op_cost(0);
5777   format %{ %}
5778   interface(CONST_INTER);
5779 %}
5780 
5781 // 64 bit integer valid for logical immediate
5782 operand immLLog()
5783 %{
5784   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (unsigned long)n->get_long()));
5785   match(ConL);
5786   op_cost(0);
5787   format %{ %}
5788   interface(CONST_INTER);
5789 %}
5790 
5791 // Long Immediate: low 32-bit mask
5792 operand immL_32bits()
5793 %{
5794   predicate(n->get_long() == 0xFFFFFFFFL);
5795   match(ConL);
5796   op_cost(0);
5797   format %{ %}
5798   interface(CONST_INTER);
5799 %}
5800 
5801 // Pointer operands
5802 // Pointer Immediate
5803 operand immP()
5804 %{
5805   match(ConP);
5806 
5807   op_cost(0);
5808   format %{ %}
5809   interface(CONST_INTER);
5810 %}
5811 
5812 // NULL Pointer Immediate
5813 operand immP0()
5814 %{
5815   predicate(n->get_ptr() == 0);
5816   match(ConP);
5817 
5818   op_cost(0);
5819   format %{ %}
5820   interface(CONST_INTER);
5821 %}
5822 
5823 // Pointer Immediate One
5824 // this is used in object initialization (initial object header)
5825 operand immP_1()
5826 %{
5827   predicate(n->get_ptr() == 1);
5828   match(ConP);
5829 
5830   op_cost(0);
5831   format %{ %}
5832   interface(CONST_INTER);
5833 %}
5834 
5835 // Polling Page Pointer Immediate
5836 operand immPollPage()
5837 %{
5838   predicate((address)n->get_ptr() == os::get_polling_page());
5839   match(ConP);
5840 
5841   op_cost(0);
5842   format %{ %}
5843   interface(CONST_INTER);
5844 %}
5845 
5846 // Card Table Byte Map Base
5847 operand immByteMapBase()
5848 %{
5849   // Get base of card map
5850   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
5851             (jbyte*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
5852   match(ConP);
5853 
5854   op_cost(0);
5855   format %{ %}
5856   interface(CONST_INTER);
5857 %}
5858 
5859 // Pointer Immediate Minus One
5860 // this is used when we want to write the current PC to the thread anchor
5861 operand immP_M1()
5862 %{
5863   predicate(n->get_ptr() == -1);
5864   match(ConP);
5865 
5866   op_cost(0);
5867   format %{ %}
5868   interface(CONST_INTER);
5869 %}
5870 
5871 // Pointer Immediate Minus Two
5872 // this is used when we want to write the current PC to the thread anchor
5873 operand immP_M2()
5874 %{
5875   predicate(n->get_ptr() == -2);
5876   match(ConP);
5877 
5878   op_cost(0);
5879   format %{ %}
5880   interface(CONST_INTER);
5881 %}
5882 
5883 // Float and Double operands
5884 // Double Immediate
5885 operand immD()
5886 %{
5887   match(ConD);
5888   op_cost(0);
5889   format %{ %}
5890   interface(CONST_INTER);
5891 %}
5892 
5893 // Double Immediate: +0.0d
5894 operand immD0()
5895 %{
5896   predicate(jlong_cast(n->getd()) == 0);
5897   match(ConD);
5898 
5899   op_cost(0);
5900   format %{ %}
5901   interface(CONST_INTER);
5902 %}
5903 
5904 // constant 'double +0.0'.
5905 operand immDPacked()
5906 %{
5907   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
5908   match(ConD);
5909   op_cost(0);
5910   format %{ %}
5911   interface(CONST_INTER);
5912 %}
5913 
5914 // Float Immediate
5915 operand immF()
5916 %{
5917   match(ConF);
5918   op_cost(0);
5919   format %{ %}
5920   interface(CONST_INTER);
5921 %}
5922 
5923 // Float Immediate: +0.0f.
5924 operand immF0()
5925 %{
5926   predicate(jint_cast(n->getf()) == 0);
5927   match(ConF);
5928 
5929   op_cost(0);
5930   format %{ %}
5931   interface(CONST_INTER);
5932 %}
5933 
5934 //
5935 operand immFPacked()
5936 %{
5937   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
5938   match(ConF);
5939   op_cost(0);
5940   format %{ %}
5941   interface(CONST_INTER);
5942 %}
5943 
5944 // Narrow pointer operands
5945 // Narrow Pointer Immediate
5946 operand immN()
5947 %{
5948   match(ConN);
5949 
5950   op_cost(0);
5951   format %{ %}
5952   interface(CONST_INTER);
5953 %}
5954 
5955 // Narrow NULL Pointer Immediate
5956 operand immN0()
5957 %{
5958   predicate(n->get_narrowcon() == 0);
5959   match(ConN);
5960 
5961   op_cost(0);
5962   format %{ %}
5963   interface(CONST_INTER);
5964 %}
5965 
5966 operand immNKlass()
5967 %{
5968   match(ConNKlass);
5969 
5970   op_cost(0);
5971   format %{ %}
5972   interface(CONST_INTER);
5973 %}
5974 
5975 // Integer 32 bit Register Operands
5976 // Integer 32 bitRegister (excludes SP)
5977 operand iRegI()
5978 %{
5979   constraint(ALLOC_IN_RC(any_reg32));
5980   match(RegI);
5981   match(iRegINoSp);
5982   op_cost(0);
5983   format %{ %}
5984   interface(REG_INTER);
5985 %}
5986 
5987 // Integer 32 bit Register not Special
5988 operand iRegINoSp()
5989 %{
5990   constraint(ALLOC_IN_RC(no_special_reg32));
5991   match(RegI);
5992   op_cost(0);
5993   format %{ %}
5994   interface(REG_INTER);
5995 %}
5996 
5997 // Integer 64 bit Register Operands
5998 // Integer 64 bit Register (includes SP)
5999 operand iRegL()
6000 %{
6001   constraint(ALLOC_IN_RC(any_reg));
6002   match(RegL);
6003   match(iRegLNoSp);
6004   op_cost(0);
6005   format %{ %}
6006   interface(REG_INTER);
6007 %}
6008 
6009 // Integer 64 bit Register not Special
6010 operand iRegLNoSp()
6011 %{
6012   constraint(ALLOC_IN_RC(no_special_reg));
6013   match(RegL);
6014   match(iRegL_R0);
6015   format %{ %}
6016   interface(REG_INTER);
6017 %}
6018 
6019 // Pointer Register Operands
6020 // Pointer Register
6021 operand iRegP()
6022 %{
6023   constraint(ALLOC_IN_RC(ptr_reg));
6024   match(RegP);
6025   match(iRegPNoSp);
6026   match(iRegP_R0);
6027   //match(iRegP_R2);
6028   //match(iRegP_R4);
6029   //match(iRegP_R5);
6030   match(thread_RegP);
6031   op_cost(0);
6032   format %{ %}
6033   interface(REG_INTER);
6034 %}
6035 
6036 // Pointer 64 bit Register not Special
6037 operand iRegPNoSp()
6038 %{
6039   constraint(ALLOC_IN_RC(no_special_ptr_reg));
6040   match(RegP);
6041   // match(iRegP);
6042   // match(iRegP_R0);
6043   // match(iRegP_R2);
6044   // match(iRegP_R4);
6045   // match(iRegP_R5);
6046   // match(thread_RegP);
6047   op_cost(0);
6048   format %{ %}
6049   interface(REG_INTER);
6050 %}
6051 
6052 // Pointer 64 bit Register R0 only
6053 operand iRegP_R0()
6054 %{
6055   constraint(ALLOC_IN_RC(r0_reg));
6056   match(RegP);
6057   // match(iRegP);
6058   match(iRegPNoSp);
6059   op_cost(0);
6060   format %{ %}
6061   interface(REG_INTER);
6062 %}
6063 
6064 // Pointer 64 bit Register R1 only
6065 operand iRegP_R1()
6066 %{
6067   constraint(ALLOC_IN_RC(r1_reg));
6068   match(RegP);
6069   // match(iRegP);
6070   match(iRegPNoSp);
6071   op_cost(0);
6072   format %{ %}
6073   interface(REG_INTER);
6074 %}
6075 
6076 // Pointer 64 bit Register R2 only
6077 operand iRegP_R2()
6078 %{
6079   constraint(ALLOC_IN_RC(r2_reg));
6080   match(RegP);
6081   // match(iRegP);
6082   match(iRegPNoSp);
6083   op_cost(0);
6084   format %{ %}
6085   interface(REG_INTER);
6086 %}
6087 
6088 // Pointer 64 bit Register R3 only
6089 operand iRegP_R3()
6090 %{
6091   constraint(ALLOC_IN_RC(r3_reg));
6092   match(RegP);
6093   // match(iRegP);
6094   match(iRegPNoSp);
6095   op_cost(0);
6096   format %{ %}
6097   interface(REG_INTER);
6098 %}
6099 
6100 // Pointer 64 bit Register R4 only
6101 operand iRegP_R4()
6102 %{
6103   constraint(ALLOC_IN_RC(r4_reg));
6104   match(RegP);
6105   // match(iRegP);
6106   match(iRegPNoSp);
6107   op_cost(0);
6108   format %{ %}
6109   interface(REG_INTER);
6110 %}
6111 
6112 // Pointer 64 bit Register R5 only
6113 operand iRegP_R5()
6114 %{
6115   constraint(ALLOC_IN_RC(r5_reg));
6116   match(RegP);
6117   // match(iRegP);
6118   match(iRegPNoSp);
6119   op_cost(0);
6120   format %{ %}
6121   interface(REG_INTER);
6122 %}
6123 
6124 // Pointer 64 bit Register R10 only
6125 operand iRegP_R10()
6126 %{
6127   constraint(ALLOC_IN_RC(r10_reg));
6128   match(RegP);
6129   // match(iRegP);
6130   match(iRegPNoSp);
6131   op_cost(0);
6132   format %{ %}
6133   interface(REG_INTER);
6134 %}
6135 
6136 // Long 64 bit Register R0 only
6137 operand iRegL_R0()
6138 %{
6139   constraint(ALLOC_IN_RC(r0_reg));
6140   match(RegL);
6141   match(iRegLNoSp);
6142   op_cost(0);
6143   format %{ %}
6144   interface(REG_INTER);
6145 %}
6146 
6147 // Long 64 bit Register R2 only
6148 operand iRegL_R2()
6149 %{
6150   constraint(ALLOC_IN_RC(r2_reg));
6151   match(RegL);
6152   match(iRegLNoSp);
6153   op_cost(0);
6154   format %{ %}
6155   interface(REG_INTER);
6156 %}
6157 
6158 // Long 64 bit Register R3 only
6159 operand iRegL_R3()
6160 %{
6161   constraint(ALLOC_IN_RC(r3_reg));
6162   match(RegL);
6163   match(iRegLNoSp);
6164   op_cost(0);
6165   format %{ %}
6166   interface(REG_INTER);
6167 %}
6168 
6169 // Long 64 bit Register R11 only
6170 operand iRegL_R11()
6171 %{
6172   constraint(ALLOC_IN_RC(r11_reg));
6173   match(RegL);
6174   match(iRegLNoSp);
6175   op_cost(0);
6176   format %{ %}
6177   interface(REG_INTER);
6178 %}
6179 
6180 // Pointer 64 bit Register FP only
6181 operand iRegP_FP()
6182 %{
6183   constraint(ALLOC_IN_RC(fp_reg));
6184   match(RegP);
6185   // match(iRegP);
6186   op_cost(0);
6187   format %{ %}
6188   interface(REG_INTER);
6189 %}
6190 
6191 // Register R0 only
6192 operand iRegI_R0()
6193 %{
6194   constraint(ALLOC_IN_RC(int_r0_reg));
6195   match(RegI);
6196   match(iRegINoSp);
6197   op_cost(0);
6198   format %{ %}
6199   interface(REG_INTER);
6200 %}
6201 
6202 // Register R2 only
6203 operand iRegI_R2()
6204 %{
6205   constraint(ALLOC_IN_RC(int_r2_reg));
6206   match(RegI);
6207   match(iRegINoSp);
6208   op_cost(0);
6209   format %{ %}
6210   interface(REG_INTER);
6211 %}
6212 
6213 // Register R3 only
6214 operand iRegI_R3()
6215 %{
6216   constraint(ALLOC_IN_RC(int_r3_reg));
6217   match(RegI);
6218   match(iRegINoSp);
6219   op_cost(0);
6220   format %{ %}
6221   interface(REG_INTER);
6222 %}
6223 
6224 
6225 // Register R4 only
6226 operand iRegI_R4()
6227 %{
6228   constraint(ALLOC_IN_RC(int_r4_reg));
6229   match(RegI);
6230   match(iRegINoSp);
6231   op_cost(0);
6232   format %{ %}
6233   interface(REG_INTER);
6234 %}
6235 
6236 
6237 // Pointer Register Operands
6238 // Narrow Pointer Register
6239 operand iRegN()
6240 %{
6241   constraint(ALLOC_IN_RC(any_reg32));
6242   match(RegN);
6243   match(iRegNNoSp);
6244   op_cost(0);
6245   format %{ %}
6246   interface(REG_INTER);
6247 %}
6248 
6249 operand iRegN_R0()
6250 %{
6251   constraint(ALLOC_IN_RC(r0_reg));
6252   match(iRegN);
6253   op_cost(0);
6254   format %{ %}
6255   interface(REG_INTER);
6256 %}
6257 
6258 operand iRegN_R2()
6259 %{
6260   constraint(ALLOC_IN_RC(r2_reg));
6261   match(iRegN);
6262   op_cost(0);
6263   format %{ %}
6264   interface(REG_INTER);
6265 %}
6266 
6267 operand iRegN_R3()
6268 %{
6269   constraint(ALLOC_IN_RC(r3_reg));
6270   match(iRegN);
6271   op_cost(0);
6272   format %{ %}
6273   interface(REG_INTER);
6274 %}
6275 
6276 // Integer 64 bit Register not Special
6277 operand iRegNNoSp()
6278 %{
6279   constraint(ALLOC_IN_RC(no_special_reg32));
6280   match(RegN);
6281   op_cost(0);
6282   format %{ %}
6283   interface(REG_INTER);
6284 %}
6285 
6286 // heap base register -- used for encoding immN0
6287 
6288 operand iRegIHeapbase()
6289 %{
6290   constraint(ALLOC_IN_RC(heapbase_reg));
6291   match(RegI);
6292   op_cost(0);
6293   format %{ %}
6294   interface(REG_INTER);
6295 %}
6296 
6297 // Float Register
6298 // Float register operands
6299 operand vRegF()
6300 %{
6301   constraint(ALLOC_IN_RC(float_reg));
6302   match(RegF);
6303 
6304   op_cost(0);
6305   format %{ %}
6306   interface(REG_INTER);
6307 %}
6308 
6309 // Double Register
6310 // Double register operands
6311 operand vRegD()
6312 %{
6313   constraint(ALLOC_IN_RC(double_reg));
6314   match(RegD);
6315 
6316   op_cost(0);
6317   format %{ %}
6318   interface(REG_INTER);
6319 %}
6320 
6321 operand vecD()
6322 %{
6323   constraint(ALLOC_IN_RC(vectord_reg));
6324   match(VecD);
6325 
6326   op_cost(0);
6327   format %{ %}
6328   interface(REG_INTER);
6329 %}
6330 
6331 operand vecX()
6332 %{
6333   constraint(ALLOC_IN_RC(vectorx_reg));
6334   match(VecX);
6335 
6336   op_cost(0);
6337   format %{ %}
6338   interface(REG_INTER);
6339 %}
6340 
6341 operand vRegD_V0()
6342 %{
6343   constraint(ALLOC_IN_RC(v0_reg));
6344   match(RegD);
6345   op_cost(0);
6346   format %{ %}
6347   interface(REG_INTER);
6348 %}
6349 
6350 operand vRegD_V1()
6351 %{
6352   constraint(ALLOC_IN_RC(v1_reg));
6353   match(RegD);
6354   op_cost(0);
6355   format %{ %}
6356   interface(REG_INTER);
6357 %}
6358 
6359 operand vRegD_V2()
6360 %{
6361   constraint(ALLOC_IN_RC(v2_reg));
6362   match(RegD);
6363   op_cost(0);
6364   format %{ %}
6365   interface(REG_INTER);
6366 %}
6367 
6368 operand vRegD_V3()
6369 %{
6370   constraint(ALLOC_IN_RC(v3_reg));
6371   match(RegD);
6372   op_cost(0);
6373   format %{ %}
6374   interface(REG_INTER);
6375 %}
6376 
6377 // Flags register, used as output of signed compare instructions
6378 
6379 // note that on AArch64 we also use this register as the output for
6380 // for floating point compare instructions (CmpF CmpD). this ensures
6381 // that ordered inequality tests use GT, GE, LT or LE none of which
6382 // pass through cases where the result is unordered i.e. one or both
6383 // inputs to the compare is a NaN. this means that the ideal code can
6384 // replace e.g. a GT with an LE and not end up capturing the NaN case
6385 // (where the comparison should always fail). EQ and NE tests are
6386 // always generated in ideal code so that unordered folds into the NE
6387 // case, matching the behaviour of AArch64 NE.
6388 //
6389 // This differs from x86 where the outputs of FP compares use a
6390 // special FP flags registers and where compares based on this
6391 // register are distinguished into ordered inequalities (cmpOpUCF) and
6392 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
6393 // to explicitly handle the unordered case in branches. x86 also has
6394 // to include extra CMoveX rules to accept a cmpOpUCF input.
6395 
6396 operand rFlagsReg()
6397 %{
6398   constraint(ALLOC_IN_RC(int_flags));
6399   match(RegFlags);
6400 
6401   op_cost(0);
6402   format %{ "RFLAGS" %}
6403   interface(REG_INTER);
6404 %}
6405 
6406 // Flags register, used as output of unsigned compare instructions
6407 operand rFlagsRegU()
6408 %{
6409   constraint(ALLOC_IN_RC(int_flags));
6410   match(RegFlags);
6411 
6412   op_cost(0);
6413   format %{ "RFLAGSU" %}
6414   interface(REG_INTER);
6415 %}
6416 
6417 // Special Registers
6418 
6419 // Method Register
6420 operand inline_cache_RegP(iRegP reg)
6421 %{
6422   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
6423   match(reg);
6424   match(iRegPNoSp);
6425   op_cost(0);
6426   format %{ %}
6427   interface(REG_INTER);
6428 %}
6429 
6430 operand interpreter_method_oop_RegP(iRegP reg)
6431 %{
6432   constraint(ALLOC_IN_RC(method_reg)); // interpreter_method_oop_reg
6433   match(reg);
6434   match(iRegPNoSp);
6435   op_cost(0);
6436   format %{ %}
6437   interface(REG_INTER);
6438 %}
6439 
6440 // Thread Register
6441 operand thread_RegP(iRegP reg)
6442 %{
6443   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
6444   match(reg);
6445   op_cost(0);
6446   format %{ %}
6447   interface(REG_INTER);
6448 %}
6449 
6450 operand lr_RegP(iRegP reg)
6451 %{
6452   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
6453   match(reg);
6454   op_cost(0);
6455   format %{ %}
6456   interface(REG_INTER);
6457 %}
6458 
6459 //----------Memory Operands----------------------------------------------------
6460 
6461 operand indirect(iRegP reg)
6462 %{
6463   constraint(ALLOC_IN_RC(ptr_reg));
6464   match(reg);
6465   op_cost(0);
6466   format %{ "[$reg]" %}
6467   interface(MEMORY_INTER) %{
6468     base($reg);
6469     index(0xffffffff);
6470     scale(0x0);
6471     disp(0x0);
6472   %}
6473 %}
6474 
6475 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
6476 %{
6477   constraint(ALLOC_IN_RC(ptr_reg));
6478   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6479   match(AddP reg (LShiftL (ConvI2L ireg) scale));
6480   op_cost(0);
6481   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
6482   interface(MEMORY_INTER) %{
6483     base($reg);
6484     index($ireg);
6485     scale($scale);
6486     disp(0x0);
6487   %}
6488 %}
6489 
6490 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
6491 %{
6492   constraint(ALLOC_IN_RC(ptr_reg));
6493   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6494   match(AddP reg (LShiftL lreg scale));
6495   op_cost(0);
6496   format %{ "$reg, $lreg lsl($scale)" %}
6497   interface(MEMORY_INTER) %{
6498     base($reg);
6499     index($lreg);
6500     scale($scale);
6501     disp(0x0);
6502   %}
6503 %}
6504 
6505 operand indIndexI2L(iRegP reg, iRegI ireg)
6506 %{
6507   constraint(ALLOC_IN_RC(ptr_reg));
6508   match(AddP reg (ConvI2L ireg));
6509   op_cost(0);
6510   format %{ "$reg, $ireg, 0, I2L" %}
6511   interface(MEMORY_INTER) %{
6512     base($reg);
6513     index($ireg);
6514     scale(0x0);
6515     disp(0x0);
6516   %}
6517 %}
6518 
6519 operand indIndex(iRegP reg, iRegL lreg)
6520 %{
6521   constraint(ALLOC_IN_RC(ptr_reg));
6522   match(AddP reg lreg);
6523   op_cost(0);
6524   format %{ "$reg, $lreg" %}
6525   interface(MEMORY_INTER) %{
6526     base($reg);
6527     index($lreg);
6528     scale(0x0);
6529     disp(0x0);
6530   %}
6531 %}
6532 
6533 operand indOffI(iRegP reg, immIOffset off)
6534 %{
6535   constraint(ALLOC_IN_RC(ptr_reg));
6536   match(AddP reg off);
6537   op_cost(0);
6538   format %{ "[$reg, $off]" %}
6539   interface(MEMORY_INTER) %{
6540     base($reg);
6541     index(0xffffffff);
6542     scale(0x0);
6543     disp($off);
6544   %}
6545 %}
6546 
6547 operand indOffI4(iRegP reg, immIOffset4 off)
6548 %{
6549   constraint(ALLOC_IN_RC(ptr_reg));
6550   match(AddP reg off);
6551   op_cost(0);
6552   format %{ "[$reg, $off]" %}
6553   interface(MEMORY_INTER) %{
6554     base($reg);
6555     index(0xffffffff);
6556     scale(0x0);
6557     disp($off);
6558   %}
6559 %}
6560 
6561 operand indOffI8(iRegP reg, immIOffset8 off)
6562 %{
6563   constraint(ALLOC_IN_RC(ptr_reg));
6564   match(AddP reg off);
6565   op_cost(0);
6566   format %{ "[$reg, $off]" %}
6567   interface(MEMORY_INTER) %{
6568     base($reg);
6569     index(0xffffffff);
6570     scale(0x0);
6571     disp($off);
6572   %}
6573 %}
6574 
6575 operand indOffI16(iRegP reg, immIOffset16 off)
6576 %{
6577   constraint(ALLOC_IN_RC(ptr_reg));
6578   match(AddP reg off);
6579   op_cost(0);
6580   format %{ "[$reg, $off]" %}
6581   interface(MEMORY_INTER) %{
6582     base($reg);
6583     index(0xffffffff);
6584     scale(0x0);
6585     disp($off);
6586   %}
6587 %}
6588 
6589 operand indOffL(iRegP reg, immLoffset off)
6590 %{
6591   constraint(ALLOC_IN_RC(ptr_reg));
6592   match(AddP reg off);
6593   op_cost(0);
6594   format %{ "[$reg, $off]" %}
6595   interface(MEMORY_INTER) %{
6596     base($reg);
6597     index(0xffffffff);
6598     scale(0x0);
6599     disp($off);
6600   %}
6601 %}
6602 
6603 operand indOffL4(iRegP reg, immLoffset4 off)
6604 %{
6605   constraint(ALLOC_IN_RC(ptr_reg));
6606   match(AddP reg off);
6607   op_cost(0);
6608   format %{ "[$reg, $off]" %}
6609   interface(MEMORY_INTER) %{
6610     base($reg);
6611     index(0xffffffff);
6612     scale(0x0);
6613     disp($off);
6614   %}
6615 %}
6616 
6617 operand indOffL8(iRegP reg, immLoffset8 off)
6618 %{
6619   constraint(ALLOC_IN_RC(ptr_reg));
6620   match(AddP reg off);
6621   op_cost(0);
6622   format %{ "[$reg, $off]" %}
6623   interface(MEMORY_INTER) %{
6624     base($reg);
6625     index(0xffffffff);
6626     scale(0x0);
6627     disp($off);
6628   %}
6629 %}
6630 
6631 operand indOffL16(iRegP reg, immLoffset16 off)
6632 %{
6633   constraint(ALLOC_IN_RC(ptr_reg));
6634   match(AddP reg off);
6635   op_cost(0);
6636   format %{ "[$reg, $off]" %}
6637   interface(MEMORY_INTER) %{
6638     base($reg);
6639     index(0xffffffff);
6640     scale(0x0);
6641     disp($off);
6642   %}
6643 %}
6644 
6645 operand indirectN(iRegN reg)
6646 %{
6647   predicate(Universe::narrow_oop_shift() == 0);
6648   constraint(ALLOC_IN_RC(ptr_reg));
6649   match(DecodeN reg);
6650   op_cost(0);
6651   format %{ "[$reg]\t# narrow" %}
6652   interface(MEMORY_INTER) %{
6653     base($reg);
6654     index(0xffffffff);
6655     scale(0x0);
6656     disp(0x0);
6657   %}
6658 %}
6659 
6660 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
6661 %{
6662   predicate(Universe::narrow_oop_shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6663   constraint(ALLOC_IN_RC(ptr_reg));
6664   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
6665   op_cost(0);
6666   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
6667   interface(MEMORY_INTER) %{
6668     base($reg);
6669     index($ireg);
6670     scale($scale);
6671     disp(0x0);
6672   %}
6673 %}
6674 
6675 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
6676 %{
6677   predicate(Universe::narrow_oop_shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6678   constraint(ALLOC_IN_RC(ptr_reg));
6679   match(AddP (DecodeN reg) (LShiftL lreg scale));
6680   op_cost(0);
6681   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
6682   interface(MEMORY_INTER) %{
6683     base($reg);
6684     index($lreg);
6685     scale($scale);
6686     disp(0x0);
6687   %}
6688 %}
6689 
6690 operand indIndexI2LN(iRegN reg, iRegI ireg)
6691 %{
6692   predicate(Universe::narrow_oop_shift() == 0);
6693   constraint(ALLOC_IN_RC(ptr_reg));
6694   match(AddP (DecodeN reg) (ConvI2L ireg));
6695   op_cost(0);
6696   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
6697   interface(MEMORY_INTER) %{
6698     base($reg);
6699     index($ireg);
6700     scale(0x0);
6701     disp(0x0);
6702   %}
6703 %}
6704 
6705 operand indIndexN(iRegN reg, iRegL lreg)
6706 %{
6707   predicate(Universe::narrow_oop_shift() == 0);
6708   constraint(ALLOC_IN_RC(ptr_reg));
6709   match(AddP (DecodeN reg) lreg);
6710   op_cost(0);
6711   format %{ "$reg, $lreg\t# narrow" %}
6712   interface(MEMORY_INTER) %{
6713     base($reg);
6714     index($lreg);
6715     scale(0x0);
6716     disp(0x0);
6717   %}
6718 %}
6719 
6720 operand indOffIN(iRegN reg, immIOffset off)
6721 %{
6722   predicate(Universe::narrow_oop_shift() == 0);
6723   constraint(ALLOC_IN_RC(ptr_reg));
6724   match(AddP (DecodeN reg) off);
6725   op_cost(0);
6726   format %{ "[$reg, $off]\t# narrow" %}
6727   interface(MEMORY_INTER) %{
6728     base($reg);
6729     index(0xffffffff);
6730     scale(0x0);
6731     disp($off);
6732   %}
6733 %}
6734 
6735 operand indOffLN(iRegN reg, immLoffset off)
6736 %{
6737   predicate(Universe::narrow_oop_shift() == 0);
6738   constraint(ALLOC_IN_RC(ptr_reg));
6739   match(AddP (DecodeN reg) off);
6740   op_cost(0);
6741   format %{ "[$reg, $off]\t# narrow" %}
6742   interface(MEMORY_INTER) %{
6743     base($reg);
6744     index(0xffffffff);
6745     scale(0x0);
6746     disp($off);
6747   %}
6748 %}
6749 
6750 
6751 
6752 // AArch64 opto stubs need to write to the pc slot in the thread anchor
6753 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
6754 %{
6755   constraint(ALLOC_IN_RC(ptr_reg));
6756   match(AddP reg off);
6757   op_cost(0);
6758   format %{ "[$reg, $off]" %}
6759   interface(MEMORY_INTER) %{
6760     base($reg);
6761     index(0xffffffff);
6762     scale(0x0);
6763     disp($off);
6764   %}
6765 %}
6766 
6767 //----------Special Memory Operands--------------------------------------------
6768 // Stack Slot Operand - This operand is used for loading and storing temporary
6769 //                      values on the stack where a match requires a value to
6770 //                      flow through memory.
6771 operand stackSlotP(sRegP reg)
6772 %{
6773   constraint(ALLOC_IN_RC(stack_slots));
6774   op_cost(100);
6775   // No match rule because this operand is only generated in matching
6776   // match(RegP);
6777   format %{ "[$reg]" %}
6778   interface(MEMORY_INTER) %{
6779     base(0x1e);  // RSP
6780     index(0x0);  // No Index
6781     scale(0x0);  // No Scale
6782     disp($reg);  // Stack Offset
6783   %}
6784 %}
6785 
6786 operand stackSlotI(sRegI reg)
6787 %{
6788   constraint(ALLOC_IN_RC(stack_slots));
6789   // No match rule because this operand is only generated in matching
6790   // match(RegI);
6791   format %{ "[$reg]" %}
6792   interface(MEMORY_INTER) %{
6793     base(0x1e);  // RSP
6794     index(0x0);  // No Index
6795     scale(0x0);  // No Scale
6796     disp($reg);  // Stack Offset
6797   %}
6798 %}
6799 
6800 operand stackSlotF(sRegF reg)
6801 %{
6802   constraint(ALLOC_IN_RC(stack_slots));
6803   // No match rule because this operand is only generated in matching
6804   // match(RegF);
6805   format %{ "[$reg]" %}
6806   interface(MEMORY_INTER) %{
6807     base(0x1e);  // RSP
6808     index(0x0);  // No Index
6809     scale(0x0);  // No Scale
6810     disp($reg);  // Stack Offset
6811   %}
6812 %}
6813 
6814 operand stackSlotD(sRegD reg)
6815 %{
6816   constraint(ALLOC_IN_RC(stack_slots));
6817   // No match rule because this operand is only generated in matching
6818   // match(RegD);
6819   format %{ "[$reg]" %}
6820   interface(MEMORY_INTER) %{
6821     base(0x1e);  // RSP
6822     index(0x0);  // No Index
6823     scale(0x0);  // No Scale
6824     disp($reg);  // Stack Offset
6825   %}
6826 %}
6827 
6828 operand stackSlotL(sRegL reg)
6829 %{
6830   constraint(ALLOC_IN_RC(stack_slots));
6831   // No match rule because this operand is only generated in matching
6832   // match(RegL);
6833   format %{ "[$reg]" %}
6834   interface(MEMORY_INTER) %{
6835     base(0x1e);  // RSP
6836     index(0x0);  // No Index
6837     scale(0x0);  // No Scale
6838     disp($reg);  // Stack Offset
6839   %}
6840 %}
6841 
6842 // Operands for expressing Control Flow
6843 // NOTE: Label is a predefined operand which should not be redefined in
6844 //       the AD file. It is generically handled within the ADLC.
6845 
6846 //----------Conditional Branch Operands----------------------------------------
6847 // Comparison Op  - This is the operation of the comparison, and is limited to
6848 //                  the following set of codes:
6849 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6850 //
6851 // Other attributes of the comparison, such as unsignedness, are specified
6852 // by the comparison instruction that sets a condition code flags register.
6853 // That result is represented by a flags operand whose subtype is appropriate
6854 // to the unsignedness (etc.) of the comparison.
6855 //
6856 // Later, the instruction which matches both the Comparison Op (a Bool) and
6857 // the flags (produced by the Cmp) specifies the coding of the comparison op
6858 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6859 
6860 // used for signed integral comparisons and fp comparisons
6861 
6862 operand cmpOp()
6863 %{
6864   match(Bool);
6865 
6866   format %{ "" %}
6867   interface(COND_INTER) %{
6868     equal(0x0, "eq");
6869     not_equal(0x1, "ne");
6870     less(0xb, "lt");
6871     greater_equal(0xa, "ge");
6872     less_equal(0xd, "le");
6873     greater(0xc, "gt");
6874     overflow(0x6, "vs");
6875     no_overflow(0x7, "vc");
6876   %}
6877 %}
6878 
6879 // used for unsigned integral comparisons
6880 
6881 operand cmpOpU()
6882 %{
6883   match(Bool);
6884 
6885   format %{ "" %}
6886   interface(COND_INTER) %{
6887     equal(0x0, "eq");
6888     not_equal(0x1, "ne");
6889     less(0x3, "lo");
6890     greater_equal(0x2, "hs");
6891     less_equal(0x9, "ls");
6892     greater(0x8, "hi");
6893     overflow(0x6, "vs");
6894     no_overflow(0x7, "vc");
6895   %}
6896 %}
6897 
6898 // used for certain integral comparisons which can be
6899 // converted to cbxx or tbxx instructions
6900 
6901 operand cmpOpEqNe()
6902 %{
6903   match(Bool);
6904   match(CmpOp);
6905   op_cost(0);
6906   predicate(n->as_Bool()->_test._test == BoolTest::ne
6907             || n->as_Bool()->_test._test == BoolTest::eq);
6908 
6909   format %{ "" %}
6910   interface(COND_INTER) %{
6911     equal(0x0, "eq");
6912     not_equal(0x1, "ne");
6913     less(0xb, "lt");
6914     greater_equal(0xa, "ge");
6915     less_equal(0xd, "le");
6916     greater(0xc, "gt");
6917     overflow(0x6, "vs");
6918     no_overflow(0x7, "vc");
6919   %}
6920 %}
6921 
6922 // used for certain integral comparisons which can be
6923 // converted to cbxx or tbxx instructions
6924 
6925 operand cmpOpLtGe()
6926 %{
6927   match(Bool);
6928   match(CmpOp);
6929   op_cost(0);
6930 
6931   predicate(n->as_Bool()->_test._test == BoolTest::lt
6932             || n->as_Bool()->_test._test == BoolTest::ge);
6933 
6934   format %{ "" %}
6935   interface(COND_INTER) %{
6936     equal(0x0, "eq");
6937     not_equal(0x1, "ne");
6938     less(0xb, "lt");
6939     greater_equal(0xa, "ge");
6940     less_equal(0xd, "le");
6941     greater(0xc, "gt");
6942     overflow(0x6, "vs");
6943     no_overflow(0x7, "vc");
6944   %}
6945 %}
6946 
6947 // used for certain unsigned integral comparisons which can be
6948 // converted to cbxx or tbxx instructions
6949 
6950 operand cmpOpUEqNeLtGe()
6951 %{
6952   match(Bool);
6953   match(CmpOp);
6954   op_cost(0);
6955 
6956   predicate(n->as_Bool()->_test._test == BoolTest::eq
6957             || n->as_Bool()->_test._test == BoolTest::ne
6958             || n->as_Bool()->_test._test == BoolTest::lt
6959             || n->as_Bool()->_test._test == BoolTest::ge);
6960 
6961   format %{ "" %}
6962   interface(COND_INTER) %{
6963     equal(0x0, "eq");
6964     not_equal(0x1, "ne");
6965     less(0xb, "lt");
6966     greater_equal(0xa, "ge");
6967     less_equal(0xd, "le");
6968     greater(0xc, "gt");
6969     overflow(0x6, "vs");
6970     no_overflow(0x7, "vc");
6971   %}
6972 %}
6973 
6974 // Special operand allowing long args to int ops to be truncated for free
6975 
6976 operand iRegL2I(iRegL reg) %{
6977 
6978   op_cost(0);
6979 
6980   match(ConvL2I reg);
6981 
6982   format %{ "l2i($reg)" %}
6983 
6984   interface(REG_INTER)
6985 %}
6986 
6987 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6988 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6989 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6990 
6991 //----------OPERAND CLASSES----------------------------------------------------
6992 // Operand Classes are groups of operands that are used as to simplify
6993 // instruction definitions by not requiring the AD writer to specify
6994 // separate instructions for every form of operand when the
6995 // instruction accepts multiple operand types with the same basic
6996 // encoding and format. The classic case of this is memory operands.
6997 
6998 // memory is used to define read/write location for load/store
6999 // instruction defs. we can turn a memory op into an Address
7000 
7001 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI, indOffL,
7002                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
7003 
7004 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
7005 // operations. it allows the src to be either an iRegI or a (ConvL2I
7006 // iRegL). in the latter case the l2i normally planted for a ConvL2I
7007 // can be elided because the 32-bit instruction will just employ the
7008 // lower 32 bits anyway.
7009 //
7010 // n.b. this does not elide all L2I conversions. if the truncated
7011 // value is consumed by more than one operation then the ConvL2I
7012 // cannot be bundled into the consuming nodes so an l2i gets planted
7013 // (actually a movw $dst $src) and the downstream instructions consume
7014 // the result of the l2i as an iRegI input. That's a shame since the
7015 // movw is actually redundant but its not too costly.
7016 
7017 opclass iRegIorL2I(iRegI, iRegL2I);
7018 
7019 //----------PIPELINE-----------------------------------------------------------
7020 // Rules which define the behavior of the target architectures pipeline.
7021 
7022 // For specific pipelines, eg A53, define the stages of that pipeline
7023 //pipe_desc(ISS, EX1, EX2, WR);
7024 #define ISS S0
7025 #define EX1 S1
7026 #define EX2 S2
7027 #define WR  S3
7028 
7029 // Integer ALU reg operation
7030 pipeline %{
7031 
7032 attributes %{
7033   // ARM instructions are of fixed length
7034   fixed_size_instructions;        // Fixed size instructions TODO does
7035   max_instructions_per_bundle = 2;   // A53 = 2, A57 = 4
7036   // ARM instructions come in 32-bit word units
7037   instruction_unit_size = 4;         // An instruction is 4 bytes long
7038   instruction_fetch_unit_size = 64;  // The processor fetches one line
7039   instruction_fetch_units = 1;       // of 64 bytes
7040 
7041   // List of nop instructions
7042   nops( MachNop );
7043 %}
7044 
7045 // We don't use an actual pipeline model so don't care about resources
7046 // or description. we do use pipeline classes to introduce fixed
7047 // latencies
7048 
7049 //----------RESOURCES----------------------------------------------------------
7050 // Resources are the functional units available to the machine
7051 
7052 resources( INS0, INS1, INS01 = INS0 | INS1,
7053            ALU0, ALU1, ALU = ALU0 | ALU1,
7054            MAC,
7055            DIV,
7056            BRANCH,
7057            LDST,
7058            NEON_FP);
7059 
7060 //----------PIPELINE DESCRIPTION-----------------------------------------------
7061 // Pipeline Description specifies the stages in the machine's pipeline
7062 
7063 // Define the pipeline as a generic 6 stage pipeline
7064 pipe_desc(S0, S1, S2, S3, S4, S5);
7065 
7066 //----------PIPELINE CLASSES---------------------------------------------------
7067 // Pipeline Classes describe the stages in which input and output are
7068 // referenced by the hardware pipeline.
7069 
7070 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
7071 %{
7072   single_instruction;
7073   src1   : S1(read);
7074   src2   : S2(read);
7075   dst    : S5(write);
7076   INS01  : ISS;
7077   NEON_FP : S5;
7078 %}
7079 
7080 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
7081 %{
7082   single_instruction;
7083   src1   : S1(read);
7084   src2   : S2(read);
7085   dst    : S5(write);
7086   INS01  : ISS;
7087   NEON_FP : S5;
7088 %}
7089 
7090 pipe_class fp_uop_s(vRegF dst, vRegF src)
7091 %{
7092   single_instruction;
7093   src    : S1(read);
7094   dst    : S5(write);
7095   INS01  : ISS;
7096   NEON_FP : S5;
7097 %}
7098 
7099 pipe_class fp_uop_d(vRegD dst, vRegD src)
7100 %{
7101   single_instruction;
7102   src    : S1(read);
7103   dst    : S5(write);
7104   INS01  : ISS;
7105   NEON_FP : S5;
7106 %}
7107 
7108 pipe_class fp_d2f(vRegF dst, vRegD src)
7109 %{
7110   single_instruction;
7111   src    : S1(read);
7112   dst    : S5(write);
7113   INS01  : ISS;
7114   NEON_FP : S5;
7115 %}
7116 
7117 pipe_class fp_f2d(vRegD dst, vRegF src)
7118 %{
7119   single_instruction;
7120   src    : S1(read);
7121   dst    : S5(write);
7122   INS01  : ISS;
7123   NEON_FP : S5;
7124 %}
7125 
7126 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
7127 %{
7128   single_instruction;
7129   src    : S1(read);
7130   dst    : S5(write);
7131   INS01  : ISS;
7132   NEON_FP : S5;
7133 %}
7134 
7135 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
7136 %{
7137   single_instruction;
7138   src    : S1(read);
7139   dst    : S5(write);
7140   INS01  : ISS;
7141   NEON_FP : S5;
7142 %}
7143 
7144 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
7145 %{
7146   single_instruction;
7147   src    : S1(read);
7148   dst    : S5(write);
7149   INS01  : ISS;
7150   NEON_FP : S5;
7151 %}
7152 
7153 pipe_class fp_l2f(vRegF dst, iRegL src)
7154 %{
7155   single_instruction;
7156   src    : S1(read);
7157   dst    : S5(write);
7158   INS01  : ISS;
7159   NEON_FP : S5;
7160 %}
7161 
7162 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
7163 %{
7164   single_instruction;
7165   src    : S1(read);
7166   dst    : S5(write);
7167   INS01  : ISS;
7168   NEON_FP : S5;
7169 %}
7170 
7171 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
7172 %{
7173   single_instruction;
7174   src    : S1(read);
7175   dst    : S5(write);
7176   INS01  : ISS;
7177   NEON_FP : S5;
7178 %}
7179 
7180 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
7181 %{
7182   single_instruction;
7183   src    : S1(read);
7184   dst    : S5(write);
7185   INS01  : ISS;
7186   NEON_FP : S5;
7187 %}
7188 
7189 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
7190 %{
7191   single_instruction;
7192   src    : S1(read);
7193   dst    : S5(write);
7194   INS01  : ISS;
7195   NEON_FP : S5;
7196 %}
7197 
7198 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
7199 %{
7200   single_instruction;
7201   src1   : S1(read);
7202   src2   : S2(read);
7203   dst    : S5(write);
7204   INS0   : ISS;
7205   NEON_FP : S5;
7206 %}
7207 
7208 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
7209 %{
7210   single_instruction;
7211   src1   : S1(read);
7212   src2   : S2(read);
7213   dst    : S5(write);
7214   INS0   : ISS;
7215   NEON_FP : S5;
7216 %}
7217 
7218 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
7219 %{
7220   single_instruction;
7221   cr     : S1(read);
7222   src1   : S1(read);
7223   src2   : S1(read);
7224   dst    : S3(write);
7225   INS01  : ISS;
7226   NEON_FP : S3;
7227 %}
7228 
7229 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
7230 %{
7231   single_instruction;
7232   cr     : S1(read);
7233   src1   : S1(read);
7234   src2   : S1(read);
7235   dst    : S3(write);
7236   INS01  : ISS;
7237   NEON_FP : S3;
7238 %}
7239 
7240 pipe_class fp_imm_s(vRegF dst)
7241 %{
7242   single_instruction;
7243   dst    : S3(write);
7244   INS01  : ISS;
7245   NEON_FP : S3;
7246 %}
7247 
7248 pipe_class fp_imm_d(vRegD dst)
7249 %{
7250   single_instruction;
7251   dst    : S3(write);
7252   INS01  : ISS;
7253   NEON_FP : S3;
7254 %}
7255 
7256 pipe_class fp_load_constant_s(vRegF dst)
7257 %{
7258   single_instruction;
7259   dst    : S4(write);
7260   INS01  : ISS;
7261   NEON_FP : S4;
7262 %}
7263 
7264 pipe_class fp_load_constant_d(vRegD dst)
7265 %{
7266   single_instruction;
7267   dst    : S4(write);
7268   INS01  : ISS;
7269   NEON_FP : S4;
7270 %}
7271 
7272 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
7273 %{
7274   single_instruction;
7275   dst    : S5(write);
7276   src1   : S1(read);
7277   src2   : S1(read);
7278   INS01  : ISS;
7279   NEON_FP : S5;
7280 %}
7281 
7282 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
7283 %{
7284   single_instruction;
7285   dst    : S5(write);
7286   src1   : S1(read);
7287   src2   : S1(read);
7288   INS0   : ISS;
7289   NEON_FP : S5;
7290 %}
7291 
7292 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
7293 %{
7294   single_instruction;
7295   dst    : S5(write);
7296   src1   : S1(read);
7297   src2   : S1(read);
7298   dst    : S1(read);
7299   INS01  : ISS;
7300   NEON_FP : S5;
7301 %}
7302 
7303 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
7304 %{
7305   single_instruction;
7306   dst    : S5(write);
7307   src1   : S1(read);
7308   src2   : S1(read);
7309   dst    : S1(read);
7310   INS0   : ISS;
7311   NEON_FP : S5;
7312 %}
7313 
7314 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
7315 %{
7316   single_instruction;
7317   dst    : S4(write);
7318   src1   : S2(read);
7319   src2   : S2(read);
7320   INS01  : ISS;
7321   NEON_FP : S4;
7322 %}
7323 
7324 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
7325 %{
7326   single_instruction;
7327   dst    : S4(write);
7328   src1   : S2(read);
7329   src2   : S2(read);
7330   INS0   : ISS;
7331   NEON_FP : S4;
7332 %}
7333 
7334 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
7335 %{
7336   single_instruction;
7337   dst    : S3(write);
7338   src1   : S2(read);
7339   src2   : S2(read);
7340   INS01  : ISS;
7341   NEON_FP : S3;
7342 %}
7343 
7344 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
7345 %{
7346   single_instruction;
7347   dst    : S3(write);
7348   src1   : S2(read);
7349   src2   : S2(read);
7350   INS0   : ISS;
7351   NEON_FP : S3;
7352 %}
7353 
7354 pipe_class vshift64(vecD dst, vecD src, vecX shift)
7355 %{
7356   single_instruction;
7357   dst    : S3(write);
7358   src    : S1(read);
7359   shift  : S1(read);
7360   INS01  : ISS;
7361   NEON_FP : S3;
7362 %}
7363 
7364 pipe_class vshift128(vecX dst, vecX src, vecX shift)
7365 %{
7366   single_instruction;
7367   dst    : S3(write);
7368   src    : S1(read);
7369   shift  : S1(read);
7370   INS0   : ISS;
7371   NEON_FP : S3;
7372 %}
7373 
7374 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
7375 %{
7376   single_instruction;
7377   dst    : S3(write);
7378   src    : S1(read);
7379   INS01  : ISS;
7380   NEON_FP : S3;
7381 %}
7382 
7383 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
7384 %{
7385   single_instruction;
7386   dst    : S3(write);
7387   src    : S1(read);
7388   INS0   : ISS;
7389   NEON_FP : S3;
7390 %}
7391 
7392 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
7393 %{
7394   single_instruction;
7395   dst    : S5(write);
7396   src1   : S1(read);
7397   src2   : S1(read);
7398   INS01  : ISS;
7399   NEON_FP : S5;
7400 %}
7401 
7402 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
7403 %{
7404   single_instruction;
7405   dst    : S5(write);
7406   src1   : S1(read);
7407   src2   : S1(read);
7408   INS0   : ISS;
7409   NEON_FP : S5;
7410 %}
7411 
7412 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
7413 %{
7414   single_instruction;
7415   dst    : S5(write);
7416   src1   : S1(read);
7417   src2   : S1(read);
7418   INS0   : ISS;
7419   NEON_FP : S5;
7420 %}
7421 
7422 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
7423 %{
7424   single_instruction;
7425   dst    : S5(write);
7426   src1   : S1(read);
7427   src2   : S1(read);
7428   INS0   : ISS;
7429   NEON_FP : S5;
7430 %}
7431 
7432 pipe_class vsqrt_fp128(vecX dst, vecX src)
7433 %{
7434   single_instruction;
7435   dst    : S5(write);
7436   src    : S1(read);
7437   INS0   : ISS;
7438   NEON_FP : S5;
7439 %}
7440 
7441 pipe_class vunop_fp64(vecD dst, vecD src)
7442 %{
7443   single_instruction;
7444   dst    : S5(write);
7445   src    : S1(read);
7446   INS01  : ISS;
7447   NEON_FP : S5;
7448 %}
7449 
7450 pipe_class vunop_fp128(vecX dst, vecX src)
7451 %{
7452   single_instruction;
7453   dst    : S5(write);
7454   src    : S1(read);
7455   INS0   : ISS;
7456   NEON_FP : S5;
7457 %}
7458 
7459 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
7460 %{
7461   single_instruction;
7462   dst    : S3(write);
7463   src    : S1(read);
7464   INS01  : ISS;
7465   NEON_FP : S3;
7466 %}
7467 
7468 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
7469 %{
7470   single_instruction;
7471   dst    : S3(write);
7472   src    : S1(read);
7473   INS01  : ISS;
7474   NEON_FP : S3;
7475 %}
7476 
7477 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
7478 %{
7479   single_instruction;
7480   dst    : S3(write);
7481   src    : S1(read);
7482   INS01  : ISS;
7483   NEON_FP : S3;
7484 %}
7485 
7486 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
7487 %{
7488   single_instruction;
7489   dst    : S3(write);
7490   src    : S1(read);
7491   INS01  : ISS;
7492   NEON_FP : S3;
7493 %}
7494 
7495 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
7496 %{
7497   single_instruction;
7498   dst    : S3(write);
7499   src    : S1(read);
7500   INS01  : ISS;
7501   NEON_FP : S3;
7502 %}
7503 
7504 pipe_class vmovi_reg_imm64(vecD dst)
7505 %{
7506   single_instruction;
7507   dst    : S3(write);
7508   INS01  : ISS;
7509   NEON_FP : S3;
7510 %}
7511 
7512 pipe_class vmovi_reg_imm128(vecX dst)
7513 %{
7514   single_instruction;
7515   dst    : S3(write);
7516   INS0   : ISS;
7517   NEON_FP : S3;
7518 %}
7519 
7520 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
7521 %{
7522   single_instruction;
7523   dst    : S5(write);
7524   mem    : ISS(read);
7525   INS01  : ISS;
7526   NEON_FP : S3;
7527 %}
7528 
7529 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
7530 %{
7531   single_instruction;
7532   dst    : S5(write);
7533   mem    : ISS(read);
7534   INS01  : ISS;
7535   NEON_FP : S3;
7536 %}
7537 
7538 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
7539 %{
7540   single_instruction;
7541   mem    : ISS(read);
7542   src    : S2(read);
7543   INS01  : ISS;
7544   NEON_FP : S3;
7545 %}
7546 
7547 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
7548 %{
7549   single_instruction;
7550   mem    : ISS(read);
7551   src    : S2(read);
7552   INS01  : ISS;
7553   NEON_FP : S3;
7554 %}
7555 
7556 //------- Integer ALU operations --------------------------
7557 
7558 // Integer ALU reg-reg operation
7559 // Operands needed in EX1, result generated in EX2
7560 // Eg.  ADD     x0, x1, x2
7561 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7562 %{
7563   single_instruction;
7564   dst    : EX2(write);
7565   src1   : EX1(read);
7566   src2   : EX1(read);
7567   INS01  : ISS; // Dual issue as instruction 0 or 1
7568   ALU    : EX2;
7569 %}
7570 
7571 // Integer ALU reg-reg operation with constant shift
7572 // Shifted register must be available in LATE_ISS instead of EX1
7573 // Eg.  ADD     x0, x1, x2, LSL #2
7574 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
7575 %{
7576   single_instruction;
7577   dst    : EX2(write);
7578   src1   : EX1(read);
7579   src2   : ISS(read);
7580   INS01  : ISS;
7581   ALU    : EX2;
7582 %}
7583 
7584 // Integer ALU reg operation with constant shift
7585 // Eg.  LSL     x0, x1, #shift
7586 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
7587 %{
7588   single_instruction;
7589   dst    : EX2(write);
7590   src1   : ISS(read);
7591   INS01  : ISS;
7592   ALU    : EX2;
7593 %}
7594 
7595 // Integer ALU reg-reg operation with variable shift
7596 // Both operands must be available in LATE_ISS instead of EX1
7597 // Result is available in EX1 instead of EX2
7598 // Eg.  LSLV    x0, x1, x2
7599 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
7600 %{
7601   single_instruction;
7602   dst    : EX1(write);
7603   src1   : ISS(read);
7604   src2   : ISS(read);
7605   INS01  : ISS;
7606   ALU    : EX1;
7607 %}
7608 
7609 // Integer ALU reg-reg operation with extract
7610 // As for _vshift above, but result generated in EX2
7611 // Eg.  EXTR    x0, x1, x2, #N
7612 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
7613 %{
7614   single_instruction;
7615   dst    : EX2(write);
7616   src1   : ISS(read);
7617   src2   : ISS(read);
7618   INS1   : ISS; // Can only dual issue as Instruction 1
7619   ALU    : EX1;
7620 %}
7621 
7622 // Integer ALU reg operation
7623 // Eg.  NEG     x0, x1
7624 pipe_class ialu_reg(iRegI dst, iRegI src)
7625 %{
7626   single_instruction;
7627   dst    : EX2(write);
7628   src    : EX1(read);
7629   INS01  : ISS;
7630   ALU    : EX2;
7631 %}
7632 
7633 // Integer ALU reg mmediate operation
7634 // Eg.  ADD     x0, x1, #N
7635 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
7636 %{
7637   single_instruction;
7638   dst    : EX2(write);
7639   src1   : EX1(read);
7640   INS01  : ISS;
7641   ALU    : EX2;
7642 %}
7643 
7644 // Integer ALU immediate operation (no source operands)
7645 // Eg.  MOV     x0, #N
7646 pipe_class ialu_imm(iRegI dst)
7647 %{
7648   single_instruction;
7649   dst    : EX1(write);
7650   INS01  : ISS;
7651   ALU    : EX1;
7652 %}
7653 
7654 //------- Compare operation -------------------------------
7655 
7656 // Compare reg-reg
7657 // Eg.  CMP     x0, x1
7658 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
7659 %{
7660   single_instruction;
7661 //  fixed_latency(16);
7662   cr     : EX2(write);
7663   op1    : EX1(read);
7664   op2    : EX1(read);
7665   INS01  : ISS;
7666   ALU    : EX2;
7667 %}
7668 
7669 // Compare reg-reg
7670 // Eg.  CMP     x0, #N
7671 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
7672 %{
7673   single_instruction;
7674 //  fixed_latency(16);
7675   cr     : EX2(write);
7676   op1    : EX1(read);
7677   INS01  : ISS;
7678   ALU    : EX2;
7679 %}
7680 
7681 //------- Conditional instructions ------------------------
7682 
7683 // Conditional no operands
7684 // Eg.  CSINC   x0, zr, zr, <cond>
7685 pipe_class icond_none(iRegI dst, rFlagsReg cr)
7686 %{
7687   single_instruction;
7688   cr     : EX1(read);
7689   dst    : EX2(write);
7690   INS01  : ISS;
7691   ALU    : EX2;
7692 %}
7693 
7694 // Conditional 2 operand
7695 // EG.  CSEL    X0, X1, X2, <cond>
7696 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
7697 %{
7698   single_instruction;
7699   cr     : EX1(read);
7700   src1   : EX1(read);
7701   src2   : EX1(read);
7702   dst    : EX2(write);
7703   INS01  : ISS;
7704   ALU    : EX2;
7705 %}
7706 
7707 // Conditional 2 operand
7708 // EG.  CSEL    X0, X1, X2, <cond>
7709 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
7710 %{
7711   single_instruction;
7712   cr     : EX1(read);
7713   src    : EX1(read);
7714   dst    : EX2(write);
7715   INS01  : ISS;
7716   ALU    : EX2;
7717 %}
7718 
7719 //------- Multiply pipeline operations --------------------
7720 
7721 // Multiply reg-reg
7722 // Eg.  MUL     w0, w1, w2
7723 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7724 %{
7725   single_instruction;
7726   dst    : WR(write);
7727   src1   : ISS(read);
7728   src2   : ISS(read);
7729   INS01  : ISS;
7730   MAC    : WR;
7731 %}
7732 
7733 // Multiply accumulate
7734 // Eg.  MADD    w0, w1, w2, w3
7735 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
7736 %{
7737   single_instruction;
7738   dst    : WR(write);
7739   src1   : ISS(read);
7740   src2   : ISS(read);
7741   src3   : ISS(read);
7742   INS01  : ISS;
7743   MAC    : WR;
7744 %}
7745 
7746 // Eg.  MUL     w0, w1, w2
7747 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7748 %{
7749   single_instruction;
7750   fixed_latency(3); // Maximum latency for 64 bit mul
7751   dst    : WR(write);
7752   src1   : ISS(read);
7753   src2   : ISS(read);
7754   INS01  : ISS;
7755   MAC    : WR;
7756 %}
7757 
7758 // Multiply accumulate
7759 // Eg.  MADD    w0, w1, w2, w3
7760 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
7761 %{
7762   single_instruction;
7763   fixed_latency(3); // Maximum latency for 64 bit mul
7764   dst    : WR(write);
7765   src1   : ISS(read);
7766   src2   : ISS(read);
7767   src3   : ISS(read);
7768   INS01  : ISS;
7769   MAC    : WR;
7770 %}
7771 
7772 //------- Divide pipeline operations --------------------
7773 
7774 // Eg.  SDIV    w0, w1, w2
7775 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7776 %{
7777   single_instruction;
7778   fixed_latency(8); // Maximum latency for 32 bit divide
7779   dst    : WR(write);
7780   src1   : ISS(read);
7781   src2   : ISS(read);
7782   INS0   : ISS; // Can only dual issue as instruction 0
7783   DIV    : WR;
7784 %}
7785 
7786 // Eg.  SDIV    x0, x1, x2
7787 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7788 %{
7789   single_instruction;
7790   fixed_latency(16); // Maximum latency for 64 bit divide
7791   dst    : WR(write);
7792   src1   : ISS(read);
7793   src2   : ISS(read);
7794   INS0   : ISS; // Can only dual issue as instruction 0
7795   DIV    : WR;
7796 %}
7797 
7798 //------- Load pipeline operations ------------------------
7799 
7800 // Load - prefetch
7801 // Eg.  PFRM    <mem>
7802 pipe_class iload_prefetch(memory mem)
7803 %{
7804   single_instruction;
7805   mem    : ISS(read);
7806   INS01  : ISS;
7807   LDST   : WR;
7808 %}
7809 
7810 // Load - reg, mem
7811 // Eg.  LDR     x0, <mem>
7812 pipe_class iload_reg_mem(iRegI dst, memory mem)
7813 %{
7814   single_instruction;
7815   dst    : WR(write);
7816   mem    : ISS(read);
7817   INS01  : ISS;
7818   LDST   : WR;
7819 %}
7820 
7821 // Load - reg, reg
7822 // Eg.  LDR     x0, [sp, x1]
7823 pipe_class iload_reg_reg(iRegI dst, iRegI src)
7824 %{
7825   single_instruction;
7826   dst    : WR(write);
7827   src    : ISS(read);
7828   INS01  : ISS;
7829   LDST   : WR;
7830 %}
7831 
7832 //------- Store pipeline operations -----------------------
7833 
7834 // Store - zr, mem
7835 // Eg.  STR     zr, <mem>
7836 pipe_class istore_mem(memory mem)
7837 %{
7838   single_instruction;
7839   mem    : ISS(read);
7840   INS01  : ISS;
7841   LDST   : WR;
7842 %}
7843 
7844 // Store - reg, mem
7845 // Eg.  STR     x0, <mem>
7846 pipe_class istore_reg_mem(iRegI src, memory mem)
7847 %{
7848   single_instruction;
7849   mem    : ISS(read);
7850   src    : EX2(read);
7851   INS01  : ISS;
7852   LDST   : WR;
7853 %}
7854 
7855 // Store - reg, reg
7856 // Eg. STR      x0, [sp, x1]
7857 pipe_class istore_reg_reg(iRegI dst, iRegI src)
7858 %{
7859   single_instruction;
7860   dst    : ISS(read);
7861   src    : EX2(read);
7862   INS01  : ISS;
7863   LDST   : WR;
7864 %}
7865 
7866 //------- Store pipeline operations -----------------------
7867 
7868 // Branch
7869 pipe_class pipe_branch()
7870 %{
7871   single_instruction;
7872   INS01  : ISS;
7873   BRANCH : EX1;
7874 %}
7875 
7876 // Conditional branch
7877 pipe_class pipe_branch_cond(rFlagsReg cr)
7878 %{
7879   single_instruction;
7880   cr     : EX1(read);
7881   INS01  : ISS;
7882   BRANCH : EX1;
7883 %}
7884 
7885 // Compare & Branch
7886 // EG.  CBZ/CBNZ
7887 pipe_class pipe_cmp_branch(iRegI op1)
7888 %{
7889   single_instruction;
7890   op1    : EX1(read);
7891   INS01  : ISS;
7892   BRANCH : EX1;
7893 %}
7894 
7895 //------- Synchronisation operations ----------------------
7896 
7897 // Any operation requiring serialization.
7898 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
7899 pipe_class pipe_serial()
7900 %{
7901   single_instruction;
7902   force_serialization;
7903   fixed_latency(16);
7904   INS01  : ISS(2); // Cannot dual issue with any other instruction
7905   LDST   : WR;
7906 %}
7907 
7908 // Generic big/slow expanded idiom - also serialized
7909 pipe_class pipe_slow()
7910 %{
7911   instruction_count(10);
7912   multiple_bundles;
7913   force_serialization;
7914   fixed_latency(16);
7915   INS01  : ISS(2); // Cannot dual issue with any other instruction
7916   LDST   : WR;
7917 %}
7918 
7919 // Empty pipeline class
7920 pipe_class pipe_class_empty()
7921 %{
7922   single_instruction;
7923   fixed_latency(0);
7924 %}
7925 
7926 // Default pipeline class.
7927 pipe_class pipe_class_default()
7928 %{
7929   single_instruction;
7930   fixed_latency(2);
7931 %}
7932 
7933 // Pipeline class for compares.
7934 pipe_class pipe_class_compare()
7935 %{
7936   single_instruction;
7937   fixed_latency(16);
7938 %}
7939 
7940 // Pipeline class for memory operations.
7941 pipe_class pipe_class_memory()
7942 %{
7943   single_instruction;
7944   fixed_latency(16);
7945 %}
7946 
7947 // Pipeline class for call.
7948 pipe_class pipe_class_call()
7949 %{
7950   single_instruction;
7951   fixed_latency(100);
7952 %}
7953 
7954 // Define the class for the Nop node.
7955 define %{
7956    MachNop = pipe_class_empty;
7957 %}
7958 
7959 %}
7960 //----------INSTRUCTIONS-------------------------------------------------------
7961 //
7962 // match      -- States which machine-independent subtree may be replaced
7963 //               by this instruction.
7964 // ins_cost   -- The estimated cost of this instruction is used by instruction
7965 //               selection to identify a minimum cost tree of machine
7966 //               instructions that matches a tree of machine-independent
7967 //               instructions.
7968 // format     -- A string providing the disassembly for this instruction.
7969 //               The value of an instruction's operand may be inserted
7970 //               by referring to it with a '$' prefix.
7971 // opcode     -- Three instruction opcodes may be provided.  These are referred
7972 //               to within an encode class as $primary, $secondary, and $tertiary
7973 //               rrspectively.  The primary opcode is commonly used to
7974 //               indicate the type of machine instruction, while secondary
7975 //               and tertiary are often used for prefix options or addressing
7976 //               modes.
7977 // ins_encode -- A list of encode classes with parameters. The encode class
7978 //               name must have been defined in an 'enc_class' specification
7979 //               in the encode section of the architecture description.
7980 
7981 // ============================================================================
7982 // Memory (Load/Store) Instructions
7983 
7984 // Load Instructions
7985 
7986 // Load Byte (8 bit signed)
7987 instruct loadB(iRegINoSp dst, memory mem)
7988 %{
7989   match(Set dst (LoadB mem));
7990   predicate(!needs_acquiring_load(n));
7991 
7992   ins_cost(4 * INSN_COST);
7993   format %{ "ldrsbw  $dst, $mem\t# byte" %}
7994 
7995   ins_encode(aarch64_enc_ldrsbw(dst, mem));
7996 
7997   ins_pipe(iload_reg_mem);
7998 %}
7999 
8000 // Load Byte (8 bit signed) into long
8001 instruct loadB2L(iRegLNoSp dst, memory mem)
8002 %{
8003   match(Set dst (ConvI2L (LoadB mem)));
8004   predicate(!needs_acquiring_load(n->in(1)));
8005 
8006   ins_cost(4 * INSN_COST);
8007   format %{ "ldrsb  $dst, $mem\t# byte" %}
8008 
8009   ins_encode(aarch64_enc_ldrsb(dst, mem));
8010 
8011   ins_pipe(iload_reg_mem);
8012 %}
8013 
8014 // Load Byte (8 bit unsigned)
8015 instruct loadUB(iRegINoSp dst, memory mem)
8016 %{
8017   match(Set dst (LoadUB mem));
8018   predicate(!needs_acquiring_load(n));
8019 
8020   ins_cost(4 * INSN_COST);
8021   format %{ "ldrbw  $dst, $mem\t# byte" %}
8022 
8023   ins_encode(aarch64_enc_ldrb(dst, mem));
8024 
8025   ins_pipe(iload_reg_mem);
8026 %}
8027 
8028 // Load Byte (8 bit unsigned) into long
8029 instruct loadUB2L(iRegLNoSp dst, memory mem)
8030 %{
8031   match(Set dst (ConvI2L (LoadUB mem)));
8032   predicate(!needs_acquiring_load(n->in(1)));
8033 
8034   ins_cost(4 * INSN_COST);
8035   format %{ "ldrb  $dst, $mem\t# byte" %}
8036 
8037   ins_encode(aarch64_enc_ldrb(dst, mem));
8038 
8039   ins_pipe(iload_reg_mem);
8040 %}
8041 
8042 // Load Short (16 bit signed)
8043 instruct loadS(iRegINoSp dst, memory mem)
8044 %{
8045   match(Set dst (LoadS mem));
8046   predicate(!needs_acquiring_load(n));
8047 
8048   ins_cost(4 * INSN_COST);
8049   format %{ "ldrshw  $dst, $mem\t# short" %}
8050 
8051   ins_encode(aarch64_enc_ldrshw(dst, mem));
8052 
8053   ins_pipe(iload_reg_mem);
8054 %}
8055 
8056 // Load Short (16 bit signed) into long
8057 instruct loadS2L(iRegLNoSp dst, memory mem)
8058 %{
8059   match(Set dst (ConvI2L (LoadS mem)));
8060   predicate(!needs_acquiring_load(n->in(1)));
8061 
8062   ins_cost(4 * INSN_COST);
8063   format %{ "ldrsh  $dst, $mem\t# short" %}
8064 
8065   ins_encode(aarch64_enc_ldrsh(dst, mem));
8066 
8067   ins_pipe(iload_reg_mem);
8068 %}
8069 
8070 // Load Char (16 bit unsigned)
8071 instruct loadUS(iRegINoSp dst, memory mem)
8072 %{
8073   match(Set dst (LoadUS mem));
8074   predicate(!needs_acquiring_load(n));
8075 
8076   ins_cost(4 * INSN_COST);
8077   format %{ "ldrh  $dst, $mem\t# short" %}
8078 
8079   ins_encode(aarch64_enc_ldrh(dst, mem));
8080 
8081   ins_pipe(iload_reg_mem);
8082 %}
8083 
8084 // Load Short/Char (16 bit unsigned) into long
8085 instruct loadUS2L(iRegLNoSp dst, memory mem)
8086 %{
8087   match(Set dst (ConvI2L (LoadUS mem)));
8088   predicate(!needs_acquiring_load(n->in(1)));
8089 
8090   ins_cost(4 * INSN_COST);
8091   format %{ "ldrh  $dst, $mem\t# short" %}
8092 
8093   ins_encode(aarch64_enc_ldrh(dst, mem));
8094 
8095   ins_pipe(iload_reg_mem);
8096 %}
8097 
8098 // Load Integer (32 bit signed)
8099 instruct loadI(iRegINoSp dst, memory mem)
8100 %{
8101   match(Set dst (LoadI mem));
8102   predicate(!needs_acquiring_load(n));
8103 
8104   ins_cost(4 * INSN_COST);
8105   format %{ "ldrw  $dst, $mem\t# int" %}
8106 
8107   ins_encode(aarch64_enc_ldrw(dst, mem));
8108 
8109   ins_pipe(iload_reg_mem);
8110 %}
8111 
8112 // Load Integer (32 bit signed) into long
8113 instruct loadI2L(iRegLNoSp dst, memory mem)
8114 %{
8115   match(Set dst (ConvI2L (LoadI mem)));
8116   predicate(!needs_acquiring_load(n->in(1)));
8117 
8118   ins_cost(4 * INSN_COST);
8119   format %{ "ldrsw  $dst, $mem\t# int" %}
8120 
8121   ins_encode(aarch64_enc_ldrsw(dst, mem));
8122 
8123   ins_pipe(iload_reg_mem);
8124 %}
8125 
8126 // Load Integer (32 bit unsigned) into long
8127 instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
8128 %{
8129   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
8130   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
8131 
8132   ins_cost(4 * INSN_COST);
8133   format %{ "ldrw  $dst, $mem\t# int" %}
8134 
8135   ins_encode(aarch64_enc_ldrw(dst, mem));
8136 
8137   ins_pipe(iload_reg_mem);
8138 %}
8139 
8140 // Load Long (64 bit signed)
8141 instruct loadL(iRegLNoSp dst, memory mem)
8142 %{
8143   match(Set dst (LoadL mem));
8144   predicate(!needs_acquiring_load(n));
8145 
8146   ins_cost(4 * INSN_COST);
8147   format %{ "ldr  $dst, $mem\t# int" %}
8148 
8149   ins_encode(aarch64_enc_ldr(dst, mem));
8150 
8151   ins_pipe(iload_reg_mem);
8152 %}
8153 
8154 // Load Range
8155 instruct loadRange(iRegINoSp dst, memory mem)
8156 %{
8157   match(Set dst (LoadRange mem));
8158 
8159   ins_cost(4 * INSN_COST);
8160   format %{ "ldrw  $dst, $mem\t# range" %}
8161 
8162   ins_encode(aarch64_enc_ldrw(dst, mem));
8163 
8164   ins_pipe(iload_reg_mem);
8165 %}
8166 
8167 // Load Pointer
8168 instruct loadP(iRegPNoSp dst, memory mem)
8169 %{
8170   match(Set dst (LoadP mem));
8171   predicate(!needs_acquiring_load(n));
8172 
8173   ins_cost(4 * INSN_COST);
8174   format %{ "ldr  $dst, $mem\t# ptr" %}
8175 
8176   ins_encode(aarch64_enc_ldr(dst, mem));
8177 
8178   ins_pipe(iload_reg_mem);
8179 %}
8180 
8181 // Load Compressed Pointer
8182 instruct loadN(iRegNNoSp dst, memory mem)
8183 %{
8184   match(Set dst (LoadN mem));
8185   predicate(!needs_acquiring_load(n));
8186 
8187   ins_cost(4 * INSN_COST);
8188   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
8189 
8190   ins_encode(aarch64_enc_ldrw(dst, mem));
8191 
8192   ins_pipe(iload_reg_mem);
8193 %}
8194 
8195 // Load Klass Pointer
8196 instruct loadKlass(iRegPNoSp dst, memory mem)
8197 %{
8198   match(Set dst (LoadKlass mem));
8199   predicate(!needs_acquiring_load(n));
8200 
8201   ins_cost(4 * INSN_COST);
8202   format %{ "ldr  $dst, $mem\t# class" %}
8203 
8204   ins_encode(aarch64_enc_ldr(dst, mem));
8205 
8206   ins_pipe(iload_reg_mem);
8207 %}
8208 
8209 // Load Narrow Klass Pointer
8210 instruct loadNKlass(iRegNNoSp dst, memory mem)
8211 %{
8212   match(Set dst (LoadNKlass mem));
8213   predicate(!needs_acquiring_load(n));
8214 
8215   ins_cost(4 * INSN_COST);
8216   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
8217 
8218   ins_encode(aarch64_enc_ldrw(dst, mem));
8219 
8220   ins_pipe(iload_reg_mem);
8221 %}
8222 
8223 // Load Float
8224 instruct loadF(vRegF dst, memory mem)
8225 %{
8226   match(Set dst (LoadF mem));
8227   predicate(!needs_acquiring_load(n));
8228 
8229   ins_cost(4 * INSN_COST);
8230   format %{ "ldrs  $dst, $mem\t# float" %}
8231 
8232   ins_encode( aarch64_enc_ldrs(dst, mem) );
8233 
8234   ins_pipe(pipe_class_memory);
8235 %}
8236 
8237 // Load Double
8238 instruct loadD(vRegD dst, memory mem)
8239 %{
8240   match(Set dst (LoadD mem));
8241   predicate(!needs_acquiring_load(n));
8242 
8243   ins_cost(4 * INSN_COST);
8244   format %{ "ldrd  $dst, $mem\t# double" %}
8245 
8246   ins_encode( aarch64_enc_ldrd(dst, mem) );
8247 
8248   ins_pipe(pipe_class_memory);
8249 %}
8250 
8251 
8252 // Load Int Constant
8253 instruct loadConI(iRegINoSp dst, immI src)
8254 %{
8255   match(Set dst src);
8256 
8257   ins_cost(INSN_COST);
8258   format %{ "mov $dst, $src\t# int" %}
8259 
8260   ins_encode( aarch64_enc_movw_imm(dst, src) );
8261 
8262   ins_pipe(ialu_imm);
8263 %}
8264 
8265 // Load Long Constant
8266 instruct loadConL(iRegLNoSp dst, immL src)
8267 %{
8268   match(Set dst src);
8269 
8270   ins_cost(INSN_COST);
8271   format %{ "mov $dst, $src\t# long" %}
8272 
8273   ins_encode( aarch64_enc_mov_imm(dst, src) );
8274 
8275   ins_pipe(ialu_imm);
8276 %}
8277 
8278 // Load Pointer Constant
8279 
8280 instruct loadConP(iRegPNoSp dst, immP con)
8281 %{
8282   match(Set dst con);
8283 
8284   ins_cost(INSN_COST * 4);
8285   format %{
8286     "mov  $dst, $con\t# ptr\n\t"
8287   %}
8288 
8289   ins_encode(aarch64_enc_mov_p(dst, con));
8290 
8291   ins_pipe(ialu_imm);
8292 %}
8293 
8294 // Load Null Pointer Constant
8295 
8296 instruct loadConP0(iRegPNoSp dst, immP0 con)
8297 %{
8298   match(Set dst con);
8299 
8300   ins_cost(INSN_COST);
8301   format %{ "mov  $dst, $con\t# NULL ptr" %}
8302 
8303   ins_encode(aarch64_enc_mov_p0(dst, con));
8304 
8305   ins_pipe(ialu_imm);
8306 %}
8307 
8308 // Load Pointer Constant One
8309 
8310 instruct loadConP1(iRegPNoSp dst, immP_1 con)
8311 %{
8312   match(Set dst con);
8313 
8314   ins_cost(INSN_COST);
8315   format %{ "mov  $dst, $con\t# NULL ptr" %}
8316 
8317   ins_encode(aarch64_enc_mov_p1(dst, con));
8318 
8319   ins_pipe(ialu_imm);
8320 %}
8321 
8322 // Load Poll Page Constant
8323 
8324 instruct loadConPollPage(iRegPNoSp dst, immPollPage con)
8325 %{
8326   match(Set dst con);
8327 
8328   ins_cost(INSN_COST);
8329   format %{ "adr  $dst, $con\t# Poll Page Ptr" %}
8330 
8331   ins_encode(aarch64_enc_mov_poll_page(dst, con));
8332 
8333   ins_pipe(ialu_imm);
8334 %}
8335 
8336 // Load Byte Map Base Constant
8337 
8338 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
8339 %{
8340   match(Set dst con);
8341 
8342   ins_cost(INSN_COST);
8343   format %{ "adr  $dst, $con\t# Byte Map Base" %}
8344 
8345   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
8346 
8347   ins_pipe(ialu_imm);
8348 %}
8349 
8350 // Load Narrow Pointer Constant
8351 
8352 instruct loadConN(iRegNNoSp dst, immN con)
8353 %{
8354   match(Set dst con);
8355 
8356   ins_cost(INSN_COST * 4);
8357   format %{ "mov  $dst, $con\t# compressed ptr" %}
8358 
8359   ins_encode(aarch64_enc_mov_n(dst, con));
8360 
8361   ins_pipe(ialu_imm);
8362 %}
8363 
8364 // Load Narrow Null Pointer Constant
8365 
8366 instruct loadConN0(iRegNNoSp dst, immN0 con)
8367 %{
8368   match(Set dst con);
8369 
8370   ins_cost(INSN_COST);
8371   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
8372 
8373   ins_encode(aarch64_enc_mov_n0(dst, con));
8374 
8375   ins_pipe(ialu_imm);
8376 %}
8377 
8378 // Load Narrow Klass Constant
8379 
8380 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
8381 %{
8382   match(Set dst con);
8383 
8384   ins_cost(INSN_COST);
8385   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
8386 
8387   ins_encode(aarch64_enc_mov_nk(dst, con));
8388 
8389   ins_pipe(ialu_imm);
8390 %}
8391 
8392 // Load Packed Float Constant
8393 
8394 instruct loadConF_packed(vRegF dst, immFPacked con) %{
8395   match(Set dst con);
8396   ins_cost(INSN_COST * 4);
8397   format %{ "fmovs  $dst, $con"%}
8398   ins_encode %{
8399     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
8400   %}
8401 
8402   ins_pipe(fp_imm_s);
8403 %}
8404 
8405 // Load Float Constant
8406 
8407 instruct loadConF(vRegF dst, immF con) %{
8408   match(Set dst con);
8409 
8410   ins_cost(INSN_COST * 4);
8411 
8412   format %{
8413     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
8414   %}
8415 
8416   ins_encode %{
8417     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
8418   %}
8419 
8420   ins_pipe(fp_load_constant_s);
8421 %}
8422 
8423 // Load Packed Double Constant
8424 
8425 instruct loadConD_packed(vRegD dst, immDPacked con) %{
8426   match(Set dst con);
8427   ins_cost(INSN_COST);
8428   format %{ "fmovd  $dst, $con"%}
8429   ins_encode %{
8430     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
8431   %}
8432 
8433   ins_pipe(fp_imm_d);
8434 %}
8435 
8436 // Load Double Constant
8437 
8438 instruct loadConD(vRegD dst, immD con) %{
8439   match(Set dst con);
8440 
8441   ins_cost(INSN_COST * 5);
8442   format %{
8443     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
8444   %}
8445 
8446   ins_encode %{
8447     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
8448   %}
8449 
8450   ins_pipe(fp_load_constant_d);
8451 %}
8452 
8453 // Store Instructions
8454 
8455 // Store CMS card-mark Immediate
8456 instruct storeimmCM0(immI0 zero, memory mem)
8457 %{
8458   match(Set mem (StoreCM mem zero));
8459   predicate(unnecessary_storestore(n));
8460 
8461   ins_cost(INSN_COST);
8462   format %{ "strb zr, $mem\t# byte" %}
8463 
8464   ins_encode(aarch64_enc_strb0(mem));
8465 
8466   ins_pipe(istore_mem);
8467 %}
8468 
8469 // Store CMS card-mark Immediate with intervening StoreStore
8470 // needed when using CMS with no conditional card marking
8471 instruct storeimmCM0_ordered(immI0 zero, memory mem)
8472 %{
8473   match(Set mem (StoreCM mem zero));
8474 
8475   ins_cost(INSN_COST * 2);
8476   format %{ "dmb ishst"
8477       "\n\tstrb zr, $mem\t# byte" %}
8478 
8479   ins_encode(aarch64_enc_strb0_ordered(mem));
8480 
8481   ins_pipe(istore_mem);
8482 %}
8483 
8484 // Store Byte
8485 instruct storeB(iRegIorL2I src, memory mem)
8486 %{
8487   match(Set mem (StoreB mem src));
8488   predicate(!needs_releasing_store(n));
8489 
8490   ins_cost(INSN_COST);
8491   format %{ "strb  $src, $mem\t# byte" %}
8492 
8493   ins_encode(aarch64_enc_strb(src, mem));
8494 
8495   ins_pipe(istore_reg_mem);
8496 %}
8497 
8498 
8499 instruct storeimmB0(immI0 zero, memory mem)
8500 %{
8501   match(Set mem (StoreB mem zero));
8502   predicate(!needs_releasing_store(n));
8503 
8504   ins_cost(INSN_COST);
8505   format %{ "strb rscractch2, $mem\t# byte" %}
8506 
8507   ins_encode(aarch64_enc_strb0(mem));
8508 
8509   ins_pipe(istore_mem);
8510 %}
8511 
8512 // Store Char/Short
8513 instruct storeC(iRegIorL2I src, memory mem)
8514 %{
8515   match(Set mem (StoreC mem src));
8516   predicate(!needs_releasing_store(n));
8517 
8518   ins_cost(INSN_COST);
8519   format %{ "strh  $src, $mem\t# short" %}
8520 
8521   ins_encode(aarch64_enc_strh(src, mem));
8522 
8523   ins_pipe(istore_reg_mem);
8524 %}
8525 
8526 instruct storeimmC0(immI0 zero, memory mem)
8527 %{
8528   match(Set mem (StoreC mem zero));
8529   predicate(!needs_releasing_store(n));
8530 
8531   ins_cost(INSN_COST);
8532   format %{ "strh  zr, $mem\t# short" %}
8533 
8534   ins_encode(aarch64_enc_strh0(mem));
8535 
8536   ins_pipe(istore_mem);
8537 %}
8538 
8539 // Store Integer
8540 
8541 instruct storeI(iRegIorL2I src, memory mem)
8542 %{
8543   match(Set mem(StoreI mem src));
8544   predicate(!needs_releasing_store(n));
8545 
8546   ins_cost(INSN_COST);
8547   format %{ "strw  $src, $mem\t# int" %}
8548 
8549   ins_encode(aarch64_enc_strw(src, mem));
8550 
8551   ins_pipe(istore_reg_mem);
8552 %}
8553 
8554 instruct storeimmI0(immI0 zero, memory mem)
8555 %{
8556   match(Set mem(StoreI mem zero));
8557   predicate(!needs_releasing_store(n));
8558 
8559   ins_cost(INSN_COST);
8560   format %{ "strw  zr, $mem\t# int" %}
8561 
8562   ins_encode(aarch64_enc_strw0(mem));
8563 
8564   ins_pipe(istore_mem);
8565 %}
8566 
8567 // Store Long (64 bit signed)
8568 instruct storeL(iRegL src, memory mem)
8569 %{
8570   match(Set mem (StoreL mem src));
8571   predicate(!needs_releasing_store(n));
8572 
8573   ins_cost(INSN_COST);
8574   format %{ "str  $src, $mem\t# int" %}
8575 
8576   ins_encode(aarch64_enc_str(src, mem));
8577 
8578   ins_pipe(istore_reg_mem);
8579 %}
8580 
8581 // Store Long (64 bit signed)
8582 instruct storeimmL0(immL0 zero, memory mem)
8583 %{
8584   match(Set mem (StoreL mem zero));
8585   predicate(!needs_releasing_store(n));
8586 
8587   ins_cost(INSN_COST);
8588   format %{ "str  zr, $mem\t# int" %}
8589 
8590   ins_encode(aarch64_enc_str0(mem));
8591 
8592   ins_pipe(istore_mem);
8593 %}
8594 
8595 // Store Pointer
8596 instruct storeP(iRegP src, memory mem)
8597 %{
8598   match(Set mem (StoreP mem src));
8599   predicate(!needs_releasing_store(n));
8600 
8601   ins_cost(INSN_COST);
8602   format %{ "str  $src, $mem\t# ptr" %}
8603 
8604   ins_encode(aarch64_enc_str(src, mem));
8605 
8606   ins_pipe(istore_reg_mem);
8607 %}
8608 
8609 // Store Pointer
8610 instruct storeimmP0(immP0 zero, memory mem)
8611 %{
8612   match(Set mem (StoreP mem zero));
8613   predicate(!needs_releasing_store(n));
8614 
8615   ins_cost(INSN_COST);
8616   format %{ "str zr, $mem\t# ptr" %}
8617 
8618   ins_encode(aarch64_enc_str0(mem));
8619 
8620   ins_pipe(istore_mem);
8621 %}
8622 
8623 // Store Compressed Pointer
8624 instruct storeN(iRegN src, memory mem)
8625 %{
8626   match(Set mem (StoreN mem src));
8627   predicate(!needs_releasing_store(n));
8628 
8629   ins_cost(INSN_COST);
8630   format %{ "strw  $src, $mem\t# compressed ptr" %}
8631 
8632   ins_encode(aarch64_enc_strw(src, mem));
8633 
8634   ins_pipe(istore_reg_mem);
8635 %}
8636 
8637 instruct storeImmN0(iRegIHeapbase heapbase, immN0 zero, memory mem)
8638 %{
8639   match(Set mem (StoreN mem zero));
8640   predicate(Universe::narrow_oop_base() == NULL &&
8641             Universe::narrow_klass_base() == NULL &&
8642             (!needs_releasing_store(n)));
8643 
8644   ins_cost(INSN_COST);
8645   format %{ "strw  rheapbase, $mem\t# compressed ptr (rheapbase==0)" %}
8646 
8647   ins_encode(aarch64_enc_strw(heapbase, mem));
8648 
8649   ins_pipe(istore_reg_mem);
8650 %}
8651 
8652 // Store Float
8653 instruct storeF(vRegF src, memory mem)
8654 %{
8655   match(Set mem (StoreF mem src));
8656   predicate(!needs_releasing_store(n));
8657 
8658   ins_cost(INSN_COST);
8659   format %{ "strs  $src, $mem\t# float" %}
8660 
8661   ins_encode( aarch64_enc_strs(src, mem) );
8662 
8663   ins_pipe(pipe_class_memory);
8664 %}
8665 
8666 // TODO
8667 // implement storeImmF0 and storeFImmPacked
8668 
8669 // Store Double
8670 instruct storeD(vRegD src, memory mem)
8671 %{
8672   match(Set mem (StoreD mem src));
8673   predicate(!needs_releasing_store(n));
8674 
8675   ins_cost(INSN_COST);
8676   format %{ "strd  $src, $mem\t# double" %}
8677 
8678   ins_encode( aarch64_enc_strd(src, mem) );
8679 
8680   ins_pipe(pipe_class_memory);
8681 %}
8682 
8683 // Store Compressed Klass Pointer
8684 instruct storeNKlass(iRegN src, memory mem)
8685 %{
8686   predicate(!needs_releasing_store(n));
8687   match(Set mem (StoreNKlass mem src));
8688 
8689   ins_cost(INSN_COST);
8690   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
8691 
8692   ins_encode(aarch64_enc_strw(src, mem));
8693 
8694   ins_pipe(istore_reg_mem);
8695 %}
8696 
8697 // TODO
8698 // implement storeImmD0 and storeDImmPacked
8699 
8700 // prefetch instructions
8701 // Must be safe to execute with invalid address (cannot fault).
8702 
8703 instruct prefetchalloc( memory mem ) %{
8704   match(PrefetchAllocation mem);
8705 
8706   ins_cost(INSN_COST);
8707   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
8708 
8709   ins_encode( aarch64_enc_prefetchw(mem) );
8710 
8711   ins_pipe(iload_prefetch);
8712 %}
8713 
8714 //  ---------------- volatile loads and stores ----------------
8715 
8716 // Load Byte (8 bit signed)
8717 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8718 %{
8719   match(Set dst (LoadB mem));
8720 
8721   ins_cost(VOLATILE_REF_COST);
8722   format %{ "ldarsb  $dst, $mem\t# byte" %}
8723 
8724   ins_encode(aarch64_enc_ldarsb(dst, mem));
8725 
8726   ins_pipe(pipe_serial);
8727 %}
8728 
8729 // Load Byte (8 bit signed) into long
8730 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8731 %{
8732   match(Set dst (ConvI2L (LoadB mem)));
8733 
8734   ins_cost(VOLATILE_REF_COST);
8735   format %{ "ldarsb  $dst, $mem\t# byte" %}
8736 
8737   ins_encode(aarch64_enc_ldarsb(dst, mem));
8738 
8739   ins_pipe(pipe_serial);
8740 %}
8741 
8742 // Load Byte (8 bit unsigned)
8743 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8744 %{
8745   match(Set dst (LoadUB mem));
8746 
8747   ins_cost(VOLATILE_REF_COST);
8748   format %{ "ldarb  $dst, $mem\t# byte" %}
8749 
8750   ins_encode(aarch64_enc_ldarb(dst, mem));
8751 
8752   ins_pipe(pipe_serial);
8753 %}
8754 
8755 // Load Byte (8 bit unsigned) into long
8756 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8757 %{
8758   match(Set dst (ConvI2L (LoadUB mem)));
8759 
8760   ins_cost(VOLATILE_REF_COST);
8761   format %{ "ldarb  $dst, $mem\t# byte" %}
8762 
8763   ins_encode(aarch64_enc_ldarb(dst, mem));
8764 
8765   ins_pipe(pipe_serial);
8766 %}
8767 
8768 // Load Short (16 bit signed)
8769 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8770 %{
8771   match(Set dst (LoadS mem));
8772 
8773   ins_cost(VOLATILE_REF_COST);
8774   format %{ "ldarshw  $dst, $mem\t# short" %}
8775 
8776   ins_encode(aarch64_enc_ldarshw(dst, mem));
8777 
8778   ins_pipe(pipe_serial);
8779 %}
8780 
8781 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8782 %{
8783   match(Set dst (LoadUS mem));
8784 
8785   ins_cost(VOLATILE_REF_COST);
8786   format %{ "ldarhw  $dst, $mem\t# short" %}
8787 
8788   ins_encode(aarch64_enc_ldarhw(dst, mem));
8789 
8790   ins_pipe(pipe_serial);
8791 %}
8792 
8793 // Load Short/Char (16 bit unsigned) into long
8794 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8795 %{
8796   match(Set dst (ConvI2L (LoadUS mem)));
8797 
8798   ins_cost(VOLATILE_REF_COST);
8799   format %{ "ldarh  $dst, $mem\t# short" %}
8800 
8801   ins_encode(aarch64_enc_ldarh(dst, mem));
8802 
8803   ins_pipe(pipe_serial);
8804 %}
8805 
8806 // Load Short/Char (16 bit signed) into long
8807 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8808 %{
8809   match(Set dst (ConvI2L (LoadS mem)));
8810 
8811   ins_cost(VOLATILE_REF_COST);
8812   format %{ "ldarh  $dst, $mem\t# short" %}
8813 
8814   ins_encode(aarch64_enc_ldarsh(dst, mem));
8815 
8816   ins_pipe(pipe_serial);
8817 %}
8818 
8819 // Load Integer (32 bit signed)
8820 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8821 %{
8822   match(Set dst (LoadI mem));
8823 
8824   ins_cost(VOLATILE_REF_COST);
8825   format %{ "ldarw  $dst, $mem\t# int" %}
8826 
8827   ins_encode(aarch64_enc_ldarw(dst, mem));
8828 
8829   ins_pipe(pipe_serial);
8830 %}
8831 
8832 // Load Integer (32 bit unsigned) into long
8833 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
8834 %{
8835   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
8836 
8837   ins_cost(VOLATILE_REF_COST);
8838   format %{ "ldarw  $dst, $mem\t# int" %}
8839 
8840   ins_encode(aarch64_enc_ldarw(dst, mem));
8841 
8842   ins_pipe(pipe_serial);
8843 %}
8844 
8845 // Load Long (64 bit signed)
8846 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8847 %{
8848   match(Set dst (LoadL mem));
8849 
8850   ins_cost(VOLATILE_REF_COST);
8851   format %{ "ldar  $dst, $mem\t# int" %}
8852 
8853   ins_encode(aarch64_enc_ldar(dst, mem));
8854 
8855   ins_pipe(pipe_serial);
8856 %}
8857 
8858 // Load Pointer
8859 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
8860 %{
8861   match(Set dst (LoadP mem));
8862 
8863   ins_cost(VOLATILE_REF_COST);
8864   format %{ "ldar  $dst, $mem\t# ptr" %}
8865 
8866   ins_encode(aarch64_enc_ldar(dst, mem));
8867 
8868   ins_pipe(pipe_serial);
8869 %}
8870 
8871 // Load Compressed Pointer
8872 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
8873 %{
8874   match(Set dst (LoadN mem));
8875 
8876   ins_cost(VOLATILE_REF_COST);
8877   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
8878 
8879   ins_encode(aarch64_enc_ldarw(dst, mem));
8880 
8881   ins_pipe(pipe_serial);
8882 %}
8883 
8884 // Load Float
8885 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
8886 %{
8887   match(Set dst (LoadF mem));
8888 
8889   ins_cost(VOLATILE_REF_COST);
8890   format %{ "ldars  $dst, $mem\t# float" %}
8891 
8892   ins_encode( aarch64_enc_fldars(dst, mem) );
8893 
8894   ins_pipe(pipe_serial);
8895 %}
8896 
8897 // Load Double
8898 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
8899 %{
8900   match(Set dst (LoadD mem));
8901 
8902   ins_cost(VOLATILE_REF_COST);
8903   format %{ "ldard  $dst, $mem\t# double" %}
8904 
8905   ins_encode( aarch64_enc_fldard(dst, mem) );
8906 
8907   ins_pipe(pipe_serial);
8908 %}
8909 
8910 // Store Byte
8911 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8912 %{
8913   match(Set mem (StoreB mem src));
8914 
8915   ins_cost(VOLATILE_REF_COST);
8916   format %{ "stlrb  $src, $mem\t# byte" %}
8917 
8918   ins_encode(aarch64_enc_stlrb(src, mem));
8919 
8920   ins_pipe(pipe_class_memory);
8921 %}
8922 
8923 // Store Char/Short
8924 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8925 %{
8926   match(Set mem (StoreC mem src));
8927 
8928   ins_cost(VOLATILE_REF_COST);
8929   format %{ "stlrh  $src, $mem\t# short" %}
8930 
8931   ins_encode(aarch64_enc_stlrh(src, mem));
8932 
8933   ins_pipe(pipe_class_memory);
8934 %}
8935 
8936 // Store Integer
8937 
8938 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8939 %{
8940   match(Set mem(StoreI mem src));
8941 
8942   ins_cost(VOLATILE_REF_COST);
8943   format %{ "stlrw  $src, $mem\t# int" %}
8944 
8945   ins_encode(aarch64_enc_stlrw(src, mem));
8946 
8947   ins_pipe(pipe_class_memory);
8948 %}
8949 
8950 // Store Long (64 bit signed)
8951 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
8952 %{
8953   match(Set mem (StoreL mem src));
8954 
8955   ins_cost(VOLATILE_REF_COST);
8956   format %{ "stlr  $src, $mem\t# int" %}
8957 
8958   ins_encode(aarch64_enc_stlr(src, mem));
8959 
8960   ins_pipe(pipe_class_memory);
8961 %}
8962 
8963 // Store Pointer
8964 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
8965 %{
8966   match(Set mem (StoreP mem src));
8967 
8968   ins_cost(VOLATILE_REF_COST);
8969   format %{ "stlr  $src, $mem\t# ptr" %}
8970 
8971   ins_encode(aarch64_enc_stlr(src, mem));
8972 
8973   ins_pipe(pipe_class_memory);
8974 %}
8975 
8976 // Store Compressed Pointer
8977 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8978 %{
8979   match(Set mem (StoreN mem src));
8980 
8981   ins_cost(VOLATILE_REF_COST);
8982   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
8983 
8984   ins_encode(aarch64_enc_stlrw(src, mem));
8985 
8986   ins_pipe(pipe_class_memory);
8987 %}
8988 
8989 // Store Float
8990 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8991 %{
8992   match(Set mem (StoreF mem src));
8993 
8994   ins_cost(VOLATILE_REF_COST);
8995   format %{ "stlrs  $src, $mem\t# float" %}
8996 
8997   ins_encode( aarch64_enc_fstlrs(src, mem) );
8998 
8999   ins_pipe(pipe_class_memory);
9000 %}
9001 
9002 // TODO
9003 // implement storeImmF0 and storeFImmPacked
9004 
9005 // Store Double
9006 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
9007 %{
9008   match(Set mem (StoreD mem src));
9009 
9010   ins_cost(VOLATILE_REF_COST);
9011   format %{ "stlrd  $src, $mem\t# double" %}
9012 
9013   ins_encode( aarch64_enc_fstlrd(src, mem) );
9014 
9015   ins_pipe(pipe_class_memory);
9016 %}
9017 
9018 //  ---------------- end of volatile loads and stores ----------------
9019 
9020 // ============================================================================
9021 // BSWAP Instructions
9022 
9023 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
9024   match(Set dst (ReverseBytesI src));
9025 
9026   ins_cost(INSN_COST);
9027   format %{ "revw  $dst, $src" %}
9028 
9029   ins_encode %{
9030     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
9031   %}
9032 
9033   ins_pipe(ialu_reg);
9034 %}
9035 
9036 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
9037   match(Set dst (ReverseBytesL src));
9038 
9039   ins_cost(INSN_COST);
9040   format %{ "rev  $dst, $src" %}
9041 
9042   ins_encode %{
9043     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
9044   %}
9045 
9046   ins_pipe(ialu_reg);
9047 %}
9048 
9049 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
9050   match(Set dst (ReverseBytesUS src));
9051 
9052   ins_cost(INSN_COST);
9053   format %{ "rev16w  $dst, $src" %}
9054 
9055   ins_encode %{
9056     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
9057   %}
9058 
9059   ins_pipe(ialu_reg);
9060 %}
9061 
9062 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
9063   match(Set dst (ReverseBytesS src));
9064 
9065   ins_cost(INSN_COST);
9066   format %{ "rev16w  $dst, $src\n\t"
9067             "sbfmw $dst, $dst, #0, #15" %}
9068 
9069   ins_encode %{
9070     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
9071     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
9072   %}
9073 
9074   ins_pipe(ialu_reg);
9075 %}
9076 
9077 // ============================================================================
9078 // Zero Count Instructions
9079 
9080 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
9081   match(Set dst (CountLeadingZerosI src));
9082 
9083   ins_cost(INSN_COST);
9084   format %{ "clzw  $dst, $src" %}
9085   ins_encode %{
9086     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
9087   %}
9088 
9089   ins_pipe(ialu_reg);
9090 %}
9091 
9092 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
9093   match(Set dst (CountLeadingZerosL src));
9094 
9095   ins_cost(INSN_COST);
9096   format %{ "clz   $dst, $src" %}
9097   ins_encode %{
9098     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
9099   %}
9100 
9101   ins_pipe(ialu_reg);
9102 %}
9103 
9104 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
9105   match(Set dst (CountTrailingZerosI src));
9106 
9107   ins_cost(INSN_COST * 2);
9108   format %{ "rbitw  $dst, $src\n\t"
9109             "clzw   $dst, $dst" %}
9110   ins_encode %{
9111     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
9112     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
9113   %}
9114 
9115   ins_pipe(ialu_reg);
9116 %}
9117 
9118 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
9119   match(Set dst (CountTrailingZerosL src));
9120 
9121   ins_cost(INSN_COST * 2);
9122   format %{ "rbit   $dst, $src\n\t"
9123             "clz    $dst, $dst" %}
9124   ins_encode %{
9125     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
9126     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
9127   %}
9128 
9129   ins_pipe(ialu_reg);
9130 %}
9131 
9132 //---------- Population Count Instructions -------------------------------------
9133 //
9134 
9135 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
9136   predicate(UsePopCountInstruction);
9137   match(Set dst (PopCountI src));
9138   effect(TEMP tmp);
9139   ins_cost(INSN_COST * 13);
9140 
9141   format %{ "movw   $src, $src\n\t"
9142             "mov    $tmp, $src\t# vector (1D)\n\t"
9143             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
9144             "addv   $tmp, $tmp\t# vector (8B)\n\t"
9145             "mov    $dst, $tmp\t# vector (1D)" %}
9146   ins_encode %{
9147     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
9148     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
9149     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
9150     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
9151     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
9152   %}
9153 
9154   ins_pipe(pipe_class_default);
9155 %}
9156 
9157 instruct popCountI_mem(iRegINoSp dst, memory mem, vRegF tmp) %{
9158   predicate(UsePopCountInstruction);
9159   match(Set dst (PopCountI (LoadI mem)));
9160   effect(TEMP tmp);
9161   ins_cost(INSN_COST * 13);
9162 
9163   format %{ "ldrs   $tmp, $mem\n\t"
9164             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
9165             "addv   $tmp, $tmp\t# vector (8B)\n\t"
9166             "mov    $dst, $tmp\t# vector (1D)" %}
9167   ins_encode %{
9168     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
9169     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
9170                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
9171     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
9172     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
9173     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
9174   %}
9175 
9176   ins_pipe(pipe_class_default);
9177 %}
9178 
9179 // Note: Long.bitCount(long) returns an int.
9180 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
9181   predicate(UsePopCountInstruction);
9182   match(Set dst (PopCountL src));
9183   effect(TEMP tmp);
9184   ins_cost(INSN_COST * 13);
9185 
9186   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
9187             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
9188             "addv   $tmp, $tmp\t# vector (8B)\n\t"
9189             "mov    $dst, $tmp\t# vector (1D)" %}
9190   ins_encode %{
9191     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
9192     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
9193     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
9194     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
9195   %}
9196 
9197   ins_pipe(pipe_class_default);
9198 %}
9199 
9200 instruct popCountL_mem(iRegINoSp dst, memory mem, vRegD tmp) %{
9201   predicate(UsePopCountInstruction);
9202   match(Set dst (PopCountL (LoadL mem)));
9203   effect(TEMP tmp);
9204   ins_cost(INSN_COST * 13);
9205 
9206   format %{ "ldrd   $tmp, $mem\n\t"
9207             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
9208             "addv   $tmp, $tmp\t# vector (8B)\n\t"
9209             "mov    $dst, $tmp\t# vector (1D)" %}
9210   ins_encode %{
9211     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
9212     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
9213                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
9214     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
9215     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
9216     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
9217   %}
9218 
9219   ins_pipe(pipe_class_default);
9220 %}
9221 
9222 // ============================================================================
9223 // MemBar Instruction
9224 
9225 instruct load_fence() %{
9226   match(LoadFence);
9227   ins_cost(VOLATILE_REF_COST);
9228 
9229   format %{ "load_fence" %}
9230 
9231   ins_encode %{
9232     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
9233   %}
9234   ins_pipe(pipe_serial);
9235 %}
9236 
9237 instruct unnecessary_membar_acquire() %{
9238   predicate(unnecessary_acquire(n));
9239   match(MemBarAcquire);
9240   ins_cost(0);
9241 
9242   format %{ "membar_acquire (elided)" %}
9243 
9244   ins_encode %{
9245     __ block_comment("membar_acquire (elided)");
9246   %}
9247 
9248   ins_pipe(pipe_class_empty);
9249 %}
9250 
9251 instruct membar_acquire() %{
9252   match(MemBarAcquire);
9253   ins_cost(VOLATILE_REF_COST);
9254 
9255   format %{ "membar_acquire" %}
9256 
9257   ins_encode %{
9258     __ block_comment("membar_acquire");
9259     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
9260   %}
9261 
9262   ins_pipe(pipe_serial);
9263 %}
9264 
9265 
9266 instruct membar_acquire_lock() %{
9267   match(MemBarAcquireLock);
9268   ins_cost(VOLATILE_REF_COST);
9269 
9270   format %{ "membar_acquire_lock (elided)" %}
9271 
9272   ins_encode %{
9273     __ block_comment("membar_acquire_lock (elided)");
9274   %}
9275 
9276   ins_pipe(pipe_serial);
9277 %}
9278 
9279 instruct store_fence() %{
9280   match(StoreFence);
9281   ins_cost(VOLATILE_REF_COST);
9282 
9283   format %{ "store_fence" %}
9284 
9285   ins_encode %{
9286     __ membar(Assembler::LoadStore|Assembler::StoreStore);
9287   %}
9288   ins_pipe(pipe_serial);
9289 %}
9290 
9291 instruct unnecessary_membar_release() %{
9292   predicate(unnecessary_release(n));
9293   match(MemBarRelease);
9294   ins_cost(0);
9295 
9296   format %{ "membar_release (elided)" %}
9297 
9298   ins_encode %{
9299     __ block_comment("membar_release (elided)");
9300   %}
9301   ins_pipe(pipe_serial);
9302 %}
9303 
9304 instruct membar_release() %{
9305   match(MemBarRelease);
9306   ins_cost(VOLATILE_REF_COST);
9307 
9308   format %{ "membar_release" %}
9309 
9310   ins_encode %{
9311     __ block_comment("membar_release");
9312     __ membar(Assembler::LoadStore|Assembler::StoreStore);
9313   %}
9314   ins_pipe(pipe_serial);
9315 %}
9316 
9317 instruct membar_storestore() %{
9318   match(MemBarStoreStore);
9319   ins_cost(VOLATILE_REF_COST);
9320 
9321   format %{ "MEMBAR-store-store" %}
9322 
9323   ins_encode %{
9324     __ membar(Assembler::StoreStore);
9325   %}
9326   ins_pipe(pipe_serial);
9327 %}
9328 
9329 instruct membar_release_lock() %{
9330   match(MemBarReleaseLock);
9331   ins_cost(VOLATILE_REF_COST);
9332 
9333   format %{ "membar_release_lock (elided)" %}
9334 
9335   ins_encode %{
9336     __ block_comment("membar_release_lock (elided)");
9337   %}
9338 
9339   ins_pipe(pipe_serial);
9340 %}
9341 
9342 instruct unnecessary_membar_volatile() %{
9343   predicate(unnecessary_volatile(n));
9344   match(MemBarVolatile);
9345   ins_cost(0);
9346 
9347   format %{ "membar_volatile (elided)" %}
9348 
9349   ins_encode %{
9350     __ block_comment("membar_volatile (elided)");
9351   %}
9352 
9353   ins_pipe(pipe_serial);
9354 %}
9355 
9356 instruct membar_volatile() %{
9357   match(MemBarVolatile);
9358   ins_cost(VOLATILE_REF_COST*100);
9359 
9360   format %{ "membar_volatile" %}
9361 
9362   ins_encode %{
9363     __ block_comment("membar_volatile");
9364     __ membar(Assembler::StoreLoad);
9365   %}
9366 
9367   ins_pipe(pipe_serial);
9368 %}
9369 
9370 // ============================================================================
9371 // Cast/Convert Instructions
9372 
9373 instruct castX2P(iRegPNoSp dst, iRegL src) %{
9374   match(Set dst (CastX2P src));
9375 
9376   ins_cost(INSN_COST);
9377   format %{ "mov $dst, $src\t# long -> ptr" %}
9378 
9379   ins_encode %{
9380     if ($dst$$reg != $src$$reg) {
9381       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
9382     }
9383   %}
9384 
9385   ins_pipe(ialu_reg);
9386 %}
9387 
9388 instruct castP2X(iRegLNoSp dst, iRegP src) %{
9389   match(Set dst (CastP2X src));
9390 
9391   ins_cost(INSN_COST);
9392   format %{ "mov $dst, $src\t# ptr -> long" %}
9393 
9394   ins_encode %{
9395     if ($dst$$reg != $src$$reg) {
9396       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
9397     }
9398   %}
9399 
9400   ins_pipe(ialu_reg);
9401 %}
9402 
9403 // Convert oop into int for vectors alignment masking
9404 instruct convP2I(iRegINoSp dst, iRegP src) %{
9405   match(Set dst (ConvL2I (CastP2X src)));
9406 
9407   ins_cost(INSN_COST);
9408   format %{ "movw $dst, $src\t# ptr -> int" %}
9409   ins_encode %{
9410     __ movw($dst$$Register, $src$$Register);
9411   %}
9412 
9413   ins_pipe(ialu_reg);
9414 %}
9415 
9416 // Convert compressed oop into int for vectors alignment masking
9417 // in case of 32bit oops (heap < 4Gb).
9418 instruct convN2I(iRegINoSp dst, iRegN src)
9419 %{
9420   predicate(Universe::narrow_oop_shift() == 0);
9421   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
9422 
9423   ins_cost(INSN_COST);
9424   format %{ "mov dst, $src\t# compressed ptr -> int" %}
9425   ins_encode %{
9426     __ movw($dst$$Register, $src$$Register);
9427   %}
9428 
9429   ins_pipe(ialu_reg);
9430 %}
9431 
9432 
9433 // Convert oop pointer into compressed form
9434 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
9435   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
9436   match(Set dst (EncodeP src));
9437   effect(KILL cr);
9438   ins_cost(INSN_COST * 3);
9439   format %{ "encode_heap_oop $dst, $src" %}
9440   ins_encode %{
9441     Register s = $src$$Register;
9442     Register d = $dst$$Register;
9443     __ encode_heap_oop(d, s);
9444   %}
9445   ins_pipe(ialu_reg);
9446 %}
9447 
9448 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
9449   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
9450   match(Set dst (EncodeP src));
9451   ins_cost(INSN_COST * 3);
9452   format %{ "encode_heap_oop_not_null $dst, $src" %}
9453   ins_encode %{
9454     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
9455   %}
9456   ins_pipe(ialu_reg);
9457 %}
9458 
9459 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
9460   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
9461             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
9462   match(Set dst (DecodeN src));
9463   ins_cost(INSN_COST * 3);
9464   format %{ "decode_heap_oop $dst, $src" %}
9465   ins_encode %{
9466     Register s = $src$$Register;
9467     Register d = $dst$$Register;
9468     __ decode_heap_oop(d, s);
9469   %}
9470   ins_pipe(ialu_reg);
9471 %}
9472 
9473 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
9474   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
9475             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
9476   match(Set dst (DecodeN src));
9477   ins_cost(INSN_COST * 3);
9478   format %{ "decode_heap_oop_not_null $dst, $src" %}
9479   ins_encode %{
9480     Register s = $src$$Register;
9481     Register d = $dst$$Register;
9482     __ decode_heap_oop_not_null(d, s);
9483   %}
9484   ins_pipe(ialu_reg);
9485 %}
9486 
9487 // n.b. AArch64 implementations of encode_klass_not_null and
9488 // decode_klass_not_null do not modify the flags register so, unlike
9489 // Intel, we don't kill CR as a side effect here
9490 
9491 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
9492   match(Set dst (EncodePKlass src));
9493 
9494   ins_cost(INSN_COST * 3);
9495   format %{ "encode_klass_not_null $dst,$src" %}
9496 
9497   ins_encode %{
9498     Register src_reg = as_Register($src$$reg);
9499     Register dst_reg = as_Register($dst$$reg);
9500     __ encode_klass_not_null(dst_reg, src_reg);
9501   %}
9502 
9503    ins_pipe(ialu_reg);
9504 %}
9505 
9506 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
9507   match(Set dst (DecodeNKlass src));
9508 
9509   ins_cost(INSN_COST * 3);
9510   format %{ "decode_klass_not_null $dst,$src" %}
9511 
9512   ins_encode %{
9513     Register src_reg = as_Register($src$$reg);
9514     Register dst_reg = as_Register($dst$$reg);
9515     if (dst_reg != src_reg) {
9516       __ decode_klass_not_null(dst_reg, src_reg);
9517     } else {
9518       __ decode_klass_not_null(dst_reg);
9519     }
9520   %}
9521 
9522    ins_pipe(ialu_reg);
9523 %}
9524 
9525 instruct checkCastPP(iRegPNoSp dst)
9526 %{
9527   match(Set dst (CheckCastPP dst));
9528 
9529   size(0);
9530   format %{ "# checkcastPP of $dst" %}
9531   ins_encode(/* empty encoding */);
9532   ins_pipe(pipe_class_empty);
9533 %}
9534 
9535 instruct castPP(iRegPNoSp dst)
9536 %{
9537   match(Set dst (CastPP dst));
9538 
9539   size(0);
9540   format %{ "# castPP of $dst" %}
9541   ins_encode(/* empty encoding */);
9542   ins_pipe(pipe_class_empty);
9543 %}
9544 
9545 instruct castII(iRegI dst)
9546 %{
9547   match(Set dst (CastII dst));
9548 
9549   size(0);
9550   format %{ "# castII of $dst" %}
9551   ins_encode(/* empty encoding */);
9552   ins_cost(0);
9553   ins_pipe(pipe_class_empty);
9554 %}
9555 
9556 // ============================================================================
9557 // Atomic operation instructions
9558 //
9559 // Intel and SPARC both implement Ideal Node LoadPLocked and
9560 // Store{PIL}Conditional instructions using a normal load for the
9561 // LoadPLocked and a CAS for the Store{PIL}Conditional.
9562 //
9563 // The ideal code appears only to use LoadPLocked/StorePLocked as a
9564 // pair to lock object allocations from Eden space when not using
9565 // TLABs.
9566 //
9567 // There does not appear to be a Load{IL}Locked Ideal Node and the
9568 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
9569 // and to use StoreIConditional only for 32-bit and StoreLConditional
9570 // only for 64-bit.
9571 //
9572 // We implement LoadPLocked and StorePLocked instructions using,
9573 // respectively the AArch64 hw load-exclusive and store-conditional
9574 // instructions. Whereas we must implement each of
9575 // Store{IL}Conditional using a CAS which employs a pair of
9576 // instructions comprising a load-exclusive followed by a
9577 // store-conditional.
9578 
9579 
9580 // Locked-load (linked load) of the current heap-top
9581 // used when updating the eden heap top
9582 // implemented using ldaxr on AArch64
9583 
9584 instruct loadPLocked(iRegPNoSp dst, indirect mem)
9585 %{
9586   match(Set dst (LoadPLocked mem));
9587 
9588   ins_cost(VOLATILE_REF_COST);
9589 
9590   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
9591 
9592   ins_encode(aarch64_enc_ldaxr(dst, mem));
9593 
9594   ins_pipe(pipe_serial);
9595 %}
9596 
9597 // Conditional-store of the updated heap-top.
9598 // Used during allocation of the shared heap.
9599 // Sets flag (EQ) on success.
9600 // implemented using stlxr on AArch64.
9601 
9602 instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
9603 %{
9604   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
9605 
9606   ins_cost(VOLATILE_REF_COST);
9607 
9608  // TODO
9609  // do we need to do a store-conditional release or can we just use a
9610  // plain store-conditional?
9611 
9612   format %{
9613     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
9614     "cmpw rscratch1, zr\t# EQ on successful write"
9615   %}
9616 
9617   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
9618 
9619   ins_pipe(pipe_serial);
9620 %}
9621 
9622 
9623 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
9624 // when attempting to rebias a lock towards the current thread.  We
9625 // must use the acquire form of cmpxchg in order to guarantee acquire
9626 // semantics in this case.
9627 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
9628 %{
9629   match(Set cr (StoreLConditional mem (Binary oldval newval)));
9630 
9631   ins_cost(VOLATILE_REF_COST);
9632 
9633   format %{
9634     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
9635     "cmpw rscratch1, zr\t# EQ on successful write"
9636   %}
9637 
9638   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
9639 
9640   ins_pipe(pipe_slow);
9641 %}
9642 
9643 // storeIConditional also has acquire semantics, for no better reason
9644 // than matching storeLConditional.  At the time of writing this
9645 // comment storeIConditional was not used anywhere by AArch64.
9646 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
9647 %{
9648   match(Set cr (StoreIConditional mem (Binary oldval newval)));
9649 
9650   ins_cost(VOLATILE_REF_COST);
9651 
9652   format %{
9653     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
9654     "cmpw rscratch1, zr\t# EQ on successful write"
9655   %}
9656 
9657   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
9658 
9659   ins_pipe(pipe_slow);
9660 %}
9661 
9662 // standard CompareAndSwapX when we are using barriers
9663 // these have higher priority than the rules selected by a predicate
9664 
9665 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
9666 // can't match them
9667 
9668 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9669 
9670   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
9671   ins_cost(2 * VOLATILE_REF_COST);
9672 
9673   effect(KILL cr);
9674 
9675   format %{
9676     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9677     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9678   %}
9679 
9680   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
9681             aarch64_enc_cset_eq(res));
9682 
9683   ins_pipe(pipe_slow);
9684 %}
9685 
9686 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9687 
9688   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
9689   ins_cost(2 * VOLATILE_REF_COST);
9690 
9691   effect(KILL cr);
9692 
9693   format %{
9694     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9695     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9696   %}
9697 
9698   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
9699             aarch64_enc_cset_eq(res));
9700 
9701   ins_pipe(pipe_slow);
9702 %}
9703 
9704 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9705 
9706   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9707   ins_cost(2 * VOLATILE_REF_COST);
9708 
9709   effect(KILL cr);
9710 
9711  format %{
9712     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9713     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9714  %}
9715 
9716  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9717             aarch64_enc_cset_eq(res));
9718 
9719   ins_pipe(pipe_slow);
9720 %}
9721 
9722 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9723 
9724   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9725   ins_cost(2 * VOLATILE_REF_COST);
9726 
9727   effect(KILL cr);
9728 
9729  format %{
9730     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9731     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9732  %}
9733 
9734  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9735             aarch64_enc_cset_eq(res));
9736 
9737   ins_pipe(pipe_slow);
9738 %}
9739 
9740 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9741 
9742   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9743   ins_cost(2 * VOLATILE_REF_COST);
9744 
9745   effect(KILL cr);
9746 
9747  format %{
9748     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9749     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9750  %}
9751 
9752  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9753             aarch64_enc_cset_eq(res));
9754 
9755   ins_pipe(pipe_slow);
9756 %}
9757 
9758 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9759 
9760   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9761   ins_cost(2 * VOLATILE_REF_COST);
9762 
9763   effect(KILL cr);
9764 
9765  format %{
9766     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9767     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9768  %}
9769 
9770  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9771             aarch64_enc_cset_eq(res));
9772 
9773   ins_pipe(pipe_slow);
9774 %}
9775 
9776 // alternative CompareAndSwapX when we are eliding barriers
9777 
9778 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9779 
9780   predicate(needs_acquiring_load_exclusive(n));
9781   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9782   ins_cost(VOLATILE_REF_COST);
9783 
9784   effect(KILL cr);
9785 
9786  format %{
9787     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9788     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9789  %}
9790 
9791  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9792             aarch64_enc_cset_eq(res));
9793 
9794   ins_pipe(pipe_slow);
9795 %}
9796 
9797 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9798 
9799   predicate(needs_acquiring_load_exclusive(n));
9800   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9801   ins_cost(VOLATILE_REF_COST);
9802 
9803   effect(KILL cr);
9804 
9805  format %{
9806     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9807     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9808  %}
9809 
9810  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9811             aarch64_enc_cset_eq(res));
9812 
9813   ins_pipe(pipe_slow);
9814 %}
9815 
9816 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9817 
9818   predicate(needs_acquiring_load_exclusive(n));
9819   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9820   ins_cost(VOLATILE_REF_COST);
9821 
9822   effect(KILL cr);
9823 
9824  format %{
9825     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9826     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9827  %}
9828 
9829  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9830             aarch64_enc_cset_eq(res));
9831 
9832   ins_pipe(pipe_slow);
9833 %}
9834 
9835 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9836 
9837   predicate(needs_acquiring_load_exclusive(n));
9838   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9839   ins_cost(VOLATILE_REF_COST);
9840 
9841   effect(KILL cr);
9842 
9843  format %{
9844     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9845     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9846  %}
9847 
9848  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9849             aarch64_enc_cset_eq(res));
9850 
9851   ins_pipe(pipe_slow);
9852 %}
9853 
9854 
9855 // ---------------------------------------------------------------------
9856 
9857 
9858 // BEGIN This section of the file is automatically generated. Do not edit --------------
9859 
9860 // Sundry CAS operations.  Note that release is always true,
9861 // regardless of the memory ordering of the CAS.  This is because we
9862 // need the volatile case to be sequentially consistent but there is
9863 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
9864 // can't check the type of memory ordering here, so we always emit a
9865 // STLXR.
9866 
9867 // This section is generated from aarch64_ad_cas.m4
9868 
9869 
9870 
9871 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9872   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9873   ins_cost(2 * VOLATILE_REF_COST);
9874   effect(TEMP_DEF res, KILL cr);
9875   format %{
9876     "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9877   %}
9878   ins_encode %{
9879     __ uxtbw(rscratch2, $oldval$$Register);
9880     __ cmpxchg($mem$$Register, rscratch2, $newval$$Register,
9881                Assembler::byte, /*acquire*/ false, /*release*/ true,
9882                /*weak*/ false, $res$$Register);
9883     __ sxtbw($res$$Register, $res$$Register);
9884   %}
9885   ins_pipe(pipe_slow);
9886 %}
9887 
9888 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9889   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9890   ins_cost(2 * VOLATILE_REF_COST);
9891   effect(TEMP_DEF res, KILL cr);
9892   format %{
9893     "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9894   %}
9895   ins_encode %{
9896     __ uxthw(rscratch2, $oldval$$Register);
9897     __ cmpxchg($mem$$Register, rscratch2, $newval$$Register,
9898                Assembler::halfword, /*acquire*/ false, /*release*/ true,
9899                /*weak*/ false, $res$$Register);
9900     __ sxthw($res$$Register, $res$$Register);
9901   %}
9902   ins_pipe(pipe_slow);
9903 %}
9904 
9905 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9906   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9907   ins_cost(2 * VOLATILE_REF_COST);
9908   effect(TEMP_DEF res, KILL cr);
9909   format %{
9910     "cmpxchg $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9911   %}
9912   ins_encode %{
9913     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9914                Assembler::word, /*acquire*/ false, /*release*/ true,
9915                /*weak*/ false, $res$$Register);
9916   %}
9917   ins_pipe(pipe_slow);
9918 %}
9919 
9920 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9921   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9922   ins_cost(2 * VOLATILE_REF_COST);
9923   effect(TEMP_DEF res, KILL cr);
9924   format %{
9925     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9926   %}
9927   ins_encode %{
9928     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9929                Assembler::xword, /*acquire*/ false, /*release*/ true,
9930                /*weak*/ false, $res$$Register);
9931   %}
9932   ins_pipe(pipe_slow);
9933 %}
9934 
9935 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9936   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9937   ins_cost(2 * VOLATILE_REF_COST);
9938   effect(TEMP_DEF res, KILL cr);
9939   format %{
9940     "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9941   %}
9942   ins_encode %{
9943     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9944                Assembler::word, /*acquire*/ false, /*release*/ true,
9945                /*weak*/ false, $res$$Register);
9946   %}
9947   ins_pipe(pipe_slow);
9948 %}
9949 
9950 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9951   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9952   ins_cost(2 * VOLATILE_REF_COST);
9953   effect(TEMP_DEF res, KILL cr);
9954   format %{
9955     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9956   %}
9957   ins_encode %{
9958     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9959                Assembler::xword, /*acquire*/ false, /*release*/ true,
9960                /*weak*/ false, $res$$Register);
9961   %}
9962   ins_pipe(pipe_slow);
9963 %}
9964 
9965 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9966   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9967   ins_cost(2 * VOLATILE_REF_COST);
9968   effect(KILL cr);
9969   format %{
9970     "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9971     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9972   %}
9973   ins_encode %{
9974     __ uxtbw(rscratch2, $oldval$$Register);
9975     __ cmpxchg($mem$$Register, rscratch2, $newval$$Register,
9976                Assembler::byte, /*acquire*/ false, /*release*/ true,
9977                /*weak*/ true, noreg);
9978     __ csetw($res$$Register, Assembler::EQ);
9979   %}
9980   ins_pipe(pipe_slow);
9981 %}
9982 
9983 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9984   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9985   ins_cost(2 * VOLATILE_REF_COST);
9986   effect(KILL cr);
9987   format %{
9988     "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9989     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9990   %}
9991   ins_encode %{
9992     __ uxthw(rscratch2, $oldval$$Register);
9993     __ cmpxchg($mem$$Register, rscratch2, $newval$$Register,
9994                Assembler::halfword, /*acquire*/ false, /*release*/ true,
9995                /*weak*/ true, noreg);
9996     __ csetw($res$$Register, Assembler::EQ);
9997   %}
9998   ins_pipe(pipe_slow);
9999 %}
10000 
10001 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
10002   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
10003   ins_cost(2 * VOLATILE_REF_COST);
10004   effect(KILL cr);
10005   format %{
10006     "cmpxchg $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
10007     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10008   %}
10009   ins_encode %{
10010     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10011                Assembler::word, /*acquire*/ false, /*release*/ true,
10012                /*weak*/ true, noreg);
10013     __ csetw($res$$Register, Assembler::EQ);
10014   %}
10015   ins_pipe(pipe_slow);
10016 %}
10017 
10018 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
10019   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
10020   ins_cost(2 * VOLATILE_REF_COST);
10021   effect(KILL cr);
10022   format %{
10023     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
10024     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10025   %}
10026   ins_encode %{
10027     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10028                Assembler::xword, /*acquire*/ false, /*release*/ true,
10029                /*weak*/ true, noreg);
10030     __ csetw($res$$Register, Assembler::EQ);
10031   %}
10032   ins_pipe(pipe_slow);
10033 %}
10034 
10035 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
10036   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
10037   ins_cost(2 * VOLATILE_REF_COST);
10038   effect(KILL cr);
10039   format %{
10040     "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
10041     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10042   %}
10043   ins_encode %{
10044     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10045                Assembler::word, /*acquire*/ false, /*release*/ true,
10046                /*weak*/ true, noreg);
10047     __ csetw($res$$Register, Assembler::EQ);
10048   %}
10049   ins_pipe(pipe_slow);
10050 %}
10051 
10052 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
10053   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
10054   ins_cost(2 * VOLATILE_REF_COST);
10055   effect(KILL cr);
10056   format %{
10057     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
10058     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10059   %}
10060   ins_encode %{
10061     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10062                Assembler::xword, /*acquire*/ false, /*release*/ true,
10063                /*weak*/ true, noreg);
10064     __ csetw($res$$Register, Assembler::EQ);
10065   %}
10066   ins_pipe(pipe_slow);
10067 %}
10068 
10069 // END This section of the file is automatically generated. Do not edit --------------
10070 // ---------------------------------------------------------------------
10071 
10072 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
10073   match(Set prev (GetAndSetI mem newv));
10074   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
10075   ins_encode %{
10076     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10077   %}
10078   ins_pipe(pipe_serial);
10079 %}
10080 
10081 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
10082   match(Set prev (GetAndSetL mem newv));
10083   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
10084   ins_encode %{
10085     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
10086   %}
10087   ins_pipe(pipe_serial);
10088 %}
10089 
10090 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
10091   match(Set prev (GetAndSetN mem newv));
10092   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
10093   ins_encode %{
10094     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10095   %}
10096   ins_pipe(pipe_serial);
10097 %}
10098 
10099 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
10100   match(Set prev (GetAndSetP mem newv));
10101   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
10102   ins_encode %{
10103     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
10104   %}
10105   ins_pipe(pipe_serial);
10106 %}
10107 
10108 
10109 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
10110   match(Set newval (GetAndAddL mem incr));
10111   ins_cost(INSN_COST * 10);
10112   format %{ "get_and_addL $newval, [$mem], $incr" %}
10113   ins_encode %{
10114     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
10115   %}
10116   ins_pipe(pipe_serial);
10117 %}
10118 
10119 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
10120   predicate(n->as_LoadStore()->result_not_used());
10121   match(Set dummy (GetAndAddL mem incr));
10122   ins_cost(INSN_COST * 9);
10123   format %{ "get_and_addL [$mem], $incr" %}
10124   ins_encode %{
10125     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
10126   %}
10127   ins_pipe(pipe_serial);
10128 %}
10129 
10130 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10131   match(Set newval (GetAndAddL mem incr));
10132   ins_cost(INSN_COST * 10);
10133   format %{ "get_and_addL $newval, [$mem], $incr" %}
10134   ins_encode %{
10135     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
10136   %}
10137   ins_pipe(pipe_serial);
10138 %}
10139 
10140 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
10141   predicate(n->as_LoadStore()->result_not_used());
10142   match(Set dummy (GetAndAddL mem incr));
10143   ins_cost(INSN_COST * 9);
10144   format %{ "get_and_addL [$mem], $incr" %}
10145   ins_encode %{
10146     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
10147   %}
10148   ins_pipe(pipe_serial);
10149 %}
10150 
10151 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10152   match(Set newval (GetAndAddI mem incr));
10153   ins_cost(INSN_COST * 10);
10154   format %{ "get_and_addI $newval, [$mem], $incr" %}
10155   ins_encode %{
10156     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10157   %}
10158   ins_pipe(pipe_serial);
10159 %}
10160 
10161 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
10162   predicate(n->as_LoadStore()->result_not_used());
10163   match(Set dummy (GetAndAddI mem incr));
10164   ins_cost(INSN_COST * 9);
10165   format %{ "get_and_addI [$mem], $incr" %}
10166   ins_encode %{
10167     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
10168   %}
10169   ins_pipe(pipe_serial);
10170 %}
10171 
10172 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10173   match(Set newval (GetAndAddI mem incr));
10174   ins_cost(INSN_COST * 10);
10175   format %{ "get_and_addI $newval, [$mem], $incr" %}
10176   ins_encode %{
10177     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10178   %}
10179   ins_pipe(pipe_serial);
10180 %}
10181 
10182 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
10183   predicate(n->as_LoadStore()->result_not_used());
10184   match(Set dummy (GetAndAddI mem incr));
10185   ins_cost(INSN_COST * 9);
10186   format %{ "get_and_addI [$mem], $incr" %}
10187   ins_encode %{
10188     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
10189   %}
10190   ins_pipe(pipe_serial);
10191 %}
10192 
10193 // Manifest a CmpL result in an integer register.
10194 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10195 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10196 %{
10197   match(Set dst (CmpL3 src1 src2));
10198   effect(KILL flags);
10199 
10200   ins_cost(INSN_COST * 6);
10201   format %{
10202       "cmp $src1, $src2"
10203       "csetw $dst, ne"
10204       "cnegw $dst, lt"
10205   %}
10206   // format %{ "CmpL3 $dst, $src1, $src2" %}
10207   ins_encode %{
10208     __ cmp($src1$$Register, $src2$$Register);
10209     __ csetw($dst$$Register, Assembler::NE);
10210     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10211   %}
10212 
10213   ins_pipe(pipe_class_default);
10214 %}
10215 
10216 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10217 %{
10218   match(Set dst (CmpL3 src1 src2));
10219   effect(KILL flags);
10220 
10221   ins_cost(INSN_COST * 6);
10222   format %{
10223       "cmp $src1, $src2"
10224       "csetw $dst, ne"
10225       "cnegw $dst, lt"
10226   %}
10227   ins_encode %{
10228     int32_t con = (int32_t)$src2$$constant;
10229      if (con < 0) {
10230       __ adds(zr, $src1$$Register, -con);
10231     } else {
10232       __ subs(zr, $src1$$Register, con);
10233     }
10234     __ csetw($dst$$Register, Assembler::NE);
10235     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10236   %}
10237 
10238   ins_pipe(pipe_class_default);
10239 %}
10240 
10241 // ============================================================================
10242 // Conditional Move Instructions
10243 
10244 // n.b. we have identical rules for both a signed compare op (cmpOp)
10245 // and an unsigned compare op (cmpOpU). it would be nice if we could
10246 // define an op class which merged both inputs and use it to type the
10247 // argument to a single rule. unfortunatelyt his fails because the
10248 // opclass does not live up to the COND_INTER interface of its
10249 // component operands. When the generic code tries to negate the
10250 // operand it ends up running the generci Machoper::negate method
10251 // which throws a ShouldNotHappen. So, we have to provide two flavours
10252 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10253 
10254 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10255   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10256 
10257   ins_cost(INSN_COST * 2);
10258   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10259 
10260   ins_encode %{
10261     __ cselw(as_Register($dst$$reg),
10262              as_Register($src2$$reg),
10263              as_Register($src1$$reg),
10264              (Assembler::Condition)$cmp$$cmpcode);
10265   %}
10266 
10267   ins_pipe(icond_reg_reg);
10268 %}
10269 
10270 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10271   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10272 
10273   ins_cost(INSN_COST * 2);
10274   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10275 
10276   ins_encode %{
10277     __ cselw(as_Register($dst$$reg),
10278              as_Register($src2$$reg),
10279              as_Register($src1$$reg),
10280              (Assembler::Condition)$cmp$$cmpcode);
10281   %}
10282 
10283   ins_pipe(icond_reg_reg);
10284 %}
10285 
10286 // special cases where one arg is zero
10287 
10288 // n.b. this is selected in preference to the rule above because it
10289 // avoids loading constant 0 into a source register
10290 
10291 // TODO
10292 // we ought only to be able to cull one of these variants as the ideal
10293 // transforms ought always to order the zero consistently (to left/right?)
10294 
10295 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10296   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10297 
10298   ins_cost(INSN_COST * 2);
10299   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10300 
10301   ins_encode %{
10302     __ cselw(as_Register($dst$$reg),
10303              as_Register($src$$reg),
10304              zr,
10305              (Assembler::Condition)$cmp$$cmpcode);
10306   %}
10307 
10308   ins_pipe(icond_reg);
10309 %}
10310 
10311 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10312   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10313 
10314   ins_cost(INSN_COST * 2);
10315   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10316 
10317   ins_encode %{
10318     __ cselw(as_Register($dst$$reg),
10319              as_Register($src$$reg),
10320              zr,
10321              (Assembler::Condition)$cmp$$cmpcode);
10322   %}
10323 
10324   ins_pipe(icond_reg);
10325 %}
10326 
10327 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10328   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10329 
10330   ins_cost(INSN_COST * 2);
10331   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10332 
10333   ins_encode %{
10334     __ cselw(as_Register($dst$$reg),
10335              zr,
10336              as_Register($src$$reg),
10337              (Assembler::Condition)$cmp$$cmpcode);
10338   %}
10339 
10340   ins_pipe(icond_reg);
10341 %}
10342 
10343 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10344   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10345 
10346   ins_cost(INSN_COST * 2);
10347   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10348 
10349   ins_encode %{
10350     __ cselw(as_Register($dst$$reg),
10351              zr,
10352              as_Register($src$$reg),
10353              (Assembler::Condition)$cmp$$cmpcode);
10354   %}
10355 
10356   ins_pipe(icond_reg);
10357 %}
10358 
10359 // special case for creating a boolean 0 or 1
10360 
10361 // n.b. this is selected in preference to the rule above because it
10362 // avoids loading constants 0 and 1 into a source register
10363 
10364 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10365   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10366 
10367   ins_cost(INSN_COST * 2);
10368   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10369 
10370   ins_encode %{
10371     // equivalently
10372     // cset(as_Register($dst$$reg),
10373     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10374     __ csincw(as_Register($dst$$reg),
10375              zr,
10376              zr,
10377              (Assembler::Condition)$cmp$$cmpcode);
10378   %}
10379 
10380   ins_pipe(icond_none);
10381 %}
10382 
10383 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10384   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10385 
10386   ins_cost(INSN_COST * 2);
10387   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10388 
10389   ins_encode %{
10390     // equivalently
10391     // cset(as_Register($dst$$reg),
10392     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10393     __ csincw(as_Register($dst$$reg),
10394              zr,
10395              zr,
10396              (Assembler::Condition)$cmp$$cmpcode);
10397   %}
10398 
10399   ins_pipe(icond_none);
10400 %}
10401 
10402 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10403   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10404 
10405   ins_cost(INSN_COST * 2);
10406   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10407 
10408   ins_encode %{
10409     __ csel(as_Register($dst$$reg),
10410             as_Register($src2$$reg),
10411             as_Register($src1$$reg),
10412             (Assembler::Condition)$cmp$$cmpcode);
10413   %}
10414 
10415   ins_pipe(icond_reg_reg);
10416 %}
10417 
10418 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10419   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10420 
10421   ins_cost(INSN_COST * 2);
10422   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10423 
10424   ins_encode %{
10425     __ csel(as_Register($dst$$reg),
10426             as_Register($src2$$reg),
10427             as_Register($src1$$reg),
10428             (Assembler::Condition)$cmp$$cmpcode);
10429   %}
10430 
10431   ins_pipe(icond_reg_reg);
10432 %}
10433 
10434 // special cases where one arg is zero
10435 
10436 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10437   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10438 
10439   ins_cost(INSN_COST * 2);
10440   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10441 
10442   ins_encode %{
10443     __ csel(as_Register($dst$$reg),
10444             zr,
10445             as_Register($src$$reg),
10446             (Assembler::Condition)$cmp$$cmpcode);
10447   %}
10448 
10449   ins_pipe(icond_reg);
10450 %}
10451 
10452 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10453   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10454 
10455   ins_cost(INSN_COST * 2);
10456   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10457 
10458   ins_encode %{
10459     __ csel(as_Register($dst$$reg),
10460             zr,
10461             as_Register($src$$reg),
10462             (Assembler::Condition)$cmp$$cmpcode);
10463   %}
10464 
10465   ins_pipe(icond_reg);
10466 %}
10467 
10468 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10469   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10470 
10471   ins_cost(INSN_COST * 2);
10472   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10473 
10474   ins_encode %{
10475     __ csel(as_Register($dst$$reg),
10476             as_Register($src$$reg),
10477             zr,
10478             (Assembler::Condition)$cmp$$cmpcode);
10479   %}
10480 
10481   ins_pipe(icond_reg);
10482 %}
10483 
10484 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10485   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10486 
10487   ins_cost(INSN_COST * 2);
10488   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10489 
10490   ins_encode %{
10491     __ csel(as_Register($dst$$reg),
10492             as_Register($src$$reg),
10493             zr,
10494             (Assembler::Condition)$cmp$$cmpcode);
10495   %}
10496 
10497   ins_pipe(icond_reg);
10498 %}
10499 
10500 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10501   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10502 
10503   ins_cost(INSN_COST * 2);
10504   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10505 
10506   ins_encode %{
10507     __ csel(as_Register($dst$$reg),
10508             as_Register($src2$$reg),
10509             as_Register($src1$$reg),
10510             (Assembler::Condition)$cmp$$cmpcode);
10511   %}
10512 
10513   ins_pipe(icond_reg_reg);
10514 %}
10515 
10516 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10517   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10518 
10519   ins_cost(INSN_COST * 2);
10520   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10521 
10522   ins_encode %{
10523     __ csel(as_Register($dst$$reg),
10524             as_Register($src2$$reg),
10525             as_Register($src1$$reg),
10526             (Assembler::Condition)$cmp$$cmpcode);
10527   %}
10528 
10529   ins_pipe(icond_reg_reg);
10530 %}
10531 
10532 // special cases where one arg is zero
10533 
10534 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10535   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10536 
10537   ins_cost(INSN_COST * 2);
10538   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10539 
10540   ins_encode %{
10541     __ csel(as_Register($dst$$reg),
10542             zr,
10543             as_Register($src$$reg),
10544             (Assembler::Condition)$cmp$$cmpcode);
10545   %}
10546 
10547   ins_pipe(icond_reg);
10548 %}
10549 
10550 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10551   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10552 
10553   ins_cost(INSN_COST * 2);
10554   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10555 
10556   ins_encode %{
10557     __ csel(as_Register($dst$$reg),
10558             zr,
10559             as_Register($src$$reg),
10560             (Assembler::Condition)$cmp$$cmpcode);
10561   %}
10562 
10563   ins_pipe(icond_reg);
10564 %}
10565 
10566 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10567   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10568 
10569   ins_cost(INSN_COST * 2);
10570   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10571 
10572   ins_encode %{
10573     __ csel(as_Register($dst$$reg),
10574             as_Register($src$$reg),
10575             zr,
10576             (Assembler::Condition)$cmp$$cmpcode);
10577   %}
10578 
10579   ins_pipe(icond_reg);
10580 %}
10581 
10582 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10583   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10584 
10585   ins_cost(INSN_COST * 2);
10586   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10587 
10588   ins_encode %{
10589     __ csel(as_Register($dst$$reg),
10590             as_Register($src$$reg),
10591             zr,
10592             (Assembler::Condition)$cmp$$cmpcode);
10593   %}
10594 
10595   ins_pipe(icond_reg);
10596 %}
10597 
10598 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10599   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10600 
10601   ins_cost(INSN_COST * 2);
10602   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10603 
10604   ins_encode %{
10605     __ cselw(as_Register($dst$$reg),
10606              as_Register($src2$$reg),
10607              as_Register($src1$$reg),
10608              (Assembler::Condition)$cmp$$cmpcode);
10609   %}
10610 
10611   ins_pipe(icond_reg_reg);
10612 %}
10613 
10614 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10615   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10616 
10617   ins_cost(INSN_COST * 2);
10618   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10619 
10620   ins_encode %{
10621     __ cselw(as_Register($dst$$reg),
10622              as_Register($src2$$reg),
10623              as_Register($src1$$reg),
10624              (Assembler::Condition)$cmp$$cmpcode);
10625   %}
10626 
10627   ins_pipe(icond_reg_reg);
10628 %}
10629 
10630 // special cases where one arg is zero
10631 
10632 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10633   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10634 
10635   ins_cost(INSN_COST * 2);
10636   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10637 
10638   ins_encode %{
10639     __ cselw(as_Register($dst$$reg),
10640              zr,
10641              as_Register($src$$reg),
10642              (Assembler::Condition)$cmp$$cmpcode);
10643   %}
10644 
10645   ins_pipe(icond_reg);
10646 %}
10647 
10648 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10649   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10650 
10651   ins_cost(INSN_COST * 2);
10652   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10653 
10654   ins_encode %{
10655     __ cselw(as_Register($dst$$reg),
10656              zr,
10657              as_Register($src$$reg),
10658              (Assembler::Condition)$cmp$$cmpcode);
10659   %}
10660 
10661   ins_pipe(icond_reg);
10662 %}
10663 
10664 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10665   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10666 
10667   ins_cost(INSN_COST * 2);
10668   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10669 
10670   ins_encode %{
10671     __ cselw(as_Register($dst$$reg),
10672              as_Register($src$$reg),
10673              zr,
10674              (Assembler::Condition)$cmp$$cmpcode);
10675   %}
10676 
10677   ins_pipe(icond_reg);
10678 %}
10679 
10680 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10681   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10682 
10683   ins_cost(INSN_COST * 2);
10684   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10685 
10686   ins_encode %{
10687     __ cselw(as_Register($dst$$reg),
10688              as_Register($src$$reg),
10689              zr,
10690              (Assembler::Condition)$cmp$$cmpcode);
10691   %}
10692 
10693   ins_pipe(icond_reg);
10694 %}
10695 
10696 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10697 %{
10698   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10699 
10700   ins_cost(INSN_COST * 3);
10701 
10702   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10703   ins_encode %{
10704     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10705     __ fcsels(as_FloatRegister($dst$$reg),
10706               as_FloatRegister($src2$$reg),
10707               as_FloatRegister($src1$$reg),
10708               cond);
10709   %}
10710 
10711   ins_pipe(fp_cond_reg_reg_s);
10712 %}
10713 
10714 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10715 %{
10716   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10717 
10718   ins_cost(INSN_COST * 3);
10719 
10720   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10721   ins_encode %{
10722     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10723     __ fcsels(as_FloatRegister($dst$$reg),
10724               as_FloatRegister($src2$$reg),
10725               as_FloatRegister($src1$$reg),
10726               cond);
10727   %}
10728 
10729   ins_pipe(fp_cond_reg_reg_s);
10730 %}
10731 
10732 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10733 %{
10734   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10735 
10736   ins_cost(INSN_COST * 3);
10737 
10738   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10739   ins_encode %{
10740     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10741     __ fcseld(as_FloatRegister($dst$$reg),
10742               as_FloatRegister($src2$$reg),
10743               as_FloatRegister($src1$$reg),
10744               cond);
10745   %}
10746 
10747   ins_pipe(fp_cond_reg_reg_d);
10748 %}
10749 
10750 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10751 %{
10752   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10753 
10754   ins_cost(INSN_COST * 3);
10755 
10756   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10757   ins_encode %{
10758     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10759     __ fcseld(as_FloatRegister($dst$$reg),
10760               as_FloatRegister($src2$$reg),
10761               as_FloatRegister($src1$$reg),
10762               cond);
10763   %}
10764 
10765   ins_pipe(fp_cond_reg_reg_d);
10766 %}
10767 
10768 // ============================================================================
10769 // Arithmetic Instructions
10770 //
10771 
10772 // Integer Addition
10773 
10774 // TODO
10775 // these currently employ operations which do not set CR and hence are
10776 // not flagged as killing CR but we would like to isolate the cases
10777 // where we want to set flags from those where we don't. need to work
10778 // out how to do that.
10779 
10780 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10781   match(Set dst (AddI src1 src2));
10782 
10783   ins_cost(INSN_COST);
10784   format %{ "addw  $dst, $src1, $src2" %}
10785 
10786   ins_encode %{
10787     __ addw(as_Register($dst$$reg),
10788             as_Register($src1$$reg),
10789             as_Register($src2$$reg));
10790   %}
10791 
10792   ins_pipe(ialu_reg_reg);
10793 %}
10794 
10795 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10796   match(Set dst (AddI src1 src2));
10797 
10798   ins_cost(INSN_COST);
10799   format %{ "addw $dst, $src1, $src2" %}
10800 
10801   // use opcode to indicate that this is an add not a sub
10802   opcode(0x0);
10803 
10804   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10805 
10806   ins_pipe(ialu_reg_imm);
10807 %}
10808 
10809 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10810   match(Set dst (AddI (ConvL2I src1) src2));
10811 
10812   ins_cost(INSN_COST);
10813   format %{ "addw $dst, $src1, $src2" %}
10814 
10815   // use opcode to indicate that this is an add not a sub
10816   opcode(0x0);
10817 
10818   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10819 
10820   ins_pipe(ialu_reg_imm);
10821 %}
10822 
10823 // Pointer Addition
10824 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10825   match(Set dst (AddP src1 src2));
10826 
10827   ins_cost(INSN_COST);
10828   format %{ "add $dst, $src1, $src2\t# ptr" %}
10829 
10830   ins_encode %{
10831     __ add(as_Register($dst$$reg),
10832            as_Register($src1$$reg),
10833            as_Register($src2$$reg));
10834   %}
10835 
10836   ins_pipe(ialu_reg_reg);
10837 %}
10838 
10839 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10840   match(Set dst (AddP src1 (ConvI2L src2)));
10841 
10842   ins_cost(1.9 * INSN_COST);
10843   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10844 
10845   ins_encode %{
10846     __ add(as_Register($dst$$reg),
10847            as_Register($src1$$reg),
10848            as_Register($src2$$reg), ext::sxtw);
10849   %}
10850 
10851   ins_pipe(ialu_reg_reg);
10852 %}
10853 
10854 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10855   match(Set dst (AddP src1 (LShiftL src2 scale)));
10856 
10857   ins_cost(1.9 * INSN_COST);
10858   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10859 
10860   ins_encode %{
10861     __ lea(as_Register($dst$$reg),
10862            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10863                    Address::lsl($scale$$constant)));
10864   %}
10865 
10866   ins_pipe(ialu_reg_reg_shift);
10867 %}
10868 
10869 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10870   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10871 
10872   ins_cost(1.9 * INSN_COST);
10873   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10874 
10875   ins_encode %{
10876     __ lea(as_Register($dst$$reg),
10877            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10878                    Address::sxtw($scale$$constant)));
10879   %}
10880 
10881   ins_pipe(ialu_reg_reg_shift);
10882 %}
10883 
10884 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10885   match(Set dst (LShiftL (ConvI2L src) scale));
10886 
10887   ins_cost(INSN_COST);
10888   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10889 
10890   ins_encode %{
10891     __ sbfiz(as_Register($dst$$reg),
10892           as_Register($src$$reg),
10893           $scale$$constant & 63, MIN(32, (-$scale$$constant) & 63));
10894   %}
10895 
10896   ins_pipe(ialu_reg_shift);
10897 %}
10898 
10899 // Pointer Immediate Addition
10900 // n.b. this needs to be more expensive than using an indirect memory
10901 // operand
10902 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10903   match(Set dst (AddP src1 src2));
10904 
10905   ins_cost(INSN_COST);
10906   format %{ "add $dst, $src1, $src2\t# ptr" %}
10907 
10908   // use opcode to indicate that this is an add not a sub
10909   opcode(0x0);
10910 
10911   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10912 
10913   ins_pipe(ialu_reg_imm);
10914 %}
10915 
10916 // Long Addition
10917 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10918 
10919   match(Set dst (AddL src1 src2));
10920 
10921   ins_cost(INSN_COST);
10922   format %{ "add  $dst, $src1, $src2" %}
10923 
10924   ins_encode %{
10925     __ add(as_Register($dst$$reg),
10926            as_Register($src1$$reg),
10927            as_Register($src2$$reg));
10928   %}
10929 
10930   ins_pipe(ialu_reg_reg);
10931 %}
10932 
10933 // No constant pool entries requiredLong Immediate Addition.
10934 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10935   match(Set dst (AddL src1 src2));
10936 
10937   ins_cost(INSN_COST);
10938   format %{ "add $dst, $src1, $src2" %}
10939 
10940   // use opcode to indicate that this is an add not a sub
10941   opcode(0x0);
10942 
10943   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10944 
10945   ins_pipe(ialu_reg_imm);
10946 %}
10947 
10948 // Integer Subtraction
10949 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10950   match(Set dst (SubI src1 src2));
10951 
10952   ins_cost(INSN_COST);
10953   format %{ "subw  $dst, $src1, $src2" %}
10954 
10955   ins_encode %{
10956     __ subw(as_Register($dst$$reg),
10957             as_Register($src1$$reg),
10958             as_Register($src2$$reg));
10959   %}
10960 
10961   ins_pipe(ialu_reg_reg);
10962 %}
10963 
10964 // Immediate Subtraction
10965 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10966   match(Set dst (SubI src1 src2));
10967 
10968   ins_cost(INSN_COST);
10969   format %{ "subw $dst, $src1, $src2" %}
10970 
10971   // use opcode to indicate that this is a sub not an add
10972   opcode(0x1);
10973 
10974   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10975 
10976   ins_pipe(ialu_reg_imm);
10977 %}
10978 
10979 // Long Subtraction
10980 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10981 
10982   match(Set dst (SubL src1 src2));
10983 
10984   ins_cost(INSN_COST);
10985   format %{ "sub  $dst, $src1, $src2" %}
10986 
10987   ins_encode %{
10988     __ sub(as_Register($dst$$reg),
10989            as_Register($src1$$reg),
10990            as_Register($src2$$reg));
10991   %}
10992 
10993   ins_pipe(ialu_reg_reg);
10994 %}
10995 
10996 // No constant pool entries requiredLong Immediate Subtraction.
10997 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10998   match(Set dst (SubL src1 src2));
10999 
11000   ins_cost(INSN_COST);
11001   format %{ "sub$dst, $src1, $src2" %}
11002 
11003   // use opcode to indicate that this is a sub not an add
11004   opcode(0x1);
11005 
11006   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
11007 
11008   ins_pipe(ialu_reg_imm);
11009 %}
11010 
11011 // Integer Negation (special case for sub)
11012 
11013 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
11014   match(Set dst (SubI zero src));
11015 
11016   ins_cost(INSN_COST);
11017   format %{ "negw $dst, $src\t# int" %}
11018 
11019   ins_encode %{
11020     __ negw(as_Register($dst$$reg),
11021             as_Register($src$$reg));
11022   %}
11023 
11024   ins_pipe(ialu_reg);
11025 %}
11026 
11027 // Long Negation
11028 
11029 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
11030   match(Set dst (SubL zero src));
11031 
11032   ins_cost(INSN_COST);
11033   format %{ "neg $dst, $src\t# long" %}
11034 
11035   ins_encode %{
11036     __ neg(as_Register($dst$$reg),
11037            as_Register($src$$reg));
11038   %}
11039 
11040   ins_pipe(ialu_reg);
11041 %}
11042 
11043 // Integer Multiply
11044 
11045 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11046   match(Set dst (MulI src1 src2));
11047 
11048   ins_cost(INSN_COST * 3);
11049   format %{ "mulw  $dst, $src1, $src2" %}
11050 
11051   ins_encode %{
11052     __ mulw(as_Register($dst$$reg),
11053             as_Register($src1$$reg),
11054             as_Register($src2$$reg));
11055   %}
11056 
11057   ins_pipe(imul_reg_reg);
11058 %}
11059 
11060 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11061   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
11062 
11063   ins_cost(INSN_COST * 3);
11064   format %{ "smull  $dst, $src1, $src2" %}
11065 
11066   ins_encode %{
11067     __ smull(as_Register($dst$$reg),
11068              as_Register($src1$$reg),
11069              as_Register($src2$$reg));
11070   %}
11071 
11072   ins_pipe(imul_reg_reg);
11073 %}
11074 
11075 // Long Multiply
11076 
11077 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11078   match(Set dst (MulL src1 src2));
11079 
11080   ins_cost(INSN_COST * 5);
11081   format %{ "mul  $dst, $src1, $src2" %}
11082 
11083   ins_encode %{
11084     __ mul(as_Register($dst$$reg),
11085            as_Register($src1$$reg),
11086            as_Register($src2$$reg));
11087   %}
11088 
11089   ins_pipe(lmul_reg_reg);
11090 %}
11091 
11092 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
11093 %{
11094   match(Set dst (MulHiL src1 src2));
11095 
11096   ins_cost(INSN_COST * 7);
11097   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
11098 
11099   ins_encode %{
11100     __ smulh(as_Register($dst$$reg),
11101              as_Register($src1$$reg),
11102              as_Register($src2$$reg));
11103   %}
11104 
11105   ins_pipe(lmul_reg_reg);
11106 %}
11107 
11108 // Combined Integer Multiply & Add/Sub
11109 
11110 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11111   match(Set dst (AddI src3 (MulI src1 src2)));
11112 
11113   ins_cost(INSN_COST * 3);
11114   format %{ "madd  $dst, $src1, $src2, $src3" %}
11115 
11116   ins_encode %{
11117     __ maddw(as_Register($dst$$reg),
11118              as_Register($src1$$reg),
11119              as_Register($src2$$reg),
11120              as_Register($src3$$reg));
11121   %}
11122 
11123   ins_pipe(imac_reg_reg);
11124 %}
11125 
11126 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11127   match(Set dst (SubI src3 (MulI src1 src2)));
11128 
11129   ins_cost(INSN_COST * 3);
11130   format %{ "msub  $dst, $src1, $src2, $src3" %}
11131 
11132   ins_encode %{
11133     __ msubw(as_Register($dst$$reg),
11134              as_Register($src1$$reg),
11135              as_Register($src2$$reg),
11136              as_Register($src3$$reg));
11137   %}
11138 
11139   ins_pipe(imac_reg_reg);
11140 %}
11141 
11142 // Combined Long Multiply & Add/Sub
11143 
11144 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11145   match(Set dst (AddL src3 (MulL src1 src2)));
11146 
11147   ins_cost(INSN_COST * 5);
11148   format %{ "madd  $dst, $src1, $src2, $src3" %}
11149 
11150   ins_encode %{
11151     __ madd(as_Register($dst$$reg),
11152             as_Register($src1$$reg),
11153             as_Register($src2$$reg),
11154             as_Register($src3$$reg));
11155   %}
11156 
11157   ins_pipe(lmac_reg_reg);
11158 %}
11159 
11160 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11161   match(Set dst (SubL src3 (MulL src1 src2)));
11162 
11163   ins_cost(INSN_COST * 5);
11164   format %{ "msub  $dst, $src1, $src2, $src3" %}
11165 
11166   ins_encode %{
11167     __ msub(as_Register($dst$$reg),
11168             as_Register($src1$$reg),
11169             as_Register($src2$$reg),
11170             as_Register($src3$$reg));
11171   %}
11172 
11173   ins_pipe(lmac_reg_reg);
11174 %}
11175 
11176 // Integer Divide
11177 
11178 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11179   match(Set dst (DivI src1 src2));
11180 
11181   ins_cost(INSN_COST * 19);
11182   format %{ "sdivw  $dst, $src1, $src2" %}
11183 
11184   ins_encode(aarch64_enc_divw(dst, src1, src2));
11185   ins_pipe(idiv_reg_reg);
11186 %}
11187 
11188 instruct signExtract(iRegINoSp dst, iRegIorL2I src1, immI_31 div1, immI_31 div2) %{
11189   match(Set dst (URShiftI (RShiftI src1 div1) div2));
11190   ins_cost(INSN_COST);
11191   format %{ "lsrw $dst, $src1, $div1" %}
11192   ins_encode %{
11193     __ lsrw(as_Register($dst$$reg), as_Register($src1$$reg), 31);
11194   %}
11195   ins_pipe(ialu_reg_shift);
11196 %}
11197 
11198 instruct div2Round(iRegINoSp dst, iRegIorL2I src, immI_31 div1, immI_31 div2) %{
11199   match(Set dst (AddI src (URShiftI (RShiftI src div1) div2)));
11200   ins_cost(INSN_COST);
11201   format %{ "addw $dst, $src, LSR $div1" %}
11202 
11203   ins_encode %{
11204     __ addw(as_Register($dst$$reg),
11205               as_Register($src$$reg),
11206               as_Register($src$$reg),
11207               Assembler::LSR, 31);
11208   %}
11209   ins_pipe(ialu_reg);
11210 %}
11211 
11212 // Long Divide
11213 
11214 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11215   match(Set dst (DivL src1 src2));
11216 
11217   ins_cost(INSN_COST * 35);
11218   format %{ "sdiv   $dst, $src1, $src2" %}
11219 
11220   ins_encode(aarch64_enc_div(dst, src1, src2));
11221   ins_pipe(ldiv_reg_reg);
11222 %}
11223 
11224 instruct signExtractL(iRegLNoSp dst, iRegL src1, immI_63 div1, immI_63 div2) %{
11225   match(Set dst (URShiftL (RShiftL src1 div1) div2));
11226   ins_cost(INSN_COST);
11227   format %{ "lsr $dst, $src1, $div1" %}
11228   ins_encode %{
11229     __ lsr(as_Register($dst$$reg), as_Register($src1$$reg), 63);
11230   %}
11231   ins_pipe(ialu_reg_shift);
11232 %}
11233 
11234 instruct div2RoundL(iRegLNoSp dst, iRegL src, immI_63 div1, immI_63 div2) %{
11235   match(Set dst (AddL src (URShiftL (RShiftL src div1) div2)));
11236   ins_cost(INSN_COST);
11237   format %{ "add $dst, $src, $div1" %}
11238 
11239   ins_encode %{
11240     __ add(as_Register($dst$$reg),
11241               as_Register($src$$reg),
11242               as_Register($src$$reg),
11243               Assembler::LSR, 63);
11244   %}
11245   ins_pipe(ialu_reg);
11246 %}
11247 
11248 // Integer Remainder
11249 
11250 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11251   match(Set dst (ModI src1 src2));
11252 
11253   ins_cost(INSN_COST * 22);
11254   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11255             "msubw($dst, rscratch1, $src2, $src1" %}
11256 
11257   ins_encode(aarch64_enc_modw(dst, src1, src2));
11258   ins_pipe(idiv_reg_reg);
11259 %}
11260 
11261 // Long Remainder
11262 
11263 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11264   match(Set dst (ModL src1 src2));
11265 
11266   ins_cost(INSN_COST * 38);
11267   format %{ "sdiv   rscratch1, $src1, $src2\n"
11268             "msub($dst, rscratch1, $src2, $src1" %}
11269 
11270   ins_encode(aarch64_enc_mod(dst, src1, src2));
11271   ins_pipe(ldiv_reg_reg);
11272 %}
11273 
11274 // Integer Shifts
11275 
11276 // Shift Left Register
11277 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11278   match(Set dst (LShiftI src1 src2));
11279 
11280   ins_cost(INSN_COST * 2);
11281   format %{ "lslvw  $dst, $src1, $src2" %}
11282 
11283   ins_encode %{
11284     __ lslvw(as_Register($dst$$reg),
11285              as_Register($src1$$reg),
11286              as_Register($src2$$reg));
11287   %}
11288 
11289   ins_pipe(ialu_reg_reg_vshift);
11290 %}
11291 
11292 // Shift Left Immediate
11293 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11294   match(Set dst (LShiftI src1 src2));
11295 
11296   ins_cost(INSN_COST);
11297   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11298 
11299   ins_encode %{
11300     __ lslw(as_Register($dst$$reg),
11301             as_Register($src1$$reg),
11302             $src2$$constant & 0x1f);
11303   %}
11304 
11305   ins_pipe(ialu_reg_shift);
11306 %}
11307 
11308 // Shift Right Logical Register
11309 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11310   match(Set dst (URShiftI src1 src2));
11311 
11312   ins_cost(INSN_COST * 2);
11313   format %{ "lsrvw  $dst, $src1, $src2" %}
11314 
11315   ins_encode %{
11316     __ lsrvw(as_Register($dst$$reg),
11317              as_Register($src1$$reg),
11318              as_Register($src2$$reg));
11319   %}
11320 
11321   ins_pipe(ialu_reg_reg_vshift);
11322 %}
11323 
11324 // Shift Right Logical Immediate
11325 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11326   match(Set dst (URShiftI src1 src2));
11327 
11328   ins_cost(INSN_COST);
11329   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11330 
11331   ins_encode %{
11332     __ lsrw(as_Register($dst$$reg),
11333             as_Register($src1$$reg),
11334             $src2$$constant & 0x1f);
11335   %}
11336 
11337   ins_pipe(ialu_reg_shift);
11338 %}
11339 
11340 // Shift Right Arithmetic Register
11341 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11342   match(Set dst (RShiftI src1 src2));
11343 
11344   ins_cost(INSN_COST * 2);
11345   format %{ "asrvw  $dst, $src1, $src2" %}
11346 
11347   ins_encode %{
11348     __ asrvw(as_Register($dst$$reg),
11349              as_Register($src1$$reg),
11350              as_Register($src2$$reg));
11351   %}
11352 
11353   ins_pipe(ialu_reg_reg_vshift);
11354 %}
11355 
11356 // Shift Right Arithmetic Immediate
11357 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11358   match(Set dst (RShiftI src1 src2));
11359 
11360   ins_cost(INSN_COST);
11361   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11362 
11363   ins_encode %{
11364     __ asrw(as_Register($dst$$reg),
11365             as_Register($src1$$reg),
11366             $src2$$constant & 0x1f);
11367   %}
11368 
11369   ins_pipe(ialu_reg_shift);
11370 %}
11371 
11372 // Combined Int Mask and Right Shift (using UBFM)
11373 // TODO
11374 
11375 // Long Shifts
11376 
11377 // Shift Left Register
11378 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11379   match(Set dst (LShiftL src1 src2));
11380 
11381   ins_cost(INSN_COST * 2);
11382   format %{ "lslv  $dst, $src1, $src2" %}
11383 
11384   ins_encode %{
11385     __ lslv(as_Register($dst$$reg),
11386             as_Register($src1$$reg),
11387             as_Register($src2$$reg));
11388   %}
11389 
11390   ins_pipe(ialu_reg_reg_vshift);
11391 %}
11392 
11393 // Shift Left Immediate
11394 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11395   match(Set dst (LShiftL src1 src2));
11396 
11397   ins_cost(INSN_COST);
11398   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11399 
11400   ins_encode %{
11401     __ lsl(as_Register($dst$$reg),
11402             as_Register($src1$$reg),
11403             $src2$$constant & 0x3f);
11404   %}
11405 
11406   ins_pipe(ialu_reg_shift);
11407 %}
11408 
11409 // Shift Right Logical Register
11410 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11411   match(Set dst (URShiftL src1 src2));
11412 
11413   ins_cost(INSN_COST * 2);
11414   format %{ "lsrv  $dst, $src1, $src2" %}
11415 
11416   ins_encode %{
11417     __ lsrv(as_Register($dst$$reg),
11418             as_Register($src1$$reg),
11419             as_Register($src2$$reg));
11420   %}
11421 
11422   ins_pipe(ialu_reg_reg_vshift);
11423 %}
11424 
11425 // Shift Right Logical Immediate
11426 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11427   match(Set dst (URShiftL src1 src2));
11428 
11429   ins_cost(INSN_COST);
11430   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11431 
11432   ins_encode %{
11433     __ lsr(as_Register($dst$$reg),
11434            as_Register($src1$$reg),
11435            $src2$$constant & 0x3f);
11436   %}
11437 
11438   ins_pipe(ialu_reg_shift);
11439 %}
11440 
11441 // A special-case pattern for card table stores.
11442 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11443   match(Set dst (URShiftL (CastP2X src1) src2));
11444 
11445   ins_cost(INSN_COST);
11446   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11447 
11448   ins_encode %{
11449     __ lsr(as_Register($dst$$reg),
11450            as_Register($src1$$reg),
11451            $src2$$constant & 0x3f);
11452   %}
11453 
11454   ins_pipe(ialu_reg_shift);
11455 %}
11456 
11457 // Shift Right Arithmetic Register
11458 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11459   match(Set dst (RShiftL src1 src2));
11460 
11461   ins_cost(INSN_COST * 2);
11462   format %{ "asrv  $dst, $src1, $src2" %}
11463 
11464   ins_encode %{
11465     __ asrv(as_Register($dst$$reg),
11466             as_Register($src1$$reg),
11467             as_Register($src2$$reg));
11468   %}
11469 
11470   ins_pipe(ialu_reg_reg_vshift);
11471 %}
11472 
11473 // Shift Right Arithmetic Immediate
11474 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11475   match(Set dst (RShiftL src1 src2));
11476 
11477   ins_cost(INSN_COST);
11478   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11479 
11480   ins_encode %{
11481     __ asr(as_Register($dst$$reg),
11482            as_Register($src1$$reg),
11483            $src2$$constant & 0x3f);
11484   %}
11485 
11486   ins_pipe(ialu_reg_shift);
11487 %}
11488 
11489 // BEGIN This section of the file is automatically generated. Do not edit --------------
11490 
11491 instruct regL_not_reg(iRegLNoSp dst,
11492                          iRegL src1, immL_M1 m1,
11493                          rFlagsReg cr) %{
11494   match(Set dst (XorL src1 m1));
11495   ins_cost(INSN_COST);
11496   format %{ "eon  $dst, $src1, zr" %}
11497 
11498   ins_encode %{
11499     __ eon(as_Register($dst$$reg),
11500               as_Register($src1$$reg),
11501               zr,
11502               Assembler::LSL, 0);
11503   %}
11504 
11505   ins_pipe(ialu_reg);
11506 %}
11507 instruct regI_not_reg(iRegINoSp dst,
11508                          iRegIorL2I src1, immI_M1 m1,
11509                          rFlagsReg cr) %{
11510   match(Set dst (XorI src1 m1));
11511   ins_cost(INSN_COST);
11512   format %{ "eonw  $dst, $src1, zr" %}
11513 
11514   ins_encode %{
11515     __ eonw(as_Register($dst$$reg),
11516               as_Register($src1$$reg),
11517               zr,
11518               Assembler::LSL, 0);
11519   %}
11520 
11521   ins_pipe(ialu_reg);
11522 %}
11523 
11524 instruct AndI_reg_not_reg(iRegINoSp dst,
11525                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
11526                          rFlagsReg cr) %{
11527   match(Set dst (AndI src1 (XorI src2 m1)));
11528   ins_cost(INSN_COST);
11529   format %{ "bicw  $dst, $src1, $src2" %}
11530 
11531   ins_encode %{
11532     __ bicw(as_Register($dst$$reg),
11533               as_Register($src1$$reg),
11534               as_Register($src2$$reg),
11535               Assembler::LSL, 0);
11536   %}
11537 
11538   ins_pipe(ialu_reg_reg);
11539 %}
11540 
11541 instruct AndL_reg_not_reg(iRegLNoSp dst,
11542                          iRegL src1, iRegL src2, immL_M1 m1,
11543                          rFlagsReg cr) %{
11544   match(Set dst (AndL src1 (XorL src2 m1)));
11545   ins_cost(INSN_COST);
11546   format %{ "bic  $dst, $src1, $src2" %}
11547 
11548   ins_encode %{
11549     __ bic(as_Register($dst$$reg),
11550               as_Register($src1$$reg),
11551               as_Register($src2$$reg),
11552               Assembler::LSL, 0);
11553   %}
11554 
11555   ins_pipe(ialu_reg_reg);
11556 %}
11557 
11558 instruct OrI_reg_not_reg(iRegINoSp dst,
11559                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
11560                          rFlagsReg cr) %{
11561   match(Set dst (OrI src1 (XorI src2 m1)));
11562   ins_cost(INSN_COST);
11563   format %{ "ornw  $dst, $src1, $src2" %}
11564 
11565   ins_encode %{
11566     __ ornw(as_Register($dst$$reg),
11567               as_Register($src1$$reg),
11568               as_Register($src2$$reg),
11569               Assembler::LSL, 0);
11570   %}
11571 
11572   ins_pipe(ialu_reg_reg);
11573 %}
11574 
11575 instruct OrL_reg_not_reg(iRegLNoSp dst,
11576                          iRegL src1, iRegL src2, immL_M1 m1,
11577                          rFlagsReg cr) %{
11578   match(Set dst (OrL src1 (XorL src2 m1)));
11579   ins_cost(INSN_COST);
11580   format %{ "orn  $dst, $src1, $src2" %}
11581 
11582   ins_encode %{
11583     __ orn(as_Register($dst$$reg),
11584               as_Register($src1$$reg),
11585               as_Register($src2$$reg),
11586               Assembler::LSL, 0);
11587   %}
11588 
11589   ins_pipe(ialu_reg_reg);
11590 %}
11591 
11592 instruct XorI_reg_not_reg(iRegINoSp dst,
11593                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
11594                          rFlagsReg cr) %{
11595   match(Set dst (XorI m1 (XorI src2 src1)));
11596   ins_cost(INSN_COST);
11597   format %{ "eonw  $dst, $src1, $src2" %}
11598 
11599   ins_encode %{
11600     __ eonw(as_Register($dst$$reg),
11601               as_Register($src1$$reg),
11602               as_Register($src2$$reg),
11603               Assembler::LSL, 0);
11604   %}
11605 
11606   ins_pipe(ialu_reg_reg);
11607 %}
11608 
11609 instruct XorL_reg_not_reg(iRegLNoSp dst,
11610                          iRegL src1, iRegL src2, immL_M1 m1,
11611                          rFlagsReg cr) %{
11612   match(Set dst (XorL m1 (XorL src2 src1)));
11613   ins_cost(INSN_COST);
11614   format %{ "eon  $dst, $src1, $src2" %}
11615 
11616   ins_encode %{
11617     __ eon(as_Register($dst$$reg),
11618               as_Register($src1$$reg),
11619               as_Register($src2$$reg),
11620               Assembler::LSL, 0);
11621   %}
11622 
11623   ins_pipe(ialu_reg_reg);
11624 %}
11625 
11626 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11627                          iRegIorL2I src1, iRegIorL2I src2,
11628                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11629   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11630   ins_cost(1.9 * INSN_COST);
11631   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11632 
11633   ins_encode %{
11634     __ bicw(as_Register($dst$$reg),
11635               as_Register($src1$$reg),
11636               as_Register($src2$$reg),
11637               Assembler::LSR,
11638               $src3$$constant & 0x1f);
11639   %}
11640 
11641   ins_pipe(ialu_reg_reg_shift);
11642 %}
11643 
11644 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11645                          iRegL src1, iRegL src2,
11646                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11647   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11648   ins_cost(1.9 * INSN_COST);
11649   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11650 
11651   ins_encode %{
11652     __ bic(as_Register($dst$$reg),
11653               as_Register($src1$$reg),
11654               as_Register($src2$$reg),
11655               Assembler::LSR,
11656               $src3$$constant & 0x3f);
11657   %}
11658 
11659   ins_pipe(ialu_reg_reg_shift);
11660 %}
11661 
11662 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11663                          iRegIorL2I src1, iRegIorL2I src2,
11664                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11665   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11666   ins_cost(1.9 * INSN_COST);
11667   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11668 
11669   ins_encode %{
11670     __ bicw(as_Register($dst$$reg),
11671               as_Register($src1$$reg),
11672               as_Register($src2$$reg),
11673               Assembler::ASR,
11674               $src3$$constant & 0x1f);
11675   %}
11676 
11677   ins_pipe(ialu_reg_reg_shift);
11678 %}
11679 
11680 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11681                          iRegL src1, iRegL src2,
11682                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11683   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11684   ins_cost(1.9 * INSN_COST);
11685   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11686 
11687   ins_encode %{
11688     __ bic(as_Register($dst$$reg),
11689               as_Register($src1$$reg),
11690               as_Register($src2$$reg),
11691               Assembler::ASR,
11692               $src3$$constant & 0x3f);
11693   %}
11694 
11695   ins_pipe(ialu_reg_reg_shift);
11696 %}
11697 
11698 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11699                          iRegIorL2I src1, iRegIorL2I src2,
11700                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11701   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11702   ins_cost(1.9 * INSN_COST);
11703   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11704 
11705   ins_encode %{
11706     __ bicw(as_Register($dst$$reg),
11707               as_Register($src1$$reg),
11708               as_Register($src2$$reg),
11709               Assembler::LSL,
11710               $src3$$constant & 0x1f);
11711   %}
11712 
11713   ins_pipe(ialu_reg_reg_shift);
11714 %}
11715 
11716 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11717                          iRegL src1, iRegL src2,
11718                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11719   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11720   ins_cost(1.9 * INSN_COST);
11721   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11722 
11723   ins_encode %{
11724     __ bic(as_Register($dst$$reg),
11725               as_Register($src1$$reg),
11726               as_Register($src2$$reg),
11727               Assembler::LSL,
11728               $src3$$constant & 0x3f);
11729   %}
11730 
11731   ins_pipe(ialu_reg_reg_shift);
11732 %}
11733 
11734 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11735                          iRegIorL2I src1, iRegIorL2I src2,
11736                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11737   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11738   ins_cost(1.9 * INSN_COST);
11739   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11740 
11741   ins_encode %{
11742     __ eonw(as_Register($dst$$reg),
11743               as_Register($src1$$reg),
11744               as_Register($src2$$reg),
11745               Assembler::LSR,
11746               $src3$$constant & 0x1f);
11747   %}
11748 
11749   ins_pipe(ialu_reg_reg_shift);
11750 %}
11751 
11752 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11753                          iRegL src1, iRegL src2,
11754                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11755   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11756   ins_cost(1.9 * INSN_COST);
11757   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11758 
11759   ins_encode %{
11760     __ eon(as_Register($dst$$reg),
11761               as_Register($src1$$reg),
11762               as_Register($src2$$reg),
11763               Assembler::LSR,
11764               $src3$$constant & 0x3f);
11765   %}
11766 
11767   ins_pipe(ialu_reg_reg_shift);
11768 %}
11769 
11770 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11771                          iRegIorL2I src1, iRegIorL2I src2,
11772                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11773   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11774   ins_cost(1.9 * INSN_COST);
11775   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11776 
11777   ins_encode %{
11778     __ eonw(as_Register($dst$$reg),
11779               as_Register($src1$$reg),
11780               as_Register($src2$$reg),
11781               Assembler::ASR,
11782               $src3$$constant & 0x1f);
11783   %}
11784 
11785   ins_pipe(ialu_reg_reg_shift);
11786 %}
11787 
11788 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11789                          iRegL src1, iRegL src2,
11790                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11791   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11792   ins_cost(1.9 * INSN_COST);
11793   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11794 
11795   ins_encode %{
11796     __ eon(as_Register($dst$$reg),
11797               as_Register($src1$$reg),
11798               as_Register($src2$$reg),
11799               Assembler::ASR,
11800               $src3$$constant & 0x3f);
11801   %}
11802 
11803   ins_pipe(ialu_reg_reg_shift);
11804 %}
11805 
11806 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11807                          iRegIorL2I src1, iRegIorL2I src2,
11808                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11809   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11810   ins_cost(1.9 * INSN_COST);
11811   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11812 
11813   ins_encode %{
11814     __ eonw(as_Register($dst$$reg),
11815               as_Register($src1$$reg),
11816               as_Register($src2$$reg),
11817               Assembler::LSL,
11818               $src3$$constant & 0x1f);
11819   %}
11820 
11821   ins_pipe(ialu_reg_reg_shift);
11822 %}
11823 
11824 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11825                          iRegL src1, iRegL src2,
11826                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11827   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11828   ins_cost(1.9 * INSN_COST);
11829   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11830 
11831   ins_encode %{
11832     __ eon(as_Register($dst$$reg),
11833               as_Register($src1$$reg),
11834               as_Register($src2$$reg),
11835               Assembler::LSL,
11836               $src3$$constant & 0x3f);
11837   %}
11838 
11839   ins_pipe(ialu_reg_reg_shift);
11840 %}
11841 
11842 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11843                          iRegIorL2I src1, iRegIorL2I src2,
11844                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11845   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11846   ins_cost(1.9 * INSN_COST);
11847   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11848 
11849   ins_encode %{
11850     __ ornw(as_Register($dst$$reg),
11851               as_Register($src1$$reg),
11852               as_Register($src2$$reg),
11853               Assembler::LSR,
11854               $src3$$constant & 0x1f);
11855   %}
11856 
11857   ins_pipe(ialu_reg_reg_shift);
11858 %}
11859 
11860 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11861                          iRegL src1, iRegL src2,
11862                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11863   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11864   ins_cost(1.9 * INSN_COST);
11865   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11866 
11867   ins_encode %{
11868     __ orn(as_Register($dst$$reg),
11869               as_Register($src1$$reg),
11870               as_Register($src2$$reg),
11871               Assembler::LSR,
11872               $src3$$constant & 0x3f);
11873   %}
11874 
11875   ins_pipe(ialu_reg_reg_shift);
11876 %}
11877 
11878 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11879                          iRegIorL2I src1, iRegIorL2I src2,
11880                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11881   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11882   ins_cost(1.9 * INSN_COST);
11883   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11884 
11885   ins_encode %{
11886     __ ornw(as_Register($dst$$reg),
11887               as_Register($src1$$reg),
11888               as_Register($src2$$reg),
11889               Assembler::ASR,
11890               $src3$$constant & 0x1f);
11891   %}
11892 
11893   ins_pipe(ialu_reg_reg_shift);
11894 %}
11895 
11896 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11897                          iRegL src1, iRegL src2,
11898                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11899   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11900   ins_cost(1.9 * INSN_COST);
11901   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11902 
11903   ins_encode %{
11904     __ orn(as_Register($dst$$reg),
11905               as_Register($src1$$reg),
11906               as_Register($src2$$reg),
11907               Assembler::ASR,
11908               $src3$$constant & 0x3f);
11909   %}
11910 
11911   ins_pipe(ialu_reg_reg_shift);
11912 %}
11913 
11914 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11915                          iRegIorL2I src1, iRegIorL2I src2,
11916                          immI src3, immI_M1 src4, rFlagsReg cr) %{
11917   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11918   ins_cost(1.9 * INSN_COST);
11919   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11920 
11921   ins_encode %{
11922     __ ornw(as_Register($dst$$reg),
11923               as_Register($src1$$reg),
11924               as_Register($src2$$reg),
11925               Assembler::LSL,
11926               $src3$$constant & 0x1f);
11927   %}
11928 
11929   ins_pipe(ialu_reg_reg_shift);
11930 %}
11931 
11932 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11933                          iRegL src1, iRegL src2,
11934                          immI src3, immL_M1 src4, rFlagsReg cr) %{
11935   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11936   ins_cost(1.9 * INSN_COST);
11937   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11938 
11939   ins_encode %{
11940     __ orn(as_Register($dst$$reg),
11941               as_Register($src1$$reg),
11942               as_Register($src2$$reg),
11943               Assembler::LSL,
11944               $src3$$constant & 0x3f);
11945   %}
11946 
11947   ins_pipe(ialu_reg_reg_shift);
11948 %}
11949 
11950 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11951                          iRegIorL2I src1, iRegIorL2I src2,
11952                          immI src3, rFlagsReg cr) %{
11953   match(Set dst (AndI src1 (URShiftI src2 src3)));
11954 
11955   ins_cost(1.9 * INSN_COST);
11956   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11957 
11958   ins_encode %{
11959     __ andw(as_Register($dst$$reg),
11960               as_Register($src1$$reg),
11961               as_Register($src2$$reg),
11962               Assembler::LSR,
11963               $src3$$constant & 0x1f);
11964   %}
11965 
11966   ins_pipe(ialu_reg_reg_shift);
11967 %}
11968 
11969 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11970                          iRegL src1, iRegL src2,
11971                          immI src3, rFlagsReg cr) %{
11972   match(Set dst (AndL src1 (URShiftL src2 src3)));
11973 
11974   ins_cost(1.9 * INSN_COST);
11975   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11976 
11977   ins_encode %{
11978     __ andr(as_Register($dst$$reg),
11979               as_Register($src1$$reg),
11980               as_Register($src2$$reg),
11981               Assembler::LSR,
11982               $src3$$constant & 0x3f);
11983   %}
11984 
11985   ins_pipe(ialu_reg_reg_shift);
11986 %}
11987 
11988 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11989                          iRegIorL2I src1, iRegIorL2I src2,
11990                          immI src3, rFlagsReg cr) %{
11991   match(Set dst (AndI src1 (RShiftI src2 src3)));
11992 
11993   ins_cost(1.9 * INSN_COST);
11994   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11995 
11996   ins_encode %{
11997     __ andw(as_Register($dst$$reg),
11998               as_Register($src1$$reg),
11999               as_Register($src2$$reg),
12000               Assembler::ASR,
12001               $src3$$constant & 0x1f);
12002   %}
12003 
12004   ins_pipe(ialu_reg_reg_shift);
12005 %}
12006 
12007 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12008                          iRegL src1, iRegL src2,
12009                          immI src3, rFlagsReg cr) %{
12010   match(Set dst (AndL src1 (RShiftL src2 src3)));
12011 
12012   ins_cost(1.9 * INSN_COST);
12013   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12014 
12015   ins_encode %{
12016     __ andr(as_Register($dst$$reg),
12017               as_Register($src1$$reg),
12018               as_Register($src2$$reg),
12019               Assembler::ASR,
12020               $src3$$constant & 0x3f);
12021   %}
12022 
12023   ins_pipe(ialu_reg_reg_shift);
12024 %}
12025 
12026 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12027                          iRegIorL2I src1, iRegIorL2I src2,
12028                          immI src3, rFlagsReg cr) %{
12029   match(Set dst (AndI src1 (LShiftI src2 src3)));
12030 
12031   ins_cost(1.9 * INSN_COST);
12032   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12033 
12034   ins_encode %{
12035     __ andw(as_Register($dst$$reg),
12036               as_Register($src1$$reg),
12037               as_Register($src2$$reg),
12038               Assembler::LSL,
12039               $src3$$constant & 0x1f);
12040   %}
12041 
12042   ins_pipe(ialu_reg_reg_shift);
12043 %}
12044 
12045 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12046                          iRegL src1, iRegL src2,
12047                          immI src3, rFlagsReg cr) %{
12048   match(Set dst (AndL src1 (LShiftL src2 src3)));
12049 
12050   ins_cost(1.9 * INSN_COST);
12051   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12052 
12053   ins_encode %{
12054     __ andr(as_Register($dst$$reg),
12055               as_Register($src1$$reg),
12056               as_Register($src2$$reg),
12057               Assembler::LSL,
12058               $src3$$constant & 0x3f);
12059   %}
12060 
12061   ins_pipe(ialu_reg_reg_shift);
12062 %}
12063 
12064 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12065                          iRegIorL2I src1, iRegIorL2I src2,
12066                          immI src3, rFlagsReg cr) %{
12067   match(Set dst (XorI src1 (URShiftI src2 src3)));
12068 
12069   ins_cost(1.9 * INSN_COST);
12070   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12071 
12072   ins_encode %{
12073     __ eorw(as_Register($dst$$reg),
12074               as_Register($src1$$reg),
12075               as_Register($src2$$reg),
12076               Assembler::LSR,
12077               $src3$$constant & 0x1f);
12078   %}
12079 
12080   ins_pipe(ialu_reg_reg_shift);
12081 %}
12082 
12083 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12084                          iRegL src1, iRegL src2,
12085                          immI src3, rFlagsReg cr) %{
12086   match(Set dst (XorL src1 (URShiftL src2 src3)));
12087 
12088   ins_cost(1.9 * INSN_COST);
12089   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12090 
12091   ins_encode %{
12092     __ eor(as_Register($dst$$reg),
12093               as_Register($src1$$reg),
12094               as_Register($src2$$reg),
12095               Assembler::LSR,
12096               $src3$$constant & 0x3f);
12097   %}
12098 
12099   ins_pipe(ialu_reg_reg_shift);
12100 %}
12101 
12102 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12103                          iRegIorL2I src1, iRegIorL2I src2,
12104                          immI src3, rFlagsReg cr) %{
12105   match(Set dst (XorI src1 (RShiftI src2 src3)));
12106 
12107   ins_cost(1.9 * INSN_COST);
12108   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12109 
12110   ins_encode %{
12111     __ eorw(as_Register($dst$$reg),
12112               as_Register($src1$$reg),
12113               as_Register($src2$$reg),
12114               Assembler::ASR,
12115               $src3$$constant & 0x1f);
12116   %}
12117 
12118   ins_pipe(ialu_reg_reg_shift);
12119 %}
12120 
12121 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12122                          iRegL src1, iRegL src2,
12123                          immI src3, rFlagsReg cr) %{
12124   match(Set dst (XorL src1 (RShiftL src2 src3)));
12125 
12126   ins_cost(1.9 * INSN_COST);
12127   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12128 
12129   ins_encode %{
12130     __ eor(as_Register($dst$$reg),
12131               as_Register($src1$$reg),
12132               as_Register($src2$$reg),
12133               Assembler::ASR,
12134               $src3$$constant & 0x3f);
12135   %}
12136 
12137   ins_pipe(ialu_reg_reg_shift);
12138 %}
12139 
12140 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12141                          iRegIorL2I src1, iRegIorL2I src2,
12142                          immI src3, rFlagsReg cr) %{
12143   match(Set dst (XorI src1 (LShiftI src2 src3)));
12144 
12145   ins_cost(1.9 * INSN_COST);
12146   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12147 
12148   ins_encode %{
12149     __ eorw(as_Register($dst$$reg),
12150               as_Register($src1$$reg),
12151               as_Register($src2$$reg),
12152               Assembler::LSL,
12153               $src3$$constant & 0x1f);
12154   %}
12155 
12156   ins_pipe(ialu_reg_reg_shift);
12157 %}
12158 
12159 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12160                          iRegL src1, iRegL src2,
12161                          immI src3, rFlagsReg cr) %{
12162   match(Set dst (XorL src1 (LShiftL src2 src3)));
12163 
12164   ins_cost(1.9 * INSN_COST);
12165   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12166 
12167   ins_encode %{
12168     __ eor(as_Register($dst$$reg),
12169               as_Register($src1$$reg),
12170               as_Register($src2$$reg),
12171               Assembler::LSL,
12172               $src3$$constant & 0x3f);
12173   %}
12174 
12175   ins_pipe(ialu_reg_reg_shift);
12176 %}
12177 
12178 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12179                          iRegIorL2I src1, iRegIorL2I src2,
12180                          immI src3, rFlagsReg cr) %{
12181   match(Set dst (OrI src1 (URShiftI src2 src3)));
12182 
12183   ins_cost(1.9 * INSN_COST);
12184   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12185 
12186   ins_encode %{
12187     __ orrw(as_Register($dst$$reg),
12188               as_Register($src1$$reg),
12189               as_Register($src2$$reg),
12190               Assembler::LSR,
12191               $src3$$constant & 0x1f);
12192   %}
12193 
12194   ins_pipe(ialu_reg_reg_shift);
12195 %}
12196 
12197 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12198                          iRegL src1, iRegL src2,
12199                          immI src3, rFlagsReg cr) %{
12200   match(Set dst (OrL src1 (URShiftL src2 src3)));
12201 
12202   ins_cost(1.9 * INSN_COST);
12203   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12204 
12205   ins_encode %{
12206     __ orr(as_Register($dst$$reg),
12207               as_Register($src1$$reg),
12208               as_Register($src2$$reg),
12209               Assembler::LSR,
12210               $src3$$constant & 0x3f);
12211   %}
12212 
12213   ins_pipe(ialu_reg_reg_shift);
12214 %}
12215 
12216 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12217                          iRegIorL2I src1, iRegIorL2I src2,
12218                          immI src3, rFlagsReg cr) %{
12219   match(Set dst (OrI src1 (RShiftI src2 src3)));
12220 
12221   ins_cost(1.9 * INSN_COST);
12222   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12223 
12224   ins_encode %{
12225     __ orrw(as_Register($dst$$reg),
12226               as_Register($src1$$reg),
12227               as_Register($src2$$reg),
12228               Assembler::ASR,
12229               $src3$$constant & 0x1f);
12230   %}
12231 
12232   ins_pipe(ialu_reg_reg_shift);
12233 %}
12234 
12235 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12236                          iRegL src1, iRegL src2,
12237                          immI src3, rFlagsReg cr) %{
12238   match(Set dst (OrL src1 (RShiftL src2 src3)));
12239 
12240   ins_cost(1.9 * INSN_COST);
12241   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12242 
12243   ins_encode %{
12244     __ orr(as_Register($dst$$reg),
12245               as_Register($src1$$reg),
12246               as_Register($src2$$reg),
12247               Assembler::ASR,
12248               $src3$$constant & 0x3f);
12249   %}
12250 
12251   ins_pipe(ialu_reg_reg_shift);
12252 %}
12253 
12254 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12255                          iRegIorL2I src1, iRegIorL2I src2,
12256                          immI src3, rFlagsReg cr) %{
12257   match(Set dst (OrI src1 (LShiftI src2 src3)));
12258 
12259   ins_cost(1.9 * INSN_COST);
12260   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12261 
12262   ins_encode %{
12263     __ orrw(as_Register($dst$$reg),
12264               as_Register($src1$$reg),
12265               as_Register($src2$$reg),
12266               Assembler::LSL,
12267               $src3$$constant & 0x1f);
12268   %}
12269 
12270   ins_pipe(ialu_reg_reg_shift);
12271 %}
12272 
12273 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12274                          iRegL src1, iRegL src2,
12275                          immI src3, rFlagsReg cr) %{
12276   match(Set dst (OrL src1 (LShiftL src2 src3)));
12277 
12278   ins_cost(1.9 * INSN_COST);
12279   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12280 
12281   ins_encode %{
12282     __ orr(as_Register($dst$$reg),
12283               as_Register($src1$$reg),
12284               as_Register($src2$$reg),
12285               Assembler::LSL,
12286               $src3$$constant & 0x3f);
12287   %}
12288 
12289   ins_pipe(ialu_reg_reg_shift);
12290 %}
12291 
12292 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12293                          iRegIorL2I src1, iRegIorL2I src2,
12294                          immI src3, rFlagsReg cr) %{
12295   match(Set dst (AddI src1 (URShiftI src2 src3)));
12296 
12297   ins_cost(1.9 * INSN_COST);
12298   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12299 
12300   ins_encode %{
12301     __ addw(as_Register($dst$$reg),
12302               as_Register($src1$$reg),
12303               as_Register($src2$$reg),
12304               Assembler::LSR,
12305               $src3$$constant & 0x1f);
12306   %}
12307 
12308   ins_pipe(ialu_reg_reg_shift);
12309 %}
12310 
12311 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12312                          iRegL src1, iRegL src2,
12313                          immI src3, rFlagsReg cr) %{
12314   match(Set dst (AddL src1 (URShiftL src2 src3)));
12315 
12316   ins_cost(1.9 * INSN_COST);
12317   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12318 
12319   ins_encode %{
12320     __ add(as_Register($dst$$reg),
12321               as_Register($src1$$reg),
12322               as_Register($src2$$reg),
12323               Assembler::LSR,
12324               $src3$$constant & 0x3f);
12325   %}
12326 
12327   ins_pipe(ialu_reg_reg_shift);
12328 %}
12329 
12330 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12331                          iRegIorL2I src1, iRegIorL2I src2,
12332                          immI src3, rFlagsReg cr) %{
12333   match(Set dst (AddI src1 (RShiftI src2 src3)));
12334 
12335   ins_cost(1.9 * INSN_COST);
12336   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12337 
12338   ins_encode %{
12339     __ addw(as_Register($dst$$reg),
12340               as_Register($src1$$reg),
12341               as_Register($src2$$reg),
12342               Assembler::ASR,
12343               $src3$$constant & 0x1f);
12344   %}
12345 
12346   ins_pipe(ialu_reg_reg_shift);
12347 %}
12348 
12349 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12350                          iRegL src1, iRegL src2,
12351                          immI src3, rFlagsReg cr) %{
12352   match(Set dst (AddL src1 (RShiftL src2 src3)));
12353 
12354   ins_cost(1.9 * INSN_COST);
12355   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12356 
12357   ins_encode %{
12358     __ add(as_Register($dst$$reg),
12359               as_Register($src1$$reg),
12360               as_Register($src2$$reg),
12361               Assembler::ASR,
12362               $src3$$constant & 0x3f);
12363   %}
12364 
12365   ins_pipe(ialu_reg_reg_shift);
12366 %}
12367 
12368 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12369                          iRegIorL2I src1, iRegIorL2I src2,
12370                          immI src3, rFlagsReg cr) %{
12371   match(Set dst (AddI src1 (LShiftI src2 src3)));
12372 
12373   ins_cost(1.9 * INSN_COST);
12374   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12375 
12376   ins_encode %{
12377     __ addw(as_Register($dst$$reg),
12378               as_Register($src1$$reg),
12379               as_Register($src2$$reg),
12380               Assembler::LSL,
12381               $src3$$constant & 0x1f);
12382   %}
12383 
12384   ins_pipe(ialu_reg_reg_shift);
12385 %}
12386 
12387 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12388                          iRegL src1, iRegL src2,
12389                          immI src3, rFlagsReg cr) %{
12390   match(Set dst (AddL src1 (LShiftL src2 src3)));
12391 
12392   ins_cost(1.9 * INSN_COST);
12393   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12394 
12395   ins_encode %{
12396     __ add(as_Register($dst$$reg),
12397               as_Register($src1$$reg),
12398               as_Register($src2$$reg),
12399               Assembler::LSL,
12400               $src3$$constant & 0x3f);
12401   %}
12402 
12403   ins_pipe(ialu_reg_reg_shift);
12404 %}
12405 
12406 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12407                          iRegIorL2I src1, iRegIorL2I src2,
12408                          immI src3, rFlagsReg cr) %{
12409   match(Set dst (SubI src1 (URShiftI src2 src3)));
12410 
12411   ins_cost(1.9 * INSN_COST);
12412   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12413 
12414   ins_encode %{
12415     __ subw(as_Register($dst$$reg),
12416               as_Register($src1$$reg),
12417               as_Register($src2$$reg),
12418               Assembler::LSR,
12419               $src3$$constant & 0x1f);
12420   %}
12421 
12422   ins_pipe(ialu_reg_reg_shift);
12423 %}
12424 
12425 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12426                          iRegL src1, iRegL src2,
12427                          immI src3, rFlagsReg cr) %{
12428   match(Set dst (SubL src1 (URShiftL src2 src3)));
12429 
12430   ins_cost(1.9 * INSN_COST);
12431   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12432 
12433   ins_encode %{
12434     __ sub(as_Register($dst$$reg),
12435               as_Register($src1$$reg),
12436               as_Register($src2$$reg),
12437               Assembler::LSR,
12438               $src3$$constant & 0x3f);
12439   %}
12440 
12441   ins_pipe(ialu_reg_reg_shift);
12442 %}
12443 
12444 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12445                          iRegIorL2I src1, iRegIorL2I src2,
12446                          immI src3, rFlagsReg cr) %{
12447   match(Set dst (SubI src1 (RShiftI src2 src3)));
12448 
12449   ins_cost(1.9 * INSN_COST);
12450   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12451 
12452   ins_encode %{
12453     __ subw(as_Register($dst$$reg),
12454               as_Register($src1$$reg),
12455               as_Register($src2$$reg),
12456               Assembler::ASR,
12457               $src3$$constant & 0x1f);
12458   %}
12459 
12460   ins_pipe(ialu_reg_reg_shift);
12461 %}
12462 
12463 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12464                          iRegL src1, iRegL src2,
12465                          immI src3, rFlagsReg cr) %{
12466   match(Set dst (SubL src1 (RShiftL src2 src3)));
12467 
12468   ins_cost(1.9 * INSN_COST);
12469   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12470 
12471   ins_encode %{
12472     __ sub(as_Register($dst$$reg),
12473               as_Register($src1$$reg),
12474               as_Register($src2$$reg),
12475               Assembler::ASR,
12476               $src3$$constant & 0x3f);
12477   %}
12478 
12479   ins_pipe(ialu_reg_reg_shift);
12480 %}
12481 
12482 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12483                          iRegIorL2I src1, iRegIorL2I src2,
12484                          immI src3, rFlagsReg cr) %{
12485   match(Set dst (SubI src1 (LShiftI src2 src3)));
12486 
12487   ins_cost(1.9 * INSN_COST);
12488   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12489 
12490   ins_encode %{
12491     __ subw(as_Register($dst$$reg),
12492               as_Register($src1$$reg),
12493               as_Register($src2$$reg),
12494               Assembler::LSL,
12495               $src3$$constant & 0x1f);
12496   %}
12497 
12498   ins_pipe(ialu_reg_reg_shift);
12499 %}
12500 
12501 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12502                          iRegL src1, iRegL src2,
12503                          immI src3, rFlagsReg cr) %{
12504   match(Set dst (SubL src1 (LShiftL src2 src3)));
12505 
12506   ins_cost(1.9 * INSN_COST);
12507   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12508 
12509   ins_encode %{
12510     __ sub(as_Register($dst$$reg),
12511               as_Register($src1$$reg),
12512               as_Register($src2$$reg),
12513               Assembler::LSL,
12514               $src3$$constant & 0x3f);
12515   %}
12516 
12517   ins_pipe(ialu_reg_reg_shift);
12518 %}
12519 
12520 
12521 
12522 // Shift Left followed by Shift Right.
12523 // This idiom is used by the compiler for the i2b bytecode etc.
12524 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12525 %{
12526   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12527   // Make sure we are not going to exceed what sbfm can do.
12528   predicate((unsigned int)n->in(2)->get_int() <= 63
12529             && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
12530 
12531   ins_cost(INSN_COST * 2);
12532   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12533   ins_encode %{
12534     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
12535     int s = 63 - lshift;
12536     int r = (rshift - lshift) & 63;
12537     __ sbfm(as_Register($dst$$reg),
12538             as_Register($src$$reg),
12539             r, s);
12540   %}
12541 
12542   ins_pipe(ialu_reg_shift);
12543 %}
12544 
12545 // Shift Left followed by Shift Right.
12546 // This idiom is used by the compiler for the i2b bytecode etc.
12547 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12548 %{
12549   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12550   // Make sure we are not going to exceed what sbfmw can do.
12551   predicate((unsigned int)n->in(2)->get_int() <= 31
12552             && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
12553 
12554   ins_cost(INSN_COST * 2);
12555   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12556   ins_encode %{
12557     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
12558     int s = 31 - lshift;
12559     int r = (rshift - lshift) & 31;
12560     __ sbfmw(as_Register($dst$$reg),
12561             as_Register($src$$reg),
12562             r, s);
12563   %}
12564 
12565   ins_pipe(ialu_reg_shift);
12566 %}
12567 
12568 // Shift Left followed by Shift Right.
12569 // This idiom is used by the compiler for the i2b bytecode etc.
12570 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12571 %{
12572   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12573   // Make sure we are not going to exceed what ubfm can do.
12574   predicate((unsigned int)n->in(2)->get_int() <= 63
12575             && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
12576 
12577   ins_cost(INSN_COST * 2);
12578   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12579   ins_encode %{
12580     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
12581     int s = 63 - lshift;
12582     int r = (rshift - lshift) & 63;
12583     __ ubfm(as_Register($dst$$reg),
12584             as_Register($src$$reg),
12585             r, s);
12586   %}
12587 
12588   ins_pipe(ialu_reg_shift);
12589 %}
12590 
12591 // Shift Left followed by Shift Right.
12592 // This idiom is used by the compiler for the i2b bytecode etc.
12593 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12594 %{
12595   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12596   // Make sure we are not going to exceed what ubfmw can do.
12597   predicate((unsigned int)n->in(2)->get_int() <= 31
12598             && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
12599 
12600   ins_cost(INSN_COST * 2);
12601   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12602   ins_encode %{
12603     int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
12604     int s = 31 - lshift;
12605     int r = (rshift - lshift) & 31;
12606     __ ubfmw(as_Register($dst$$reg),
12607             as_Register($src$$reg),
12608             r, s);
12609   %}
12610 
12611   ins_pipe(ialu_reg_shift);
12612 %}
12613 // Bitfield extract with shift & mask
12614 
12615 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12616 %{
12617   match(Set dst (AndI (URShiftI src rshift) mask));
12618 
12619   ins_cost(INSN_COST);
12620   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12621   ins_encode %{
12622     int rshift = $rshift$$constant;
12623     long mask = $mask$$constant;
12624     int width = exact_log2(mask+1);
12625     __ ubfxw(as_Register($dst$$reg),
12626             as_Register($src$$reg), rshift, width);
12627   %}
12628   ins_pipe(ialu_reg_shift);
12629 %}
12630 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12631 %{
12632   match(Set dst (AndL (URShiftL src rshift) mask));
12633 
12634   ins_cost(INSN_COST);
12635   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12636   ins_encode %{
12637     int rshift = $rshift$$constant;
12638     long mask = $mask$$constant;
12639     int width = exact_log2(mask+1);
12640     __ ubfx(as_Register($dst$$reg),
12641             as_Register($src$$reg), rshift, width);
12642   %}
12643   ins_pipe(ialu_reg_shift);
12644 %}
12645 
12646 // We can use ubfx when extending an And with a mask when we know mask
12647 // is positive.  We know that because immI_bitmask guarantees it.
12648 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12649 %{
12650   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12651 
12652   ins_cost(INSN_COST * 2);
12653   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12654   ins_encode %{
12655     int rshift = $rshift$$constant;
12656     long mask = $mask$$constant;
12657     int width = exact_log2(mask+1);
12658     __ ubfx(as_Register($dst$$reg),
12659             as_Register($src$$reg), rshift, width);
12660   %}
12661   ins_pipe(ialu_reg_shift);
12662 %}
12663 
12664 // We can use ubfiz when masking by a positive number and then left shifting the result.
12665 // We know that the mask is positive because immI_bitmask guarantees it.
12666 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12667 %{
12668   match(Set dst (LShiftI (AndI src mask) lshift));
12669   predicate((unsigned int)n->in(2)->get_int() <= 31 &&
12670     (exact_log2(n->in(1)->in(2)->get_int()+1) + (unsigned int)n->in(2)->get_int()) <= (31+1));
12671 
12672   ins_cost(INSN_COST);
12673   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12674   ins_encode %{
12675     int lshift = $lshift$$constant;
12676     long mask = $mask$$constant;
12677     int width = exact_log2(mask+1);
12678     __ ubfizw(as_Register($dst$$reg),
12679           as_Register($src$$reg), lshift, width);
12680   %}
12681   ins_pipe(ialu_reg_shift);
12682 %}
12683 // We can use ubfiz when masking by a positive number and then left shifting the result.
12684 // We know that the mask is positive because immL_bitmask guarantees it.
12685 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12686 %{
12687   match(Set dst (LShiftL (AndL src mask) lshift));
12688   predicate((unsigned int)n->in(2)->get_int() <= 63 &&
12689     (exact_log2_long(n->in(1)->in(2)->get_long()+1) + (unsigned int)n->in(2)->get_int()) <= (63+1));
12690 
12691   ins_cost(INSN_COST);
12692   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12693   ins_encode %{
12694     int lshift = $lshift$$constant;
12695     long mask = $mask$$constant;
12696     int width = exact_log2(mask+1);
12697     __ ubfiz(as_Register($dst$$reg),
12698           as_Register($src$$reg), lshift, width);
12699   %}
12700   ins_pipe(ialu_reg_shift);
12701 %}
12702 
12703 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12704 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12705 %{
12706   match(Set dst (LShiftL (ConvI2L(AndI src mask)) lshift));
12707   predicate((unsigned int)n->in(2)->get_int() <= 31 &&
12708     (exact_log2((unsigned int)n->in(1)->in(1)->in(2)->get_int()+1) + (unsigned int)n->in(2)->get_int()) <= 32);
12709 
12710   ins_cost(INSN_COST);
12711   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12712   ins_encode %{
12713     int lshift = $lshift$$constant;
12714     long mask = $mask$$constant;
12715     int width = exact_log2(mask+1);
12716     __ ubfiz(as_Register($dst$$reg),
12717              as_Register($src$$reg), lshift, width);
12718   %}
12719   ins_pipe(ialu_reg_shift);
12720 %}
12721 
12722 // Rotations
12723 
12724 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12725 %{
12726   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12727   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
12728 
12729   ins_cost(INSN_COST);
12730   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12731 
12732   ins_encode %{
12733     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12734             $rshift$$constant & 63);
12735   %}
12736   ins_pipe(ialu_reg_reg_extr);
12737 %}
12738 
12739 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12740 %{
12741   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12742   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
12743 
12744   ins_cost(INSN_COST);
12745   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12746 
12747   ins_encode %{
12748     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12749             $rshift$$constant & 31);
12750   %}
12751   ins_pipe(ialu_reg_reg_extr);
12752 %}
12753 
12754 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12755 %{
12756   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12757   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
12758 
12759   ins_cost(INSN_COST);
12760   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12761 
12762   ins_encode %{
12763     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12764             $rshift$$constant & 63);
12765   %}
12766   ins_pipe(ialu_reg_reg_extr);
12767 %}
12768 
12769 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12770 %{
12771   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12772   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
12773 
12774   ins_cost(INSN_COST);
12775   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12776 
12777   ins_encode %{
12778     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12779             $rshift$$constant & 31);
12780   %}
12781   ins_pipe(ialu_reg_reg_extr);
12782 %}
12783 
12784 
12785 // rol expander
12786 
12787 instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
12788 %{
12789   effect(DEF dst, USE src, USE shift);
12790 
12791   format %{ "rol    $dst, $src, $shift" %}
12792   ins_cost(INSN_COST * 3);
12793   ins_encode %{
12794     __ subw(rscratch1, zr, as_Register($shift$$reg));
12795     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
12796             rscratch1);
12797     %}
12798   ins_pipe(ialu_reg_reg_vshift);
12799 %}
12800 
12801 // rol expander
12802 
12803 instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
12804 %{
12805   effect(DEF dst, USE src, USE shift);
12806 
12807   format %{ "rol    $dst, $src, $shift" %}
12808   ins_cost(INSN_COST * 3);
12809   ins_encode %{
12810     __ subw(rscratch1, zr, as_Register($shift$$reg));
12811     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
12812             rscratch1);
12813     %}
12814   ins_pipe(ialu_reg_reg_vshift);
12815 %}
12816 
12817 instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
12818 %{
12819   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
12820 
12821   expand %{
12822     rolL_rReg(dst, src, shift, cr);
12823   %}
12824 %}
12825 
12826 instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
12827 %{
12828   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
12829 
12830   expand %{
12831     rolL_rReg(dst, src, shift, cr);
12832   %}
12833 %}
12834 
12835 instruct rolI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
12836 %{
12837   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
12838 
12839   expand %{
12840     rolI_rReg(dst, src, shift, cr);
12841   %}
12842 %}
12843 
12844 instruct rolI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
12845 %{
12846   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
12847 
12848   expand %{
12849     rolI_rReg(dst, src, shift, cr);
12850   %}
12851 %}
12852 
12853 // ror expander
12854 
12855 instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
12856 %{
12857   effect(DEF dst, USE src, USE shift);
12858 
12859   format %{ "ror    $dst, $src, $shift" %}
12860   ins_cost(INSN_COST);
12861   ins_encode %{
12862     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
12863             as_Register($shift$$reg));
12864     %}
12865   ins_pipe(ialu_reg_reg_vshift);
12866 %}
12867 
12868 // ror expander
12869 
12870 instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
12871 %{
12872   effect(DEF dst, USE src, USE shift);
12873 
12874   format %{ "ror    $dst, $src, $shift" %}
12875   ins_cost(INSN_COST);
12876   ins_encode %{
12877     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
12878             as_Register($shift$$reg));
12879     %}
12880   ins_pipe(ialu_reg_reg_vshift);
12881 %}
12882 
12883 instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
12884 %{
12885   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
12886 
12887   expand %{
12888     rorL_rReg(dst, src, shift, cr);
12889   %}
12890 %}
12891 
12892 instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
12893 %{
12894   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
12895 
12896   expand %{
12897     rorL_rReg(dst, src, shift, cr);
12898   %}
12899 %}
12900 
12901 instruct rorI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
12902 %{
12903   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
12904 
12905   expand %{
12906     rorI_rReg(dst, src, shift, cr);
12907   %}
12908 %}
12909 
12910 instruct rorI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
12911 %{
12912   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
12913 
12914   expand %{
12915     rorI_rReg(dst, src, shift, cr);
12916   %}
12917 %}
12918 
12919 // Add/subtract (extended)
12920 
12921 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12922 %{
12923   match(Set dst (AddL src1 (ConvI2L src2)));
12924   ins_cost(INSN_COST);
12925   format %{ "add  $dst, $src1, $src2, sxtw" %}
12926 
12927    ins_encode %{
12928      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12929             as_Register($src2$$reg), ext::sxtw);
12930    %}
12931   ins_pipe(ialu_reg_reg);
12932 %};
12933 
12934 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12935 %{
12936   match(Set dst (SubL src1 (ConvI2L src2)));
12937   ins_cost(INSN_COST);
12938   format %{ "sub  $dst, $src1, $src2, sxtw" %}
12939 
12940    ins_encode %{
12941      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12942             as_Register($src2$$reg), ext::sxtw);
12943    %}
12944   ins_pipe(ialu_reg_reg);
12945 %};
12946 
12947 
12948 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12949 %{
12950   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12951   ins_cost(INSN_COST);
12952   format %{ "add  $dst, $src1, $src2, sxth" %}
12953 
12954    ins_encode %{
12955      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12956             as_Register($src2$$reg), ext::sxth);
12957    %}
12958   ins_pipe(ialu_reg_reg);
12959 %}
12960 
12961 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12962 %{
12963   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12964   ins_cost(INSN_COST);
12965   format %{ "add  $dst, $src1, $src2, sxtb" %}
12966 
12967    ins_encode %{
12968      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12969             as_Register($src2$$reg), ext::sxtb);
12970    %}
12971   ins_pipe(ialu_reg_reg);
12972 %}
12973 
12974 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12975 %{
12976   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12977   ins_cost(INSN_COST);
12978   format %{ "add  $dst, $src1, $src2, uxtb" %}
12979 
12980    ins_encode %{
12981      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12982             as_Register($src2$$reg), ext::uxtb);
12983    %}
12984   ins_pipe(ialu_reg_reg);
12985 %}
12986 
12987 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12988 %{
12989   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12990   ins_cost(INSN_COST);
12991   format %{ "add  $dst, $src1, $src2, sxth" %}
12992 
12993    ins_encode %{
12994      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12995             as_Register($src2$$reg), ext::sxth);
12996    %}
12997   ins_pipe(ialu_reg_reg);
12998 %}
12999 
13000 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13001 %{
13002   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13003   ins_cost(INSN_COST);
13004   format %{ "add  $dst, $src1, $src2, sxtw" %}
13005 
13006    ins_encode %{
13007      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13008             as_Register($src2$$reg), ext::sxtw);
13009    %}
13010   ins_pipe(ialu_reg_reg);
13011 %}
13012 
13013 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13014 %{
13015   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13016   ins_cost(INSN_COST);
13017   format %{ "add  $dst, $src1, $src2, sxtb" %}
13018 
13019    ins_encode %{
13020      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13021             as_Register($src2$$reg), ext::sxtb);
13022    %}
13023   ins_pipe(ialu_reg_reg);
13024 %}
13025 
13026 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13027 %{
13028   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13029   ins_cost(INSN_COST);
13030   format %{ "add  $dst, $src1, $src2, uxtb" %}
13031 
13032    ins_encode %{
13033      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13034             as_Register($src2$$reg), ext::uxtb);
13035    %}
13036   ins_pipe(ialu_reg_reg);
13037 %}
13038 
13039 
13040 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13041 %{
13042   match(Set dst (AddI src1 (AndI src2 mask)));
13043   ins_cost(INSN_COST);
13044   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13045 
13046    ins_encode %{
13047      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13048             as_Register($src2$$reg), ext::uxtb);
13049    %}
13050   ins_pipe(ialu_reg_reg);
13051 %}
13052 
13053 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13054 %{
13055   match(Set dst (AddI src1 (AndI src2 mask)));
13056   ins_cost(INSN_COST);
13057   format %{ "addw  $dst, $src1, $src2, uxth" %}
13058 
13059    ins_encode %{
13060      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13061             as_Register($src2$$reg), ext::uxth);
13062    %}
13063   ins_pipe(ialu_reg_reg);
13064 %}
13065 
13066 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13067 %{
13068   match(Set dst (AddL src1 (AndL src2 mask)));
13069   ins_cost(INSN_COST);
13070   format %{ "add  $dst, $src1, $src2, uxtb" %}
13071 
13072    ins_encode %{
13073      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13074             as_Register($src2$$reg), ext::uxtb);
13075    %}
13076   ins_pipe(ialu_reg_reg);
13077 %}
13078 
13079 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13080 %{
13081   match(Set dst (AddL src1 (AndL src2 mask)));
13082   ins_cost(INSN_COST);
13083   format %{ "add  $dst, $src1, $src2, uxth" %}
13084 
13085    ins_encode %{
13086      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13087             as_Register($src2$$reg), ext::uxth);
13088    %}
13089   ins_pipe(ialu_reg_reg);
13090 %}
13091 
13092 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13093 %{
13094   match(Set dst (AddL src1 (AndL src2 mask)));
13095   ins_cost(INSN_COST);
13096   format %{ "add  $dst, $src1, $src2, uxtw" %}
13097 
13098    ins_encode %{
13099      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13100             as_Register($src2$$reg), ext::uxtw);
13101    %}
13102   ins_pipe(ialu_reg_reg);
13103 %}
13104 
13105 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13106 %{
13107   match(Set dst (SubI src1 (AndI src2 mask)));
13108   ins_cost(INSN_COST);
13109   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13110 
13111    ins_encode %{
13112      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13113             as_Register($src2$$reg), ext::uxtb);
13114    %}
13115   ins_pipe(ialu_reg_reg);
13116 %}
13117 
13118 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13119 %{
13120   match(Set dst (SubI src1 (AndI src2 mask)));
13121   ins_cost(INSN_COST);
13122   format %{ "subw  $dst, $src1, $src2, uxth" %}
13123 
13124    ins_encode %{
13125      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13126             as_Register($src2$$reg), ext::uxth);
13127    %}
13128   ins_pipe(ialu_reg_reg);
13129 %}
13130 
13131 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13132 %{
13133   match(Set dst (SubL src1 (AndL src2 mask)));
13134   ins_cost(INSN_COST);
13135   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13136 
13137    ins_encode %{
13138      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13139             as_Register($src2$$reg), ext::uxtb);
13140    %}
13141   ins_pipe(ialu_reg_reg);
13142 %}
13143 
13144 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13145 %{
13146   match(Set dst (SubL src1 (AndL src2 mask)));
13147   ins_cost(INSN_COST);
13148   format %{ "sub  $dst, $src1, $src2, uxth" %}
13149 
13150    ins_encode %{
13151      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13152             as_Register($src2$$reg), ext::uxth);
13153    %}
13154   ins_pipe(ialu_reg_reg);
13155 %}
13156 
13157 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13158 %{
13159   match(Set dst (SubL src1 (AndL src2 mask)));
13160   ins_cost(INSN_COST);
13161   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13162 
13163    ins_encode %{
13164      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13165             as_Register($src2$$reg), ext::uxtw);
13166    %}
13167   ins_pipe(ialu_reg_reg);
13168 %}
13169 
13170 
13171 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13172 %{
13173   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13174   ins_cost(1.9 * INSN_COST);
13175   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13176 
13177    ins_encode %{
13178      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13179             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13180    %}
13181   ins_pipe(ialu_reg_reg_shift);
13182 %}
13183 
13184 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13185 %{
13186   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13187   ins_cost(1.9 * INSN_COST);
13188   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13189 
13190    ins_encode %{
13191      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13192             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13193    %}
13194   ins_pipe(ialu_reg_reg_shift);
13195 %}
13196 
13197 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13198 %{
13199   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13200   ins_cost(1.9 * INSN_COST);
13201   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13202 
13203    ins_encode %{
13204      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13205             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13206    %}
13207   ins_pipe(ialu_reg_reg_shift);
13208 %}
13209 
13210 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13211 %{
13212   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13213   ins_cost(1.9 * INSN_COST);
13214   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13215 
13216    ins_encode %{
13217      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13218             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13219    %}
13220   ins_pipe(ialu_reg_reg_shift);
13221 %}
13222 
13223 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13224 %{
13225   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13226   ins_cost(1.9 * INSN_COST);
13227   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13228 
13229    ins_encode %{
13230      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13231             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13232    %}
13233   ins_pipe(ialu_reg_reg_shift);
13234 %}
13235 
13236 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13237 %{
13238   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13239   ins_cost(1.9 * INSN_COST);
13240   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13241 
13242    ins_encode %{
13243      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13244             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13245    %}
13246   ins_pipe(ialu_reg_reg_shift);
13247 %}
13248 
13249 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13250 %{
13251   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13252   ins_cost(1.9 * INSN_COST);
13253   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13254 
13255    ins_encode %{
13256      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13257             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13258    %}
13259   ins_pipe(ialu_reg_reg_shift);
13260 %}
13261 
13262 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13263 %{
13264   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13265   ins_cost(1.9 * INSN_COST);
13266   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13267 
13268    ins_encode %{
13269      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13270             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13271    %}
13272   ins_pipe(ialu_reg_reg_shift);
13273 %}
13274 
13275 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13276 %{
13277   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13278   ins_cost(1.9 * INSN_COST);
13279   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13280 
13281    ins_encode %{
13282      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13283             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13284    %}
13285   ins_pipe(ialu_reg_reg_shift);
13286 %}
13287 
13288 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13289 %{
13290   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13291   ins_cost(1.9 * INSN_COST);
13292   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13293 
13294    ins_encode %{
13295      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13296             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13297    %}
13298   ins_pipe(ialu_reg_reg_shift);
13299 %}
13300 
13301 
13302 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13303 %{
13304   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13305   ins_cost(1.9 * INSN_COST);
13306   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13307 
13308    ins_encode %{
13309      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13310             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13311    %}
13312   ins_pipe(ialu_reg_reg_shift);
13313 %};
13314 
13315 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13316 %{
13317   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13318   ins_cost(1.9 * INSN_COST);
13319   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13320 
13321    ins_encode %{
13322      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13323             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13324    %}
13325   ins_pipe(ialu_reg_reg_shift);
13326 %};
13327 
13328 
13329 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13330 %{
13331   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13332   ins_cost(1.9 * INSN_COST);
13333   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13334 
13335    ins_encode %{
13336      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13337             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13338    %}
13339   ins_pipe(ialu_reg_reg_shift);
13340 %}
13341 
13342 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13343 %{
13344   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13345   ins_cost(1.9 * INSN_COST);
13346   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13347 
13348    ins_encode %{
13349      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13350             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13351    %}
13352   ins_pipe(ialu_reg_reg_shift);
13353 %}
13354 
13355 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13356 %{
13357   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13358   ins_cost(1.9 * INSN_COST);
13359   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13360 
13361    ins_encode %{
13362      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13363             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13364    %}
13365   ins_pipe(ialu_reg_reg_shift);
13366 %}
13367 
13368 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13369 %{
13370   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13371   ins_cost(1.9 * INSN_COST);
13372   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13373 
13374    ins_encode %{
13375      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13376             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13377    %}
13378   ins_pipe(ialu_reg_reg_shift);
13379 %}
13380 
13381 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13382 %{
13383   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13384   ins_cost(1.9 * INSN_COST);
13385   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13386 
13387    ins_encode %{
13388      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13389             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13390    %}
13391   ins_pipe(ialu_reg_reg_shift);
13392 %}
13393 
13394 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13395 %{
13396   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13397   ins_cost(1.9 * INSN_COST);
13398   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13399 
13400    ins_encode %{
13401      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13402             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13403    %}
13404   ins_pipe(ialu_reg_reg_shift);
13405 %}
13406 
13407 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13408 %{
13409   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13410   ins_cost(1.9 * INSN_COST);
13411   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13412 
13413    ins_encode %{
13414      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13415             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13416    %}
13417   ins_pipe(ialu_reg_reg_shift);
13418 %}
13419 
13420 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13421 %{
13422   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13423   ins_cost(1.9 * INSN_COST);
13424   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13425 
13426    ins_encode %{
13427      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13428             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13429    %}
13430   ins_pipe(ialu_reg_reg_shift);
13431 %}
13432 
13433 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13434 %{
13435   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13436   ins_cost(1.9 * INSN_COST);
13437   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13438 
13439    ins_encode %{
13440      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13441             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13442    %}
13443   ins_pipe(ialu_reg_reg_shift);
13444 %}
13445 
13446 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13447 %{
13448   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13449   ins_cost(1.9 * INSN_COST);
13450   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13451 
13452    ins_encode %{
13453      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13454             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13455    %}
13456   ins_pipe(ialu_reg_reg_shift);
13457 %}
13458 // END This section of the file is automatically generated. Do not edit --------------
13459 
13460 // ============================================================================
13461 // Floating Point Arithmetic Instructions
13462 
13463 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13464   match(Set dst (AddF src1 src2));
13465 
13466   ins_cost(INSN_COST * 5);
13467   format %{ "fadds   $dst, $src1, $src2" %}
13468 
13469   ins_encode %{
13470     __ fadds(as_FloatRegister($dst$$reg),
13471              as_FloatRegister($src1$$reg),
13472              as_FloatRegister($src2$$reg));
13473   %}
13474 
13475   ins_pipe(fp_dop_reg_reg_s);
13476 %}
13477 
13478 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13479   match(Set dst (AddD src1 src2));
13480 
13481   ins_cost(INSN_COST * 5);
13482   format %{ "faddd   $dst, $src1, $src2" %}
13483 
13484   ins_encode %{
13485     __ faddd(as_FloatRegister($dst$$reg),
13486              as_FloatRegister($src1$$reg),
13487              as_FloatRegister($src2$$reg));
13488   %}
13489 
13490   ins_pipe(fp_dop_reg_reg_d);
13491 %}
13492 
13493 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13494   match(Set dst (SubF src1 src2));
13495 
13496   ins_cost(INSN_COST * 5);
13497   format %{ "fsubs   $dst, $src1, $src2" %}
13498 
13499   ins_encode %{
13500     __ fsubs(as_FloatRegister($dst$$reg),
13501              as_FloatRegister($src1$$reg),
13502              as_FloatRegister($src2$$reg));
13503   %}
13504 
13505   ins_pipe(fp_dop_reg_reg_s);
13506 %}
13507 
13508 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13509   match(Set dst (SubD src1 src2));
13510 
13511   ins_cost(INSN_COST * 5);
13512   format %{ "fsubd   $dst, $src1, $src2" %}
13513 
13514   ins_encode %{
13515     __ fsubd(as_FloatRegister($dst$$reg),
13516              as_FloatRegister($src1$$reg),
13517              as_FloatRegister($src2$$reg));
13518   %}
13519 
13520   ins_pipe(fp_dop_reg_reg_d);
13521 %}
13522 
13523 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13524   match(Set dst (MulF src1 src2));
13525 
13526   ins_cost(INSN_COST * 6);
13527   format %{ "fmuls   $dst, $src1, $src2" %}
13528 
13529   ins_encode %{
13530     __ fmuls(as_FloatRegister($dst$$reg),
13531              as_FloatRegister($src1$$reg),
13532              as_FloatRegister($src2$$reg));
13533   %}
13534 
13535   ins_pipe(fp_dop_reg_reg_s);
13536 %}
13537 
13538 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13539   match(Set dst (MulD src1 src2));
13540 
13541   ins_cost(INSN_COST * 6);
13542   format %{ "fmuld   $dst, $src1, $src2" %}
13543 
13544   ins_encode %{
13545     __ fmuld(as_FloatRegister($dst$$reg),
13546              as_FloatRegister($src1$$reg),
13547              as_FloatRegister($src2$$reg));
13548   %}
13549 
13550   ins_pipe(fp_dop_reg_reg_d);
13551 %}
13552 
13553 // src1 * src2 + src3
13554 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13555   predicate(UseFMA);
13556   match(Set dst (FmaF src3 (Binary src1 src2)));
13557 
13558   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13559 
13560   ins_encode %{
13561     __ fmadds(as_FloatRegister($dst$$reg),
13562              as_FloatRegister($src1$$reg),
13563              as_FloatRegister($src2$$reg),
13564              as_FloatRegister($src3$$reg));
13565   %}
13566 
13567   ins_pipe(pipe_class_default);
13568 %}
13569 
13570 // src1 * src2 + src3
13571 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13572   predicate(UseFMA);
13573   match(Set dst (FmaD src3 (Binary src1 src2)));
13574 
13575   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13576 
13577   ins_encode %{
13578     __ fmaddd(as_FloatRegister($dst$$reg),
13579              as_FloatRegister($src1$$reg),
13580              as_FloatRegister($src2$$reg),
13581              as_FloatRegister($src3$$reg));
13582   %}
13583 
13584   ins_pipe(pipe_class_default);
13585 %}
13586 
13587 // -src1 * src2 + src3
13588 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13589   predicate(UseFMA);
13590   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13591   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13592 
13593   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13594 
13595   ins_encode %{
13596     __ fmsubs(as_FloatRegister($dst$$reg),
13597               as_FloatRegister($src1$$reg),
13598               as_FloatRegister($src2$$reg),
13599               as_FloatRegister($src3$$reg));
13600   %}
13601 
13602   ins_pipe(pipe_class_default);
13603 %}
13604 
13605 // -src1 * src2 + src3
13606 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13607   predicate(UseFMA);
13608   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13609   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13610 
13611   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13612 
13613   ins_encode %{
13614     __ fmsubd(as_FloatRegister($dst$$reg),
13615               as_FloatRegister($src1$$reg),
13616               as_FloatRegister($src2$$reg),
13617               as_FloatRegister($src3$$reg));
13618   %}
13619 
13620   ins_pipe(pipe_class_default);
13621 %}
13622 
13623 // -src1 * src2 - src3
13624 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13625   predicate(UseFMA);
13626   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13627   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13628 
13629   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13630 
13631   ins_encode %{
13632     __ fnmadds(as_FloatRegister($dst$$reg),
13633                as_FloatRegister($src1$$reg),
13634                as_FloatRegister($src2$$reg),
13635                as_FloatRegister($src3$$reg));
13636   %}
13637 
13638   ins_pipe(pipe_class_default);
13639 %}
13640 
13641 // -src1 * src2 - src3
13642 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13643   predicate(UseFMA);
13644   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
13645   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13646 
13647   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
13648 
13649   ins_encode %{
13650     __ fnmaddd(as_FloatRegister($dst$$reg),
13651                as_FloatRegister($src1$$reg),
13652                as_FloatRegister($src2$$reg),
13653                as_FloatRegister($src3$$reg));
13654   %}
13655 
13656   ins_pipe(pipe_class_default);
13657 %}
13658 
13659 // src1 * src2 - src3
13660 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13661   predicate(UseFMA);
13662   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13663 
13664   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
13665 
13666   ins_encode %{
13667     __ fnmsubs(as_FloatRegister($dst$$reg),
13668                as_FloatRegister($src1$$reg),
13669                as_FloatRegister($src2$$reg),
13670                as_FloatRegister($src3$$reg));
13671   %}
13672 
13673   ins_pipe(pipe_class_default);
13674 %}
13675 
13676 // src1 * src2 - src3
13677 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13678   predicate(UseFMA);
13679   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13680 
13681   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
13682 
13683   ins_encode %{
13684   // n.b. insn name should be fnmsubd
13685     __ fnmsub(as_FloatRegister($dst$$reg),
13686               as_FloatRegister($src1$$reg),
13687               as_FloatRegister($src2$$reg),
13688               as_FloatRegister($src3$$reg));
13689   %}
13690 
13691   ins_pipe(pipe_class_default);
13692 %}
13693 
13694 
13695 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13696   match(Set dst (DivF src1  src2));
13697 
13698   ins_cost(INSN_COST * 18);
13699   format %{ "fdivs   $dst, $src1, $src2" %}
13700 
13701   ins_encode %{
13702     __ fdivs(as_FloatRegister($dst$$reg),
13703              as_FloatRegister($src1$$reg),
13704              as_FloatRegister($src2$$reg));
13705   %}
13706 
13707   ins_pipe(fp_div_s);
13708 %}
13709 
13710 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13711   match(Set dst (DivD src1  src2));
13712 
13713   ins_cost(INSN_COST * 32);
13714   format %{ "fdivd   $dst, $src1, $src2" %}
13715 
13716   ins_encode %{
13717     __ fdivd(as_FloatRegister($dst$$reg),
13718              as_FloatRegister($src1$$reg),
13719              as_FloatRegister($src2$$reg));
13720   %}
13721 
13722   ins_pipe(fp_div_d);
13723 %}
13724 
13725 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13726   match(Set dst (NegF src));
13727 
13728   ins_cost(INSN_COST * 3);
13729   format %{ "fneg   $dst, $src" %}
13730 
13731   ins_encode %{
13732     __ fnegs(as_FloatRegister($dst$$reg),
13733              as_FloatRegister($src$$reg));
13734   %}
13735 
13736   ins_pipe(fp_uop_s);
13737 %}
13738 
13739 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13740   match(Set dst (NegD src));
13741 
13742   ins_cost(INSN_COST * 3);
13743   format %{ "fnegd   $dst, $src" %}
13744 
13745   ins_encode %{
13746     __ fnegd(as_FloatRegister($dst$$reg),
13747              as_FloatRegister($src$$reg));
13748   %}
13749 
13750   ins_pipe(fp_uop_d);
13751 %}
13752 
13753 instruct absF_reg(vRegF dst, vRegF src) %{
13754   match(Set dst (AbsF src));
13755 
13756   ins_cost(INSN_COST * 3);
13757   format %{ "fabss   $dst, $src" %}
13758   ins_encode %{
13759     __ fabss(as_FloatRegister($dst$$reg),
13760              as_FloatRegister($src$$reg));
13761   %}
13762 
13763   ins_pipe(fp_uop_s);
13764 %}
13765 
13766 instruct absD_reg(vRegD dst, vRegD src) %{
13767   match(Set dst (AbsD src));
13768 
13769   ins_cost(INSN_COST * 3);
13770   format %{ "fabsd   $dst, $src" %}
13771   ins_encode %{
13772     __ fabsd(as_FloatRegister($dst$$reg),
13773              as_FloatRegister($src$$reg));
13774   %}
13775 
13776   ins_pipe(fp_uop_d);
13777 %}
13778 
13779 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13780   match(Set dst (SqrtD src));
13781 
13782   ins_cost(INSN_COST * 50);
13783   format %{ "fsqrtd  $dst, $src" %}
13784   ins_encode %{
13785     __ fsqrtd(as_FloatRegister($dst$$reg),
13786              as_FloatRegister($src$$reg));
13787   %}
13788 
13789   ins_pipe(fp_div_s);
13790 %}
13791 
13792 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13793   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
13794 
13795   ins_cost(INSN_COST * 50);
13796   format %{ "fsqrts  $dst, $src" %}
13797   ins_encode %{
13798     __ fsqrts(as_FloatRegister($dst$$reg),
13799              as_FloatRegister($src$$reg));
13800   %}
13801 
13802   ins_pipe(fp_div_d);
13803 %}
13804 
13805 // ============================================================================
13806 // Logical Instructions
13807 
13808 // Integer Logical Instructions
13809 
13810 // And Instructions
13811 
13812 
13813 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13814   match(Set dst (AndI src1 src2));
13815 
13816   format %{ "andw  $dst, $src1, $src2\t# int" %}
13817 
13818   ins_cost(INSN_COST);
13819   ins_encode %{
13820     __ andw(as_Register($dst$$reg),
13821             as_Register($src1$$reg),
13822             as_Register($src2$$reg));
13823   %}
13824 
13825   ins_pipe(ialu_reg_reg);
13826 %}
13827 
13828 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13829   match(Set dst (AndI src1 src2));
13830 
13831   format %{ "andsw  $dst, $src1, $src2\t# int" %}
13832 
13833   ins_cost(INSN_COST);
13834   ins_encode %{
13835     __ andw(as_Register($dst$$reg),
13836             as_Register($src1$$reg),
13837             (unsigned long)($src2$$constant));
13838   %}
13839 
13840   ins_pipe(ialu_reg_imm);
13841 %}
13842 
13843 // Or Instructions
13844 
13845 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13846   match(Set dst (OrI src1 src2));
13847 
13848   format %{ "orrw  $dst, $src1, $src2\t# int" %}
13849 
13850   ins_cost(INSN_COST);
13851   ins_encode %{
13852     __ orrw(as_Register($dst$$reg),
13853             as_Register($src1$$reg),
13854             as_Register($src2$$reg));
13855   %}
13856 
13857   ins_pipe(ialu_reg_reg);
13858 %}
13859 
13860 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13861   match(Set dst (OrI src1 src2));
13862 
13863   format %{ "orrw  $dst, $src1, $src2\t# int" %}
13864 
13865   ins_cost(INSN_COST);
13866   ins_encode %{
13867     __ orrw(as_Register($dst$$reg),
13868             as_Register($src1$$reg),
13869             (unsigned long)($src2$$constant));
13870   %}
13871 
13872   ins_pipe(ialu_reg_imm);
13873 %}
13874 
13875 // Xor Instructions
13876 
13877 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13878   match(Set dst (XorI src1 src2));
13879 
13880   format %{ "eorw  $dst, $src1, $src2\t# int" %}
13881 
13882   ins_cost(INSN_COST);
13883   ins_encode %{
13884     __ eorw(as_Register($dst$$reg),
13885             as_Register($src1$$reg),
13886             as_Register($src2$$reg));
13887   %}
13888 
13889   ins_pipe(ialu_reg_reg);
13890 %}
13891 
13892 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13893   match(Set dst (XorI src1 src2));
13894 
13895   format %{ "eorw  $dst, $src1, $src2\t# int" %}
13896 
13897   ins_cost(INSN_COST);
13898   ins_encode %{
13899     __ eorw(as_Register($dst$$reg),
13900             as_Register($src1$$reg),
13901             (unsigned long)($src2$$constant));
13902   %}
13903 
13904   ins_pipe(ialu_reg_imm);
13905 %}
13906 
13907 // Long Logical Instructions
13908 // TODO
13909 
13910 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13911   match(Set dst (AndL src1 src2));
13912 
13913   format %{ "and  $dst, $src1, $src2\t# int" %}
13914 
13915   ins_cost(INSN_COST);
13916   ins_encode %{
13917     __ andr(as_Register($dst$$reg),
13918             as_Register($src1$$reg),
13919             as_Register($src2$$reg));
13920   %}
13921 
13922   ins_pipe(ialu_reg_reg);
13923 %}
13924 
13925 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13926   match(Set dst (AndL src1 src2));
13927 
13928   format %{ "and  $dst, $src1, $src2\t# int" %}
13929 
13930   ins_cost(INSN_COST);
13931   ins_encode %{
13932     __ andr(as_Register($dst$$reg),
13933             as_Register($src1$$reg),
13934             (unsigned long)($src2$$constant));
13935   %}
13936 
13937   ins_pipe(ialu_reg_imm);
13938 %}
13939 
13940 // Or Instructions
13941 
13942 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13943   match(Set dst (OrL src1 src2));
13944 
13945   format %{ "orr  $dst, $src1, $src2\t# int" %}
13946 
13947   ins_cost(INSN_COST);
13948   ins_encode %{
13949     __ orr(as_Register($dst$$reg),
13950            as_Register($src1$$reg),
13951            as_Register($src2$$reg));
13952   %}
13953 
13954   ins_pipe(ialu_reg_reg);
13955 %}
13956 
13957 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13958   match(Set dst (OrL src1 src2));
13959 
13960   format %{ "orr  $dst, $src1, $src2\t# int" %}
13961 
13962   ins_cost(INSN_COST);
13963   ins_encode %{
13964     __ orr(as_Register($dst$$reg),
13965            as_Register($src1$$reg),
13966            (unsigned long)($src2$$constant));
13967   %}
13968 
13969   ins_pipe(ialu_reg_imm);
13970 %}
13971 
13972 // Xor Instructions
13973 
13974 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13975   match(Set dst (XorL src1 src2));
13976 
13977   format %{ "eor  $dst, $src1, $src2\t# int" %}
13978 
13979   ins_cost(INSN_COST);
13980   ins_encode %{
13981     __ eor(as_Register($dst$$reg),
13982            as_Register($src1$$reg),
13983            as_Register($src2$$reg));
13984   %}
13985 
13986   ins_pipe(ialu_reg_reg);
13987 %}
13988 
13989 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13990   match(Set dst (XorL src1 src2));
13991 
13992   ins_cost(INSN_COST);
13993   format %{ "eor  $dst, $src1, $src2\t# int" %}
13994 
13995   ins_encode %{
13996     __ eor(as_Register($dst$$reg),
13997            as_Register($src1$$reg),
13998            (unsigned long)($src2$$constant));
13999   %}
14000 
14001   ins_pipe(ialu_reg_imm);
14002 %}
14003 
14004 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14005 %{
14006   match(Set dst (ConvI2L src));
14007 
14008   ins_cost(INSN_COST);
14009   format %{ "sxtw  $dst, $src\t# i2l" %}
14010   ins_encode %{
14011     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14012   %}
14013   ins_pipe(ialu_reg_shift);
14014 %}
14015 
14016 // this pattern occurs in bigmath arithmetic
14017 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14018 %{
14019   match(Set dst (AndL (ConvI2L src) mask));
14020 
14021   ins_cost(INSN_COST);
14022   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14023   ins_encode %{
14024     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14025   %}
14026 
14027   ins_pipe(ialu_reg_shift);
14028 %}
14029 
14030 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14031   match(Set dst (ConvL2I src));
14032 
14033   ins_cost(INSN_COST);
14034   format %{ "movw  $dst, $src \t// l2i" %}
14035 
14036   ins_encode %{
14037     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14038   %}
14039 
14040   ins_pipe(ialu_reg);
14041 %}
14042 
14043 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14044 %{
14045   match(Set dst (Conv2B src));
14046   effect(KILL cr);
14047 
14048   format %{
14049     "cmpw $src, zr\n\t"
14050     "cset $dst, ne"
14051   %}
14052 
14053   ins_encode %{
14054     __ cmpw(as_Register($src$$reg), zr);
14055     __ cset(as_Register($dst$$reg), Assembler::NE);
14056   %}
14057 
14058   ins_pipe(ialu_reg);
14059 %}
14060 
14061 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14062 %{
14063   match(Set dst (Conv2B src));
14064   effect(KILL cr);
14065 
14066   format %{
14067     "cmp  $src, zr\n\t"
14068     "cset $dst, ne"
14069   %}
14070 
14071   ins_encode %{
14072     __ cmp(as_Register($src$$reg), zr);
14073     __ cset(as_Register($dst$$reg), Assembler::NE);
14074   %}
14075 
14076   ins_pipe(ialu_reg);
14077 %}
14078 
14079 instruct convD2F_reg(vRegF dst, vRegD src) %{
14080   match(Set dst (ConvD2F src));
14081 
14082   ins_cost(INSN_COST * 5);
14083   format %{ "fcvtd  $dst, $src \t// d2f" %}
14084 
14085   ins_encode %{
14086     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14087   %}
14088 
14089   ins_pipe(fp_d2f);
14090 %}
14091 
14092 instruct convF2D_reg(vRegD dst, vRegF src) %{
14093   match(Set dst (ConvF2D src));
14094 
14095   ins_cost(INSN_COST * 5);
14096   format %{ "fcvts  $dst, $src \t// f2d" %}
14097 
14098   ins_encode %{
14099     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14100   %}
14101 
14102   ins_pipe(fp_f2d);
14103 %}
14104 
14105 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14106   match(Set dst (ConvF2I src));
14107 
14108   ins_cost(INSN_COST * 5);
14109   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14110 
14111   ins_encode %{
14112     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14113   %}
14114 
14115   ins_pipe(fp_f2i);
14116 %}
14117 
14118 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14119   match(Set dst (ConvF2L src));
14120 
14121   ins_cost(INSN_COST * 5);
14122   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14123 
14124   ins_encode %{
14125     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14126   %}
14127 
14128   ins_pipe(fp_f2l);
14129 %}
14130 
14131 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14132   match(Set dst (ConvI2F src));
14133 
14134   ins_cost(INSN_COST * 5);
14135   format %{ "scvtfws  $dst, $src \t// i2f" %}
14136 
14137   ins_encode %{
14138     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14139   %}
14140 
14141   ins_pipe(fp_i2f);
14142 %}
14143 
14144 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14145   match(Set dst (ConvL2F src));
14146 
14147   ins_cost(INSN_COST * 5);
14148   format %{ "scvtfs  $dst, $src \t// l2f" %}
14149 
14150   ins_encode %{
14151     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14152   %}
14153 
14154   ins_pipe(fp_l2f);
14155 %}
14156 
14157 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14158   match(Set dst (ConvD2I src));
14159 
14160   ins_cost(INSN_COST * 5);
14161   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14162 
14163   ins_encode %{
14164     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14165   %}
14166 
14167   ins_pipe(fp_d2i);
14168 %}
14169 
14170 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14171   match(Set dst (ConvD2L src));
14172 
14173   ins_cost(INSN_COST * 5);
14174   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14175 
14176   ins_encode %{
14177     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14178   %}
14179 
14180   ins_pipe(fp_d2l);
14181 %}
14182 
14183 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14184   match(Set dst (ConvI2D src));
14185 
14186   ins_cost(INSN_COST * 5);
14187   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14188 
14189   ins_encode %{
14190     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14191   %}
14192 
14193   ins_pipe(fp_i2d);
14194 %}
14195 
14196 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14197   match(Set dst (ConvL2D src));
14198 
14199   ins_cost(INSN_COST * 5);
14200   format %{ "scvtfd  $dst, $src \t// l2d" %}
14201 
14202   ins_encode %{
14203     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14204   %}
14205 
14206   ins_pipe(fp_l2d);
14207 %}
14208 
14209 // stack <-> reg and reg <-> reg shuffles with no conversion
14210 
14211 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14212 
14213   match(Set dst (MoveF2I src));
14214 
14215   effect(DEF dst, USE src);
14216 
14217   ins_cost(4 * INSN_COST);
14218 
14219   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14220 
14221   ins_encode %{
14222     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14223   %}
14224 
14225   ins_pipe(iload_reg_reg);
14226 
14227 %}
14228 
14229 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14230 
14231   match(Set dst (MoveI2F src));
14232 
14233   effect(DEF dst, USE src);
14234 
14235   ins_cost(4 * INSN_COST);
14236 
14237   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14238 
14239   ins_encode %{
14240     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14241   %}
14242 
14243   ins_pipe(pipe_class_memory);
14244 
14245 %}
14246 
14247 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14248 
14249   match(Set dst (MoveD2L src));
14250 
14251   effect(DEF dst, USE src);
14252 
14253   ins_cost(4 * INSN_COST);
14254 
14255   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14256 
14257   ins_encode %{
14258     __ ldr($dst$$Register, Address(sp, $src$$disp));
14259   %}
14260 
14261   ins_pipe(iload_reg_reg);
14262 
14263 %}
14264 
14265 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14266 
14267   match(Set dst (MoveL2D src));
14268 
14269   effect(DEF dst, USE src);
14270 
14271   ins_cost(4 * INSN_COST);
14272 
14273   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14274 
14275   ins_encode %{
14276     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14277   %}
14278 
14279   ins_pipe(pipe_class_memory);
14280 
14281 %}
14282 
14283 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14284 
14285   match(Set dst (MoveF2I src));
14286 
14287   effect(DEF dst, USE src);
14288 
14289   ins_cost(INSN_COST);
14290 
14291   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14292 
14293   ins_encode %{
14294     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14295   %}
14296 
14297   ins_pipe(pipe_class_memory);
14298 
14299 %}
14300 
14301 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14302 
14303   match(Set dst (MoveI2F src));
14304 
14305   effect(DEF dst, USE src);
14306 
14307   ins_cost(INSN_COST);
14308 
14309   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14310 
14311   ins_encode %{
14312     __ strw($src$$Register, Address(sp, $dst$$disp));
14313   %}
14314 
14315   ins_pipe(istore_reg_reg);
14316 
14317 %}
14318 
14319 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14320 
14321   match(Set dst (MoveD2L src));
14322 
14323   effect(DEF dst, USE src);
14324 
14325   ins_cost(INSN_COST);
14326 
14327   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14328 
14329   ins_encode %{
14330     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14331   %}
14332 
14333   ins_pipe(pipe_class_memory);
14334 
14335 %}
14336 
14337 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14338 
14339   match(Set dst (MoveL2D src));
14340 
14341   effect(DEF dst, USE src);
14342 
14343   ins_cost(INSN_COST);
14344 
14345   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14346 
14347   ins_encode %{
14348     __ str($src$$Register, Address(sp, $dst$$disp));
14349   %}
14350 
14351   ins_pipe(istore_reg_reg);
14352 
14353 %}
14354 
14355 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14356 
14357   match(Set dst (MoveF2I src));
14358 
14359   effect(DEF dst, USE src);
14360 
14361   ins_cost(INSN_COST);
14362 
14363   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14364 
14365   ins_encode %{
14366     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14367   %}
14368 
14369   ins_pipe(fp_f2i);
14370 
14371 %}
14372 
14373 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14374 
14375   match(Set dst (MoveI2F src));
14376 
14377   effect(DEF dst, USE src);
14378 
14379   ins_cost(INSN_COST);
14380 
14381   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14382 
14383   ins_encode %{
14384     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14385   %}
14386 
14387   ins_pipe(fp_i2f);
14388 
14389 %}
14390 
14391 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14392 
14393   match(Set dst (MoveD2L src));
14394 
14395   effect(DEF dst, USE src);
14396 
14397   ins_cost(INSN_COST);
14398 
14399   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14400 
14401   ins_encode %{
14402     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14403   %}
14404 
14405   ins_pipe(fp_d2l);
14406 
14407 %}
14408 
14409 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14410 
14411   match(Set dst (MoveL2D src));
14412 
14413   effect(DEF dst, USE src);
14414 
14415   ins_cost(INSN_COST);
14416 
14417   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14418 
14419   ins_encode %{
14420     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14421   %}
14422 
14423   ins_pipe(fp_l2d);
14424 
14425 %}
14426 
14427 // ============================================================================
14428 // clearing of an array
14429 
14430 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14431 %{
14432   match(Set dummy (ClearArray cnt base));
14433   effect(USE_KILL cnt, USE_KILL base);
14434 
14435   ins_cost(4 * INSN_COST);
14436   format %{ "ClearArray $cnt, $base" %}
14437 
14438   ins_encode %{
14439     __ zero_words($base$$Register, $cnt$$Register);
14440   %}
14441 
14442   ins_pipe(pipe_class_memory);
14443 %}
14444 
14445 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14446 %{
14447   predicate((u_int64_t)n->in(2)->get_long()
14448             < (u_int64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14449   match(Set dummy (ClearArray cnt base));
14450   effect(USE_KILL base);
14451 
14452   ins_cost(4 * INSN_COST);
14453   format %{ "ClearArray $cnt, $base" %}
14454 
14455   ins_encode %{
14456     __ zero_words($base$$Register, (u_int64_t)$cnt$$constant);
14457   %}
14458 
14459   ins_pipe(pipe_class_memory);
14460 %}
14461 
14462 // ============================================================================
14463 // Overflow Math Instructions
14464 
14465 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14466 %{
14467   match(Set cr (OverflowAddI op1 op2));
14468 
14469   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14470   ins_cost(INSN_COST);
14471   ins_encode %{
14472     __ cmnw($op1$$Register, $op2$$Register);
14473   %}
14474 
14475   ins_pipe(icmp_reg_reg);
14476 %}
14477 
14478 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14479 %{
14480   match(Set cr (OverflowAddI op1 op2));
14481 
14482   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14483   ins_cost(INSN_COST);
14484   ins_encode %{
14485     __ cmnw($op1$$Register, $op2$$constant);
14486   %}
14487 
14488   ins_pipe(icmp_reg_imm);
14489 %}
14490 
14491 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14492 %{
14493   match(Set cr (OverflowAddL op1 op2));
14494 
14495   format %{ "cmn   $op1, $op2\t# overflow check long" %}
14496   ins_cost(INSN_COST);
14497   ins_encode %{
14498     __ cmn($op1$$Register, $op2$$Register);
14499   %}
14500 
14501   ins_pipe(icmp_reg_reg);
14502 %}
14503 
14504 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14505 %{
14506   match(Set cr (OverflowAddL op1 op2));
14507 
14508   format %{ "cmn   $op1, $op2\t# overflow check long" %}
14509   ins_cost(INSN_COST);
14510   ins_encode %{
14511     __ cmn($op1$$Register, $op2$$constant);
14512   %}
14513 
14514   ins_pipe(icmp_reg_imm);
14515 %}
14516 
14517 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14518 %{
14519   match(Set cr (OverflowSubI op1 op2));
14520 
14521   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
14522   ins_cost(INSN_COST);
14523   ins_encode %{
14524     __ cmpw($op1$$Register, $op2$$Register);
14525   %}
14526 
14527   ins_pipe(icmp_reg_reg);
14528 %}
14529 
14530 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14531 %{
14532   match(Set cr (OverflowSubI op1 op2));
14533 
14534   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
14535   ins_cost(INSN_COST);
14536   ins_encode %{
14537     __ cmpw($op1$$Register, $op2$$constant);
14538   %}
14539 
14540   ins_pipe(icmp_reg_imm);
14541 %}
14542 
14543 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14544 %{
14545   match(Set cr (OverflowSubL op1 op2));
14546 
14547   format %{ "cmp   $op1, $op2\t# overflow check long" %}
14548   ins_cost(INSN_COST);
14549   ins_encode %{
14550     __ cmp($op1$$Register, $op2$$Register);
14551   %}
14552 
14553   ins_pipe(icmp_reg_reg);
14554 %}
14555 
14556 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14557 %{
14558   match(Set cr (OverflowSubL op1 op2));
14559 
14560   format %{ "cmp   $op1, $op2\t# overflow check long" %}
14561   ins_cost(INSN_COST);
14562   ins_encode %{
14563     __ cmp($op1$$Register, $op2$$constant);
14564   %}
14565 
14566   ins_pipe(icmp_reg_imm);
14567 %}
14568 
14569 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14570 %{
14571   match(Set cr (OverflowSubI zero op1));
14572 
14573   format %{ "cmpw  zr, $op1\t# overflow check int" %}
14574   ins_cost(INSN_COST);
14575   ins_encode %{
14576     __ cmpw(zr, $op1$$Register);
14577   %}
14578 
14579   ins_pipe(icmp_reg_imm);
14580 %}
14581 
14582 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14583 %{
14584   match(Set cr (OverflowSubL zero op1));
14585 
14586   format %{ "cmp   zr, $op1\t# overflow check long" %}
14587   ins_cost(INSN_COST);
14588   ins_encode %{
14589     __ cmp(zr, $op1$$Register);
14590   %}
14591 
14592   ins_pipe(icmp_reg_imm);
14593 %}
14594 
14595 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14596 %{
14597   match(Set cr (OverflowMulI op1 op2));
14598 
14599   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14600             "cmp   rscratch1, rscratch1, sxtw\n\t"
14601             "movw  rscratch1, #0x80000000\n\t"
14602             "cselw rscratch1, rscratch1, zr, NE\n\t"
14603             "cmpw  rscratch1, #1" %}
14604   ins_cost(5 * INSN_COST);
14605   ins_encode %{
14606     __ smull(rscratch1, $op1$$Register, $op2$$Register);
14607     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
14608     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
14609     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14610     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
14611   %}
14612 
14613   ins_pipe(pipe_slow);
14614 %}
14615 
14616 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14617 %{
14618   match(If cmp (OverflowMulI op1 op2));
14619   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14620             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14621   effect(USE labl, KILL cr);
14622 
14623   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14624             "cmp   rscratch1, rscratch1, sxtw\n\t"
14625             "b$cmp   $labl" %}
14626   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14627   ins_encode %{
14628     Label* L = $labl$$label;
14629     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14630     __ smull(rscratch1, $op1$$Register, $op2$$Register);
14631     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
14632     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14633   %}
14634 
14635   ins_pipe(pipe_serial);
14636 %}
14637 
14638 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14639 %{
14640   match(Set cr (OverflowMulL op1 op2));
14641 
14642   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
14643             "smulh rscratch2, $op1, $op2\n\t"
14644             "cmp   rscratch2, rscratch1, ASR #63\n\t"
14645             "movw  rscratch1, #0x80000000\n\t"
14646             "cselw rscratch1, rscratch1, zr, NE\n\t"
14647             "cmpw  rscratch1, #1" %}
14648   ins_cost(6 * INSN_COST);
14649   ins_encode %{
14650     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
14651     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14652     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
14653     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
14654     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14655     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
14656   %}
14657 
14658   ins_pipe(pipe_slow);
14659 %}
14660 
14661 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14662 %{
14663   match(If cmp (OverflowMulL op1 op2));
14664   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14665             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14666   effect(USE labl, KILL cr);
14667 
14668   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
14669             "smulh rscratch2, $op1, $op2\n\t"
14670             "cmp   rscratch2, rscratch1, ASR #63\n\t"
14671             "b$cmp $labl" %}
14672   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14673   ins_encode %{
14674     Label* L = $labl$$label;
14675     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14676     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
14677     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14678     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
14679     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14680   %}
14681 
14682   ins_pipe(pipe_serial);
14683 %}
14684 
14685 // ============================================================================
14686 // Compare Instructions
14687 
14688 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14689 %{
14690   match(Set cr (CmpI op1 op2));
14691 
14692   effect(DEF cr, USE op1, USE op2);
14693 
14694   ins_cost(INSN_COST);
14695   format %{ "cmpw  $op1, $op2" %}
14696 
14697   ins_encode(aarch64_enc_cmpw(op1, op2));
14698 
14699   ins_pipe(icmp_reg_reg);
14700 %}
14701 
14702 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14703 %{
14704   match(Set cr (CmpI op1 zero));
14705 
14706   effect(DEF cr, USE op1);
14707 
14708   ins_cost(INSN_COST);
14709   format %{ "cmpw $op1, 0" %}
14710 
14711   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14712 
14713   ins_pipe(icmp_reg_imm);
14714 %}
14715 
14716 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14717 %{
14718   match(Set cr (CmpI op1 op2));
14719 
14720   effect(DEF cr, USE op1);
14721 
14722   ins_cost(INSN_COST);
14723   format %{ "cmpw  $op1, $op2" %}
14724 
14725   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14726 
14727   ins_pipe(icmp_reg_imm);
14728 %}
14729 
14730 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14731 %{
14732   match(Set cr (CmpI op1 op2));
14733 
14734   effect(DEF cr, USE op1);
14735 
14736   ins_cost(INSN_COST * 2);
14737   format %{ "cmpw  $op1, $op2" %}
14738 
14739   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14740 
14741   ins_pipe(icmp_reg_imm);
14742 %}
14743 
14744 // Unsigned compare Instructions; really, same as signed compare
14745 // except it should only be used to feed an If or a CMovI which takes a
14746 // cmpOpU.
14747 
14748 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14749 %{
14750   match(Set cr (CmpU op1 op2));
14751 
14752   effect(DEF cr, USE op1, USE op2);
14753 
14754   ins_cost(INSN_COST);
14755   format %{ "cmpw  $op1, $op2\t# unsigned" %}
14756 
14757   ins_encode(aarch64_enc_cmpw(op1, op2));
14758 
14759   ins_pipe(icmp_reg_reg);
14760 %}
14761 
14762 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14763 %{
14764   match(Set cr (CmpU op1 zero));
14765 
14766   effect(DEF cr, USE op1);
14767 
14768   ins_cost(INSN_COST);
14769   format %{ "cmpw $op1, #0\t# unsigned" %}
14770 
14771   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14772 
14773   ins_pipe(icmp_reg_imm);
14774 %}
14775 
14776 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14777 %{
14778   match(Set cr (CmpU op1 op2));
14779 
14780   effect(DEF cr, USE op1);
14781 
14782   ins_cost(INSN_COST);
14783   format %{ "cmpw  $op1, $op2\t# unsigned" %}
14784 
14785   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14786 
14787   ins_pipe(icmp_reg_imm);
14788 %}
14789 
14790 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14791 %{
14792   match(Set cr (CmpU op1 op2));
14793 
14794   effect(DEF cr, USE op1);
14795 
14796   ins_cost(INSN_COST * 2);
14797   format %{ "cmpw  $op1, $op2\t# unsigned" %}
14798 
14799   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14800 
14801   ins_pipe(icmp_reg_imm);
14802 %}
14803 
14804 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14805 %{
14806   match(Set cr (CmpL op1 op2));
14807 
14808   effect(DEF cr, USE op1, USE op2);
14809 
14810   ins_cost(INSN_COST);
14811   format %{ "cmp  $op1, $op2" %}
14812 
14813   ins_encode(aarch64_enc_cmp(op1, op2));
14814 
14815   ins_pipe(icmp_reg_reg);
14816 %}
14817 
14818 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14819 %{
14820   match(Set cr (CmpL op1 zero));
14821 
14822   effect(DEF cr, USE op1);
14823 
14824   ins_cost(INSN_COST);
14825   format %{ "tst  $op1" %}
14826 
14827   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14828 
14829   ins_pipe(icmp_reg_imm);
14830 %}
14831 
14832 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14833 %{
14834   match(Set cr (CmpL op1 op2));
14835 
14836   effect(DEF cr, USE op1);
14837 
14838   ins_cost(INSN_COST);
14839   format %{ "cmp  $op1, $op2" %}
14840 
14841   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14842 
14843   ins_pipe(icmp_reg_imm);
14844 %}
14845 
14846 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14847 %{
14848   match(Set cr (CmpL op1 op2));
14849 
14850   effect(DEF cr, USE op1);
14851 
14852   ins_cost(INSN_COST * 2);
14853   format %{ "cmp  $op1, $op2" %}
14854 
14855   ins_encode(aarch64_enc_cmp_imm(op1, op2));
14856 
14857   ins_pipe(icmp_reg_imm);
14858 %}
14859 
14860 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14861 %{
14862   match(Set cr (CmpUL op1 op2));
14863 
14864   effect(DEF cr, USE op1, USE op2);
14865 
14866   ins_cost(INSN_COST);
14867   format %{ "cmp  $op1, $op2" %}
14868 
14869   ins_encode(aarch64_enc_cmp(op1, op2));
14870 
14871   ins_pipe(icmp_reg_reg);
14872 %}
14873 
14874 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14875 %{
14876   match(Set cr (CmpUL op1 zero));
14877 
14878   effect(DEF cr, USE op1);
14879 
14880   ins_cost(INSN_COST);
14881   format %{ "tst  $op1" %}
14882 
14883   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14884 
14885   ins_pipe(icmp_reg_imm);
14886 %}
14887 
14888 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14889 %{
14890   match(Set cr (CmpUL op1 op2));
14891 
14892   effect(DEF cr, USE op1);
14893 
14894   ins_cost(INSN_COST);
14895   format %{ "cmp  $op1, $op2" %}
14896 
14897   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14898 
14899   ins_pipe(icmp_reg_imm);
14900 %}
14901 
14902 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14903 %{
14904   match(Set cr (CmpUL op1 op2));
14905 
14906   effect(DEF cr, USE op1);
14907 
14908   ins_cost(INSN_COST * 2);
14909   format %{ "cmp  $op1, $op2" %}
14910 
14911   ins_encode(aarch64_enc_cmp_imm(op1, op2));
14912 
14913   ins_pipe(icmp_reg_imm);
14914 %}
14915 
14916 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14917 %{
14918   match(Set cr (CmpP op1 op2));
14919 
14920   effect(DEF cr, USE op1, USE op2);
14921 
14922   ins_cost(INSN_COST);
14923   format %{ "cmp  $op1, $op2\t // ptr" %}
14924 
14925   ins_encode(aarch64_enc_cmpp(op1, op2));
14926 
14927   ins_pipe(icmp_reg_reg);
14928 %}
14929 
14930 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14931 %{
14932   match(Set cr (CmpN op1 op2));
14933 
14934   effect(DEF cr, USE op1, USE op2);
14935 
14936   ins_cost(INSN_COST);
14937   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
14938 
14939   ins_encode(aarch64_enc_cmpn(op1, op2));
14940 
14941   ins_pipe(icmp_reg_reg);
14942 %}
14943 
14944 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14945 %{
14946   match(Set cr (CmpP op1 zero));
14947 
14948   effect(DEF cr, USE op1, USE zero);
14949 
14950   ins_cost(INSN_COST);
14951   format %{ "cmp  $op1, 0\t // ptr" %}
14952 
14953   ins_encode(aarch64_enc_testp(op1));
14954 
14955   ins_pipe(icmp_reg_imm);
14956 %}
14957 
14958 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14959 %{
14960   match(Set cr (CmpN op1 zero));
14961 
14962   effect(DEF cr, USE op1, USE zero);
14963 
14964   ins_cost(INSN_COST);
14965   format %{ "cmp  $op1, 0\t // compressed ptr" %}
14966 
14967   ins_encode(aarch64_enc_testn(op1));
14968 
14969   ins_pipe(icmp_reg_imm);
14970 %}
14971 
14972 // FP comparisons
14973 //
14974 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14975 // using normal cmpOp. See declaration of rFlagsReg for details.
14976 
14977 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14978 %{
14979   match(Set cr (CmpF src1 src2));
14980 
14981   ins_cost(3 * INSN_COST);
14982   format %{ "fcmps $src1, $src2" %}
14983 
14984   ins_encode %{
14985     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14986   %}
14987 
14988   ins_pipe(pipe_class_compare);
14989 %}
14990 
14991 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14992 %{
14993   match(Set cr (CmpF src1 src2));
14994 
14995   ins_cost(3 * INSN_COST);
14996   format %{ "fcmps $src1, 0.0" %}
14997 
14998   ins_encode %{
14999     __ fcmps(as_FloatRegister($src1$$reg), 0.0D);
15000   %}
15001 
15002   ins_pipe(pipe_class_compare);
15003 %}
15004 // FROM HERE
15005 
15006 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15007 %{
15008   match(Set cr (CmpD src1 src2));
15009 
15010   ins_cost(3 * INSN_COST);
15011   format %{ "fcmpd $src1, $src2" %}
15012 
15013   ins_encode %{
15014     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15015   %}
15016 
15017   ins_pipe(pipe_class_compare);
15018 %}
15019 
15020 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15021 %{
15022   match(Set cr (CmpD src1 src2));
15023 
15024   ins_cost(3 * INSN_COST);
15025   format %{ "fcmpd $src1, 0.0" %}
15026 
15027   ins_encode %{
15028     __ fcmpd(as_FloatRegister($src1$$reg), 0.0D);
15029   %}
15030 
15031   ins_pipe(pipe_class_compare);
15032 %}
15033 
15034 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15035 %{
15036   match(Set dst (CmpF3 src1 src2));
15037   effect(KILL cr);
15038 
15039   ins_cost(5 * INSN_COST);
15040   format %{ "fcmps $src1, $src2\n\t"
15041             "csinvw($dst, zr, zr, eq\n\t"
15042             "csnegw($dst, $dst, $dst, lt)"
15043   %}
15044 
15045   ins_encode %{
15046     Label done;
15047     FloatRegister s1 = as_FloatRegister($src1$$reg);
15048     FloatRegister s2 = as_FloatRegister($src2$$reg);
15049     Register d = as_Register($dst$$reg);
15050     __ fcmps(s1, s2);
15051     // installs 0 if EQ else -1
15052     __ csinvw(d, zr, zr, Assembler::EQ);
15053     // keeps -1 if less or unordered else installs 1
15054     __ csnegw(d, d, d, Assembler::LT);
15055     __ bind(done);
15056   %}
15057 
15058   ins_pipe(pipe_class_default);
15059 
15060 %}
15061 
15062 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15063 %{
15064   match(Set dst (CmpD3 src1 src2));
15065   effect(KILL cr);
15066 
15067   ins_cost(5 * INSN_COST);
15068   format %{ "fcmpd $src1, $src2\n\t"
15069             "csinvw($dst, zr, zr, eq\n\t"
15070             "csnegw($dst, $dst, $dst, lt)"
15071   %}
15072 
15073   ins_encode %{
15074     Label done;
15075     FloatRegister s1 = as_FloatRegister($src1$$reg);
15076     FloatRegister s2 = as_FloatRegister($src2$$reg);
15077     Register d = as_Register($dst$$reg);
15078     __ fcmpd(s1, s2);
15079     // installs 0 if EQ else -1
15080     __ csinvw(d, zr, zr, Assembler::EQ);
15081     // keeps -1 if less or unordered else installs 1
15082     __ csnegw(d, d, d, Assembler::LT);
15083     __ bind(done);
15084   %}
15085   ins_pipe(pipe_class_default);
15086 
15087 %}
15088 
15089 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15090 %{
15091   match(Set dst (CmpF3 src1 zero));
15092   effect(KILL cr);
15093 
15094   ins_cost(5 * INSN_COST);
15095   format %{ "fcmps $src1, 0.0\n\t"
15096             "csinvw($dst, zr, zr, eq\n\t"
15097             "csnegw($dst, $dst, $dst, lt)"
15098   %}
15099 
15100   ins_encode %{
15101     Label done;
15102     FloatRegister s1 = as_FloatRegister($src1$$reg);
15103     Register d = as_Register($dst$$reg);
15104     __ fcmps(s1, 0.0D);
15105     // installs 0 if EQ else -1
15106     __ csinvw(d, zr, zr, Assembler::EQ);
15107     // keeps -1 if less or unordered else installs 1
15108     __ csnegw(d, d, d, Assembler::LT);
15109     __ bind(done);
15110   %}
15111 
15112   ins_pipe(pipe_class_default);
15113 
15114 %}
15115 
15116 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15117 %{
15118   match(Set dst (CmpD3 src1 zero));
15119   effect(KILL cr);
15120 
15121   ins_cost(5 * INSN_COST);
15122   format %{ "fcmpd $src1, 0.0\n\t"
15123             "csinvw($dst, zr, zr, eq\n\t"
15124             "csnegw($dst, $dst, $dst, lt)"
15125   %}
15126 
15127   ins_encode %{
15128     Label done;
15129     FloatRegister s1 = as_FloatRegister($src1$$reg);
15130     Register d = as_Register($dst$$reg);
15131     __ fcmpd(s1, 0.0D);
15132     // installs 0 if EQ else -1
15133     __ csinvw(d, zr, zr, Assembler::EQ);
15134     // keeps -1 if less or unordered else installs 1
15135     __ csnegw(d, d, d, Assembler::LT);
15136     __ bind(done);
15137   %}
15138   ins_pipe(pipe_class_default);
15139 
15140 %}
15141 
15142 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15143 %{
15144   match(Set dst (CmpLTMask p q));
15145   effect(KILL cr);
15146 
15147   ins_cost(3 * INSN_COST);
15148 
15149   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15150             "csetw $dst, lt\n\t"
15151             "subw $dst, zr, $dst"
15152   %}
15153 
15154   ins_encode %{
15155     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15156     __ csetw(as_Register($dst$$reg), Assembler::LT);
15157     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15158   %}
15159 
15160   ins_pipe(ialu_reg_reg);
15161 %}
15162 
15163 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15164 %{
15165   match(Set dst (CmpLTMask src zero));
15166   effect(KILL cr);
15167 
15168   ins_cost(INSN_COST);
15169 
15170   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15171 
15172   ins_encode %{
15173     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15174   %}
15175 
15176   ins_pipe(ialu_reg_shift);
15177 %}
15178 
15179 // ============================================================================
15180 // Max and Min
15181 
15182 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15183 %{
15184   match(Set dst (MinI src1 src2));
15185 
15186   effect(DEF dst, USE src1, USE src2, KILL cr);
15187   size(8);
15188 
15189   ins_cost(INSN_COST * 3);
15190   format %{
15191     "cmpw $src1 $src2\t signed int\n\t"
15192     "cselw $dst, $src1, $src2 lt\t"
15193   %}
15194 
15195   ins_encode %{
15196     __ cmpw(as_Register($src1$$reg),
15197             as_Register($src2$$reg));
15198     __ cselw(as_Register($dst$$reg),
15199              as_Register($src1$$reg),
15200              as_Register($src2$$reg),
15201              Assembler::LT);
15202   %}
15203 
15204   ins_pipe(ialu_reg_reg);
15205 %}
15206 // FROM HERE
15207 
15208 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15209 %{
15210   match(Set dst (MaxI src1 src2));
15211 
15212   effect(DEF dst, USE src1, USE src2, KILL cr);
15213   size(8);
15214 
15215   ins_cost(INSN_COST * 3);
15216   format %{
15217     "cmpw $src1 $src2\t signed int\n\t"
15218     "cselw $dst, $src1, $src2 gt\t"
15219   %}
15220 
15221   ins_encode %{
15222     __ cmpw(as_Register($src1$$reg),
15223             as_Register($src2$$reg));
15224     __ cselw(as_Register($dst$$reg),
15225              as_Register($src1$$reg),
15226              as_Register($src2$$reg),
15227              Assembler::GT);
15228   %}
15229 
15230   ins_pipe(ialu_reg_reg);
15231 %}
15232 
15233 // ============================================================================
15234 // Branch Instructions
15235 
15236 // Direct Branch.
15237 instruct branch(label lbl)
15238 %{
15239   match(Goto);
15240 
15241   effect(USE lbl);
15242 
15243   ins_cost(BRANCH_COST);
15244   format %{ "b  $lbl" %}
15245 
15246   ins_encode(aarch64_enc_b(lbl));
15247 
15248   ins_pipe(pipe_branch);
15249 %}
15250 
15251 // Conditional Near Branch
15252 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15253 %{
15254   // Same match rule as `branchConFar'.
15255   match(If cmp cr);
15256 
15257   effect(USE lbl);
15258 
15259   ins_cost(BRANCH_COST);
15260   // If set to 1 this indicates that the current instruction is a
15261   // short variant of a long branch. This avoids using this
15262   // instruction in first-pass matching. It will then only be used in
15263   // the `Shorten_branches' pass.
15264   // ins_short_branch(1);
15265   format %{ "b$cmp  $lbl" %}
15266 
15267   ins_encode(aarch64_enc_br_con(cmp, lbl));
15268 
15269   ins_pipe(pipe_branch_cond);
15270 %}
15271 
15272 // Conditional Near Branch Unsigned
15273 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15274 %{
15275   // Same match rule as `branchConFar'.
15276   match(If cmp cr);
15277 
15278   effect(USE lbl);
15279 
15280   ins_cost(BRANCH_COST);
15281   // If set to 1 this indicates that the current instruction is a
15282   // short variant of a long branch. This avoids using this
15283   // instruction in first-pass matching. It will then only be used in
15284   // the `Shorten_branches' pass.
15285   // ins_short_branch(1);
15286   format %{ "b$cmp  $lbl\t# unsigned" %}
15287 
15288   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15289 
15290   ins_pipe(pipe_branch_cond);
15291 %}
15292 
15293 // Make use of CBZ and CBNZ.  These instructions, as well as being
15294 // shorter than (cmp; branch), have the additional benefit of not
15295 // killing the flags.
15296 
15297 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15298   match(If cmp (CmpI op1 op2));
15299   effect(USE labl);
15300 
15301   ins_cost(BRANCH_COST);
15302   format %{ "cbw$cmp   $op1, $labl" %}
15303   ins_encode %{
15304     Label* L = $labl$$label;
15305     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15306     if (cond == Assembler::EQ)
15307       __ cbzw($op1$$Register, *L);
15308     else
15309       __ cbnzw($op1$$Register, *L);
15310   %}
15311   ins_pipe(pipe_cmp_branch);
15312 %}
15313 
15314 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15315   match(If cmp (CmpL op1 op2));
15316   effect(USE labl);
15317 
15318   ins_cost(BRANCH_COST);
15319   format %{ "cb$cmp   $op1, $labl" %}
15320   ins_encode %{
15321     Label* L = $labl$$label;
15322     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15323     if (cond == Assembler::EQ)
15324       __ cbz($op1$$Register, *L);
15325     else
15326       __ cbnz($op1$$Register, *L);
15327   %}
15328   ins_pipe(pipe_cmp_branch);
15329 %}
15330 
15331 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15332   match(If cmp (CmpP op1 op2));
15333   effect(USE labl);
15334 
15335   ins_cost(BRANCH_COST);
15336   format %{ "cb$cmp   $op1, $labl" %}
15337   ins_encode %{
15338     Label* L = $labl$$label;
15339     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15340     if (cond == Assembler::EQ)
15341       __ cbz($op1$$Register, *L);
15342     else
15343       __ cbnz($op1$$Register, *L);
15344   %}
15345   ins_pipe(pipe_cmp_branch);
15346 %}
15347 
15348 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15349   match(If cmp (CmpN op1 op2));
15350   effect(USE labl);
15351 
15352   ins_cost(BRANCH_COST);
15353   format %{ "cbw$cmp   $op1, $labl" %}
15354   ins_encode %{
15355     Label* L = $labl$$label;
15356     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15357     if (cond == Assembler::EQ)
15358       __ cbzw($op1$$Register, *L);
15359     else
15360       __ cbnzw($op1$$Register, *L);
15361   %}
15362   ins_pipe(pipe_cmp_branch);
15363 %}
15364 
15365 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15366   match(If cmp (CmpP (DecodeN oop) zero));
15367   effect(USE labl);
15368 
15369   ins_cost(BRANCH_COST);
15370   format %{ "cb$cmp   $oop, $labl" %}
15371   ins_encode %{
15372     Label* L = $labl$$label;
15373     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15374     if (cond == Assembler::EQ)
15375       __ cbzw($oop$$Register, *L);
15376     else
15377       __ cbnzw($oop$$Register, *L);
15378   %}
15379   ins_pipe(pipe_cmp_branch);
15380 %}
15381 
15382 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
15383   match(If cmp (CmpU op1 op2));
15384   effect(USE labl);
15385 
15386   ins_cost(BRANCH_COST);
15387   format %{ "cbw$cmp   $op1, $labl" %}
15388   ins_encode %{
15389     Label* L = $labl$$label;
15390     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15391     if (cond == Assembler::EQ || cond == Assembler::LS)
15392       __ cbzw($op1$$Register, *L);
15393     else
15394       __ cbnzw($op1$$Register, *L);
15395   %}
15396   ins_pipe(pipe_cmp_branch);
15397 %}
15398 
15399 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
15400   match(If cmp (CmpUL op1 op2));
15401   effect(USE labl);
15402 
15403   ins_cost(BRANCH_COST);
15404   format %{ "cb$cmp   $op1, $labl" %}
15405   ins_encode %{
15406     Label* L = $labl$$label;
15407     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15408     if (cond == Assembler::EQ || cond == Assembler::LS)
15409       __ cbz($op1$$Register, *L);
15410     else
15411       __ cbnz($op1$$Register, *L);
15412   %}
15413   ins_pipe(pipe_cmp_branch);
15414 %}
15415 
15416 // Test bit and Branch
15417 
15418 // Patterns for short (< 32KiB) variants
15419 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15420   match(If cmp (CmpL op1 op2));
15421   effect(USE labl);
15422 
15423   ins_cost(BRANCH_COST);
15424   format %{ "cb$cmp   $op1, $labl # long" %}
15425   ins_encode %{
15426     Label* L = $labl$$label;
15427     Assembler::Condition cond =
15428       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15429     __ tbr(cond, $op1$$Register, 63, *L);
15430   %}
15431   ins_pipe(pipe_cmp_branch);
15432   ins_short_branch(1);
15433 %}
15434 
15435 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15436   match(If cmp (CmpI op1 op2));
15437   effect(USE labl);
15438 
15439   ins_cost(BRANCH_COST);
15440   format %{ "cb$cmp   $op1, $labl # int" %}
15441   ins_encode %{
15442     Label* L = $labl$$label;
15443     Assembler::Condition cond =
15444       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15445     __ tbr(cond, $op1$$Register, 31, *L);
15446   %}
15447   ins_pipe(pipe_cmp_branch);
15448   ins_short_branch(1);
15449 %}
15450 
15451 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15452   match(If cmp (CmpL (AndL op1 op2) op3));
15453   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
15454   effect(USE labl);
15455 
15456   ins_cost(BRANCH_COST);
15457   format %{ "tb$cmp   $op1, $op2, $labl" %}
15458   ins_encode %{
15459     Label* L = $labl$$label;
15460     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15461     int bit = exact_log2($op2$$constant);
15462     __ tbr(cond, $op1$$Register, bit, *L);
15463   %}
15464   ins_pipe(pipe_cmp_branch);
15465   ins_short_branch(1);
15466 %}
15467 
15468 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15469   match(If cmp (CmpI (AndI op1 op2) op3));
15470   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
15471   effect(USE labl);
15472 
15473   ins_cost(BRANCH_COST);
15474   format %{ "tb$cmp   $op1, $op2, $labl" %}
15475   ins_encode %{
15476     Label* L = $labl$$label;
15477     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15478     int bit = exact_log2($op2$$constant);
15479     __ tbr(cond, $op1$$Register, bit, *L);
15480   %}
15481   ins_pipe(pipe_cmp_branch);
15482   ins_short_branch(1);
15483 %}
15484 
15485 // And far variants
15486 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15487   match(If cmp (CmpL op1 op2));
15488   effect(USE labl);
15489 
15490   ins_cost(BRANCH_COST);
15491   format %{ "cb$cmp   $op1, $labl # long" %}
15492   ins_encode %{
15493     Label* L = $labl$$label;
15494     Assembler::Condition cond =
15495       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15496     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15497   %}
15498   ins_pipe(pipe_cmp_branch);
15499 %}
15500 
15501 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15502   match(If cmp (CmpI op1 op2));
15503   effect(USE labl);
15504 
15505   ins_cost(BRANCH_COST);
15506   format %{ "cb$cmp   $op1, $labl # int" %}
15507   ins_encode %{
15508     Label* L = $labl$$label;
15509     Assembler::Condition cond =
15510       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15511     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15512   %}
15513   ins_pipe(pipe_cmp_branch);
15514 %}
15515 
15516 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15517   match(If cmp (CmpL (AndL op1 op2) op3));
15518   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
15519   effect(USE labl);
15520 
15521   ins_cost(BRANCH_COST);
15522   format %{ "tb$cmp   $op1, $op2, $labl" %}
15523   ins_encode %{
15524     Label* L = $labl$$label;
15525     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15526     int bit = exact_log2($op2$$constant);
15527     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15528   %}
15529   ins_pipe(pipe_cmp_branch);
15530 %}
15531 
15532 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15533   match(If cmp (CmpI (AndI op1 op2) op3));
15534   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
15535   effect(USE labl);
15536 
15537   ins_cost(BRANCH_COST);
15538   format %{ "tb$cmp   $op1, $op2, $labl" %}
15539   ins_encode %{
15540     Label* L = $labl$$label;
15541     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15542     int bit = exact_log2($op2$$constant);
15543     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15544   %}
15545   ins_pipe(pipe_cmp_branch);
15546 %}
15547 
15548 // Test bits
15549 
15550 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15551   match(Set cr (CmpL (AndL op1 op2) op3));
15552   predicate(Assembler::operand_valid_for_logical_immediate
15553             (/*is_32*/false, n->in(1)->in(2)->get_long()));
15554 
15555   ins_cost(INSN_COST);
15556   format %{ "tst $op1, $op2 # long" %}
15557   ins_encode %{
15558     __ tst($op1$$Register, $op2$$constant);
15559   %}
15560   ins_pipe(ialu_reg_reg);
15561 %}
15562 
15563 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15564   match(Set cr (CmpI (AndI op1 op2) op3));
15565   predicate(Assembler::operand_valid_for_logical_immediate
15566             (/*is_32*/true, n->in(1)->in(2)->get_int()));
15567 
15568   ins_cost(INSN_COST);
15569   format %{ "tst $op1, $op2 # int" %}
15570   ins_encode %{
15571     __ tstw($op1$$Register, $op2$$constant);
15572   %}
15573   ins_pipe(ialu_reg_reg);
15574 %}
15575 
15576 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15577   match(Set cr (CmpL (AndL op1 op2) op3));
15578 
15579   ins_cost(INSN_COST);
15580   format %{ "tst $op1, $op2 # long" %}
15581   ins_encode %{
15582     __ tst($op1$$Register, $op2$$Register);
15583   %}
15584   ins_pipe(ialu_reg_reg);
15585 %}
15586 
15587 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15588   match(Set cr (CmpI (AndI op1 op2) op3));
15589 
15590   ins_cost(INSN_COST);
15591   format %{ "tstw $op1, $op2 # int" %}
15592   ins_encode %{
15593     __ tstw($op1$$Register, $op2$$Register);
15594   %}
15595   ins_pipe(ialu_reg_reg);
15596 %}
15597 
15598 
15599 // Conditional Far Branch
15600 // Conditional Far Branch Unsigned
15601 // TODO: fixme
15602 
15603 // counted loop end branch near
15604 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15605 %{
15606   match(CountedLoopEnd cmp cr);
15607 
15608   effect(USE lbl);
15609 
15610   ins_cost(BRANCH_COST);
15611   // short variant.
15612   // ins_short_branch(1);
15613   format %{ "b$cmp $lbl \t// counted loop end" %}
15614 
15615   ins_encode(aarch64_enc_br_con(cmp, lbl));
15616 
15617   ins_pipe(pipe_branch);
15618 %}
15619 
15620 // counted loop end branch near Unsigned
15621 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15622 %{
15623   match(CountedLoopEnd cmp cr);
15624 
15625   effect(USE lbl);
15626 
15627   ins_cost(BRANCH_COST);
15628   // short variant.
15629   // ins_short_branch(1);
15630   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
15631 
15632   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15633 
15634   ins_pipe(pipe_branch);
15635 %}
15636 
15637 // counted loop end branch far
15638 // counted loop end branch far unsigned
15639 // TODO: fixme
15640 
15641 // ============================================================================
15642 // inlined locking and unlocking
15643 
15644 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
15645 %{
15646   match(Set cr (FastLock object box));
15647   effect(TEMP tmp, TEMP tmp2);
15648 
15649   // TODO
15650   // identify correct cost
15651   ins_cost(5 * INSN_COST);
15652   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
15653 
15654   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
15655 
15656   ins_pipe(pipe_serial);
15657 %}
15658 
15659 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
15660 %{
15661   match(Set cr (FastUnlock object box));
15662   effect(TEMP tmp, TEMP tmp2);
15663 
15664   ins_cost(5 * INSN_COST);
15665   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
15666 
15667   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
15668 
15669   ins_pipe(pipe_serial);
15670 %}
15671 
15672 
15673 // ============================================================================
15674 // Safepoint Instructions
15675 
15676 // TODO
15677 // provide a near and far version of this code
15678 
15679 instruct safePoint(iRegP poll)
15680 %{
15681   match(SafePoint poll);
15682 
15683   format %{
15684     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15685   %}
15686   ins_encode %{
15687     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15688   %}
15689   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15690 %}
15691 
15692 
15693 // ============================================================================
15694 // Procedure Call/Return Instructions
15695 
15696 // Call Java Static Instruction
15697 
15698 instruct CallStaticJavaDirect(method meth)
15699 %{
15700   match(CallStaticJava);
15701 
15702   effect(USE meth);
15703 
15704   ins_cost(CALL_COST);
15705 
15706   format %{ "call,static $meth \t// ==> " %}
15707 
15708   ins_encode( aarch64_enc_java_static_call(meth),
15709               aarch64_enc_call_epilog );
15710 
15711   ins_pipe(pipe_class_call);
15712 %}
15713 
15714 // TO HERE
15715 
15716 // Call Java Dynamic Instruction
15717 instruct CallDynamicJavaDirect(method meth)
15718 %{
15719   match(CallDynamicJava);
15720 
15721   effect(USE meth);
15722 
15723   ins_cost(CALL_COST);
15724 
15725   format %{ "CALL,dynamic $meth \t// ==> " %}
15726 
15727   ins_encode( aarch64_enc_java_dynamic_call(meth),
15728                aarch64_enc_call_epilog );
15729 
15730   ins_pipe(pipe_class_call);
15731 %}
15732 
15733 // Call Runtime Instruction
15734 
15735 instruct CallRuntimeDirect(method meth)
15736 %{
15737   match(CallRuntime);
15738 
15739   effect(USE meth);
15740 
15741   ins_cost(CALL_COST);
15742 
15743   format %{ "CALL, runtime $meth" %}
15744 
15745   ins_encode( aarch64_enc_java_to_runtime(meth) );
15746 
15747   ins_pipe(pipe_class_call);
15748 %}
15749 
15750 // Call Runtime Instruction
15751 
15752 instruct CallLeafDirect(method meth)
15753 %{
15754   match(CallLeaf);
15755 
15756   effect(USE meth);
15757 
15758   ins_cost(CALL_COST);
15759 
15760   format %{ "CALL, runtime leaf $meth" %}
15761 
15762   ins_encode( aarch64_enc_java_to_runtime(meth) );
15763 
15764   ins_pipe(pipe_class_call);
15765 %}
15766 
15767 // Call Runtime Instruction
15768 
15769 instruct CallLeafNoFPDirect(method meth)
15770 %{
15771   match(CallLeafNoFP);
15772 
15773   effect(USE meth);
15774 
15775   ins_cost(CALL_COST);
15776 
15777   format %{ "CALL, runtime leaf nofp $meth" %}
15778 
15779   ins_encode( aarch64_enc_java_to_runtime(meth) );
15780 
15781   ins_pipe(pipe_class_call);
15782 %}
15783 
15784 // Tail Call; Jump from runtime stub to Java code.
15785 // Also known as an 'interprocedural jump'.
15786 // Target of jump will eventually return to caller.
15787 // TailJump below removes the return address.
15788 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop)
15789 %{
15790   match(TailCall jump_target method_oop);
15791 
15792   ins_cost(CALL_COST);
15793 
15794   format %{ "br $jump_target\t# $method_oop holds method oop" %}
15795 
15796   ins_encode(aarch64_enc_tail_call(jump_target));
15797 
15798   ins_pipe(pipe_class_call);
15799 %}
15800 
15801 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
15802 %{
15803   match(TailJump jump_target ex_oop);
15804 
15805   ins_cost(CALL_COST);
15806 
15807   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15808 
15809   ins_encode(aarch64_enc_tail_jmp(jump_target));
15810 
15811   ins_pipe(pipe_class_call);
15812 %}
15813 
15814 // Create exception oop: created by stack-crawling runtime code.
15815 // Created exception is now available to this handler, and is setup
15816 // just prior to jumping to this handler. No code emitted.
15817 // TODO check
15818 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15819 instruct CreateException(iRegP_R0 ex_oop)
15820 %{
15821   match(Set ex_oop (CreateEx));
15822 
15823   format %{ " -- \t// exception oop; no code emitted" %}
15824 
15825   size(0);
15826 
15827   ins_encode( /*empty*/ );
15828 
15829   ins_pipe(pipe_class_empty);
15830 %}
15831 
15832 // Rethrow exception: The exception oop will come in the first
15833 // argument position. Then JUMP (not call) to the rethrow stub code.
15834 instruct RethrowException() %{
15835   match(Rethrow);
15836   ins_cost(CALL_COST);
15837 
15838   format %{ "b rethrow_stub" %}
15839 
15840   ins_encode( aarch64_enc_rethrow() );
15841 
15842   ins_pipe(pipe_class_call);
15843 %}
15844 
15845 
15846 // Return Instruction
15847 // epilog node loads ret address into lr as part of frame pop
15848 instruct Ret()
15849 %{
15850   match(Return);
15851 
15852   format %{ "ret\t// return register" %}
15853 
15854   ins_encode( aarch64_enc_ret() );
15855 
15856   ins_pipe(pipe_branch);
15857 %}
15858 
15859 // Die now.
15860 instruct ShouldNotReachHere() %{
15861   match(Halt);
15862 
15863   ins_cost(CALL_COST);
15864   format %{ "ShouldNotReachHere" %}
15865 
15866   ins_encode %{
15867     // +1 so NativeInstruction::is_sigill_zombie_not_entrant() doesn't
15868     // return true
15869     __ dpcs1(0xdead + 1);
15870   %}
15871 
15872   ins_pipe(pipe_class_default);
15873 %}
15874 
15875 // ============================================================================
15876 // Partial Subtype Check
15877 //
15878 // superklass array for an instance of the superklass.  Set a hidden
15879 // internal cache on a hit (cache is checked with exposed code in
15880 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
15881 // encoding ALSO sets flags.
15882 
15883 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15884 %{
15885   match(Set result (PartialSubtypeCheck sub super));
15886   effect(KILL cr, KILL temp);
15887 
15888   ins_cost(1100);  // slightly larger than the next version
15889   format %{ "partialSubtypeCheck $result, $sub, $super" %}
15890 
15891   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15892 
15893   opcode(0x1); // Force zero of result reg on hit
15894 
15895   ins_pipe(pipe_class_memory);
15896 %}
15897 
15898 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
15899 %{
15900   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
15901   effect(KILL temp, KILL result);
15902 
15903   ins_cost(1100);  // slightly larger than the next version
15904   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
15905 
15906   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15907 
15908   opcode(0x0); // Don't zero result reg on hit
15909 
15910   ins_pipe(pipe_class_memory);
15911 %}
15912 
15913 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15914                         iRegI_R0 result, iRegP_R10 tmp1, rFlagsReg cr)
15915 %{
15916   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
15917   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15918   effect(KILL tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15919 
15920   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
15921   ins_encode %{
15922     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15923     __ string_compare($str1$$Register, $str2$$Register,
15924                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
15925                       $tmp1$$Register,
15926                       fnoreg, fnoreg, StrIntrinsicNode::UU);
15927   %}
15928   ins_pipe(pipe_class_memory);
15929 %}
15930 
15931 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15932                         iRegI_R0 result, iRegP_R10 tmp1, rFlagsReg cr)
15933 %{
15934   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
15935   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15936   effect(KILL tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15937 
15938   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
15939   ins_encode %{
15940     __ string_compare($str1$$Register, $str2$$Register,
15941                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
15942                       $tmp1$$Register,
15943                       fnoreg, fnoreg, StrIntrinsicNode::LL);
15944   %}
15945   ins_pipe(pipe_class_memory);
15946 %}
15947 
15948 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15949                         iRegI_R0 result, vRegD vtmp1, vRegD vtmp2, iRegP_R10 tmp1, rFlagsReg cr)
15950 %{
15951   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15952   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15953   effect(KILL tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, TEMP vtmp1, TEMP vtmp2, KILL cr);
15954 
15955   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
15956   ins_encode %{
15957     __ string_compare($str1$$Register, $str2$$Register,
15958                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
15959                       $tmp1$$Register,
15960                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, StrIntrinsicNode::UL);
15961   %}
15962   ins_pipe(pipe_class_memory);
15963 %}
15964 
15965 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15966                         iRegI_R0 result, vRegD vtmp1, vRegD vtmp2, iRegP_R10 tmp1, rFlagsReg cr)
15967 %{
15968   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
15969   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15970   effect(KILL tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, TEMP vtmp1, TEMP vtmp2, KILL cr);
15971 
15972   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
15973   ins_encode %{
15974     __ string_compare($str1$$Register, $str2$$Register,
15975                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
15976                       $tmp1$$Register,
15977                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, StrIntrinsicNode::LU);
15978   %}
15979   ins_pipe(pipe_class_memory);
15980 %}
15981 
15982 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15983        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15984 %{
15985   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15986   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15987   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15988          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15989   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
15990 
15991   ins_encode %{
15992     __ string_indexof($str1$$Register, $str2$$Register,
15993                       $cnt1$$Register, $cnt2$$Register,
15994                       $tmp1$$Register, $tmp2$$Register,
15995                       $tmp3$$Register, $tmp4$$Register,
15996                       -1, $result$$Register, StrIntrinsicNode::UU);
15997   %}
15998   ins_pipe(pipe_class_memory);
15999 %}
16000 
16001 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16002        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16003 %{
16004   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16005   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16006   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16007          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16008   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16009 
16010   ins_encode %{
16011     __ string_indexof($str1$$Register, $str2$$Register,
16012                       $cnt1$$Register, $cnt2$$Register,
16013                       $tmp1$$Register, $tmp2$$Register,
16014                       $tmp3$$Register, $tmp4$$Register,
16015                       -1, $result$$Register, StrIntrinsicNode::LL);
16016   %}
16017   ins_pipe(pipe_class_memory);
16018 %}
16019 
16020 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16021        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16022 %{
16023   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16024   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16025   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16026          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16027   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16028 
16029   ins_encode %{
16030     __ string_indexof($str1$$Register, $str2$$Register,
16031                       $cnt1$$Register, $cnt2$$Register,
16032                       $tmp1$$Register, $tmp2$$Register,
16033                       $tmp3$$Register, $tmp4$$Register,
16034                       -1, $result$$Register, StrIntrinsicNode::UL);
16035   %}
16036   ins_pipe(pipe_class_memory);
16037 %}
16038 
16039 instruct string_indexofLU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16040        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16041 %{
16042   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LU);
16043   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16044   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16045          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16046   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LU)" %}
16047 
16048   ins_encode %{
16049     __ string_indexof($str1$$Register, $str2$$Register,
16050                       $cnt1$$Register, $cnt2$$Register,
16051                       $tmp1$$Register, $tmp2$$Register,
16052                       $tmp3$$Register, $tmp4$$Register,
16053                       -1, $result$$Register, StrIntrinsicNode::LU);
16054   %}
16055   ins_pipe(pipe_class_memory);
16056 %}
16057 
16058 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16059                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16060                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16061 %{
16062   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16063   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16064   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16065          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16066   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16067 
16068   ins_encode %{
16069     int icnt2 = (int)$int_cnt2$$constant;
16070     __ string_indexof($str1$$Register, $str2$$Register,
16071                       $cnt1$$Register, zr,
16072                       $tmp1$$Register, $tmp2$$Register,
16073                       $tmp3$$Register, $tmp4$$Register,
16074                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16075   %}
16076   ins_pipe(pipe_class_memory);
16077 %}
16078 
16079 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16080                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16081                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16082 %{
16083   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16084   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16085   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16086          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16087   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16088 
16089   ins_encode %{
16090     int icnt2 = (int)$int_cnt2$$constant;
16091     __ string_indexof($str1$$Register, $str2$$Register,
16092                       $cnt1$$Register, zr,
16093                       $tmp1$$Register, $tmp2$$Register,
16094                       $tmp3$$Register, $tmp4$$Register,
16095                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16096   %}
16097   ins_pipe(pipe_class_memory);
16098 %}
16099 
16100 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16101                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16102                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16103 %{
16104   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16105   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16106   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16107          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16108   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16109 
16110   ins_encode %{
16111     int icnt2 = (int)$int_cnt2$$constant;
16112     __ string_indexof($str1$$Register, $str2$$Register,
16113                       $cnt1$$Register, zr,
16114                       $tmp1$$Register, $tmp2$$Register,
16115                       $tmp3$$Register, $tmp4$$Register,
16116                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16117   %}
16118   ins_pipe(pipe_class_memory);
16119 %}
16120 
16121 instruct string_indexof_conLU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16122                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16123                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16124 %{
16125   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LU);
16126   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16127   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16128          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16129   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LU)" %}
16130 
16131   ins_encode %{
16132     int icnt2 = (int)$int_cnt2$$constant;
16133     __ string_indexof($str1$$Register, $str2$$Register,
16134                       $cnt1$$Register, zr,
16135                       $tmp1$$Register, $tmp2$$Register,
16136                       $tmp3$$Register, $tmp4$$Register,
16137                       icnt2, $result$$Register, StrIntrinsicNode::LU);
16138   %}
16139   ins_pipe(pipe_class_memory);
16140 %}
16141 
16142 instruct string_indexofU_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16143                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16144                               iRegINoSp tmp3, rFlagsReg cr)
16145 %{
16146   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16147   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16148          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16149 
16150   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16151 
16152   ins_encode %{
16153     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16154                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16155                            $tmp3$$Register);
16156   %}
16157   ins_pipe(pipe_class_memory);
16158 %}
16159 
16160 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16161                         iRegI_R0 result, rFlagsReg cr)
16162 %{
16163   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16164   match(Set result (StrEquals (Binary str1 str2) cnt));
16165   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16166 
16167   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16168   ins_encode %{
16169     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16170     __ string_equals($str1$$Register, $str2$$Register,
16171                      $result$$Register, $cnt$$Register, 1);
16172   %}
16173   ins_pipe(pipe_class_memory);
16174 %}
16175 
16176 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16177                         iRegI_R0 result, rFlagsReg cr)
16178 %{
16179   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16180   match(Set result (StrEquals (Binary str1 str2) cnt));
16181   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16182 
16183   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16184   ins_encode %{
16185     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16186     __ string_equals($str1$$Register, $str2$$Register,
16187                      $result$$Register, $cnt$$Register, 2);
16188   %}
16189   ins_pipe(pipe_class_memory);
16190 %}
16191 
16192 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16193                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16194                        iRegP_R10 tmp, rFlagsReg cr)
16195 %{
16196   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16197   match(Set result (AryEq ary1 ary2));
16198   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16199 
16200   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16201   ins_encode %{
16202     __ arrays_equals($ary1$$Register, $ary2$$Register,
16203                      $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16204                      $result$$Register, $tmp$$Register, 1);
16205     %}
16206   ins_pipe(pipe_class_memory);
16207 %}
16208 
16209 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16210                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16211                        iRegP_R10 tmp, rFlagsReg cr)
16212 %{
16213   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16214   match(Set result (AryEq ary1 ary2));
16215   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16216 
16217   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16218   ins_encode %{
16219     __ arrays_equals($ary1$$Register, $ary2$$Register,
16220                      $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16221                      $result$$Register, $tmp$$Register, 2);
16222   %}
16223   ins_pipe(pipe_class_memory);
16224 %}
16225 
16226 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16227 %{
16228   match(Set result (HasNegatives ary1 len));
16229   effect(USE_KILL ary1, USE_KILL len, KILL cr);
16230   format %{ "has negatives byte[] $ary1,$len -> $result" %}
16231   ins_encode %{
16232     __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
16233   %}
16234   ins_pipe( pipe_slow );
16235 %}
16236 
16237 // fast char[] to byte[] compression
16238 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16239                          vRegD_V0 tmp1, vRegD_V1 tmp2,
16240                          vRegD_V2 tmp3, vRegD_V3 tmp4,
16241                          iRegI_R0 result, rFlagsReg cr)
16242 %{
16243   match(Set result (StrCompressedCopy src (Binary dst len)));
16244   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16245 
16246   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
16247   ins_encode %{
16248     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16249                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16250                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
16251                            $result$$Register);
16252   %}
16253   ins_pipe( pipe_slow );
16254 %}
16255 
16256 // fast byte[] to char[] inflation
16257 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16258                         vRegD tmp1, vRegD tmp2, vRegD tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16259 %{
16260   match(Set dummy (StrInflatedCopy src (Binary dst len)));
16261   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16262 
16263   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
16264   ins_encode %{
16265     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16266                           $tmp1$$FloatRegister, $tmp2$$FloatRegister, $tmp3$$FloatRegister, $tmp4$$Register);
16267   %}
16268   ins_pipe(pipe_class_memory);
16269 %}
16270 
16271 // encode char[] to byte[] in ISO_8859_1
16272 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16273                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
16274                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
16275                           iRegI_R0 result, rFlagsReg cr)
16276 %{
16277   match(Set result (EncodeISOArray src (Binary dst len)));
16278   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
16279          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
16280 
16281   format %{ "Encode array $src,$dst,$len -> $result" %}
16282   ins_encode %{
16283     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16284          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
16285          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
16286   %}
16287   ins_pipe( pipe_class_memory );
16288 %}
16289 
16290 // ============================================================================
16291 // This name is KNOWN by the ADLC and cannot be changed.
16292 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16293 // for this guy.
16294 instruct tlsLoadP(thread_RegP dst)
16295 %{
16296   match(Set dst (ThreadLocal));
16297 
16298   ins_cost(0);
16299 
16300   format %{ " -- \t// $dst=Thread::current(), empty" %}
16301 
16302   size(0);
16303 
16304   ins_encode( /*empty*/ );
16305 
16306   ins_pipe(pipe_class_empty);
16307 %}
16308 
16309 // ====================VECTOR INSTRUCTIONS=====================================
16310 
16311 // Load vector (32 bits)
16312 instruct loadV4(vecD dst, vmem4 mem)
16313 %{
16314   predicate(n->as_LoadVector()->memory_size() == 4);
16315   match(Set dst (LoadVector mem));
16316   ins_cost(4 * INSN_COST);
16317   format %{ "ldrs   $dst,$mem\t# vector (32 bits)" %}
16318   ins_encode( aarch64_enc_ldrvS(dst, mem) );
16319   ins_pipe(vload_reg_mem64);
16320 %}
16321 
16322 // Load vector (64 bits)
16323 instruct loadV8(vecD dst, vmem8 mem)
16324 %{
16325   predicate(n->as_LoadVector()->memory_size() == 8);
16326   match(Set dst (LoadVector mem));
16327   ins_cost(4 * INSN_COST);
16328   format %{ "ldrd   $dst,$mem\t# vector (64 bits)" %}
16329   ins_encode( aarch64_enc_ldrvD(dst, mem) );
16330   ins_pipe(vload_reg_mem64);
16331 %}
16332 
16333 // Load Vector (128 bits)
16334 instruct loadV16(vecX dst, vmem16 mem)
16335 %{
16336   predicate(n->as_LoadVector()->memory_size() == 16);
16337   match(Set dst (LoadVector mem));
16338   ins_cost(4 * INSN_COST);
16339   format %{ "ldrq   $dst,$mem\t# vector (128 bits)" %}
16340   ins_encode( aarch64_enc_ldrvQ(dst, mem) );
16341   ins_pipe(vload_reg_mem128);
16342 %}
16343 
16344 // Store Vector (32 bits)
16345 instruct storeV4(vecD src, vmem4 mem)
16346 %{
16347   predicate(n->as_StoreVector()->memory_size() == 4);
16348   match(Set mem (StoreVector mem src));
16349   ins_cost(4 * INSN_COST);
16350   format %{ "strs   $mem,$src\t# vector (32 bits)" %}
16351   ins_encode( aarch64_enc_strvS(src, mem) );
16352   ins_pipe(vstore_reg_mem64);
16353 %}
16354 
16355 // Store Vector (64 bits)
16356 instruct storeV8(vecD src, vmem8 mem)
16357 %{
16358   predicate(n->as_StoreVector()->memory_size() == 8);
16359   match(Set mem (StoreVector mem src));
16360   ins_cost(4 * INSN_COST);
16361   format %{ "strd   $mem,$src\t# vector (64 bits)" %}
16362   ins_encode( aarch64_enc_strvD(src, mem) );
16363   ins_pipe(vstore_reg_mem64);
16364 %}
16365 
16366 // Store Vector (128 bits)
16367 instruct storeV16(vecX src, vmem16 mem)
16368 %{
16369   predicate(n->as_StoreVector()->memory_size() == 16);
16370   match(Set mem (StoreVector mem src));
16371   ins_cost(4 * INSN_COST);
16372   format %{ "strq   $mem,$src\t# vector (128 bits)" %}
16373   ins_encode( aarch64_enc_strvQ(src, mem) );
16374   ins_pipe(vstore_reg_mem128);
16375 %}
16376 
16377 instruct replicate8B(vecD dst, iRegIorL2I src)
16378 %{
16379   predicate(n->as_Vector()->length() == 4 ||
16380             n->as_Vector()->length() == 8);
16381   match(Set dst (ReplicateB src));
16382   ins_cost(INSN_COST);
16383   format %{ "dup  $dst, $src\t# vector (8B)" %}
16384   ins_encode %{
16385     __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($src$$reg));
16386   %}
16387   ins_pipe(vdup_reg_reg64);
16388 %}
16389 
16390 instruct replicate16B(vecX dst, iRegIorL2I src)
16391 %{
16392   predicate(n->as_Vector()->length() == 16);
16393   match(Set dst (ReplicateB src));
16394   ins_cost(INSN_COST);
16395   format %{ "dup  $dst, $src\t# vector (16B)" %}
16396   ins_encode %{
16397     __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($src$$reg));
16398   %}
16399   ins_pipe(vdup_reg_reg128);
16400 %}
16401 
16402 instruct replicate8B_imm(vecD dst, immI con)
16403 %{
16404   predicate(n->as_Vector()->length() == 4 ||
16405             n->as_Vector()->length() == 8);
16406   match(Set dst (ReplicateB con));
16407   ins_cost(INSN_COST);
16408   format %{ "movi  $dst, $con\t# vector(8B)" %}
16409   ins_encode %{
16410     __ mov(as_FloatRegister($dst$$reg), __ T8B, $con$$constant & 0xff);
16411   %}
16412   ins_pipe(vmovi_reg_imm64);
16413 %}
16414 
16415 instruct replicate16B_imm(vecX dst, immI con)
16416 %{
16417   predicate(n->as_Vector()->length() == 16);
16418   match(Set dst (ReplicateB con));
16419   ins_cost(INSN_COST);
16420   format %{ "movi  $dst, $con\t# vector(16B)" %}
16421   ins_encode %{
16422     __ mov(as_FloatRegister($dst$$reg), __ T16B, $con$$constant & 0xff);
16423   %}
16424   ins_pipe(vmovi_reg_imm128);
16425 %}
16426 
16427 instruct replicate4S(vecD dst, iRegIorL2I src)
16428 %{
16429   predicate(n->as_Vector()->length() == 2 ||
16430             n->as_Vector()->length() == 4);
16431   match(Set dst (ReplicateS src));
16432   ins_cost(INSN_COST);
16433   format %{ "dup  $dst, $src\t# vector (4S)" %}
16434   ins_encode %{
16435     __ dup(as_FloatRegister($dst$$reg), __ T4H, as_Register($src$$reg));
16436   %}
16437   ins_pipe(vdup_reg_reg64);
16438 %}
16439 
16440 instruct replicate8S(vecX dst, iRegIorL2I src)
16441 %{
16442   predicate(n->as_Vector()->length() == 8);
16443   match(Set dst (ReplicateS src));
16444   ins_cost(INSN_COST);
16445   format %{ "dup  $dst, $src\t# vector (8S)" %}
16446   ins_encode %{
16447     __ dup(as_FloatRegister($dst$$reg), __ T8H, as_Register($src$$reg));
16448   %}
16449   ins_pipe(vdup_reg_reg128);
16450 %}
16451 
16452 instruct replicate4S_imm(vecD dst, immI con)
16453 %{
16454   predicate(n->as_Vector()->length() == 2 ||
16455             n->as_Vector()->length() == 4);
16456   match(Set dst (ReplicateS con));
16457   ins_cost(INSN_COST);
16458   format %{ "movi  $dst, $con\t# vector(4H)" %}
16459   ins_encode %{
16460     __ mov(as_FloatRegister($dst$$reg), __ T4H, $con$$constant & 0xffff);
16461   %}
16462   ins_pipe(vmovi_reg_imm64);
16463 %}
16464 
16465 instruct replicate8S_imm(vecX dst, immI con)
16466 %{
16467   predicate(n->as_Vector()->length() == 8);
16468   match(Set dst (ReplicateS con));
16469   ins_cost(INSN_COST);
16470   format %{ "movi  $dst, $con\t# vector(8H)" %}
16471   ins_encode %{
16472     __ mov(as_FloatRegister($dst$$reg), __ T8H, $con$$constant & 0xffff);
16473   %}
16474   ins_pipe(vmovi_reg_imm128);
16475 %}
16476 
16477 instruct replicate2I(vecD dst, iRegIorL2I src)
16478 %{
16479   predicate(n->as_Vector()->length() == 2);
16480   match(Set dst (ReplicateI src));
16481   ins_cost(INSN_COST);
16482   format %{ "dup  $dst, $src\t# vector (2I)" %}
16483   ins_encode %{
16484     __ dup(as_FloatRegister($dst$$reg), __ T2S, as_Register($src$$reg));
16485   %}
16486   ins_pipe(vdup_reg_reg64);
16487 %}
16488 
16489 instruct replicate4I(vecX dst, iRegIorL2I src)
16490 %{
16491   predicate(n->as_Vector()->length() == 4);
16492   match(Set dst (ReplicateI src));
16493   ins_cost(INSN_COST);
16494   format %{ "dup  $dst, $src\t# vector (4I)" %}
16495   ins_encode %{
16496     __ dup(as_FloatRegister($dst$$reg), __ T4S, as_Register($src$$reg));
16497   %}
16498   ins_pipe(vdup_reg_reg128);
16499 %}
16500 
16501 instruct replicate2I_imm(vecD dst, immI con)
16502 %{
16503   predicate(n->as_Vector()->length() == 2);
16504   match(Set dst (ReplicateI con));
16505   ins_cost(INSN_COST);
16506   format %{ "movi  $dst, $con\t# vector(2I)" %}
16507   ins_encode %{
16508     __ mov(as_FloatRegister($dst$$reg), __ T2S, $con$$constant);
16509   %}
16510   ins_pipe(vmovi_reg_imm64);
16511 %}
16512 
16513 instruct replicate4I_imm(vecX dst, immI con)
16514 %{
16515   predicate(n->as_Vector()->length() == 4);
16516   match(Set dst (ReplicateI con));
16517   ins_cost(INSN_COST);
16518   format %{ "movi  $dst, $con\t# vector(4I)" %}
16519   ins_encode %{
16520     __ mov(as_FloatRegister($dst$$reg), __ T4S, $con$$constant);
16521   %}
16522   ins_pipe(vmovi_reg_imm128);
16523 %}
16524 
16525 instruct replicate2L(vecX dst, iRegL src)
16526 %{
16527   predicate(n->as_Vector()->length() == 2);
16528   match(Set dst (ReplicateL src));
16529   ins_cost(INSN_COST);
16530   format %{ "dup  $dst, $src\t# vector (2L)" %}
16531   ins_encode %{
16532     __ dup(as_FloatRegister($dst$$reg), __ T2D, as_Register($src$$reg));
16533   %}
16534   ins_pipe(vdup_reg_reg128);
16535 %}
16536 
16537 instruct replicate2L_zero(vecX dst, immI0 zero)
16538 %{
16539   predicate(n->as_Vector()->length() == 2);
16540   match(Set dst (ReplicateI zero));
16541   ins_cost(INSN_COST);
16542   format %{ "movi  $dst, $zero\t# vector(4I)" %}
16543   ins_encode %{
16544     __ eor(as_FloatRegister($dst$$reg), __ T16B,
16545            as_FloatRegister($dst$$reg),
16546            as_FloatRegister($dst$$reg));
16547   %}
16548   ins_pipe(vmovi_reg_imm128);
16549 %}
16550 
16551 instruct replicate2F(vecD dst, vRegF src)
16552 %{
16553   predicate(n->as_Vector()->length() == 2);
16554   match(Set dst (ReplicateF src));
16555   ins_cost(INSN_COST);
16556   format %{ "dup  $dst, $src\t# vector (2F)" %}
16557   ins_encode %{
16558     __ dup(as_FloatRegister($dst$$reg), __ T2S,
16559            as_FloatRegister($src$$reg));
16560   %}
16561   ins_pipe(vdup_reg_freg64);
16562 %}
16563 
16564 instruct replicate4F(vecX dst, vRegF src)
16565 %{
16566   predicate(n->as_Vector()->length() == 4);
16567   match(Set dst (ReplicateF src));
16568   ins_cost(INSN_COST);
16569   format %{ "dup  $dst, $src\t# vector (4F)" %}
16570   ins_encode %{
16571     __ dup(as_FloatRegister($dst$$reg), __ T4S,
16572            as_FloatRegister($src$$reg));
16573   %}
16574   ins_pipe(vdup_reg_freg128);
16575 %}
16576 
16577 instruct replicate2D(vecX dst, vRegD src)
16578 %{
16579   predicate(n->as_Vector()->length() == 2);
16580   match(Set dst (ReplicateD src));
16581   ins_cost(INSN_COST);
16582   format %{ "dup  $dst, $src\t# vector (2D)" %}
16583   ins_encode %{
16584     __ dup(as_FloatRegister($dst$$reg), __ T2D,
16585            as_FloatRegister($src$$reg));
16586   %}
16587   ins_pipe(vdup_reg_dreg128);
16588 %}
16589 
16590 // ====================REDUCTION ARITHMETIC====================================
16591 
16592 instruct reduce_add2I(iRegINoSp dst, iRegIorL2I src1, vecD src2, iRegINoSp tmp, iRegINoSp tmp2)
16593 %{
16594   match(Set dst (AddReductionVI src1 src2));
16595   ins_cost(INSN_COST);
16596   effect(TEMP tmp, TEMP tmp2);
16597   format %{ "umov  $tmp, $src2, S, 0\n\t"
16598             "umov  $tmp2, $src2, S, 1\n\t"
16599             "addw  $dst, $src1, $tmp\n\t"
16600             "addw  $dst, $dst, $tmp2\t add reduction2i"
16601   %}
16602   ins_encode %{
16603     __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 0);
16604     __ umov($tmp2$$Register, as_FloatRegister($src2$$reg), __ S, 1);
16605     __ addw($dst$$Register, $src1$$Register, $tmp$$Register);
16606     __ addw($dst$$Register, $dst$$Register, $tmp2$$Register);
16607   %}
16608   ins_pipe(pipe_class_default);
16609 %}
16610 
16611 instruct reduce_add4I(iRegINoSp dst, iRegIorL2I src1, vecX src2, vecX tmp, iRegINoSp tmp2)
16612 %{
16613   match(Set dst (AddReductionVI src1 src2));
16614   ins_cost(INSN_COST);
16615   effect(TEMP tmp, TEMP tmp2);
16616   format %{ "addv  $tmp, T4S, $src2\n\t"
16617             "umov  $tmp2, $tmp, S, 0\n\t"
16618             "addw  $dst, $tmp2, $src1\t add reduction4i"
16619   %}
16620   ins_encode %{
16621     __ addv(as_FloatRegister($tmp$$reg), __ T4S,
16622             as_FloatRegister($src2$$reg));
16623     __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 0);
16624     __ addw($dst$$Register, $tmp2$$Register, $src1$$Register);
16625   %}
16626   ins_pipe(pipe_class_default);
16627 %}
16628 
16629 instruct reduce_mul2I(iRegINoSp dst, iRegIorL2I src1, vecD src2, iRegINoSp tmp)
16630 %{
16631   match(Set dst (MulReductionVI src1 src2));
16632   ins_cost(INSN_COST);
16633   effect(TEMP tmp, TEMP dst);
16634   format %{ "umov  $tmp, $src2, S, 0\n\t"
16635             "mul   $dst, $tmp, $src1\n\t"
16636             "umov  $tmp, $src2, S, 1\n\t"
16637             "mul   $dst, $tmp, $dst\t mul reduction2i\n\t"
16638   %}
16639   ins_encode %{
16640     __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 0);
16641     __ mul($dst$$Register, $tmp$$Register, $src1$$Register);
16642     __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 1);
16643     __ mul($dst$$Register, $tmp$$Register, $dst$$Register);
16644   %}
16645   ins_pipe(pipe_class_default);
16646 %}
16647 
16648 instruct reduce_mul4I(iRegINoSp dst, iRegIorL2I src1, vecX src2, vecX tmp, iRegINoSp tmp2)
16649 %{
16650   match(Set dst (MulReductionVI src1 src2));
16651   ins_cost(INSN_COST);
16652   effect(TEMP tmp, TEMP tmp2, TEMP dst);
16653   format %{ "ins   $tmp, $src2, 0, 1\n\t"
16654             "mul   $tmp, $tmp, $src2\n\t"
16655             "umov  $tmp2, $tmp, S, 0\n\t"
16656             "mul   $dst, $tmp2, $src1\n\t"
16657             "umov  $tmp2, $tmp, S, 1\n\t"
16658             "mul   $dst, $tmp2, $dst\t mul reduction4i\n\t"
16659   %}
16660   ins_encode %{
16661     __ ins(as_FloatRegister($tmp$$reg), __ D,
16662            as_FloatRegister($src2$$reg), 0, 1);
16663     __ mulv(as_FloatRegister($tmp$$reg), __ T2S,
16664            as_FloatRegister($tmp$$reg), as_FloatRegister($src2$$reg));
16665     __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 0);
16666     __ mul($dst$$Register, $tmp2$$Register, $src1$$Register);
16667     __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 1);
16668     __ mul($dst$$Register, $tmp2$$Register, $dst$$Register);
16669   %}
16670   ins_pipe(pipe_class_default);
16671 %}
16672 
16673 instruct reduce_add2F(vRegF dst, vRegF src1, vecD src2, vecD tmp)
16674 %{
16675   match(Set dst (AddReductionVF src1 src2));
16676   ins_cost(INSN_COST);
16677   effect(TEMP tmp, TEMP dst);
16678   format %{ "fadds $dst, $src1, $src2\n\t"
16679             "ins   $tmp, S, $src2, 0, 1\n\t"
16680             "fadds $dst, $dst, $tmp\t add reduction2f"
16681   %}
16682   ins_encode %{
16683     __ fadds(as_FloatRegister($dst$$reg),
16684              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16685     __ ins(as_FloatRegister($tmp$$reg), __ S,
16686            as_FloatRegister($src2$$reg), 0, 1);
16687     __ fadds(as_FloatRegister($dst$$reg),
16688              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16689   %}
16690   ins_pipe(pipe_class_default);
16691 %}
16692 
16693 instruct reduce_add4F(vRegF dst, vRegF src1, vecX src2, vecX tmp)
16694 %{
16695   match(Set dst (AddReductionVF src1 src2));
16696   ins_cost(INSN_COST);
16697   effect(TEMP tmp, TEMP dst);
16698   format %{ "fadds $dst, $src1, $src2\n\t"
16699             "ins   $tmp, S, $src2, 0, 1\n\t"
16700             "fadds $dst, $dst, $tmp\n\t"
16701             "ins   $tmp, S, $src2, 0, 2\n\t"
16702             "fadds $dst, $dst, $tmp\n\t"
16703             "ins   $tmp, S, $src2, 0, 3\n\t"
16704             "fadds $dst, $dst, $tmp\t add reduction4f"
16705   %}
16706   ins_encode %{
16707     __ fadds(as_FloatRegister($dst$$reg),
16708              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16709     __ ins(as_FloatRegister($tmp$$reg), __ S,
16710            as_FloatRegister($src2$$reg), 0, 1);
16711     __ fadds(as_FloatRegister($dst$$reg),
16712              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16713     __ ins(as_FloatRegister($tmp$$reg), __ S,
16714            as_FloatRegister($src2$$reg), 0, 2);
16715     __ fadds(as_FloatRegister($dst$$reg),
16716              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16717     __ ins(as_FloatRegister($tmp$$reg), __ S,
16718            as_FloatRegister($src2$$reg), 0, 3);
16719     __ fadds(as_FloatRegister($dst$$reg),
16720              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16721   %}
16722   ins_pipe(pipe_class_default);
16723 %}
16724 
16725 instruct reduce_mul2F(vRegF dst, vRegF src1, vecD src2, vecD tmp)
16726 %{
16727   match(Set dst (MulReductionVF src1 src2));
16728   ins_cost(INSN_COST);
16729   effect(TEMP tmp, TEMP dst);
16730   format %{ "fmuls $dst, $src1, $src2\n\t"
16731             "ins   $tmp, S, $src2, 0, 1\n\t"
16732             "fmuls $dst, $dst, $tmp\t add reduction4f"
16733   %}
16734   ins_encode %{
16735     __ fmuls(as_FloatRegister($dst$$reg),
16736              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16737     __ ins(as_FloatRegister($tmp$$reg), __ S,
16738            as_FloatRegister($src2$$reg), 0, 1);
16739     __ fmuls(as_FloatRegister($dst$$reg),
16740              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16741   %}
16742   ins_pipe(pipe_class_default);
16743 %}
16744 
16745 instruct reduce_mul4F(vRegF dst, vRegF src1, vecX src2, vecX tmp)
16746 %{
16747   match(Set dst (MulReductionVF src1 src2));
16748   ins_cost(INSN_COST);
16749   effect(TEMP tmp, TEMP dst);
16750   format %{ "fmuls $dst, $src1, $src2\n\t"
16751             "ins   $tmp, S, $src2, 0, 1\n\t"
16752             "fmuls $dst, $dst, $tmp\n\t"
16753             "ins   $tmp, S, $src2, 0, 2\n\t"
16754             "fmuls $dst, $dst, $tmp\n\t"
16755             "ins   $tmp, S, $src2, 0, 3\n\t"
16756             "fmuls $dst, $dst, $tmp\t add reduction4f"
16757   %}
16758   ins_encode %{
16759     __ fmuls(as_FloatRegister($dst$$reg),
16760              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16761     __ ins(as_FloatRegister($tmp$$reg), __ S,
16762            as_FloatRegister($src2$$reg), 0, 1);
16763     __ fmuls(as_FloatRegister($dst$$reg),
16764              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16765     __ ins(as_FloatRegister($tmp$$reg), __ S,
16766            as_FloatRegister($src2$$reg), 0, 2);
16767     __ fmuls(as_FloatRegister($dst$$reg),
16768              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16769     __ ins(as_FloatRegister($tmp$$reg), __ S,
16770            as_FloatRegister($src2$$reg), 0, 3);
16771     __ fmuls(as_FloatRegister($dst$$reg),
16772              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16773   %}
16774   ins_pipe(pipe_class_default);
16775 %}
16776 
16777 instruct reduce_add2D(vRegD dst, vRegD src1, vecX src2, vecX tmp)
16778 %{
16779   match(Set dst (AddReductionVD src1 src2));
16780   ins_cost(INSN_COST);
16781   effect(TEMP tmp, TEMP dst);
16782   format %{ "faddd $dst, $src1, $src2\n\t"
16783             "ins   $tmp, D, $src2, 0, 1\n\t"
16784             "faddd $dst, $dst, $tmp\t add reduction2d"
16785   %}
16786   ins_encode %{
16787     __ faddd(as_FloatRegister($dst$$reg),
16788              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16789     __ ins(as_FloatRegister($tmp$$reg), __ D,
16790            as_FloatRegister($src2$$reg), 0, 1);
16791     __ faddd(as_FloatRegister($dst$$reg),
16792              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16793   %}
16794   ins_pipe(pipe_class_default);
16795 %}
16796 
16797 instruct reduce_mul2D(vRegD dst, vRegD src1, vecX src2, vecX tmp)
16798 %{
16799   match(Set dst (MulReductionVD src1 src2));
16800   ins_cost(INSN_COST);
16801   effect(TEMP tmp, TEMP dst);
16802   format %{ "fmuld $dst, $src1, $src2\n\t"
16803             "ins   $tmp, D, $src2, 0, 1\n\t"
16804             "fmuld $dst, $dst, $tmp\t add reduction2d"
16805   %}
16806   ins_encode %{
16807     __ fmuld(as_FloatRegister($dst$$reg),
16808              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16809     __ ins(as_FloatRegister($tmp$$reg), __ D,
16810            as_FloatRegister($src2$$reg), 0, 1);
16811     __ fmuld(as_FloatRegister($dst$$reg),
16812              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16813   %}
16814   ins_pipe(pipe_class_default);
16815 %}
16816 
16817 // ====================VECTOR ARITHMETIC=======================================
16818 
16819 // --------------------------------- ADD --------------------------------------
16820 
16821 instruct vadd8B(vecD dst, vecD src1, vecD src2)
16822 %{
16823   predicate(n->as_Vector()->length() == 4 ||
16824             n->as_Vector()->length() == 8);
16825   match(Set dst (AddVB src1 src2));
16826   ins_cost(INSN_COST);
16827   format %{ "addv  $dst,$src1,$src2\t# vector (8B)" %}
16828   ins_encode %{
16829     __ addv(as_FloatRegister($dst$$reg), __ T8B,
16830             as_FloatRegister($src1$$reg),
16831             as_FloatRegister($src2$$reg));
16832   %}
16833   ins_pipe(vdop64);
16834 %}
16835 
16836 instruct vadd16B(vecX dst, vecX src1, vecX src2)
16837 %{
16838   predicate(n->as_Vector()->length() == 16);
16839   match(Set dst (AddVB src1 src2));
16840   ins_cost(INSN_COST);
16841   format %{ "addv  $dst,$src1,$src2\t# vector (16B)" %}
16842   ins_encode %{
16843     __ addv(as_FloatRegister($dst$$reg), __ T16B,
16844             as_FloatRegister($src1$$reg),
16845             as_FloatRegister($src2$$reg));
16846   %}
16847   ins_pipe(vdop128);
16848 %}
16849 
16850 instruct vadd4S(vecD dst, vecD src1, vecD src2)
16851 %{
16852   predicate(n->as_Vector()->length() == 2 ||
16853             n->as_Vector()->length() == 4);
16854   match(Set dst (AddVS src1 src2));
16855   ins_cost(INSN_COST);
16856   format %{ "addv  $dst,$src1,$src2\t# vector (4H)" %}
16857   ins_encode %{
16858     __ addv(as_FloatRegister($dst$$reg), __ T4H,
16859             as_FloatRegister($src1$$reg),
16860             as_FloatRegister($src2$$reg));
16861   %}
16862   ins_pipe(vdop64);
16863 %}
16864 
16865 instruct vadd8S(vecX dst, vecX src1, vecX src2)
16866 %{
16867   predicate(n->as_Vector()->length() == 8);
16868   match(Set dst (AddVS src1 src2));
16869   ins_cost(INSN_COST);
16870   format %{ "addv  $dst,$src1,$src2\t# vector (8H)" %}
16871   ins_encode %{
16872     __ addv(as_FloatRegister($dst$$reg), __ T8H,
16873             as_FloatRegister($src1$$reg),
16874             as_FloatRegister($src2$$reg));
16875   %}
16876   ins_pipe(vdop128);
16877 %}
16878 
16879 instruct vadd2I(vecD dst, vecD src1, vecD src2)
16880 %{
16881   predicate(n->as_Vector()->length() == 2);
16882   match(Set dst (AddVI src1 src2));
16883   ins_cost(INSN_COST);
16884   format %{ "addv  $dst,$src1,$src2\t# vector (2S)" %}
16885   ins_encode %{
16886     __ addv(as_FloatRegister($dst$$reg), __ T2S,
16887             as_FloatRegister($src1$$reg),
16888             as_FloatRegister($src2$$reg));
16889   %}
16890   ins_pipe(vdop64);
16891 %}
16892 
16893 instruct vadd4I(vecX dst, vecX src1, vecX src2)
16894 %{
16895   predicate(n->as_Vector()->length() == 4);
16896   match(Set dst (AddVI src1 src2));
16897   ins_cost(INSN_COST);
16898   format %{ "addv  $dst,$src1,$src2\t# vector (4S)" %}
16899   ins_encode %{
16900     __ addv(as_FloatRegister($dst$$reg), __ T4S,
16901             as_FloatRegister($src1$$reg),
16902             as_FloatRegister($src2$$reg));
16903   %}
16904   ins_pipe(vdop128);
16905 %}
16906 
16907 instruct vadd2L(vecX dst, vecX src1, vecX src2)
16908 %{
16909   predicate(n->as_Vector()->length() == 2);
16910   match(Set dst (AddVL src1 src2));
16911   ins_cost(INSN_COST);
16912   format %{ "addv  $dst,$src1,$src2\t# vector (2L)" %}
16913   ins_encode %{
16914     __ addv(as_FloatRegister($dst$$reg), __ T2D,
16915             as_FloatRegister($src1$$reg),
16916             as_FloatRegister($src2$$reg));
16917   %}
16918   ins_pipe(vdop128);
16919 %}
16920 
16921 instruct vadd2F(vecD dst, vecD src1, vecD src2)
16922 %{
16923   predicate(n->as_Vector()->length() == 2);
16924   match(Set dst (AddVF src1 src2));
16925   ins_cost(INSN_COST);
16926   format %{ "fadd  $dst,$src1,$src2\t# vector (2S)" %}
16927   ins_encode %{
16928     __ fadd(as_FloatRegister($dst$$reg), __ T2S,
16929             as_FloatRegister($src1$$reg),
16930             as_FloatRegister($src2$$reg));
16931   %}
16932   ins_pipe(vdop_fp64);
16933 %}
16934 
16935 instruct vadd4F(vecX dst, vecX src1, vecX src2)
16936 %{
16937   predicate(n->as_Vector()->length() == 4);
16938   match(Set dst (AddVF src1 src2));
16939   ins_cost(INSN_COST);
16940   format %{ "fadd  $dst,$src1,$src2\t# vector (4S)" %}
16941   ins_encode %{
16942     __ fadd(as_FloatRegister($dst$$reg), __ T4S,
16943             as_FloatRegister($src1$$reg),
16944             as_FloatRegister($src2$$reg));
16945   %}
16946   ins_pipe(vdop_fp128);
16947 %}
16948 
16949 instruct vadd2D(vecX dst, vecX src1, vecX src2)
16950 %{
16951   match(Set dst (AddVD src1 src2));
16952   ins_cost(INSN_COST);
16953   format %{ "fadd  $dst,$src1,$src2\t# vector (2D)" %}
16954   ins_encode %{
16955     __ fadd(as_FloatRegister($dst$$reg), __ T2D,
16956             as_FloatRegister($src1$$reg),
16957             as_FloatRegister($src2$$reg));
16958   %}
16959   ins_pipe(vdop_fp128);
16960 %}
16961 
16962 // --------------------------------- SUB --------------------------------------
16963 
16964 instruct vsub8B(vecD dst, vecD src1, vecD src2)
16965 %{
16966   predicate(n->as_Vector()->length() == 4 ||
16967             n->as_Vector()->length() == 8);
16968   match(Set dst (SubVB src1 src2));
16969   ins_cost(INSN_COST);
16970   format %{ "subv  $dst,$src1,$src2\t# vector (8B)" %}
16971   ins_encode %{
16972     __ subv(as_FloatRegister($dst$$reg), __ T8B,
16973             as_FloatRegister($src1$$reg),
16974             as_FloatRegister($src2$$reg));
16975   %}
16976   ins_pipe(vdop64);
16977 %}
16978 
16979 instruct vsub16B(vecX dst, vecX src1, vecX src2)
16980 %{
16981   predicate(n->as_Vector()->length() == 16);
16982   match(Set dst (SubVB src1 src2));
16983   ins_cost(INSN_COST);
16984   format %{ "subv  $dst,$src1,$src2\t# vector (16B)" %}
16985   ins_encode %{
16986     __ subv(as_FloatRegister($dst$$reg), __ T16B,
16987             as_FloatRegister($src1$$reg),
16988             as_FloatRegister($src2$$reg));
16989   %}
16990   ins_pipe(vdop128);
16991 %}
16992 
16993 instruct vsub4S(vecD dst, vecD src1, vecD src2)
16994 %{
16995   predicate(n->as_Vector()->length() == 2 ||
16996             n->as_Vector()->length() == 4);
16997   match(Set dst (SubVS src1 src2));
16998   ins_cost(INSN_COST);
16999   format %{ "subv  $dst,$src1,$src2\t# vector (4H)" %}
17000   ins_encode %{
17001     __ subv(as_FloatRegister($dst$$reg), __ T4H,
17002             as_FloatRegister($src1$$reg),
17003             as_FloatRegister($src2$$reg));
17004   %}
17005   ins_pipe(vdop64);
17006 %}
17007 
17008 instruct vsub8S(vecX dst, vecX src1, vecX src2)
17009 %{
17010   predicate(n->as_Vector()->length() == 8);
17011   match(Set dst (SubVS src1 src2));
17012   ins_cost(INSN_COST);
17013   format %{ "subv  $dst,$src1,$src2\t# vector (8H)" %}
17014   ins_encode %{
17015     __ subv(as_FloatRegister($dst$$reg), __ T8H,
17016             as_FloatRegister($src1$$reg),
17017             as_FloatRegister($src2$$reg));
17018   %}
17019   ins_pipe(vdop128);
17020 %}
17021 
17022 instruct vsub2I(vecD dst, vecD src1, vecD src2)
17023 %{
17024   predicate(n->as_Vector()->length() == 2);
17025   match(Set dst (SubVI src1 src2));
17026   ins_cost(INSN_COST);
17027   format %{ "subv  $dst,$src1,$src2\t# vector (2S)" %}
17028   ins_encode %{
17029     __ subv(as_FloatRegister($dst$$reg), __ T2S,
17030             as_FloatRegister($src1$$reg),
17031             as_FloatRegister($src2$$reg));
17032   %}
17033   ins_pipe(vdop64);
17034 %}
17035 
17036 instruct vsub4I(vecX dst, vecX src1, vecX src2)
17037 %{
17038   predicate(n->as_Vector()->length() == 4);
17039   match(Set dst (SubVI src1 src2));
17040   ins_cost(INSN_COST);
17041   format %{ "subv  $dst,$src1,$src2\t# vector (4S)" %}
17042   ins_encode %{
17043     __ subv(as_FloatRegister($dst$$reg), __ T4S,
17044             as_FloatRegister($src1$$reg),
17045             as_FloatRegister($src2$$reg));
17046   %}
17047   ins_pipe(vdop128);
17048 %}
17049 
17050 instruct vsub2L(vecX dst, vecX src1, vecX src2)
17051 %{
17052   predicate(n->as_Vector()->length() == 2);
17053   match(Set dst (SubVL src1 src2));
17054   ins_cost(INSN_COST);
17055   format %{ "subv  $dst,$src1,$src2\t# vector (2L)" %}
17056   ins_encode %{
17057     __ subv(as_FloatRegister($dst$$reg), __ T2D,
17058             as_FloatRegister($src1$$reg),
17059             as_FloatRegister($src2$$reg));
17060   %}
17061   ins_pipe(vdop128);
17062 %}
17063 
17064 instruct vsub2F(vecD dst, vecD src1, vecD src2)
17065 %{
17066   predicate(n->as_Vector()->length() == 2);
17067   match(Set dst (SubVF src1 src2));
17068   ins_cost(INSN_COST);
17069   format %{ "fsub  $dst,$src1,$src2\t# vector (2S)" %}
17070   ins_encode %{
17071     __ fsub(as_FloatRegister($dst$$reg), __ T2S,
17072             as_FloatRegister($src1$$reg),
17073             as_FloatRegister($src2$$reg));
17074   %}
17075   ins_pipe(vdop_fp64);
17076 %}
17077 
17078 instruct vsub4F(vecX dst, vecX src1, vecX src2)
17079 %{
17080   predicate(n->as_Vector()->length() == 4);
17081   match(Set dst (SubVF src1 src2));
17082   ins_cost(INSN_COST);
17083   format %{ "fsub  $dst,$src1,$src2\t# vector (4S)" %}
17084   ins_encode %{
17085     __ fsub(as_FloatRegister($dst$$reg), __ T4S,
17086             as_FloatRegister($src1$$reg),
17087             as_FloatRegister($src2$$reg));
17088   %}
17089   ins_pipe(vdop_fp128);
17090 %}
17091 
17092 instruct vsub2D(vecX dst, vecX src1, vecX src2)
17093 %{
17094   predicate(n->as_Vector()->length() == 2);
17095   match(Set dst (SubVD src1 src2));
17096   ins_cost(INSN_COST);
17097   format %{ "fsub  $dst,$src1,$src2\t# vector (2D)" %}
17098   ins_encode %{
17099     __ fsub(as_FloatRegister($dst$$reg), __ T2D,
17100             as_FloatRegister($src1$$reg),
17101             as_FloatRegister($src2$$reg));
17102   %}
17103   ins_pipe(vdop_fp128);
17104 %}
17105 
17106 // --------------------------------- MUL --------------------------------------
17107 
17108 instruct vmul4S(vecD dst, vecD src1, vecD src2)
17109 %{
17110   predicate(n->as_Vector()->length() == 2 ||
17111             n->as_Vector()->length() == 4);
17112   match(Set dst (MulVS src1 src2));
17113   ins_cost(INSN_COST);
17114   format %{ "mulv  $dst,$src1,$src2\t# vector (4H)" %}
17115   ins_encode %{
17116     __ mulv(as_FloatRegister($dst$$reg), __ T4H,
17117             as_FloatRegister($src1$$reg),
17118             as_FloatRegister($src2$$reg));
17119   %}
17120   ins_pipe(vmul64);
17121 %}
17122 
17123 instruct vmul8S(vecX dst, vecX src1, vecX src2)
17124 %{
17125   predicate(n->as_Vector()->length() == 8);
17126   match(Set dst (MulVS src1 src2));
17127   ins_cost(INSN_COST);
17128   format %{ "mulv  $dst,$src1,$src2\t# vector (8H)" %}
17129   ins_encode %{
17130     __ mulv(as_FloatRegister($dst$$reg), __ T8H,
17131             as_FloatRegister($src1$$reg),
17132             as_FloatRegister($src2$$reg));
17133   %}
17134   ins_pipe(vmul128);
17135 %}
17136 
17137 instruct vmul2I(vecD dst, vecD src1, vecD src2)
17138 %{
17139   predicate(n->as_Vector()->length() == 2);
17140   match(Set dst (MulVI src1 src2));
17141   ins_cost(INSN_COST);
17142   format %{ "mulv  $dst,$src1,$src2\t# vector (2S)" %}
17143   ins_encode %{
17144     __ mulv(as_FloatRegister($dst$$reg), __ T2S,
17145             as_FloatRegister($src1$$reg),
17146             as_FloatRegister($src2$$reg));
17147   %}
17148   ins_pipe(vmul64);
17149 %}
17150 
17151 instruct vmul4I(vecX dst, vecX src1, vecX src2)
17152 %{
17153   predicate(n->as_Vector()->length() == 4);
17154   match(Set dst (MulVI src1 src2));
17155   ins_cost(INSN_COST);
17156   format %{ "mulv  $dst,$src1,$src2\t# vector (4S)" %}
17157   ins_encode %{
17158     __ mulv(as_FloatRegister($dst$$reg), __ T4S,
17159             as_FloatRegister($src1$$reg),
17160             as_FloatRegister($src2$$reg));
17161   %}
17162   ins_pipe(vmul128);
17163 %}
17164 
17165 instruct vmul2F(vecD dst, vecD src1, vecD src2)
17166 %{
17167   predicate(n->as_Vector()->length() == 2);
17168   match(Set dst (MulVF src1 src2));
17169   ins_cost(INSN_COST);
17170   format %{ "fmul  $dst,$src1,$src2\t# vector (2S)" %}
17171   ins_encode %{
17172     __ fmul(as_FloatRegister($dst$$reg), __ T2S,
17173             as_FloatRegister($src1$$reg),
17174             as_FloatRegister($src2$$reg));
17175   %}
17176   ins_pipe(vmuldiv_fp64);
17177 %}
17178 
17179 instruct vmul4F(vecX dst, vecX src1, vecX src2)
17180 %{
17181   predicate(n->as_Vector()->length() == 4);
17182   match(Set dst (MulVF src1 src2));
17183   ins_cost(INSN_COST);
17184   format %{ "fmul  $dst,$src1,$src2\t# vector (4S)" %}
17185   ins_encode %{
17186     __ fmul(as_FloatRegister($dst$$reg), __ T4S,
17187             as_FloatRegister($src1$$reg),
17188             as_FloatRegister($src2$$reg));
17189   %}
17190   ins_pipe(vmuldiv_fp128);
17191 %}
17192 
17193 instruct vmul2D(vecX dst, vecX src1, vecX src2)
17194 %{
17195   predicate(n->as_Vector()->length() == 2);
17196   match(Set dst (MulVD src1 src2));
17197   ins_cost(INSN_COST);
17198   format %{ "fmul  $dst,$src1,$src2\t# vector (2D)" %}
17199   ins_encode %{
17200     __ fmul(as_FloatRegister($dst$$reg), __ T2D,
17201             as_FloatRegister($src1$$reg),
17202             as_FloatRegister($src2$$reg));
17203   %}
17204   ins_pipe(vmuldiv_fp128);
17205 %}
17206 
17207 // --------------------------------- MLA --------------------------------------
17208 
17209 instruct vmla4S(vecD dst, vecD src1, vecD src2)
17210 %{
17211   predicate(n->as_Vector()->length() == 2 ||
17212             n->as_Vector()->length() == 4);
17213   match(Set dst (AddVS dst (MulVS src1 src2)));
17214   ins_cost(INSN_COST);
17215   format %{ "mlav  $dst,$src1,$src2\t# vector (4H)" %}
17216   ins_encode %{
17217     __ mlav(as_FloatRegister($dst$$reg), __ T4H,
17218             as_FloatRegister($src1$$reg),
17219             as_FloatRegister($src2$$reg));
17220   %}
17221   ins_pipe(vmla64);
17222 %}
17223 
17224 instruct vmla8S(vecX dst, vecX src1, vecX src2)
17225 %{
17226   predicate(n->as_Vector()->length() == 8);
17227   match(Set dst (AddVS dst (MulVS src1 src2)));
17228   ins_cost(INSN_COST);
17229   format %{ "mlav  $dst,$src1,$src2\t# vector (8H)" %}
17230   ins_encode %{
17231     __ mlav(as_FloatRegister($dst$$reg), __ T8H,
17232             as_FloatRegister($src1$$reg),
17233             as_FloatRegister($src2$$reg));
17234   %}
17235   ins_pipe(vmla128);
17236 %}
17237 
17238 instruct vmla2I(vecD dst, vecD src1, vecD src2)
17239 %{
17240   predicate(n->as_Vector()->length() == 2);
17241   match(Set dst (AddVI dst (MulVI src1 src2)));
17242   ins_cost(INSN_COST);
17243   format %{ "mlav  $dst,$src1,$src2\t# vector (2S)" %}
17244   ins_encode %{
17245     __ mlav(as_FloatRegister($dst$$reg), __ T2S,
17246             as_FloatRegister($src1$$reg),
17247             as_FloatRegister($src2$$reg));
17248   %}
17249   ins_pipe(vmla64);
17250 %}
17251 
17252 instruct vmla4I(vecX dst, vecX src1, vecX src2)
17253 %{
17254   predicate(n->as_Vector()->length() == 4);
17255   match(Set dst (AddVI dst (MulVI src1 src2)));
17256   ins_cost(INSN_COST);
17257   format %{ "mlav  $dst,$src1,$src2\t# vector (4S)" %}
17258   ins_encode %{
17259     __ mlav(as_FloatRegister($dst$$reg), __ T4S,
17260             as_FloatRegister($src1$$reg),
17261             as_FloatRegister($src2$$reg));
17262   %}
17263   ins_pipe(vmla128);
17264 %}
17265 
17266 // dst + src1 * src2
17267 instruct vmla2F(vecD dst, vecD src1, vecD src2) %{
17268   predicate(UseFMA && n->as_Vector()->length() == 2);
17269   match(Set dst (FmaVF  dst (Binary src1 src2)));
17270   format %{ "fmla  $dst,$src1,$src2\t# vector (2S)" %}
17271   ins_cost(INSN_COST);
17272   ins_encode %{
17273     __ fmla(as_FloatRegister($dst$$reg), __ T2S,
17274             as_FloatRegister($src1$$reg),
17275             as_FloatRegister($src2$$reg));
17276   %}
17277   ins_pipe(vmuldiv_fp64);
17278 %}
17279 
17280 // dst + src1 * src2
17281 instruct vmla4F(vecX dst, vecX src1, vecX src2) %{
17282   predicate(UseFMA && n->as_Vector()->length() == 4);
17283   match(Set dst (FmaVF  dst (Binary src1 src2)));
17284   format %{ "fmla  $dst,$src1,$src2\t# vector (4S)" %}
17285   ins_cost(INSN_COST);
17286   ins_encode %{
17287     __ fmla(as_FloatRegister($dst$$reg), __ T4S,
17288             as_FloatRegister($src1$$reg),
17289             as_FloatRegister($src2$$reg));
17290   %}
17291   ins_pipe(vmuldiv_fp128);
17292 %}
17293 
17294 // dst + src1 * src2
17295 instruct vmla2D(vecX dst, vecX src1, vecX src2) %{
17296   predicate(UseFMA && n->as_Vector()->length() == 2);
17297   match(Set dst (FmaVD  dst (Binary src1 src2)));
17298   format %{ "fmla  $dst,$src1,$src2\t# vector (2D)" %}
17299   ins_cost(INSN_COST);
17300   ins_encode %{
17301     __ fmla(as_FloatRegister($dst$$reg), __ T2D,
17302             as_FloatRegister($src1$$reg),
17303             as_FloatRegister($src2$$reg));
17304   %}
17305   ins_pipe(vmuldiv_fp128);
17306 %}
17307 
17308 // --------------------------------- MLS --------------------------------------
17309 
17310 instruct vmls4S(vecD dst, vecD src1, vecD src2)
17311 %{
17312   predicate(n->as_Vector()->length() == 2 ||
17313             n->as_Vector()->length() == 4);
17314   match(Set dst (SubVS dst (MulVS src1 src2)));
17315   ins_cost(INSN_COST);
17316   format %{ "mlsv  $dst,$src1,$src2\t# vector (4H)" %}
17317   ins_encode %{
17318     __ mlsv(as_FloatRegister($dst$$reg), __ T4H,
17319             as_FloatRegister($src1$$reg),
17320             as_FloatRegister($src2$$reg));
17321   %}
17322   ins_pipe(vmla64);
17323 %}
17324 
17325 instruct vmls8S(vecX dst, vecX src1, vecX src2)
17326 %{
17327   predicate(n->as_Vector()->length() == 8);
17328   match(Set dst (SubVS dst (MulVS src1 src2)));
17329   ins_cost(INSN_COST);
17330   format %{ "mlsv  $dst,$src1,$src2\t# vector (8H)" %}
17331   ins_encode %{
17332     __ mlsv(as_FloatRegister($dst$$reg), __ T8H,
17333             as_FloatRegister($src1$$reg),
17334             as_FloatRegister($src2$$reg));
17335   %}
17336   ins_pipe(vmla128);
17337 %}
17338 
17339 instruct vmls2I(vecD dst, vecD src1, vecD src2)
17340 %{
17341   predicate(n->as_Vector()->length() == 2);
17342   match(Set dst (SubVI dst (MulVI src1 src2)));
17343   ins_cost(INSN_COST);
17344   format %{ "mlsv  $dst,$src1,$src2\t# vector (2S)" %}
17345   ins_encode %{
17346     __ mlsv(as_FloatRegister($dst$$reg), __ T2S,
17347             as_FloatRegister($src1$$reg),
17348             as_FloatRegister($src2$$reg));
17349   %}
17350   ins_pipe(vmla64);
17351 %}
17352 
17353 instruct vmls4I(vecX dst, vecX src1, vecX src2)
17354 %{
17355   predicate(n->as_Vector()->length() == 4);
17356   match(Set dst (SubVI dst (MulVI src1 src2)));
17357   ins_cost(INSN_COST);
17358   format %{ "mlsv  $dst,$src1,$src2\t# vector (4S)" %}
17359   ins_encode %{
17360     __ mlsv(as_FloatRegister($dst$$reg), __ T4S,
17361             as_FloatRegister($src1$$reg),
17362             as_FloatRegister($src2$$reg));
17363   %}
17364   ins_pipe(vmla128);
17365 %}
17366 
17367 // dst - src1 * src2
17368 instruct vmls2F(vecD dst, vecD src1, vecD src2) %{
17369   predicate(UseFMA && n->as_Vector()->length() == 2);
17370   match(Set dst (FmaVF  dst (Binary (NegVF src1) src2)));
17371   match(Set dst (FmaVF  dst (Binary src1 (NegVF src2))));
17372   format %{ "fmls  $dst,$src1,$src2\t# vector (2S)" %}
17373   ins_cost(INSN_COST);
17374   ins_encode %{
17375     __ fmls(as_FloatRegister($dst$$reg), __ T2S,
17376             as_FloatRegister($src1$$reg),
17377             as_FloatRegister($src2$$reg));
17378   %}
17379   ins_pipe(vmuldiv_fp64);
17380 %}
17381 
17382 // dst - src1 * src2
17383 instruct vmls4F(vecX dst, vecX src1, vecX src2) %{
17384   predicate(UseFMA && n->as_Vector()->length() == 4);
17385   match(Set dst (FmaVF  dst (Binary (NegVF src1) src2)));
17386   match(Set dst (FmaVF  dst (Binary src1 (NegVF src2))));
17387   format %{ "fmls  $dst,$src1,$src2\t# vector (4S)" %}
17388   ins_cost(INSN_COST);
17389   ins_encode %{
17390     __ fmls(as_FloatRegister($dst$$reg), __ T4S,
17391             as_FloatRegister($src1$$reg),
17392             as_FloatRegister($src2$$reg));
17393   %}
17394   ins_pipe(vmuldiv_fp128);
17395 %}
17396 
17397 // dst - src1 * src2
17398 instruct vmls2D(vecX dst, vecX src1, vecX src2) %{
17399   predicate(UseFMA && n->as_Vector()->length() == 2);
17400   match(Set dst (FmaVD  dst (Binary (NegVD src1) src2)));
17401   match(Set dst (FmaVD  dst (Binary src1 (NegVD src2))));
17402   format %{ "fmls  $dst,$src1,$src2\t# vector (2D)" %}
17403   ins_cost(INSN_COST);
17404   ins_encode %{
17405     __ fmls(as_FloatRegister($dst$$reg), __ T2D,
17406             as_FloatRegister($src1$$reg),
17407             as_FloatRegister($src2$$reg));
17408   %}
17409   ins_pipe(vmuldiv_fp128);
17410 %}
17411 
17412 // --------------------------------- DIV --------------------------------------
17413 
17414 instruct vdiv2F(vecD dst, vecD src1, vecD src2)
17415 %{
17416   predicate(n->as_Vector()->length() == 2);
17417   match(Set dst (DivVF src1 src2));
17418   ins_cost(INSN_COST);
17419   format %{ "fdiv  $dst,$src1,$src2\t# vector (2S)" %}
17420   ins_encode %{
17421     __ fdiv(as_FloatRegister($dst$$reg), __ T2S,
17422             as_FloatRegister($src1$$reg),
17423             as_FloatRegister($src2$$reg));
17424   %}
17425   ins_pipe(vmuldiv_fp64);
17426 %}
17427 
17428 instruct vdiv4F(vecX dst, vecX src1, vecX src2)
17429 %{
17430   predicate(n->as_Vector()->length() == 4);
17431   match(Set dst (DivVF src1 src2));
17432   ins_cost(INSN_COST);
17433   format %{ "fdiv  $dst,$src1,$src2\t# vector (4S)" %}
17434   ins_encode %{
17435     __ fdiv(as_FloatRegister($dst$$reg), __ T4S,
17436             as_FloatRegister($src1$$reg),
17437             as_FloatRegister($src2$$reg));
17438   %}
17439   ins_pipe(vmuldiv_fp128);
17440 %}
17441 
17442 instruct vdiv2D(vecX dst, vecX src1, vecX src2)
17443 %{
17444   predicate(n->as_Vector()->length() == 2);
17445   match(Set dst (DivVD src1 src2));
17446   ins_cost(INSN_COST);
17447   format %{ "fdiv  $dst,$src1,$src2\t# vector (2D)" %}
17448   ins_encode %{
17449     __ fdiv(as_FloatRegister($dst$$reg), __ T2D,
17450             as_FloatRegister($src1$$reg),
17451             as_FloatRegister($src2$$reg));
17452   %}
17453   ins_pipe(vmuldiv_fp128);
17454 %}
17455 
17456 // --------------------------------- SQRT -------------------------------------
17457 
17458 instruct vsqrt2D(vecX dst, vecX src)
17459 %{
17460   predicate(n->as_Vector()->length() == 2);
17461   match(Set dst (SqrtVD src));
17462   format %{ "fsqrt  $dst, $src\t# vector (2D)" %}
17463   ins_encode %{
17464     __ fsqrt(as_FloatRegister($dst$$reg), __ T2D,
17465              as_FloatRegister($src$$reg));
17466   %}
17467   ins_pipe(vsqrt_fp128);
17468 %}
17469 
17470 // --------------------------------- ABS --------------------------------------
17471 
17472 instruct vabs2F(vecD dst, vecD src)
17473 %{
17474   predicate(n->as_Vector()->length() == 2);
17475   match(Set dst (AbsVF src));
17476   ins_cost(INSN_COST * 3);
17477   format %{ "fabs  $dst,$src\t# vector (2S)" %}
17478   ins_encode %{
17479     __ fabs(as_FloatRegister($dst$$reg), __ T2S,
17480             as_FloatRegister($src$$reg));
17481   %}
17482   ins_pipe(vunop_fp64);
17483 %}
17484 
17485 instruct vabs4F(vecX dst, vecX src)
17486 %{
17487   predicate(n->as_Vector()->length() == 4);
17488   match(Set dst (AbsVF src));
17489   ins_cost(INSN_COST * 3);
17490   format %{ "fabs  $dst,$src\t# vector (4S)" %}
17491   ins_encode %{
17492     __ fabs(as_FloatRegister($dst$$reg), __ T4S,
17493             as_FloatRegister($src$$reg));
17494   %}
17495   ins_pipe(vunop_fp128);
17496 %}
17497 
17498 instruct vabs2D(vecX dst, vecX src)
17499 %{
17500   predicate(n->as_Vector()->length() == 2);
17501   match(Set dst (AbsVD src));
17502   ins_cost(INSN_COST * 3);
17503   format %{ "fabs  $dst,$src\t# vector (2D)" %}
17504   ins_encode %{
17505     __ fabs(as_FloatRegister($dst$$reg), __ T2D,
17506             as_FloatRegister($src$$reg));
17507   %}
17508   ins_pipe(vunop_fp128);
17509 %}
17510 
17511 // --------------------------------- NEG --------------------------------------
17512 
17513 instruct vneg2F(vecD dst, vecD src)
17514 %{
17515   predicate(n->as_Vector()->length() == 2);
17516   match(Set dst (NegVF src));
17517   ins_cost(INSN_COST * 3);
17518   format %{ "fneg  $dst,$src\t# vector (2S)" %}
17519   ins_encode %{
17520     __ fneg(as_FloatRegister($dst$$reg), __ T2S,
17521             as_FloatRegister($src$$reg));
17522   %}
17523   ins_pipe(vunop_fp64);
17524 %}
17525 
17526 instruct vneg4F(vecX dst, vecX src)
17527 %{
17528   predicate(n->as_Vector()->length() == 4);
17529   match(Set dst (NegVF src));
17530   ins_cost(INSN_COST * 3);
17531   format %{ "fneg  $dst,$src\t# vector (4S)" %}
17532   ins_encode %{
17533     __ fneg(as_FloatRegister($dst$$reg), __ T4S,
17534             as_FloatRegister($src$$reg));
17535   %}
17536   ins_pipe(vunop_fp128);
17537 %}
17538 
17539 instruct vneg2D(vecX dst, vecX src)
17540 %{
17541   predicate(n->as_Vector()->length() == 2);
17542   match(Set dst (NegVD src));
17543   ins_cost(INSN_COST * 3);
17544   format %{ "fneg  $dst,$src\t# vector (2D)" %}
17545   ins_encode %{
17546     __ fneg(as_FloatRegister($dst$$reg), __ T2D,
17547             as_FloatRegister($src$$reg));
17548   %}
17549   ins_pipe(vunop_fp128);
17550 %}
17551 
17552 // --------------------------------- AND --------------------------------------
17553 
17554 instruct vand8B(vecD dst, vecD src1, vecD src2)
17555 %{
17556   predicate(n->as_Vector()->length_in_bytes() == 4 ||
17557             n->as_Vector()->length_in_bytes() == 8);
17558   match(Set dst (AndV src1 src2));
17559   ins_cost(INSN_COST);
17560   format %{ "and  $dst,$src1,$src2\t# vector (8B)" %}
17561   ins_encode %{
17562     __ andr(as_FloatRegister($dst$$reg), __ T8B,
17563             as_FloatRegister($src1$$reg),
17564             as_FloatRegister($src2$$reg));
17565   %}
17566   ins_pipe(vlogical64);
17567 %}
17568 
17569 instruct vand16B(vecX dst, vecX src1, vecX src2)
17570 %{
17571   predicate(n->as_Vector()->length_in_bytes() == 16);
17572   match(Set dst (AndV src1 src2));
17573   ins_cost(INSN_COST);
17574   format %{ "and  $dst,$src1,$src2\t# vector (16B)" %}
17575   ins_encode %{
17576     __ andr(as_FloatRegister($dst$$reg), __ T16B,
17577             as_FloatRegister($src1$$reg),
17578             as_FloatRegister($src2$$reg));
17579   %}
17580   ins_pipe(vlogical128);
17581 %}
17582 
17583 // --------------------------------- OR ---------------------------------------
17584 
17585 instruct vor8B(vecD dst, vecD src1, vecD src2)
17586 %{
17587   predicate(n->as_Vector()->length_in_bytes() == 4 ||
17588             n->as_Vector()->length_in_bytes() == 8);
17589   match(Set dst (OrV src1 src2));
17590   ins_cost(INSN_COST);
17591   format %{ "and  $dst,$src1,$src2\t# vector (8B)" %}
17592   ins_encode %{
17593     __ orr(as_FloatRegister($dst$$reg), __ T8B,
17594             as_FloatRegister($src1$$reg),
17595             as_FloatRegister($src2$$reg));
17596   %}
17597   ins_pipe(vlogical64);
17598 %}
17599 
17600 instruct vor16B(vecX dst, vecX src1, vecX src2)
17601 %{
17602   predicate(n->as_Vector()->length_in_bytes() == 16);
17603   match(Set dst (OrV src1 src2));
17604   ins_cost(INSN_COST);
17605   format %{ "orr  $dst,$src1,$src2\t# vector (16B)" %}
17606   ins_encode %{
17607     __ orr(as_FloatRegister($dst$$reg), __ T16B,
17608             as_FloatRegister($src1$$reg),
17609             as_FloatRegister($src2$$reg));
17610   %}
17611   ins_pipe(vlogical128);
17612 %}
17613 
17614 // --------------------------------- XOR --------------------------------------
17615 
17616 instruct vxor8B(vecD dst, vecD src1, vecD src2)
17617 %{
17618   predicate(n->as_Vector()->length_in_bytes() == 4 ||
17619             n->as_Vector()->length_in_bytes() == 8);
17620   match(Set dst (XorV src1 src2));
17621   ins_cost(INSN_COST);
17622   format %{ "xor  $dst,$src1,$src2\t# vector (8B)" %}
17623   ins_encode %{
17624     __ eor(as_FloatRegister($dst$$reg), __ T8B,
17625             as_FloatRegister($src1$$reg),
17626             as_FloatRegister($src2$$reg));
17627   %}
17628   ins_pipe(vlogical64);
17629 %}
17630 
17631 instruct vxor16B(vecX dst, vecX src1, vecX src2)
17632 %{
17633   predicate(n->as_Vector()->length_in_bytes() == 16);
17634   match(Set dst (XorV src1 src2));
17635   ins_cost(INSN_COST);
17636   format %{ "xor  $dst,$src1,$src2\t# vector (16B)" %}
17637   ins_encode %{
17638     __ eor(as_FloatRegister($dst$$reg), __ T16B,
17639             as_FloatRegister($src1$$reg),
17640             as_FloatRegister($src2$$reg));
17641   %}
17642   ins_pipe(vlogical128);
17643 %}
17644 
17645 // ------------------------------ Shift ---------------------------------------
17646 
17647 instruct vshiftcntL(vecX dst, iRegIorL2I cnt) %{
17648   match(Set dst (LShiftCntV cnt));
17649   format %{ "dup  $dst, $cnt\t# shift count (vecX)" %}
17650   ins_encode %{
17651     __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($cnt$$reg));
17652   %}
17653   ins_pipe(vdup_reg_reg128);
17654 %}
17655 
17656 // Right shifts on aarch64 SIMD are implemented as left shift by -ve amount
17657 instruct vshiftcntR(vecX dst, iRegIorL2I cnt) %{
17658   match(Set dst (RShiftCntV cnt));
17659   format %{ "dup  $dst, $cnt\t# shift count (vecX)\n\tneg  $dst, $dst\t T16B" %}
17660   ins_encode %{
17661     __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($cnt$$reg));
17662     __ negr(as_FloatRegister($dst$$reg), __ T16B, as_FloatRegister($dst$$reg));
17663   %}
17664   ins_pipe(vdup_reg_reg128);
17665 %}
17666 
17667 instruct vsll8B(vecD dst, vecD src, vecX shift) %{
17668   predicate(n->as_Vector()->length() == 4 ||
17669             n->as_Vector()->length() == 8);
17670   match(Set dst (LShiftVB src shift));
17671   match(Set dst (RShiftVB src shift));
17672   ins_cost(INSN_COST);
17673   format %{ "sshl  $dst,$src,$shift\t# vector (8B)" %}
17674   ins_encode %{
17675     __ sshl(as_FloatRegister($dst$$reg), __ T8B,
17676             as_FloatRegister($src$$reg),
17677             as_FloatRegister($shift$$reg));
17678   %}
17679   ins_pipe(vshift64);
17680 %}
17681 
17682 instruct vsll16B(vecX dst, vecX src, vecX shift) %{
17683   predicate(n->as_Vector()->length() == 16);
17684   match(Set dst (LShiftVB src shift));
17685   match(Set dst (RShiftVB src shift));
17686   ins_cost(INSN_COST);
17687   format %{ "sshl  $dst,$src,$shift\t# vector (16B)" %}
17688   ins_encode %{
17689     __ sshl(as_FloatRegister($dst$$reg), __ T16B,
17690             as_FloatRegister($src$$reg),
17691             as_FloatRegister($shift$$reg));
17692   %}
17693   ins_pipe(vshift128);
17694 %}
17695 
17696 instruct vsrl8B(vecD dst, vecD src, vecX shift) %{
17697   predicate(n->as_Vector()->length() == 4 ||
17698             n->as_Vector()->length() == 8);
17699   match(Set dst (URShiftVB src shift));
17700   ins_cost(INSN_COST);
17701   format %{ "ushl  $dst,$src,$shift\t# vector (8B)" %}
17702   ins_encode %{
17703     __ ushl(as_FloatRegister($dst$$reg), __ T8B,
17704             as_FloatRegister($src$$reg),
17705             as_FloatRegister($shift$$reg));
17706   %}
17707   ins_pipe(vshift64);
17708 %}
17709 
17710 instruct vsrl16B(vecX dst, vecX src, vecX shift) %{
17711   predicate(n->as_Vector()->length() == 16);
17712   match(Set dst (URShiftVB src shift));
17713   ins_cost(INSN_COST);
17714   format %{ "ushl  $dst,$src,$shift\t# vector (16B)" %}
17715   ins_encode %{
17716     __ ushl(as_FloatRegister($dst$$reg), __ T16B,
17717             as_FloatRegister($src$$reg),
17718             as_FloatRegister($shift$$reg));
17719   %}
17720   ins_pipe(vshift128);
17721 %}
17722 
17723 instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
17724   predicate(n->as_Vector()->length() == 4 ||
17725             n->as_Vector()->length() == 8);
17726   match(Set dst (LShiftVB src shift));
17727   ins_cost(INSN_COST);
17728   format %{ "shl    $dst, $src, $shift\t# vector (8B)" %}
17729   ins_encode %{
17730     int sh = (int)$shift$$constant;
17731     if (sh >= 8) {
17732       __ eor(as_FloatRegister($dst$$reg), __ T8B,
17733              as_FloatRegister($src$$reg),
17734              as_FloatRegister($src$$reg));
17735     } else {
17736       __ shl(as_FloatRegister($dst$$reg), __ T8B,
17737              as_FloatRegister($src$$reg), sh);
17738     }
17739   %}
17740   ins_pipe(vshift64_imm);
17741 %}
17742 
17743 instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
17744   predicate(n->as_Vector()->length() == 16);
17745   match(Set dst (LShiftVB src shift));
17746   ins_cost(INSN_COST);
17747   format %{ "shl    $dst, $src, $shift\t# vector (16B)" %}
17748   ins_encode %{
17749     int sh = (int)$shift$$constant;
17750     if (sh >= 8) {
17751       __ eor(as_FloatRegister($dst$$reg), __ T16B,
17752              as_FloatRegister($src$$reg),
17753              as_FloatRegister($src$$reg));
17754     } else {
17755       __ shl(as_FloatRegister($dst$$reg), __ T16B,
17756              as_FloatRegister($src$$reg), sh);
17757     }
17758   %}
17759   ins_pipe(vshift128_imm);
17760 %}
17761 
17762 instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{
17763   predicate(n->as_Vector()->length() == 4 ||
17764             n->as_Vector()->length() == 8);
17765   match(Set dst (RShiftVB src shift));
17766   ins_cost(INSN_COST);
17767   format %{ "sshr    $dst, $src, $shift\t# vector (8B)" %}
17768   ins_encode %{
17769     int sh = (int)$shift$$constant;
17770     if (sh >= 8) sh = 7;
17771     __ sshr(as_FloatRegister($dst$$reg), __ T8B,
17772            as_FloatRegister($src$$reg), sh);
17773   %}
17774   ins_pipe(vshift64_imm);
17775 %}
17776 
17777 instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
17778   predicate(n->as_Vector()->length() == 16);
17779   match(Set dst (RShiftVB src shift));
17780   ins_cost(INSN_COST);
17781   format %{ "sshr    $dst, $src, $shift\t# vector (16B)" %}
17782   ins_encode %{
17783     int sh = (int)$shift$$constant;
17784     if (sh >= 8) sh = 7;
17785     __ sshr(as_FloatRegister($dst$$reg), __ T16B,
17786            as_FloatRegister($src$$reg), sh);
17787   %}
17788   ins_pipe(vshift128_imm);
17789 %}
17790 
17791 instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{
17792   predicate(n->as_Vector()->length() == 4 ||
17793             n->as_Vector()->length() == 8);
17794   match(Set dst (URShiftVB src shift));
17795   ins_cost(INSN_COST);
17796   format %{ "ushr    $dst, $src, $shift\t# vector (8B)" %}
17797   ins_encode %{
17798     int sh = (int)$shift$$constant;
17799     if (sh >= 8) {
17800       __ eor(as_FloatRegister($dst$$reg), __ T8B,
17801              as_FloatRegister($src$$reg),
17802              as_FloatRegister($src$$reg));
17803     } else {
17804       __ ushr(as_FloatRegister($dst$$reg), __ T8B,
17805              as_FloatRegister($src$$reg), sh);
17806     }
17807   %}
17808   ins_pipe(vshift64_imm);
17809 %}
17810 
17811 instruct vsrl16B_imm(vecX dst, vecX src, immI shift) %{
17812   predicate(n->as_Vector()->length() == 16);
17813   match(Set dst (URShiftVB src shift));
17814   ins_cost(INSN_COST);
17815   format %{ "ushr    $dst, $src, $shift\t# vector (16B)" %}
17816   ins_encode %{
17817     int sh = (int)$shift$$constant;
17818     if (sh >= 8) {
17819       __ eor(as_FloatRegister($dst$$reg), __ T16B,
17820              as_FloatRegister($src$$reg),
17821              as_FloatRegister($src$$reg));
17822     } else {
17823       __ ushr(as_FloatRegister($dst$$reg), __ T16B,
17824              as_FloatRegister($src$$reg), sh);
17825     }
17826   %}
17827   ins_pipe(vshift128_imm);
17828 %}
17829 
17830 instruct vsll4S(vecD dst, vecD src, vecX shift) %{
17831   predicate(n->as_Vector()->length() == 2 ||
17832             n->as_Vector()->length() == 4);
17833   match(Set dst (LShiftVS src shift));
17834   match(Set dst (RShiftVS src shift));
17835   ins_cost(INSN_COST);
17836   format %{ "sshl  $dst,$src,$shift\t# vector (4H)" %}
17837   ins_encode %{
17838     __ sshl(as_FloatRegister($dst$$reg), __ T4H,
17839             as_FloatRegister($src$$reg),
17840             as_FloatRegister($shift$$reg));
17841   %}
17842   ins_pipe(vshift64);
17843 %}
17844 
17845 instruct vsll8S(vecX dst, vecX src, vecX shift) %{
17846   predicate(n->as_Vector()->length() == 8);
17847   match(Set dst (LShiftVS src shift));
17848   match(Set dst (RShiftVS src shift));
17849   ins_cost(INSN_COST);
17850   format %{ "sshl  $dst,$src,$shift\t# vector (8H)" %}
17851   ins_encode %{
17852     __ sshl(as_FloatRegister($dst$$reg), __ T8H,
17853             as_FloatRegister($src$$reg),
17854             as_FloatRegister($shift$$reg));
17855   %}
17856   ins_pipe(vshift128);
17857 %}
17858 
17859 instruct vsrl4S(vecD dst, vecD src, vecX shift) %{
17860   predicate(n->as_Vector()->length() == 2 ||
17861             n->as_Vector()->length() == 4);
17862   match(Set dst (URShiftVS src shift));
17863   ins_cost(INSN_COST);
17864   format %{ "ushl  $dst,$src,$shift\t# vector (4H)" %}
17865   ins_encode %{
17866     __ ushl(as_FloatRegister($dst$$reg), __ T4H,
17867             as_FloatRegister($src$$reg),
17868             as_FloatRegister($shift$$reg));
17869   %}
17870   ins_pipe(vshift64);
17871 %}
17872 
17873 instruct vsrl8S(vecX dst, vecX src, vecX shift) %{
17874   predicate(n->as_Vector()->length() == 8);
17875   match(Set dst (URShiftVS src shift));
17876   ins_cost(INSN_COST);
17877   format %{ "ushl  $dst,$src,$shift\t# vector (8H)" %}
17878   ins_encode %{
17879     __ ushl(as_FloatRegister($dst$$reg), __ T8H,
17880             as_FloatRegister($src$$reg),
17881             as_FloatRegister($shift$$reg));
17882   %}
17883   ins_pipe(vshift128);
17884 %}
17885 
17886 instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
17887   predicate(n->as_Vector()->length() == 2 ||
17888             n->as_Vector()->length() == 4);
17889   match(Set dst (LShiftVS src shift));
17890   ins_cost(INSN_COST);
17891   format %{ "shl    $dst, $src, $shift\t# vector (4H)" %}
17892   ins_encode %{
17893     int sh = (int)$shift$$constant;
17894     if (sh >= 16) {
17895       __ eor(as_FloatRegister($dst$$reg), __ T8B,
17896              as_FloatRegister($src$$reg),
17897              as_FloatRegister($src$$reg));
17898     } else {
17899       __ shl(as_FloatRegister($dst$$reg), __ T4H,
17900              as_FloatRegister($src$$reg), sh);
17901     }
17902   %}
17903   ins_pipe(vshift64_imm);
17904 %}
17905 
17906 instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
17907   predicate(n->as_Vector()->length() == 8);
17908   match(Set dst (LShiftVS src shift));
17909   ins_cost(INSN_COST);
17910   format %{ "shl    $dst, $src, $shift\t# vector (8H)" %}
17911   ins_encode %{
17912     int sh = (int)$shift$$constant;
17913     if (sh >= 16) {
17914       __ eor(as_FloatRegister($dst$$reg), __ T16B,
17915              as_FloatRegister($src$$reg),
17916              as_FloatRegister($src$$reg));
17917     } else {
17918       __ shl(as_FloatRegister($dst$$reg), __ T8H,
17919              as_FloatRegister($src$$reg), sh);
17920     }
17921   %}
17922   ins_pipe(vshift128_imm);
17923 %}
17924 
17925 instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{
17926   predicate(n->as_Vector()->length() == 2 ||
17927             n->as_Vector()->length() == 4);
17928   match(Set dst (RShiftVS src shift));
17929   ins_cost(INSN_COST);
17930   format %{ "sshr    $dst, $src, $shift\t# vector (4H)" %}
17931   ins_encode %{
17932     int sh = (int)$shift$$constant;
17933     if (sh >= 16) sh = 15;
17934     __ sshr(as_FloatRegister($dst$$reg), __ T4H,
17935            as_FloatRegister($src$$reg), sh);
17936   %}
17937   ins_pipe(vshift64_imm);
17938 %}
17939 
17940 instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
17941   predicate(n->as_Vector()->length() == 8);
17942   match(Set dst (RShiftVS src shift));
17943   ins_cost(INSN_COST);
17944   format %{ "sshr    $dst, $src, $shift\t# vector (8H)" %}
17945   ins_encode %{
17946     int sh = (int)$shift$$constant;
17947     if (sh >= 16) sh = 15;
17948     __ sshr(as_FloatRegister($dst$$reg), __ T8H,
17949            as_FloatRegister($src$$reg), sh);
17950   %}
17951   ins_pipe(vshift128_imm);
17952 %}
17953 
17954 instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{
17955   predicate(n->as_Vector()->length() == 2 ||
17956             n->as_Vector()->length() == 4);
17957   match(Set dst (URShiftVS src shift));
17958   ins_cost(INSN_COST);
17959   format %{ "ushr    $dst, $src, $shift\t# vector (4H)" %}
17960   ins_encode %{
17961     int sh = (int)$shift$$constant;
17962     if (sh >= 16) {
17963       __ eor(as_FloatRegister($dst$$reg), __ T8B,
17964              as_FloatRegister($src$$reg),
17965              as_FloatRegister($src$$reg));
17966     } else {
17967       __ ushr(as_FloatRegister($dst$$reg), __ T4H,
17968              as_FloatRegister($src$$reg), sh);
17969     }
17970   %}
17971   ins_pipe(vshift64_imm);
17972 %}
17973 
17974 instruct vsrl8S_imm(vecX dst, vecX src, immI shift) %{
17975   predicate(n->as_Vector()->length() == 8);
17976   match(Set dst (URShiftVS src shift));
17977   ins_cost(INSN_COST);
17978   format %{ "ushr    $dst, $src, $shift\t# vector (8H)" %}
17979   ins_encode %{
17980     int sh = (int)$shift$$constant;
17981     if (sh >= 16) {
17982       __ eor(as_FloatRegister($dst$$reg), __ T16B,
17983              as_FloatRegister($src$$reg),
17984              as_FloatRegister($src$$reg));
17985     } else {
17986       __ ushr(as_FloatRegister($dst$$reg), __ T8H,
17987              as_FloatRegister($src$$reg), sh);
17988     }
17989   %}
17990   ins_pipe(vshift128_imm);
17991 %}
17992 
17993 instruct vsll2I(vecD dst, vecD src, vecX shift) %{
17994   predicate(n->as_Vector()->length() == 2);
17995   match(Set dst (LShiftVI src shift));
17996   match(Set dst (RShiftVI src shift));
17997   ins_cost(INSN_COST);
17998   format %{ "sshl  $dst,$src,$shift\t# vector (2S)" %}
17999   ins_encode %{
18000     __ sshl(as_FloatRegister($dst$$reg), __ T2S,
18001             as_FloatRegister($src$$reg),
18002             as_FloatRegister($shift$$reg));
18003   %}
18004   ins_pipe(vshift64);
18005 %}
18006 
18007 instruct vsll4I(vecX dst, vecX src, vecX shift) %{
18008   predicate(n->as_Vector()->length() == 4);
18009   match(Set dst (LShiftVI src shift));
18010   match(Set dst (RShiftVI src shift));
18011   ins_cost(INSN_COST);
18012   format %{ "sshl  $dst,$src,$shift\t# vector (4S)" %}
18013   ins_encode %{
18014     __ sshl(as_FloatRegister($dst$$reg), __ T4S,
18015             as_FloatRegister($src$$reg),
18016             as_FloatRegister($shift$$reg));
18017   %}
18018   ins_pipe(vshift128);
18019 %}
18020 
18021 instruct vsrl2I(vecD dst, vecD src, vecX shift) %{
18022   predicate(n->as_Vector()->length() == 2);
18023   match(Set dst (URShiftVI src shift));
18024   ins_cost(INSN_COST);
18025   format %{ "ushl  $dst,$src,$shift\t# vector (2S)" %}
18026   ins_encode %{
18027     __ ushl(as_FloatRegister($dst$$reg), __ T2S,
18028             as_FloatRegister($src$$reg),
18029             as_FloatRegister($shift$$reg));
18030   %}
18031   ins_pipe(vshift64);
18032 %}
18033 
18034 instruct vsrl4I(vecX dst, vecX src, vecX shift) %{
18035   predicate(n->as_Vector()->length() == 4);
18036   match(Set dst (URShiftVI src shift));
18037   ins_cost(INSN_COST);
18038   format %{ "ushl  $dst,$src,$shift\t# vector (4S)" %}
18039   ins_encode %{
18040     __ ushl(as_FloatRegister($dst$$reg), __ T4S,
18041             as_FloatRegister($src$$reg),
18042             as_FloatRegister($shift$$reg));
18043   %}
18044   ins_pipe(vshift128);
18045 %}
18046 
18047 instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
18048   predicate(n->as_Vector()->length() == 2);
18049   match(Set dst (LShiftVI src shift));
18050   ins_cost(INSN_COST);
18051   format %{ "shl    $dst, $src, $shift\t# vector (2S)" %}
18052   ins_encode %{
18053     __ shl(as_FloatRegister($dst$$reg), __ T2S,
18054            as_FloatRegister($src$$reg),
18055            (int)$shift$$constant);
18056   %}
18057   ins_pipe(vshift64_imm);
18058 %}
18059 
18060 instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{
18061   predicate(n->as_Vector()->length() == 4);
18062   match(Set dst (LShiftVI src shift));
18063   ins_cost(INSN_COST);
18064   format %{ "shl    $dst, $src, $shift\t# vector (4S)" %}
18065   ins_encode %{
18066     __ shl(as_FloatRegister($dst$$reg), __ T4S,
18067            as_FloatRegister($src$$reg),
18068            (int)$shift$$constant);
18069   %}
18070   ins_pipe(vshift128_imm);
18071 %}
18072 
18073 instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{
18074   predicate(n->as_Vector()->length() == 2);
18075   match(Set dst (RShiftVI src shift));
18076   ins_cost(INSN_COST);
18077   format %{ "sshr    $dst, $src, $shift\t# vector (2S)" %}
18078   ins_encode %{
18079     __ sshr(as_FloatRegister($dst$$reg), __ T2S,
18080             as_FloatRegister($src$$reg),
18081             (int)$shift$$constant);
18082   %}
18083   ins_pipe(vshift64_imm);
18084 %}
18085 
18086 instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{
18087   predicate(n->as_Vector()->length() == 4);
18088   match(Set dst (RShiftVI src shift));
18089   ins_cost(INSN_COST);
18090   format %{ "sshr    $dst, $src, $shift\t# vector (4S)" %}
18091   ins_encode %{
18092     __ sshr(as_FloatRegister($dst$$reg), __ T4S,
18093             as_FloatRegister($src$$reg),
18094             (int)$shift$$constant);
18095   %}
18096   ins_pipe(vshift128_imm);
18097 %}
18098 
18099 instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{
18100   predicate(n->as_Vector()->length() == 2);
18101   match(Set dst (URShiftVI src shift));
18102   ins_cost(INSN_COST);
18103   format %{ "ushr    $dst, $src, $shift\t# vector (2S)" %}
18104   ins_encode %{
18105     __ ushr(as_FloatRegister($dst$$reg), __ T2S,
18106             as_FloatRegister($src$$reg),
18107             (int)$shift$$constant);
18108   %}
18109   ins_pipe(vshift64_imm);
18110 %}
18111 
18112 instruct vsrl4I_imm(vecX dst, vecX src, immI shift) %{
18113   predicate(n->as_Vector()->length() == 4);
18114   match(Set dst (URShiftVI src shift));
18115   ins_cost(INSN_COST);
18116   format %{ "ushr    $dst, $src, $shift\t# vector (4S)" %}
18117   ins_encode %{
18118     __ ushr(as_FloatRegister($dst$$reg), __ T4S,
18119             as_FloatRegister($src$$reg),
18120             (int)$shift$$constant);
18121   %}
18122   ins_pipe(vshift128_imm);
18123 %}
18124 
18125 instruct vsll2L(vecX dst, vecX src, vecX shift) %{
18126   predicate(n->as_Vector()->length() == 2);
18127   match(Set dst (LShiftVL src shift));
18128   match(Set dst (RShiftVL src shift));
18129   ins_cost(INSN_COST);
18130   format %{ "sshl  $dst,$src,$shift\t# vector (2D)" %}
18131   ins_encode %{
18132     __ sshl(as_FloatRegister($dst$$reg), __ T2D,
18133             as_FloatRegister($src$$reg),
18134             as_FloatRegister($shift$$reg));
18135   %}
18136   ins_pipe(vshift128);
18137 %}
18138 
18139 instruct vsrl2L(vecX dst, vecX src, vecX shift) %{
18140   predicate(n->as_Vector()->length() == 2);
18141   match(Set dst (URShiftVL src shift));
18142   ins_cost(INSN_COST);
18143   format %{ "ushl  $dst,$src,$shift\t# vector (2D)" %}
18144   ins_encode %{
18145     __ ushl(as_FloatRegister($dst$$reg), __ T2D,
18146             as_FloatRegister($src$$reg),
18147             as_FloatRegister($shift$$reg));
18148   %}
18149   ins_pipe(vshift128);
18150 %}
18151 
18152 instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
18153   predicate(n->as_Vector()->length() == 2);
18154   match(Set dst (LShiftVL src shift));
18155   ins_cost(INSN_COST);
18156   format %{ "shl    $dst, $src, $shift\t# vector (2D)" %}
18157   ins_encode %{
18158     __ shl(as_FloatRegister($dst$$reg), __ T2D,
18159            as_FloatRegister($src$$reg),
18160            (int)$shift$$constant);
18161   %}
18162   ins_pipe(vshift128_imm);
18163 %}
18164 
18165 instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{
18166   predicate(n->as_Vector()->length() == 2);
18167   match(Set dst (RShiftVL src shift));
18168   ins_cost(INSN_COST);
18169   format %{ "sshr    $dst, $src, $shift\t# vector (2D)" %}
18170   ins_encode %{
18171     __ sshr(as_FloatRegister($dst$$reg), __ T2D,
18172             as_FloatRegister($src$$reg),
18173             (int)$shift$$constant);
18174   %}
18175   ins_pipe(vshift128_imm);
18176 %}
18177 
18178 instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{
18179   predicate(n->as_Vector()->length() == 2);
18180   match(Set dst (URShiftVL src shift));
18181   ins_cost(INSN_COST);
18182   format %{ "ushr    $dst, $src, $shift\t# vector (2D)" %}
18183   ins_encode %{
18184     __ ushr(as_FloatRegister($dst$$reg), __ T2D,
18185             as_FloatRegister($src$$reg),
18186             (int)$shift$$constant);
18187   %}
18188   ins_pipe(vshift128_imm);
18189 %}
18190 
18191 //----------PEEPHOLE RULES-----------------------------------------------------
18192 // These must follow all instruction definitions as they use the names
18193 // defined in the instructions definitions.
18194 //
18195 // peepmatch ( root_instr_name [preceding_instruction]* );
18196 //
18197 // peepconstraint %{
18198 // (instruction_number.operand_name relational_op instruction_number.operand_name
18199 //  [, ...] );
18200 // // instruction numbers are zero-based using left to right order in peepmatch
18201 //
18202 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
18203 // // provide an instruction_number.operand_name for each operand that appears
18204 // // in the replacement instruction's match rule
18205 //
18206 // ---------VM FLAGS---------------------------------------------------------
18207 //
18208 // All peephole optimizations can be turned off using -XX:-OptoPeephole
18209 //
18210 // Each peephole rule is given an identifying number starting with zero and
18211 // increasing by one in the order seen by the parser.  An individual peephole
18212 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
18213 // on the command-line.
18214 //
18215 // ---------CURRENT LIMITATIONS----------------------------------------------
18216 //
18217 // Only match adjacent instructions in same basic block
18218 // Only equality constraints
18219 // Only constraints between operands, not (0.dest_reg == RAX_enc)
18220 // Only one replacement instruction
18221 //
18222 // ---------EXAMPLE----------------------------------------------------------
18223 //
18224 // // pertinent parts of existing instructions in architecture description
18225 // instruct movI(iRegINoSp dst, iRegI src)
18226 // %{
18227 //   match(Set dst (CopyI src));
18228 // %}
18229 //
18230 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
18231 // %{
18232 //   match(Set dst (AddI dst src));
18233 //   effect(KILL cr);
18234 // %}
18235 //
18236 // // Change (inc mov) to lea
18237 // peephole %{
18238 //   // increment preceeded by register-register move
18239 //   peepmatch ( incI_iReg movI );
18240 //   // require that the destination register of the increment
18241 //   // match the destination register of the move
18242 //   peepconstraint ( 0.dst == 1.dst );
18243 //   // construct a replacement instruction that sets
18244 //   // the destination to ( move's source register + one )
18245 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
18246 // %}
18247 //
18248 
18249 // Implementation no longer uses movX instructions since
18250 // machine-independent system no longer uses CopyX nodes.
18251 //
18252 // peephole
18253 // %{
18254 //   peepmatch (incI_iReg movI);
18255 //   peepconstraint (0.dst == 1.dst);
18256 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
18257 // %}
18258 
18259 // peephole
18260 // %{
18261 //   peepmatch (decI_iReg movI);
18262 //   peepconstraint (0.dst == 1.dst);
18263 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
18264 // %}
18265 
18266 // peephole
18267 // %{
18268 //   peepmatch (addI_iReg_imm movI);
18269 //   peepconstraint (0.dst == 1.dst);
18270 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
18271 // %}
18272 
18273 // peephole
18274 // %{
18275 //   peepmatch (incL_iReg movL);
18276 //   peepconstraint (0.dst == 1.dst);
18277 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
18278 // %}
18279 
18280 // peephole
18281 // %{
18282 //   peepmatch (decL_iReg movL);
18283 //   peepconstraint (0.dst == 1.dst);
18284 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
18285 // %}
18286 
18287 // peephole
18288 // %{
18289 //   peepmatch (addL_iReg_imm movL);
18290 //   peepconstraint (0.dst == 1.dst);
18291 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
18292 // %}
18293 
18294 // peephole
18295 // %{
18296 //   peepmatch (addP_iReg_imm movP);
18297 //   peepconstraint (0.dst == 1.dst);
18298 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
18299 // %}
18300 
18301 // // Change load of spilled value to only a spill
18302 // instruct storeI(memory mem, iRegI src)
18303 // %{
18304 //   match(Set mem (StoreI mem src));
18305 // %}
18306 //
18307 // instruct loadI(iRegINoSp dst, memory mem)
18308 // %{
18309 //   match(Set dst (LoadI mem));
18310 // %}
18311 //
18312 
18313 //----------SMARTSPILL RULES---------------------------------------------------
18314 // These must follow all instruction definitions as they use the names
18315 // defined in the instructions definitions.
18316 
18317 // Local Variables:
18318 // mode: c++
18319 // End: