1 //
2 // Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2020, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // archtecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 invisible to the allocator (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
98 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
99 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
100 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
101 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
102 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
103 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
104 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
105 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
106 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
107 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
108 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
109 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
110 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
111 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
112 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
113 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18->as_VMReg() );
114 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
115 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
116 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
117 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
118 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
119 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
120 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
121 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
122 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
123 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
124 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
125 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
126 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
127 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
128 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
129 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
130 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
131 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
132 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
133 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
134 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
135 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
136 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
137 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
138 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
139 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
140 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
141
142 // ----------------------------
143 // Float/Double Registers
144 // ----------------------------
145
146 // Double Registers
147
148 // The rules of ADL require that double registers be defined in pairs.
149 // Each pair must be two 32-bit values, but not necessarily a pair of
150 // single float registers. In each pair, ADLC-assigned register numbers
151 // must be adjacent, with the lower number even. Finally, when the
152 // CPU stores such a register pair to memory, the word associated with
153 // the lower ADLC-assigned number must be stored to the lower address.
154
155 // AArch64 has 32 floating-point registers. Each can store a vector of
156 // single or double precision floating-point values up to 8 * 32
157 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
158 // use the first float or double element of the vector.
159
160 // for Java use float registers v0-v15 are always save on call whereas
161 // the platform ABI treats v8-v15 as callee save). float registers
162 // v16-v31 are SOC as per the platform spec
163
164 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
165 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
166 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
167 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
168
169 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
170 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
171 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
172 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
173
174 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
175 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
176 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
177 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
178
179 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
180 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
181 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
182 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
183
184 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
185 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
186 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
187 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
188
189 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
190 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
191 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
192 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
193
194 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
195 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
196 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
197 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
198
199 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
200 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
201 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
202 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
203
204 reg_def V8 ( SOC, SOC, Op_RegF, 8, v8->as_VMReg() );
205 reg_def V8_H ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next() );
206 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
207 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
208
209 reg_def V9 ( SOC, SOC, Op_RegF, 9, v9->as_VMReg() );
210 reg_def V9_H ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next() );
211 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
212 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
213
214 reg_def V10 ( SOC, SOC, Op_RegF, 10, v10->as_VMReg() );
215 reg_def V10_H( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next() );
216 reg_def V10_J( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2));
217 reg_def V10_K( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3));
218
219 reg_def V11 ( SOC, SOC, Op_RegF, 11, v11->as_VMReg() );
220 reg_def V11_H( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next() );
221 reg_def V11_J( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2));
222 reg_def V11_K( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3));
223
224 reg_def V12 ( SOC, SOC, Op_RegF, 12, v12->as_VMReg() );
225 reg_def V12_H( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next() );
226 reg_def V12_J( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2));
227 reg_def V12_K( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3));
228
229 reg_def V13 ( SOC, SOC, Op_RegF, 13, v13->as_VMReg() );
230 reg_def V13_H( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next() );
231 reg_def V13_J( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2));
232 reg_def V13_K( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3));
233
234 reg_def V14 ( SOC, SOC, Op_RegF, 14, v14->as_VMReg() );
235 reg_def V14_H( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next() );
236 reg_def V14_J( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2));
237 reg_def V14_K( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3));
238
239 reg_def V15 ( SOC, SOC, Op_RegF, 15, v15->as_VMReg() );
240 reg_def V15_H( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next() );
241 reg_def V15_J( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2));
242 reg_def V15_K( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3));
243
244 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
245 reg_def V16_H( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
246 reg_def V16_J( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2));
247 reg_def V16_K( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3));
248
249 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
250 reg_def V17_H( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
251 reg_def V17_J( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2));
252 reg_def V17_K( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3));
253
254 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
255 reg_def V18_H( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
256 reg_def V18_J( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2));
257 reg_def V18_K( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3));
258
259 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
260 reg_def V19_H( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
261 reg_def V19_J( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2));
262 reg_def V19_K( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3));
263
264 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
265 reg_def V20_H( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
266 reg_def V20_J( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2));
267 reg_def V20_K( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3));
268
269 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
270 reg_def V21_H( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
271 reg_def V21_J( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2));
272 reg_def V21_K( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3));
273
274 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
275 reg_def V22_H( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
276 reg_def V22_J( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2));
277 reg_def V22_K( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3));
278
279 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
280 reg_def V23_H( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
281 reg_def V23_J( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2));
282 reg_def V23_K( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3));
283
284 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
285 reg_def V24_H( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
286 reg_def V24_J( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2));
287 reg_def V24_K( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3));
288
289 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
290 reg_def V25_H( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
291 reg_def V25_J( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2));
292 reg_def V25_K( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3));
293
294 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
295 reg_def V26_H( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
296 reg_def V26_J( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2));
297 reg_def V26_K( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3));
298
299 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
300 reg_def V27_H( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
301 reg_def V27_J( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2));
302 reg_def V27_K( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3));
303
304 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
305 reg_def V28_H( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
306 reg_def V28_J( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2));
307 reg_def V28_K( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3));
308
309 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
310 reg_def V29_H( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
311 reg_def V29_J( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2));
312 reg_def V29_K( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3));
313
314 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
315 reg_def V30_H( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
316 reg_def V30_J( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2));
317 reg_def V30_K( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3));
318
319 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
320 reg_def V31_H( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
321 reg_def V31_J( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2));
322 reg_def V31_K( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3));
323
324 // ----------------------------
325 // Special Registers
326 // ----------------------------
327
328 // the AArch64 CSPR status flag register is not directly acessible as
329 // instruction operand. the FPSR status flag register is a system
330 // register which can be written/read using MSR/MRS but again does not
331 // appear as an operand (a code identifying the FSPR occurs as an
332 // immediate value in the instruction).
333
334 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
335
336
337 // Specify priority of register selection within phases of register
338 // allocation. Highest priority is first. A useful heuristic is to
339 // give registers a low priority when they are required by machine
340 // instructions, like EAX and EDX on I486, and choose no-save registers
341 // before save-on-call, & save-on-call before save-on-entry. Registers
342 // which participate in fixed calling sequences should come last.
343 // Registers which are used as pairs must fall on an even boundary.
344
345 alloc_class chunk0(
346 // volatiles
347 R10, R10_H,
348 R11, R11_H,
349 R12, R12_H,
350 R13, R13_H,
351 R14, R14_H,
352 R15, R15_H,
353 R16, R16_H,
354 R17, R17_H,
355 R18, R18_H,
356
357 // arg registers
358 R0, R0_H,
359 R1, R1_H,
360 R2, R2_H,
361 R3, R3_H,
362 R4, R4_H,
363 R5, R5_H,
364 R6, R6_H,
365 R7, R7_H,
366
367 // non-volatiles
368 R19, R19_H,
369 R20, R20_H,
370 R21, R21_H,
371 R22, R22_H,
372 R23, R23_H,
373 R24, R24_H,
374 R25, R25_H,
375 R26, R26_H,
376
377 // non-allocatable registers
378
379 R27, R27_H, // heapbase
380 R28, R28_H, // thread
381 R29, R29_H, // fp
382 R30, R30_H, // lr
383 R31, R31_H, // sp
384 );
385
386 alloc_class chunk1(
387
388 // no save
389 V16, V16_H, V16_J, V16_K,
390 V17, V17_H, V17_J, V17_K,
391 V18, V18_H, V18_J, V18_K,
392 V19, V19_H, V19_J, V19_K,
393 V20, V20_H, V20_J, V20_K,
394 V21, V21_H, V21_J, V21_K,
395 V22, V22_H, V22_J, V22_K,
396 V23, V23_H, V23_J, V23_K,
397 V24, V24_H, V24_J, V24_K,
398 V25, V25_H, V25_J, V25_K,
399 V26, V26_H, V26_J, V26_K,
400 V27, V27_H, V27_J, V27_K,
401 V28, V28_H, V28_J, V28_K,
402 V29, V29_H, V29_J, V29_K,
403 V30, V30_H, V30_J, V30_K,
404 V31, V31_H, V31_J, V31_K,
405
406 // arg registers
407 V0, V0_H, V0_J, V0_K,
408 V1, V1_H, V1_J, V1_K,
409 V2, V2_H, V2_J, V2_K,
410 V3, V3_H, V3_J, V3_K,
411 V4, V4_H, V4_J, V4_K,
412 V5, V5_H, V5_J, V5_K,
413 V6, V6_H, V6_J, V6_K,
414 V7, V7_H, V7_J, V7_K,
415
416 // non-volatiles
417 V8, V8_H, V8_J, V8_K,
418 V9, V9_H, V9_J, V9_K,
419 V10, V10_H, V10_J, V10_K,
420 V11, V11_H, V11_J, V11_K,
421 V12, V12_H, V12_J, V12_K,
422 V13, V13_H, V13_J, V13_K,
423 V14, V14_H, V14_J, V14_K,
424 V15, V15_H, V15_J, V15_K,
425 );
426
427 alloc_class chunk2(RFLAGS);
428
429 //----------Architecture Description Register Classes--------------------------
430 // Several register classes are automatically defined based upon information in
431 // this architecture description.
432 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
433 // 2) reg_class compiler_method_oop_reg ( /* as def'd in frame section */ )
434 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
435 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
436 //
437
438 // Class for all 32 bit general purpose registers
439 reg_class all_reg32(
440 R0,
441 R1,
442 R2,
443 R3,
444 R4,
445 R5,
446 R6,
447 R7,
448 R10,
449 R11,
450 R12,
451 R13,
452 R14,
453 R15,
454 R16,
455 R17,
456 R18,
457 R19,
458 R20,
459 R21,
460 R22,
461 R23,
462 R24,
463 R25,
464 R26,
465 R27,
466 R28,
467 R29,
468 R30,
469 R31
470 );
471
472
473 // Class for all 32 bit integer registers (excluding SP which
474 // will never be used as an integer register)
475 reg_class any_reg32 %{
476 return _ANY_REG32_mask;
477 %}
478
479 // Singleton class for R0 int register
480 reg_class int_r0_reg(R0);
481
482 // Singleton class for R2 int register
483 reg_class int_r2_reg(R2);
484
485 // Singleton class for R3 int register
486 reg_class int_r3_reg(R3);
487
488 // Singleton class for R4 int register
489 reg_class int_r4_reg(R4);
490
491 // Singleton class for R31 int register
492 reg_class int_r31_reg(R31);
493
494 // Class for all 64 bit general purpose registers
495 reg_class all_reg(
496 R0, R0_H,
497 R1, R1_H,
498 R2, R2_H,
499 R3, R3_H,
500 R4, R4_H,
501 R5, R5_H,
502 R6, R6_H,
503 R7, R7_H,
504 R10, R10_H,
505 R11, R11_H,
506 R12, R12_H,
507 R13, R13_H,
508 R14, R14_H,
509 R15, R15_H,
510 R16, R16_H,
511 R17, R17_H,
512 R18, R18_H,
513 R19, R19_H,
514 R20, R20_H,
515 R21, R21_H,
516 R22, R22_H,
517 R23, R23_H,
518 R24, R24_H,
519 R25, R25_H,
520 R26, R26_H,
521 R27, R27_H,
522 R28, R28_H,
523 R29, R29_H,
524 R30, R30_H,
525 R31, R31_H
526 );
527
528 // Class for all long integer registers (including SP)
529 reg_class any_reg %{
530 return _ANY_REG_mask;
531 %}
532
533 // Class for non-allocatable 32 bit registers
534 reg_class non_allocatable_reg32(
535 R28, // thread
536 R30, // lr
537 R31 // sp
538 );
539
540 // Class for non-allocatable 64 bit registers
541 reg_class non_allocatable_reg(
542 R28, R28_H, // thread
543 R30, R30_H, // lr
544 R31, R31_H // sp
545 );
546
547 // Class for all non-special integer registers
548 reg_class no_special_reg32 %{
549 return _NO_SPECIAL_REG32_mask;
550 %}
551
552 // Class for all non-special long integer registers
553 reg_class no_special_reg %{
554 return _NO_SPECIAL_REG_mask;
555 %}
556
557 // Class for 64 bit register r0
558 reg_class r0_reg(
559 R0, R0_H
560 );
561
562 // Class for 64 bit register r1
563 reg_class r1_reg(
564 R1, R1_H
565 );
566
567 // Class for 64 bit register r2
568 reg_class r2_reg(
569 R2, R2_H
570 );
571
572 // Class for 64 bit register r3
573 reg_class r3_reg(
574 R3, R3_H
575 );
576
577 // Class for 64 bit register r4
578 reg_class r4_reg(
579 R4, R4_H
580 );
581
582 // Class for 64 bit register r5
583 reg_class r5_reg(
584 R5, R5_H
585 );
586
587 // Class for 64 bit register r10
588 reg_class r10_reg(
589 R10, R10_H
590 );
591
592 // Class for 64 bit register r11
593 reg_class r11_reg(
594 R11, R11_H
595 );
596
597 // Class for method register
598 reg_class method_reg(
599 R12, R12_H
600 );
601
602 // Class for heapbase register
603 reg_class heapbase_reg(
604 R27, R27_H
605 );
606
607 // Class for thread register
608 reg_class thread_reg(
609 R28, R28_H
610 );
611
612 // Class for frame pointer register
613 reg_class fp_reg(
614 R29, R29_H
615 );
616
617 // Class for link register
618 reg_class lr_reg(
619 R30, R30_H
620 );
621
622 // Class for long sp register
623 reg_class sp_reg(
624 R31, R31_H
625 );
626
627 // Class for all pointer registers
628 reg_class ptr_reg %{
629 return _PTR_REG_mask;
630 %}
631
632 // Class for all non_special pointer registers
633 reg_class no_special_ptr_reg %{
634 return _NO_SPECIAL_PTR_REG_mask;
635 %}
636
637 // Class for all float registers
638 reg_class float_reg(
639 V0,
640 V1,
641 V2,
642 V3,
643 V4,
644 V5,
645 V6,
646 V7,
647 V8,
648 V9,
649 V10,
650 V11,
651 V12,
652 V13,
653 V14,
654 V15,
655 V16,
656 V17,
657 V18,
658 V19,
659 V20,
660 V21,
661 V22,
662 V23,
663 V24,
664 V25,
665 V26,
666 V27,
667 V28,
668 V29,
669 V30,
670 V31
671 );
672
673 // Double precision float registers have virtual `high halves' that
674 // are needed by the allocator.
675 // Class for all double registers
676 reg_class double_reg(
677 V0, V0_H,
678 V1, V1_H,
679 V2, V2_H,
680 V3, V3_H,
681 V4, V4_H,
682 V5, V5_H,
683 V6, V6_H,
684 V7, V7_H,
685 V8, V8_H,
686 V9, V9_H,
687 V10, V10_H,
688 V11, V11_H,
689 V12, V12_H,
690 V13, V13_H,
691 V14, V14_H,
692 V15, V15_H,
693 V16, V16_H,
694 V17, V17_H,
695 V18, V18_H,
696 V19, V19_H,
697 V20, V20_H,
698 V21, V21_H,
699 V22, V22_H,
700 V23, V23_H,
701 V24, V24_H,
702 V25, V25_H,
703 V26, V26_H,
704 V27, V27_H,
705 V28, V28_H,
706 V29, V29_H,
707 V30, V30_H,
708 V31, V31_H
709 );
710
711 // Class for all 64bit vector registers
712 reg_class vectord_reg(
713 V0, V0_H,
714 V1, V1_H,
715 V2, V2_H,
716 V3, V3_H,
717 V4, V4_H,
718 V5, V5_H,
719 V6, V6_H,
720 V7, V7_H,
721 V8, V8_H,
722 V9, V9_H,
723 V10, V10_H,
724 V11, V11_H,
725 V12, V12_H,
726 V13, V13_H,
727 V14, V14_H,
728 V15, V15_H,
729 V16, V16_H,
730 V17, V17_H,
731 V18, V18_H,
732 V19, V19_H,
733 V20, V20_H,
734 V21, V21_H,
735 V22, V22_H,
736 V23, V23_H,
737 V24, V24_H,
738 V25, V25_H,
739 V26, V26_H,
740 V27, V27_H,
741 V28, V28_H,
742 V29, V29_H,
743 V30, V30_H,
744 V31, V31_H
745 );
746
747 // Class for all 128bit vector registers
748 reg_class vectorx_reg(
749 V0, V0_H, V0_J, V0_K,
750 V1, V1_H, V1_J, V1_K,
751 V2, V2_H, V2_J, V2_K,
752 V3, V3_H, V3_J, V3_K,
753 V4, V4_H, V4_J, V4_K,
754 V5, V5_H, V5_J, V5_K,
755 V6, V6_H, V6_J, V6_K,
756 V7, V7_H, V7_J, V7_K,
757 V8, V8_H, V8_J, V8_K,
758 V9, V9_H, V9_J, V9_K,
759 V10, V10_H, V10_J, V10_K,
760 V11, V11_H, V11_J, V11_K,
761 V12, V12_H, V12_J, V12_K,
762 V13, V13_H, V13_J, V13_K,
763 V14, V14_H, V14_J, V14_K,
764 V15, V15_H, V15_J, V15_K,
765 V16, V16_H, V16_J, V16_K,
766 V17, V17_H, V17_J, V17_K,
767 V18, V18_H, V18_J, V18_K,
768 V19, V19_H, V19_J, V19_K,
769 V20, V20_H, V20_J, V20_K,
770 V21, V21_H, V21_J, V21_K,
771 V22, V22_H, V22_J, V22_K,
772 V23, V23_H, V23_J, V23_K,
773 V24, V24_H, V24_J, V24_K,
774 V25, V25_H, V25_J, V25_K,
775 V26, V26_H, V26_J, V26_K,
776 V27, V27_H, V27_J, V27_K,
777 V28, V28_H, V28_J, V28_K,
778 V29, V29_H, V29_J, V29_K,
779 V30, V30_H, V30_J, V30_K,
780 V31, V31_H, V31_J, V31_K
781 );
782
783 // Class for 128 bit register v0
784 reg_class v0_reg(
785 V0, V0_H
786 );
787
788 // Class for 128 bit register v1
789 reg_class v1_reg(
790 V1, V1_H
791 );
792
793 // Class for 128 bit register v2
794 reg_class v2_reg(
795 V2, V2_H
796 );
797
798 // Class for 128 bit register v3
799 reg_class v3_reg(
800 V3, V3_H
801 );
802
803 // Class for 128 bit register v4
804 reg_class v4_reg(
805 V4, V4_H
806 );
807
808 // Class for 128 bit register v5
809 reg_class v5_reg(
810 V5, V5_H
811 );
812
813 // Class for 128 bit register v6
814 reg_class v6_reg(
815 V6, V6_H
816 );
817
818 // Class for 128 bit register v7
819 reg_class v7_reg(
820 V7, V7_H
821 );
822
823 // Class for 128 bit register v8
824 reg_class v8_reg(
825 V8, V8_H
826 );
827
828 // Class for 128 bit register v9
829 reg_class v9_reg(
830 V9, V9_H
831 );
832
833 // Class for 128 bit register v10
834 reg_class v10_reg(
835 V10, V10_H
836 );
837
838 // Class for 128 bit register v11
839 reg_class v11_reg(
840 V11, V11_H
841 );
842
843 // Class for 128 bit register v12
844 reg_class v12_reg(
845 V12, V12_H
846 );
847
848 // Class for 128 bit register v13
849 reg_class v13_reg(
850 V13, V13_H
851 );
852
853 // Class for 128 bit register v14
854 reg_class v14_reg(
855 V14, V14_H
856 );
857
858 // Class for 128 bit register v15
859 reg_class v15_reg(
860 V15, V15_H
861 );
862
863 // Class for 128 bit register v16
864 reg_class v16_reg(
865 V16, V16_H
866 );
867
868 // Class for 128 bit register v17
869 reg_class v17_reg(
870 V17, V17_H
871 );
872
873 // Class for 128 bit register v18
874 reg_class v18_reg(
875 V18, V18_H
876 );
877
878 // Class for 128 bit register v19
879 reg_class v19_reg(
880 V19, V19_H
881 );
882
883 // Class for 128 bit register v20
884 reg_class v20_reg(
885 V20, V20_H
886 );
887
888 // Class for 128 bit register v21
889 reg_class v21_reg(
890 V21, V21_H
891 );
892
893 // Class for 128 bit register v22
894 reg_class v22_reg(
895 V22, V22_H
896 );
897
898 // Class for 128 bit register v23
899 reg_class v23_reg(
900 V23, V23_H
901 );
902
903 // Class for 128 bit register v24
904 reg_class v24_reg(
905 V24, V24_H
906 );
907
908 // Class for 128 bit register v25
909 reg_class v25_reg(
910 V25, V25_H
911 );
912
913 // Class for 128 bit register v26
914 reg_class v26_reg(
915 V26, V26_H
916 );
917
918 // Class for 128 bit register v27
919 reg_class v27_reg(
920 V27, V27_H
921 );
922
923 // Class for 128 bit register v28
924 reg_class v28_reg(
925 V28, V28_H
926 );
927
928 // Class for 128 bit register v29
929 reg_class v29_reg(
930 V29, V29_H
931 );
932
933 // Class for 128 bit register v30
934 reg_class v30_reg(
935 V30, V30_H
936 );
937
938 // Class for 128 bit register v31
939 reg_class v31_reg(
940 V31, V31_H
941 );
942
943 // Singleton class for condition codes
944 reg_class int_flags(RFLAGS);
945
946 %}
947
948 //----------DEFINITION BLOCK---------------------------------------------------
949 // Define name --> value mappings to inform the ADLC of an integer valued name
950 // Current support includes integer values in the range [0, 0x7FFFFFFF]
951 // Format:
952 // int_def <name> ( <int_value>, <expression>);
953 // Generated Code in ad_<arch>.hpp
954 // #define <name> (<expression>)
955 // // value == <int_value>
956 // Generated code in ad_<arch>.cpp adlc_verification()
957 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
958 //
959
960 // we follow the ppc-aix port in using a simple cost model which ranks
961 // register operations as cheap, memory ops as more expensive and
962 // branches as most expensive. the first two have a low as well as a
963 // normal cost. huge cost appears to be a way of saying don't do
964 // something
965
966 definitions %{
967 // The default cost (of a register move instruction).
968 int_def INSN_COST ( 100, 100);
969 int_def BRANCH_COST ( 200, 2 * INSN_COST);
970 int_def CALL_COST ( 200, 2 * INSN_COST);
971 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
972 %}
973
974
975 //----------SOURCE BLOCK-------------------------------------------------------
976 // This is a block of C++ code which provides values, functions, and
977 // definitions necessary in the rest of the architecture description
978
979 source_hpp %{
980
981 #include "asm/macroAssembler.hpp"
982 #include "gc/shared/cardTable.hpp"
983 #include "gc/shared/cardTableBarrierSet.hpp"
984 #include "gc/shared/collectedHeap.hpp"
985 #include "opto/addnode.hpp"
986 #include "opto/convertnode.hpp"
987
988 extern RegMask _ANY_REG32_mask;
989 extern RegMask _ANY_REG_mask;
990 extern RegMask _PTR_REG_mask;
991 extern RegMask _NO_SPECIAL_REG32_mask;
992 extern RegMask _NO_SPECIAL_REG_mask;
993 extern RegMask _NO_SPECIAL_PTR_REG_mask;
994
995 class CallStubImpl {
996
997 //--------------------------------------------------------------
998 //---< Used for optimization in Compile::shorten_branches >---
999 //--------------------------------------------------------------
1000
1001 public:
1002 // Size of call trampoline stub.
1003 static uint size_call_trampoline() {
1004 return 0; // no call trampolines on this platform
1005 }
1006
1007 // number of relocations needed by a call trampoline stub
1008 static uint reloc_call_trampoline() {
1009 return 0; // no call trampolines on this platform
1010 }
1011 };
1012
1013 class HandlerImpl {
1014
1015 public:
1016
1017 static int emit_exception_handler(CodeBuffer &cbuf);
1018 static int emit_deopt_handler(CodeBuffer& cbuf);
1019
1020 static uint size_exception_handler() {
1021 return MacroAssembler::far_branch_size();
1022 }
1023
1024 static uint size_deopt_handler() {
1025 // count one adr and one far branch instruction
1026 return 4 * NativeInstruction::instruction_size;
1027 }
1028 };
1029
1030 class Node::PD {
1031 public:
1032 enum NodeFlags {
1033 _last_flag = Node::_last_flag
1034 };
1035 };
1036
1037 bool is_CAS(int opcode, bool maybe_volatile);
1038
1039 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1040
1041 bool unnecessary_acquire(const Node *barrier);
1042 bool needs_acquiring_load(const Node *load);
1043
1044 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1045
1046 bool unnecessary_release(const Node *barrier);
1047 bool unnecessary_volatile(const Node *barrier);
1048 bool needs_releasing_store(const Node *store);
1049
1050 // predicate controlling translation of CompareAndSwapX
1051 bool needs_acquiring_load_exclusive(const Node *load);
1052
1053 // predicate controlling addressing modes
1054 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1055 %}
1056
1057 source %{
1058
1059 // Derived RegMask with conditionally allocatable registers
1060
1061 void PhaseOutput::pd_perform_mach_node_analysis() {
1062 }
1063
1064 int MachNode::pd_alignment_required() const {
1065 return 1;
1066 }
1067
1068 int MachNode::compute_padding(int current_offset) const {
1069 return 0;
1070 }
1071
1072 RegMask _ANY_REG32_mask;
1073 RegMask _ANY_REG_mask;
1074 RegMask _PTR_REG_mask;
1075 RegMask _NO_SPECIAL_REG32_mask;
1076 RegMask _NO_SPECIAL_REG_mask;
1077 RegMask _NO_SPECIAL_PTR_REG_mask;
1078
1079 void reg_mask_init() {
1080 // We derive below RegMask(s) from the ones which are auto-generated from
1081 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1082 // registers conditionally reserved.
1083
1084 _ANY_REG32_mask = _ALL_REG32_mask;
1085 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1086
1087 _ANY_REG_mask = _ALL_REG_mask;
1088
1089 _PTR_REG_mask = _ALL_REG_mask;
1090
1091 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1092 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1093
1094 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1095 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1096
1097 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1098 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1099
1100 // r27 is not allocatable when compressed oops is on and heapbase is not
1101 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1102 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL || UseAOT)) {
1103 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1104 _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1105 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1106 }
1107
1108 // r29 is not allocatable when PreserveFramePointer is on
1109 if (PreserveFramePointer) {
1110 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1111 _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
1112 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
1113 }
1114 }
1115
1116 // Optimizaton of volatile gets and puts
1117 // -------------------------------------
1118 //
1119 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1120 // use to implement volatile reads and writes. For a volatile read
1121 // we simply need
1122 //
1123 // ldar<x>
1124 //
1125 // and for a volatile write we need
1126 //
1127 // stlr<x>
1128 //
1129 // Alternatively, we can implement them by pairing a normal
1130 // load/store with a memory barrier. For a volatile read we need
1131 //
1132 // ldr<x>
1133 // dmb ishld
1134 //
1135 // for a volatile write
1136 //
1137 // dmb ish
1138 // str<x>
1139 // dmb ish
1140 //
1141 // We can also use ldaxr and stlxr to implement compare and swap CAS
1142 // sequences. These are normally translated to an instruction
1143 // sequence like the following
1144 //
1145 // dmb ish
1146 // retry:
1147 // ldxr<x> rval raddr
1148 // cmp rval rold
1149 // b.ne done
1150 // stlxr<x> rval, rnew, rold
1151 // cbnz rval retry
1152 // done:
1153 // cset r0, eq
1154 // dmb ishld
1155 //
1156 // Note that the exclusive store is already using an stlxr
1157 // instruction. That is required to ensure visibility to other
1158 // threads of the exclusive write (assuming it succeeds) before that
1159 // of any subsequent writes.
1160 //
1161 // The following instruction sequence is an improvement on the above
1162 //
1163 // retry:
1164 // ldaxr<x> rval raddr
1165 // cmp rval rold
1166 // b.ne done
1167 // stlxr<x> rval, rnew, rold
1168 // cbnz rval retry
1169 // done:
1170 // cset r0, eq
1171 //
1172 // We don't need the leading dmb ish since the stlxr guarantees
1173 // visibility of prior writes in the case that the swap is
1174 // successful. Crucially we don't have to worry about the case where
1175 // the swap is not successful since no valid program should be
1176 // relying on visibility of prior changes by the attempting thread
1177 // in the case where the CAS fails.
1178 //
1179 // Similarly, we don't need the trailing dmb ishld if we substitute
1180 // an ldaxr instruction since that will provide all the guarantees we
1181 // require regarding observation of changes made by other threads
1182 // before any change to the CAS address observed by the load.
1183 //
1184 // In order to generate the desired instruction sequence we need to
1185 // be able to identify specific 'signature' ideal graph node
1186 // sequences which i) occur as a translation of a volatile reads or
1187 // writes or CAS operations and ii) do not occur through any other
1188 // translation or graph transformation. We can then provide
1189 // alternative aldc matching rules which translate these node
1190 // sequences to the desired machine code sequences. Selection of the
1191 // alternative rules can be implemented by predicates which identify
1192 // the relevant node sequences.
1193 //
1194 // The ideal graph generator translates a volatile read to the node
1195 // sequence
1196 //
1197 // LoadX[mo_acquire]
1198 // MemBarAcquire
1199 //
1200 // As a special case when using the compressed oops optimization we
1201 // may also see this variant
1202 //
1203 // LoadN[mo_acquire]
1204 // DecodeN
1205 // MemBarAcquire
1206 //
1207 // A volatile write is translated to the node sequence
1208 //
1209 // MemBarRelease
1210 // StoreX[mo_release] {CardMark}-optional
1211 // MemBarVolatile
1212 //
1213 // n.b. the above node patterns are generated with a strict
1214 // 'signature' configuration of input and output dependencies (see
1215 // the predicates below for exact details). The card mark may be as
1216 // simple as a few extra nodes or, in a few GC configurations, may
1217 // include more complex control flow between the leading and
1218 // trailing memory barriers. However, whatever the card mark
1219 // configuration these signatures are unique to translated volatile
1220 // reads/stores -- they will not appear as a result of any other
1221 // bytecode translation or inlining nor as a consequence of
1222 // optimizing transforms.
1223 //
1224 // We also want to catch inlined unsafe volatile gets and puts and
1225 // be able to implement them using either ldar<x>/stlr<x> or some
1226 // combination of ldr<x>/stlr<x> and dmb instructions.
1227 //
1228 // Inlined unsafe volatiles puts manifest as a minor variant of the
1229 // normal volatile put node sequence containing an extra cpuorder
1230 // membar
1231 //
1232 // MemBarRelease
1233 // MemBarCPUOrder
1234 // StoreX[mo_release] {CardMark}-optional
1235 // MemBarCPUOrder
1236 // MemBarVolatile
1237 //
1238 // n.b. as an aside, a cpuorder membar is not itself subject to
1239 // matching and translation by adlc rules. However, the rule
1240 // predicates need to detect its presence in order to correctly
1241 // select the desired adlc rules.
1242 //
1243 // Inlined unsafe volatile gets manifest as a slightly different
1244 // node sequence to a normal volatile get because of the
1245 // introduction of some CPUOrder memory barriers to bracket the
1246 // Load. However, but the same basic skeleton of a LoadX feeding a
1247 // MemBarAcquire, possibly thorugh an optional DecodeN, is still
1248 // present
1249 //
1250 // MemBarCPUOrder
1251 // || \\
1252 // MemBarCPUOrder LoadX[mo_acquire]
1253 // || |
1254 // || {DecodeN} optional
1255 // || /
1256 // MemBarAcquire
1257 //
1258 // In this case the acquire membar does not directly depend on the
1259 // load. However, we can be sure that the load is generated from an
1260 // inlined unsafe volatile get if we see it dependent on this unique
1261 // sequence of membar nodes. Similarly, given an acquire membar we
1262 // can know that it was added because of an inlined unsafe volatile
1263 // get if it is fed and feeds a cpuorder membar and if its feed
1264 // membar also feeds an acquiring load.
1265 //
1266 // Finally an inlined (Unsafe) CAS operation is translated to the
1267 // following ideal graph
1268 //
1269 // MemBarRelease
1270 // MemBarCPUOrder
1271 // CompareAndSwapX {CardMark}-optional
1272 // MemBarCPUOrder
1273 // MemBarAcquire
1274 //
1275 // So, where we can identify these volatile read and write
1276 // signatures we can choose to plant either of the above two code
1277 // sequences. For a volatile read we can simply plant a normal
1278 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1279 // also choose to inhibit translation of the MemBarAcquire and
1280 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1281 //
1282 // When we recognise a volatile store signature we can choose to
1283 // plant at a dmb ish as a translation for the MemBarRelease, a
1284 // normal str<x> and then a dmb ish for the MemBarVolatile.
1285 // Alternatively, we can inhibit translation of the MemBarRelease
1286 // and MemBarVolatile and instead plant a simple stlr<x>
1287 // instruction.
1288 //
1289 // when we recognise a CAS signature we can choose to plant a dmb
1290 // ish as a translation for the MemBarRelease, the conventional
1291 // macro-instruction sequence for the CompareAndSwap node (which
1292 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1293 // Alternatively, we can elide generation of the dmb instructions
1294 // and plant the alternative CompareAndSwap macro-instruction
1295 // sequence (which uses ldaxr<x>).
1296 //
1297 // Of course, the above only applies when we see these signature
1298 // configurations. We still want to plant dmb instructions in any
1299 // other cases where we may see a MemBarAcquire, MemBarRelease or
1300 // MemBarVolatile. For example, at the end of a constructor which
1301 // writes final/volatile fields we will see a MemBarRelease
1302 // instruction and this needs a 'dmb ish' lest we risk the
1303 // constructed object being visible without making the
1304 // final/volatile field writes visible.
1305 //
1306 // n.b. the translation rules below which rely on detection of the
1307 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1308 // If we see anything other than the signature configurations we
1309 // always just translate the loads and stores to ldr<x> and str<x>
1310 // and translate acquire, release and volatile membars to the
1311 // relevant dmb instructions.
1312 //
1313
1314 // is_CAS(int opcode, bool maybe_volatile)
1315 //
1316 // return true if opcode is one of the possible CompareAndSwapX
1317 // values otherwise false.
1318
1319 bool is_CAS(int opcode, bool maybe_volatile)
1320 {
1321 switch(opcode) {
1322 // We handle these
1323 case Op_CompareAndSwapI:
1324 case Op_CompareAndSwapL:
1325 case Op_CompareAndSwapP:
1326 case Op_CompareAndSwapN:
1327 case Op_ShenandoahCompareAndSwapP:
1328 case Op_ShenandoahCompareAndSwapN:
1329 case Op_CompareAndSwapB:
1330 case Op_CompareAndSwapS:
1331 case Op_GetAndSetI:
1332 case Op_GetAndSetL:
1333 case Op_GetAndSetP:
1334 case Op_GetAndSetN:
1335 case Op_GetAndAddI:
1336 case Op_GetAndAddL:
1337 return true;
1338 case Op_CompareAndExchangeI:
1339 case Op_CompareAndExchangeN:
1340 case Op_CompareAndExchangeB:
1341 case Op_CompareAndExchangeS:
1342 case Op_CompareAndExchangeL:
1343 case Op_CompareAndExchangeP:
1344 case Op_WeakCompareAndSwapB:
1345 case Op_WeakCompareAndSwapS:
1346 case Op_WeakCompareAndSwapI:
1347 case Op_WeakCompareAndSwapL:
1348 case Op_WeakCompareAndSwapP:
1349 case Op_WeakCompareAndSwapN:
1350 case Op_ShenandoahWeakCompareAndSwapP:
1351 case Op_ShenandoahWeakCompareAndSwapN:
1352 case Op_ShenandoahCompareAndExchangeP:
1353 case Op_ShenandoahCompareAndExchangeN:
1354 return maybe_volatile;
1355 default:
1356 return false;
1357 }
1358 }
1359
1360 // helper to determine the maximum number of Phi nodes we may need to
1361 // traverse when searching from a card mark membar for the merge mem
1362 // feeding a trailing membar or vice versa
1363
1364 // predicates controlling emit of ldr<x>/ldar<x>
1365
1366 bool unnecessary_acquire(const Node *barrier)
1367 {
1368 assert(barrier->is_MemBar(), "expecting a membar");
1369
1370 MemBarNode* mb = barrier->as_MemBar();
1371
1372 if (mb->trailing_load()) {
1373 return true;
1374 }
1375
1376 if (mb->trailing_load_store()) {
1377 Node* load_store = mb->in(MemBarNode::Precedent);
1378 assert(load_store->is_LoadStore(), "unexpected graph shape");
1379 return is_CAS(load_store->Opcode(), true);
1380 }
1381
1382 return false;
1383 }
1384
1385 bool needs_acquiring_load(const Node *n)
1386 {
1387 assert(n->is_Load(), "expecting a load");
1388 LoadNode *ld = n->as_Load();
1389 return ld->is_acquire();
1390 }
1391
1392 bool unnecessary_release(const Node *n)
1393 {
1394 assert((n->is_MemBar() &&
1395 n->Opcode() == Op_MemBarRelease),
1396 "expecting a release membar");
1397
1398 MemBarNode *barrier = n->as_MemBar();
1399 if (!barrier->leading()) {
1400 return false;
1401 } else {
1402 Node* trailing = barrier->trailing_membar();
1403 MemBarNode* trailing_mb = trailing->as_MemBar();
1404 assert(trailing_mb->trailing(), "Not a trailing membar?");
1405 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1406
1407 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1408 if (mem->is_Store()) {
1409 assert(mem->as_Store()->is_release(), "");
1410 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1411 return true;
1412 } else {
1413 assert(mem->is_LoadStore(), "");
1414 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1415 return is_CAS(mem->Opcode(), true);
1416 }
1417 }
1418 return false;
1419 }
1420
1421 bool unnecessary_volatile(const Node *n)
1422 {
1423 // assert n->is_MemBar();
1424 MemBarNode *mbvol = n->as_MemBar();
1425
1426 bool release = mbvol->trailing_store();
1427 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1428 #ifdef ASSERT
1429 if (release) {
1430 Node* leading = mbvol->leading_membar();
1431 assert(leading->Opcode() == Op_MemBarRelease, "");
1432 assert(leading->as_MemBar()->leading_store(), "");
1433 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1434 }
1435 #endif
1436
1437 return release;
1438 }
1439
1440 // predicates controlling emit of str<x>/stlr<x>
1441
1442 bool needs_releasing_store(const Node *n)
1443 {
1444 // assert n->is_Store();
1445 StoreNode *st = n->as_Store();
1446 return st->trailing_membar() != NULL;
1447 }
1448
1449 // predicate controlling translation of CAS
1450 //
1451 // returns true if CAS needs to use an acquiring load otherwise false
1452
1453 bool needs_acquiring_load_exclusive(const Node *n)
1454 {
1455 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1456 LoadStoreNode* ldst = n->as_LoadStore();
1457 if (is_CAS(n->Opcode(), false)) {
1458 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1459 } else {
1460 return ldst->trailing_membar() != NULL;
1461 }
1462
1463 // so we can just return true here
1464 return true;
1465 }
1466
1467 #define __ _masm.
1468
1469 // advance declarations for helper functions to convert register
1470 // indices to register objects
1471
1472 // the ad file has to provide implementations of certain methods
1473 // expected by the generic code
1474 //
1475 // REQUIRED FUNCTIONALITY
1476
1477 //=============================================================================
1478
1479 // !!!!! Special hack to get all types of calls to specify the byte offset
1480 // from the start of the call to the point where the return address
1481 // will point.
1482
1483 int MachCallStaticJavaNode::ret_addr_offset()
1484 {
1485 // call should be a simple bl
1486 int off = 4;
1487 return off;
1488 }
1489
1490 int MachCallDynamicJavaNode::ret_addr_offset()
1491 {
1492 return 16; // movz, movk, movk, bl
1493 }
1494
1495 int MachCallRuntimeNode::ret_addr_offset() {
1496 // for generated stubs the call will be
1497 // far_call(addr)
1498 // for real runtime callouts it will be six instructions
1499 // see aarch64_enc_java_to_runtime
1500 // adr(rscratch2, retaddr)
1501 // lea(rscratch1, RuntimeAddress(addr)
1502 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1503 // blr(rscratch1)
1504 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1505 if (cb) {
1506 return MacroAssembler::far_branch_size();
1507 } else {
1508 return 6 * NativeInstruction::instruction_size;
1509 }
1510 }
1511
1512 // Indicate if the safepoint node needs the polling page as an input
1513
1514 // the shared code plants the oop data at the start of the generated
1515 // code for the safepoint node and that needs ot be at the load
1516 // instruction itself. so we cannot plant a mov of the safepoint poll
1517 // address followed by a load. setting this to true means the mov is
1518 // scheduled as a prior instruction. that's better for scheduling
1519 // anyway.
1520
1521 bool SafePointNode::needs_polling_address_input()
1522 {
1523 return true;
1524 }
1525
1526 //=============================================================================
1527
1528 #ifndef PRODUCT
1529 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1530 st->print("BREAKPOINT");
1531 }
1532 #endif
1533
1534 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1535 C2_MacroAssembler _masm(&cbuf);
1536 __ brk(0);
1537 }
1538
1539 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1540 return MachNode::size(ra_);
1541 }
1542
1543 //=============================================================================
1544
1545 #ifndef PRODUCT
1546 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1547 st->print("nop \t# %d bytes pad for loops and calls", _count);
1548 }
1549 #endif
1550
1551 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1552 C2_MacroAssembler _masm(&cbuf);
1553 for (int i = 0; i < _count; i++) {
1554 __ nop();
1555 }
1556 }
1557
1558 uint MachNopNode::size(PhaseRegAlloc*) const {
1559 return _count * NativeInstruction::instruction_size;
1560 }
1561
1562 //=============================================================================
1563 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1564
1565 int ConstantTable::calculate_table_base_offset() const {
1566 return 0; // absolute addressing, no offset
1567 }
1568
1569 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1570 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1571 ShouldNotReachHere();
1572 }
1573
1574 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1575 // Empty encoding
1576 }
1577
1578 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1579 return 0;
1580 }
1581
1582 #ifndef PRODUCT
1583 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1584 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1585 }
1586 #endif
1587
1588 #ifndef PRODUCT
1589 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1590 Compile* C = ra_->C;
1591
1592 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1593
1594 if (C->output()->need_stack_bang(framesize))
1595 st->print("# stack bang size=%d\n\t", framesize);
1596
1597 if (framesize < ((1 << 9) + 2 * wordSize)) {
1598 st->print("sub sp, sp, #%d\n\t", framesize);
1599 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1600 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1601 } else {
1602 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1603 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1604 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1605 st->print("sub sp, sp, rscratch1");
1606 }
1607 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1608 st->print("\n\t");
1609 st->print("ldr rscratch1, [guard]\n\t");
1610 st->print("dmb ishld\n\t");
1611 st->print("ldr rscratch2, [rthread, #thread_disarmed_offset]\n\t");
1612 st->print("cmp rscratch1, rscratch2\n\t");
1613 st->print("b.eq skip");
1614 st->print("\n\t");
1615 st->print("blr #nmethod_entry_barrier_stub\n\t");
1616 st->print("b skip\n\t");
1617 st->print("guard: int\n\t");
1618 st->print("\n\t");
1619 st->print("skip:\n\t");
1620 }
1621 }
1622 #endif
1623
1624 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1625 Compile* C = ra_->C;
1626 C2_MacroAssembler _masm(&cbuf);
1627
1628 // n.b. frame size includes space for return pc and rfp
1629 const int framesize = C->output()->frame_size_in_bytes();
1630 assert(framesize%(2*wordSize) == 0, "must preserve 2*wordSize alignment");
1631
1632 // insert a nop at the start of the prolog so we can patch in a
1633 // branch if we need to invalidate the method later
1634 __ nop();
1635
1636 if (C->clinit_barrier_on_entry()) {
1637 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1638
1639 Label L_skip_barrier;
1640
1641 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1642 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1643 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1644 __ bind(L_skip_barrier);
1645 }
1646
1647 int bangsize = C->output()->bang_size_in_bytes();
1648 if (C->output()->need_stack_bang(bangsize) && UseStackBanging)
1649 __ generate_stack_overflow_check(bangsize);
1650
1651 __ build_frame(framesize);
1652
1653 if (C->stub_function() == NULL) {
1654 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1655 bs->nmethod_entry_barrier(&_masm);
1656 }
1657
1658 if (VerifyStackAtCalls) {
1659 Unimplemented();
1660 }
1661
1662 C->output()->set_frame_complete(cbuf.insts_size());
1663
1664 if (C->has_mach_constant_base_node()) {
1665 // NOTE: We set the table base offset here because users might be
1666 // emitted before MachConstantBaseNode.
1667 ConstantTable& constant_table = C->output()->constant_table();
1668 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1669 }
1670 }
1671
1672 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1673 {
1674 return MachNode::size(ra_); // too many variables; just compute it
1675 // the hard way
1676 }
1677
1678 int MachPrologNode::reloc() const
1679 {
1680 return 0;
1681 }
1682
1683 //=============================================================================
1684
1685 #ifndef PRODUCT
1686 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1687 Compile* C = ra_->C;
1688 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1689
1690 st->print("# pop frame %d\n\t",framesize);
1691
1692 if (framesize == 0) {
1693 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1694 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1695 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1696 st->print("add sp, sp, #%d\n\t", framesize);
1697 } else {
1698 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1699 st->print("add sp, sp, rscratch1\n\t");
1700 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1701 }
1702
1703 if (do_polling() && C->is_method_compilation()) {
1704 st->print("# touch polling page\n\t");
1705 st->print("ldr rscratch1, [rthread],#polling_page_offset\n\t");
1706 st->print("ldr zr, [rscratch1]");
1707 }
1708 }
1709 #endif
1710
1711 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1712 Compile* C = ra_->C;
1713 C2_MacroAssembler _masm(&cbuf);
1714 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1715
1716 __ remove_frame(framesize);
1717
1718 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1719 __ reserved_stack_check();
1720 }
1721
1722 if (do_polling() && C->is_method_compilation()) {
1723 __ fetch_and_read_polling_page(rscratch1, relocInfo::poll_return_type);
1724 }
1725 }
1726
1727 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1728 // Variable size. Determine dynamically.
1729 return MachNode::size(ra_);
1730 }
1731
1732 int MachEpilogNode::reloc() const {
1733 // Return number of relocatable values contained in this instruction.
1734 return 1; // 1 for polling page.
1735 }
1736
1737 const Pipeline * MachEpilogNode::pipeline() const {
1738 return MachNode::pipeline_class();
1739 }
1740
1741 //=============================================================================
1742
1743 // Figure out which register class each belongs in: rc_int, rc_float or
1744 // rc_stack.
1745 enum RC { rc_bad, rc_int, rc_float, rc_stack };
1746
1747 static enum RC rc_class(OptoReg::Name reg) {
1748
1749 if (reg == OptoReg::Bad) {
1750 return rc_bad;
1751 }
1752
1753 // we have 30 int registers * 2 halves
1754 // (rscratch1 and rscratch2 are omitted)
1755 int slots_of_int_registers = RegisterImpl::max_slots_per_register * (RegisterImpl::number_of_registers - 2);
1756
1757 if (reg < slots_of_int_registers) {
1758 return rc_int;
1759 }
1760
1761 // we have 32 float register * 4 halves
1762 if (reg < slots_of_int_registers + FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers) {
1763 return rc_float;
1764 }
1765
1766 // Between float regs & stack is the flags regs.
1767 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1768
1769 return rc_stack;
1770 }
1771
1772 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1773 Compile* C = ra_->C;
1774
1775 // Get registers to move.
1776 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1777 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1778 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1779 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1780
1781 enum RC src_hi_rc = rc_class(src_hi);
1782 enum RC src_lo_rc = rc_class(src_lo);
1783 enum RC dst_hi_rc = rc_class(dst_hi);
1784 enum RC dst_lo_rc = rc_class(dst_lo);
1785
1786 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1787
1788 if (src_hi != OptoReg::Bad) {
1789 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1790 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1791 "expected aligned-adjacent pairs");
1792 }
1793
1794 if (src_lo == dst_lo && src_hi == dst_hi) {
1795 return 0; // Self copy, no move.
1796 }
1797
1798 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1799 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1800 int src_offset = ra_->reg2offset(src_lo);
1801 int dst_offset = ra_->reg2offset(dst_lo);
1802
1803 if (bottom_type()->isa_vect() != NULL) {
1804 uint ireg = ideal_reg();
1805 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1806 if (cbuf) {
1807 C2_MacroAssembler _masm(cbuf);
1808 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1809 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1810 // stack->stack
1811 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1812 if (ireg == Op_VecD) {
1813 __ unspill(rscratch1, true, src_offset);
1814 __ spill(rscratch1, true, dst_offset);
1815 } else {
1816 __ spill_copy128(src_offset, dst_offset);
1817 }
1818 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1819 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1820 ireg == Op_VecD ? __ T8B : __ T16B,
1821 as_FloatRegister(Matcher::_regEncode[src_lo]));
1822 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1823 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1824 ireg == Op_VecD ? __ D : __ Q,
1825 ra_->reg2offset(dst_lo));
1826 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1827 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1828 ireg == Op_VecD ? __ D : __ Q,
1829 ra_->reg2offset(src_lo));
1830 } else {
1831 ShouldNotReachHere();
1832 }
1833 }
1834 } else if (cbuf) {
1835 C2_MacroAssembler _masm(cbuf);
1836 switch (src_lo_rc) {
1837 case rc_int:
1838 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1839 if (is64) {
1840 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1841 as_Register(Matcher::_regEncode[src_lo]));
1842 } else {
1843 C2_MacroAssembler _masm(cbuf);
1844 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1845 as_Register(Matcher::_regEncode[src_lo]));
1846 }
1847 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1848 if (is64) {
1849 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1850 as_Register(Matcher::_regEncode[src_lo]));
1851 } else {
1852 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1853 as_Register(Matcher::_regEncode[src_lo]));
1854 }
1855 } else { // gpr --> stack spill
1856 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1857 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
1858 }
1859 break;
1860 case rc_float:
1861 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
1862 if (is64) {
1863 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
1864 as_FloatRegister(Matcher::_regEncode[src_lo]));
1865 } else {
1866 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
1867 as_FloatRegister(Matcher::_regEncode[src_lo]));
1868 }
1869 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
1870 if (cbuf) {
1871 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1872 as_FloatRegister(Matcher::_regEncode[src_lo]));
1873 } else {
1874 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1875 as_FloatRegister(Matcher::_regEncode[src_lo]));
1876 }
1877 } else { // fpr --> stack spill
1878 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1879 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1880 is64 ? __ D : __ S, dst_offset);
1881 }
1882 break;
1883 case rc_stack:
1884 if (dst_lo_rc == rc_int) { // stack --> gpr load
1885 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
1886 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
1887 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1888 is64 ? __ D : __ S, src_offset);
1889 } else { // stack --> stack copy
1890 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1891 __ unspill(rscratch1, is64, src_offset);
1892 __ spill(rscratch1, is64, dst_offset);
1893 }
1894 break;
1895 default:
1896 assert(false, "bad rc_class for spill");
1897 ShouldNotReachHere();
1898 }
1899 }
1900
1901 if (st) {
1902 st->print("spill ");
1903 if (src_lo_rc == rc_stack) {
1904 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
1905 } else {
1906 st->print("%s -> ", Matcher::regName[src_lo]);
1907 }
1908 if (dst_lo_rc == rc_stack) {
1909 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
1910 } else {
1911 st->print("%s", Matcher::regName[dst_lo]);
1912 }
1913 if (bottom_type()->isa_vect() != NULL) {
1914 st->print("\t# vector spill size = %d", ideal_reg()==Op_VecD ? 64:128);
1915 } else {
1916 st->print("\t# spill size = %d", is64 ? 64:32);
1917 }
1918 }
1919
1920 return 0;
1921
1922 }
1923
1924 #ifndef PRODUCT
1925 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1926 if (!ra_)
1927 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
1928 else
1929 implementation(NULL, ra_, false, st);
1930 }
1931 #endif
1932
1933 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1934 implementation(&cbuf, ra_, false, NULL);
1935 }
1936
1937 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1938 return MachNode::size(ra_);
1939 }
1940
1941 //=============================================================================
1942
1943 #ifndef PRODUCT
1944 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1945 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1946 int reg = ra_->get_reg_first(this);
1947 st->print("add %s, rsp, #%d]\t# box lock",
1948 Matcher::regName[reg], offset);
1949 }
1950 #endif
1951
1952 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1953 C2_MacroAssembler _masm(&cbuf);
1954
1955 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1956 int reg = ra_->get_encode(this);
1957
1958 // This add will handle any 24-bit signed offset. 24 bits allows an
1959 // 8 megabyte stack frame.
1960 __ add(as_Register(reg), sp, offset);
1961 }
1962
1963 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
1964 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
1965 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1966
1967 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
1968 return NativeInstruction::instruction_size;
1969 } else {
1970 return 2 * NativeInstruction::instruction_size;
1971 }
1972 }
1973
1974 //=============================================================================
1975
1976 #ifndef PRODUCT
1977 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1978 {
1979 st->print_cr("# MachUEPNode");
1980 if (UseCompressedClassPointers) {
1981 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1982 if (CompressedKlassPointers::shift() != 0) {
1983 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1984 }
1985 } else {
1986 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1987 }
1988 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
1989 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
1990 }
1991 #endif
1992
1993 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1994 {
1995 // This is the unverified entry point.
1996 C2_MacroAssembler _masm(&cbuf);
1997
1998 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
1999 Label skip;
2000 // TODO
2001 // can we avoid this skip and still use a reloc?
2002 __ br(Assembler::EQ, skip);
2003 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2004 __ bind(skip);
2005 }
2006
2007 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2008 {
2009 return MachNode::size(ra_);
2010 }
2011
2012 // REQUIRED EMIT CODE
2013
2014 //=============================================================================
2015
2016 // Emit exception handler code.
2017 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2018 {
2019 // mov rscratch1 #exception_blob_entry_point
2020 // br rscratch1
2021 // Note that the code buffer's insts_mark is always relative to insts.
2022 // That's why we must use the macroassembler to generate a handler.
2023 C2_MacroAssembler _masm(&cbuf);
2024 address base = __ start_a_stub(size_exception_handler());
2025 if (base == NULL) {
2026 ciEnv::current()->record_failure("CodeCache is full");
2027 return 0; // CodeBuffer::expand failed
2028 }
2029 int offset = __ offset();
2030 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2031 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2032 __ end_a_stub();
2033 return offset;
2034 }
2035
2036 // Emit deopt handler code.
2037 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2038 {
2039 // Note that the code buffer's insts_mark is always relative to insts.
2040 // That's why we must use the macroassembler to generate a handler.
2041 C2_MacroAssembler _masm(&cbuf);
2042 address base = __ start_a_stub(size_deopt_handler());
2043 if (base == NULL) {
2044 ciEnv::current()->record_failure("CodeCache is full");
2045 return 0; // CodeBuffer::expand failed
2046 }
2047 int offset = __ offset();
2048
2049 __ adr(lr, __ pc());
2050 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2051
2052 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
2053 __ end_a_stub();
2054 return offset;
2055 }
2056
2057 // REQUIRED MATCHER CODE
2058
2059 //=============================================================================
2060
2061 const bool Matcher::match_rule_supported(int opcode) {
2062 if (!has_match_rule(opcode))
2063 return false;
2064
2065 bool ret_value = true;
2066 switch (opcode) {
2067 case Op_CacheWB:
2068 case Op_CacheWBPreSync:
2069 case Op_CacheWBPostSync:
2070 if (!VM_Version::supports_data_cache_line_flush()) {
2071 ret_value = false;
2072 }
2073 break;
2074 }
2075
2076 return ret_value; // Per default match rules are supported.
2077 }
2078
2079 // Identify extra cases that we might want to provide match rules for vector nodes and
2080 // other intrinsics guarded with vector length (vlen) and element type (bt).
2081 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
2082 if (!match_rule_supported(opcode)) {
2083 return false;
2084 }
2085
2086 // Special cases which require vector length
2087 switch (opcode) {
2088 case Op_MulAddVS2VI: {
2089 if (vlen != 4) {
2090 return false;
2091 }
2092 break;
2093 }
2094 }
2095
2096 return true; // Per default match rules are supported.
2097 }
2098
2099 const bool Matcher::has_predicated_vectors(void) {
2100 return false;
2101 }
2102
2103 const int Matcher::float_pressure(int default_pressure_threshold) {
2104 return default_pressure_threshold;
2105 }
2106
2107 int Matcher::regnum_to_fpu_offset(int regnum)
2108 {
2109 Unimplemented();
2110 return 0;
2111 }
2112
2113 // Is this branch offset short enough that a short branch can be used?
2114 //
2115 // NOTE: If the platform does not provide any short branch variants, then
2116 // this method should return false for offset 0.
2117 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2118 // The passed offset is relative to address of the branch.
2119
2120 return (-32768 <= offset && offset < 32768);
2121 }
2122
2123 const bool Matcher::isSimpleConstant64(jlong value) {
2124 // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
2125 // Probably always true, even if a temp register is required.
2126 return true;
2127 }
2128
2129 // true just means we have fast l2f conversion
2130 const bool Matcher::convL2FSupported(void) {
2131 return true;
2132 }
2133
2134 // Vector width in bytes.
2135 const int Matcher::vector_width_in_bytes(BasicType bt) {
2136 int size = MIN2(16,(int)MaxVectorSize);
2137 // Minimum 2 values in vector
2138 if (size < 2*type2aelembytes(bt)) size = 0;
2139 // But never < 4
2140 if (size < 4) size = 0;
2141 return size;
2142 }
2143
2144 // Limits on vector size (number of elements) loaded into vector.
2145 const int Matcher::max_vector_size(const BasicType bt) {
2146 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2147 }
2148 const int Matcher::min_vector_size(const BasicType bt) {
2149 // For the moment limit the vector size to 8 bytes
2150 int size = 8 / type2aelembytes(bt);
2151 if (size < 2) size = 2;
2152 return size;
2153 }
2154
2155 // Vector ideal reg.
2156 const uint Matcher::vector_ideal_reg(int len) {
2157 switch(len) {
2158 case 8: return Op_VecD;
2159 case 16: return Op_VecX;
2160 }
2161 ShouldNotReachHere();
2162 return 0;
2163 }
2164
2165 // AES support not yet implemented
2166 const bool Matcher::pass_original_key_for_aes() {
2167 return false;
2168 }
2169
2170 // aarch64 supports misaligned vectors store/load.
2171 const bool Matcher::misaligned_vectors_ok() {
2172 return true;
2173 }
2174
2175 // false => size gets scaled to BytesPerLong, ok.
2176 const bool Matcher::init_array_count_is_in_bytes = false;
2177
2178 // Use conditional move (CMOVL)
2179 const int Matcher::long_cmove_cost() {
2180 // long cmoves are no more expensive than int cmoves
2181 return 0;
2182 }
2183
2184 const int Matcher::float_cmove_cost() {
2185 // float cmoves are no more expensive than int cmoves
2186 return 0;
2187 }
2188
2189 // Does the CPU require late expand (see block.cpp for description of late expand)?
2190 const bool Matcher::require_postalloc_expand = false;
2191
2192 // Do we need to mask the count passed to shift instructions or does
2193 // the cpu only look at the lower 5/6 bits anyway?
2194 const bool Matcher::need_masked_shift_count = false;
2195
2196 // No support for generic vector operands.
2197 const bool Matcher::supports_generic_vector_operands = false;
2198
2199 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
2200 ShouldNotReachHere(); // generic vector operands not supported
2201 return NULL;
2202 }
2203
2204 bool Matcher::is_generic_reg2reg_move(MachNode* m) {
2205 ShouldNotReachHere(); // generic vector operands not supported
2206 return false;
2207 }
2208
2209 bool Matcher::is_generic_vector(MachOper* opnd) {
2210 ShouldNotReachHere(); // generic vector operands not supported
2211 return false;
2212 }
2213
2214 // This affects two different things:
2215 // - how Decode nodes are matched
2216 // - how ImplicitNullCheck opportunities are recognized
2217 // If true, the matcher will try to remove all Decodes and match them
2218 // (as operands) into nodes. NullChecks are not prepared to deal with
2219 // Decodes by final_graph_reshaping().
2220 // If false, final_graph_reshaping() forces the decode behind the Cmp
2221 // for a NullCheck. The matcher matches the Decode node into a register.
2222 // Implicit_null_check optimization moves the Decode along with the
2223 // memory operation back up before the NullCheck.
2224 bool Matcher::narrow_oop_use_complex_address() {
2225 return CompressedOops::shift() == 0;
2226 }
2227
2228 bool Matcher::narrow_klass_use_complex_address() {
2229 // TODO
2230 // decide whether we need to set this to true
2231 return false;
2232 }
2233
2234 bool Matcher::const_oop_prefer_decode() {
2235 // Prefer ConN+DecodeN over ConP in simple compressed oops mode.
2236 return CompressedOops::base() == NULL;
2237 }
2238
2239 bool Matcher::const_klass_prefer_decode() {
2240 // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
2241 return CompressedKlassPointers::base() == NULL;
2242 }
2243
2244 // Is it better to copy float constants, or load them directly from
2245 // memory? Intel can load a float constant from a direct address,
2246 // requiring no extra registers. Most RISCs will have to materialize
2247 // an address into a register first, so they would do better to copy
2248 // the constant from stack.
2249 const bool Matcher::rematerialize_float_constants = false;
2250
2251 // If CPU can load and store mis-aligned doubles directly then no
2252 // fixup is needed. Else we split the double into 2 integer pieces
2253 // and move it piece-by-piece. Only happens when passing doubles into
2254 // C code as the Java calling convention forces doubles to be aligned.
2255 const bool Matcher::misaligned_doubles_ok = true;
2256
2257 // No-op on amd64
2258 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
2259 Unimplemented();
2260 }
2261
2262 // Advertise here if the CPU requires explicit rounding operations to implement strictfp mode.
2263 const bool Matcher::strict_fp_requires_explicit_rounding = false;
2264
2265 // Are floats converted to double when stored to stack during
2266 // deoptimization?
2267 bool Matcher::float_in_double() { return false; }
2268
2269 // Do ints take an entire long register or just half?
2270 // The relevant question is how the int is callee-saved:
2271 // the whole long is written but de-opt'ing will have to extract
2272 // the relevant 32 bits.
2273 const bool Matcher::int_in_long = true;
2274
2275 // Return whether or not this register is ever used as an argument.
2276 // This function is used on startup to build the trampoline stubs in
2277 // generateOptoStub. Registers not mentioned will be killed by the VM
2278 // call in the trampoline, and arguments in those registers not be
2279 // available to the callee.
2280 bool Matcher::can_be_java_arg(int reg)
2281 {
2282 return
2283 reg == R0_num || reg == R0_H_num ||
2284 reg == R1_num || reg == R1_H_num ||
2285 reg == R2_num || reg == R2_H_num ||
2286 reg == R3_num || reg == R3_H_num ||
2287 reg == R4_num || reg == R4_H_num ||
2288 reg == R5_num || reg == R5_H_num ||
2289 reg == R6_num || reg == R6_H_num ||
2290 reg == R7_num || reg == R7_H_num ||
2291 reg == V0_num || reg == V0_H_num ||
2292 reg == V1_num || reg == V1_H_num ||
2293 reg == V2_num || reg == V2_H_num ||
2294 reg == V3_num || reg == V3_H_num ||
2295 reg == V4_num || reg == V4_H_num ||
2296 reg == V5_num || reg == V5_H_num ||
2297 reg == V6_num || reg == V6_H_num ||
2298 reg == V7_num || reg == V7_H_num;
2299 }
2300
2301 bool Matcher::is_spillable_arg(int reg)
2302 {
2303 return can_be_java_arg(reg);
2304 }
2305
2306 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2307 return false;
2308 }
2309
2310 RegMask Matcher::divI_proj_mask() {
2311 ShouldNotReachHere();
2312 return RegMask();
2313 }
2314
2315 // Register for MODI projection of divmodI.
2316 RegMask Matcher::modI_proj_mask() {
2317 ShouldNotReachHere();
2318 return RegMask();
2319 }
2320
2321 // Register for DIVL projection of divmodL.
2322 RegMask Matcher::divL_proj_mask() {
2323 ShouldNotReachHere();
2324 return RegMask();
2325 }
2326
2327 // Register for MODL projection of divmodL.
2328 RegMask Matcher::modL_proj_mask() {
2329 ShouldNotReachHere();
2330 return RegMask();
2331 }
2332
2333 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2334 return FP_REG_mask();
2335 }
2336
2337 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2338 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2339 Node* u = addp->fast_out(i);
2340 if (u->is_Mem()) {
2341 int opsize = u->as_Mem()->memory_size();
2342 assert(opsize > 0, "unexpected memory operand size");
2343 if (u->as_Mem()->memory_size() != (1<<shift)) {
2344 return false;
2345 }
2346 }
2347 }
2348 return true;
2349 }
2350
2351 const bool Matcher::convi2l_type_required = false;
2352
2353 // Should the matcher clone input 'm' of node 'n'?
2354 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2355 if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
2356 mstack.push(m, Visit); // m = ShiftCntV
2357 return true;
2358 }
2359 return false;
2360 }
2361
2362 // Should the Matcher clone shifts on addressing modes, expecting them
2363 // to be subsumed into complex addressing expressions or compute them
2364 // into registers?
2365 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2366 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2367 return true;
2368 }
2369
2370 Node *off = m->in(AddPNode::Offset);
2371 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2372 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2373 // Are there other uses besides address expressions?
2374 !is_visited(off)) {
2375 address_visited.set(off->_idx); // Flag as address_visited
2376 mstack.push(off->in(2), Visit);
2377 Node *conv = off->in(1);
2378 if (conv->Opcode() == Op_ConvI2L &&
2379 // Are there other uses besides address expressions?
2380 !is_visited(conv)) {
2381 address_visited.set(conv->_idx); // Flag as address_visited
2382 mstack.push(conv->in(1), Pre_Visit);
2383 } else {
2384 mstack.push(conv, Pre_Visit);
2385 }
2386 address_visited.test_set(m->_idx); // Flag as address_visited
2387 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2388 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2389 return true;
2390 } else if (off->Opcode() == Op_ConvI2L &&
2391 // Are there other uses besides address expressions?
2392 !is_visited(off)) {
2393 address_visited.test_set(m->_idx); // Flag as address_visited
2394 address_visited.set(off->_idx); // Flag as address_visited
2395 mstack.push(off->in(1), Pre_Visit);
2396 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2397 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2398 return true;
2399 }
2400 return false;
2401 }
2402
2403 void Compile::reshape_address(AddPNode* addp) {
2404 }
2405
2406
2407 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2408 C2_MacroAssembler _masm(&cbuf); \
2409 { \
2410 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2411 guarantee(DISP == 0, "mode not permitted for volatile"); \
2412 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2413 __ INSN(REG, as_Register(BASE)); \
2414 }
2415
2416
2417 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2418 {
2419 Address::extend scale;
2420
2421 // Hooboy, this is fugly. We need a way to communicate to the
2422 // encoder that the index needs to be sign extended, so we have to
2423 // enumerate all the cases.
2424 switch (opcode) {
2425 case INDINDEXSCALEDI2L:
2426 case INDINDEXSCALEDI2LN:
2427 case INDINDEXI2L:
2428 case INDINDEXI2LN:
2429 scale = Address::sxtw(size);
2430 break;
2431 default:
2432 scale = Address::lsl(size);
2433 }
2434
2435 if (index == -1) {
2436 return Address(base, disp);
2437 } else {
2438 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2439 return Address(base, as_Register(index), scale);
2440 }
2441 }
2442
2443
2444 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2445 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2446 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2447 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2448 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2449
2450 // Used for all non-volatile memory accesses. The use of
2451 // $mem->opcode() to discover whether this pattern uses sign-extended
2452 // offsets is something of a kludge.
2453 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2454 Register reg, int opcode,
2455 Register base, int index, int scale, int disp,
2456 int size_in_memory)
2457 {
2458 Address addr = mem2address(opcode, base, index, scale, disp);
2459 if (addr.getMode() == Address::base_plus_offset) {
2460 /* If we get an out-of-range offset it is a bug in the compiler,
2461 so we assert here. */
2462 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2463 "c2 compiler bug");
2464 /* Fix up any out-of-range offsets. */
2465 assert_different_registers(rscratch1, base);
2466 assert_different_registers(rscratch1, reg);
2467 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2468 }
2469 (masm.*insn)(reg, addr);
2470 }
2471
2472 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2473 FloatRegister reg, int opcode,
2474 Register base, int index, int size, int disp,
2475 int size_in_memory)
2476 {
2477 Address::extend scale;
2478
2479 switch (opcode) {
2480 case INDINDEXSCALEDI2L:
2481 case INDINDEXSCALEDI2LN:
2482 scale = Address::sxtw(size);
2483 break;
2484 default:
2485 scale = Address::lsl(size);
2486 }
2487
2488 if (index == -1) {
2489 /* If we get an out-of-range offset it is a bug in the compiler,
2490 so we assert here. */
2491 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2492 /* Fix up any out-of-range offsets. */
2493 assert_different_registers(rscratch1, base);
2494 Address addr = Address(base, disp);
2495 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2496 (masm.*insn)(reg, addr);
2497 } else {
2498 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2499 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2500 }
2501 }
2502
2503 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2504 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2505 int opcode, Register base, int index, int size, int disp)
2506 {
2507 if (index == -1) {
2508 (masm.*insn)(reg, T, Address(base, disp));
2509 } else {
2510 assert(disp == 0, "unsupported address mode");
2511 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2512 }
2513 }
2514
2515 %}
2516
2517
2518
2519 //----------ENCODING BLOCK-----------------------------------------------------
2520 // This block specifies the encoding classes used by the compiler to
2521 // output byte streams. Encoding classes are parameterized macros
2522 // used by Machine Instruction Nodes in order to generate the bit
2523 // encoding of the instruction. Operands specify their base encoding
2524 // interface with the interface keyword. There are currently
2525 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2526 // COND_INTER. REG_INTER causes an operand to generate a function
2527 // which returns its register number when queried. CONST_INTER causes
2528 // an operand to generate a function which returns the value of the
2529 // constant when queried. MEMORY_INTER causes an operand to generate
2530 // four functions which return the Base Register, the Index Register,
2531 // the Scale Value, and the Offset Value of the operand when queried.
2532 // COND_INTER causes an operand to generate six functions which return
2533 // the encoding code (ie - encoding bits for the instruction)
2534 // associated with each basic boolean condition for a conditional
2535 // instruction.
2536 //
2537 // Instructions specify two basic values for encoding. Again, a
2538 // function is available to check if the constant displacement is an
2539 // oop. They use the ins_encode keyword to specify their encoding
2540 // classes (which must be a sequence of enc_class names, and their
2541 // parameters, specified in the encoding block), and they use the
2542 // opcode keyword to specify, in order, their primary, secondary, and
2543 // tertiary opcode. Only the opcode sections which a particular
2544 // instruction needs for encoding need to be specified.
2545 encode %{
2546 // Build emit functions for each basic byte or larger field in the
2547 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2548 // from C++ code in the enc_class source block. Emit functions will
2549 // live in the main source block for now. In future, we can
2550 // generalize this by adding a syntax that specifies the sizes of
2551 // fields in an order, so that the adlc can build the emit functions
2552 // automagically
2553
2554 // catch all for unimplemented encodings
2555 enc_class enc_unimplemented %{
2556 C2_MacroAssembler _masm(&cbuf);
2557 __ unimplemented("C2 catch all");
2558 %}
2559
2560 // BEGIN Non-volatile memory access
2561
2562 // This encoding class is generated automatically from ad_encode.m4.
2563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2564 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2565 Register dst_reg = as_Register($dst$$reg);
2566 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2567 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2568 %}
2569
2570 // This encoding class is generated automatically from ad_encode.m4.
2571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2572 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2573 Register dst_reg = as_Register($dst$$reg);
2574 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2575 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2576 %}
2577
2578 // This encoding class is generated automatically from ad_encode.m4.
2579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2580 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2581 Register dst_reg = as_Register($dst$$reg);
2582 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2583 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2584 %}
2585
2586 // This encoding class is generated automatically from ad_encode.m4.
2587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2588 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2589 Register dst_reg = as_Register($dst$$reg);
2590 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2591 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2592 %}
2593
2594 // This encoding class is generated automatically from ad_encode.m4.
2595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2596 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2597 Register dst_reg = as_Register($dst$$reg);
2598 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2599 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2600 %}
2601
2602 // This encoding class is generated automatically from ad_encode.m4.
2603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2604 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2605 Register dst_reg = as_Register($dst$$reg);
2606 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2607 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2608 %}
2609
2610 // This encoding class is generated automatically from ad_encode.m4.
2611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2612 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2613 Register dst_reg = as_Register($dst$$reg);
2614 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2615 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2616 %}
2617
2618 // This encoding class is generated automatically from ad_encode.m4.
2619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2620 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2621 Register dst_reg = as_Register($dst$$reg);
2622 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2623 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2624 %}
2625
2626 // This encoding class is generated automatically from ad_encode.m4.
2627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2628 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2629 Register dst_reg = as_Register($dst$$reg);
2630 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2631 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2632 %}
2633
2634 // This encoding class is generated automatically from ad_encode.m4.
2635 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2636 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2637 Register dst_reg = as_Register($dst$$reg);
2638 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2639 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2640 %}
2641
2642 // This encoding class is generated automatically from ad_encode.m4.
2643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2644 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2645 Register dst_reg = as_Register($dst$$reg);
2646 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2647 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2648 %}
2649
2650 // This encoding class is generated automatically from ad_encode.m4.
2651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2652 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2653 Register dst_reg = as_Register($dst$$reg);
2654 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2655 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2656 %}
2657
2658 // This encoding class is generated automatically from ad_encode.m4.
2659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2660 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2661 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2662 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2663 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2664 %}
2665
2666 // This encoding class is generated automatically from ad_encode.m4.
2667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2668 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2669 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2670 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2671 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2672 %}
2673
2674 // This encoding class is generated automatically from ad_encode.m4.
2675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2676 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2677 Register src_reg = as_Register($src$$reg);
2678 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2679 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2680 %}
2681
2682 // This encoding class is generated automatically from ad_encode.m4.
2683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2684 enc_class aarch64_enc_strb0(memory1 mem) %{
2685 C2_MacroAssembler _masm(&cbuf);
2686 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2687 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2688 %}
2689
2690 // This encoding class is generated automatically from ad_encode.m4.
2691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2692 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2693 Register src_reg = as_Register($src$$reg);
2694 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2695 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2696 %}
2697
2698 // This encoding class is generated automatically from ad_encode.m4.
2699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2700 enc_class aarch64_enc_strh0(memory2 mem) %{
2701 C2_MacroAssembler _masm(&cbuf);
2702 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2703 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2704 %}
2705
2706 // This encoding class is generated automatically from ad_encode.m4.
2707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2708 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2709 Register src_reg = as_Register($src$$reg);
2710 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2711 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2712 %}
2713
2714 // This encoding class is generated automatically from ad_encode.m4.
2715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2716 enc_class aarch64_enc_strw0(memory4 mem) %{
2717 C2_MacroAssembler _masm(&cbuf);
2718 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2719 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2720 %}
2721
2722 // This encoding class is generated automatically from ad_encode.m4.
2723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2724 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2725 Register src_reg = as_Register($src$$reg);
2726 // we sometimes get asked to store the stack pointer into the
2727 // current thread -- we cannot do that directly on AArch64
2728 if (src_reg == r31_sp) {
2729 C2_MacroAssembler _masm(&cbuf);
2730 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2731 __ mov(rscratch2, sp);
2732 src_reg = rscratch2;
2733 }
2734 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2735 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2736 %}
2737
2738 // This encoding class is generated automatically from ad_encode.m4.
2739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2740 enc_class aarch64_enc_str0(memory8 mem) %{
2741 C2_MacroAssembler _masm(&cbuf);
2742 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2743 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2744 %}
2745
2746 // This encoding class is generated automatically from ad_encode.m4.
2747 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2748 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
2749 FloatRegister src_reg = as_FloatRegister($src$$reg);
2750 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2751 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2752 %}
2753
2754 // This encoding class is generated automatically from ad_encode.m4.
2755 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2756 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
2757 FloatRegister src_reg = as_FloatRegister($src$$reg);
2758 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2759 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2760 %}
2761
2762 // This encoding class is generated automatically from ad_encode.m4.
2763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2764 enc_class aarch64_enc_strw_immn(immN src, memory1 mem) %{
2765 C2_MacroAssembler _masm(&cbuf);
2766 address con = (address)$src$$constant;
2767 // need to do this the hard way until we can manage relocs
2768 // for 32 bit constants
2769 __ movoop(rscratch2, (jobject)con);
2770 if (con) __ encode_heap_oop_not_null(rscratch2);
2771 loadStore(_masm, &MacroAssembler::strw, rscratch2, $mem->opcode(),
2772 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2773 %}
2774
2775 // This encoding class is generated automatically from ad_encode.m4.
2776 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2777 enc_class aarch64_enc_strw_immnk(immN src, memory4 mem) %{
2778 C2_MacroAssembler _masm(&cbuf);
2779 address con = (address)$src$$constant;
2780 // need to do this the hard way until we can manage relocs
2781 // for 32 bit constants
2782 __ movoop(rscratch2, (jobject)con);
2783 __ encode_klass_not_null(rscratch2);
2784 loadStore(_masm, &MacroAssembler::strw, rscratch2, $mem->opcode(),
2785 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2786 %}
2787
2788 // This encoding class is generated automatically from ad_encode.m4.
2789 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2790 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
2791 C2_MacroAssembler _masm(&cbuf);
2792 __ membar(Assembler::StoreStore);
2793 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2794 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2795 %}
2796
2797 // END Non-volatile memory access
2798
2799 // Vector loads and stores
2800 enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
2801 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2802 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
2803 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2804 %}
2805
2806 enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
2807 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2808 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
2809 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2810 %}
2811
2812 enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
2813 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2814 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
2815 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2816 %}
2817
2818 enc_class aarch64_enc_strvS(vecD src, memory mem) %{
2819 FloatRegister src_reg = as_FloatRegister($src$$reg);
2820 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
2821 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2822 %}
2823
2824 enc_class aarch64_enc_strvD(vecD src, memory mem) %{
2825 FloatRegister src_reg = as_FloatRegister($src$$reg);
2826 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
2827 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2828 %}
2829
2830 enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
2831 FloatRegister src_reg = as_FloatRegister($src$$reg);
2832 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
2833 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2834 %}
2835
2836 // volatile loads and stores
2837
2838 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
2839 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2840 rscratch1, stlrb);
2841 %}
2842
2843 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
2844 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2845 rscratch1, stlrh);
2846 %}
2847
2848 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
2849 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2850 rscratch1, stlrw);
2851 %}
2852
2853
2854 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
2855 Register dst_reg = as_Register($dst$$reg);
2856 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2857 rscratch1, ldarb);
2858 __ sxtbw(dst_reg, dst_reg);
2859 %}
2860
2861 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
2862 Register dst_reg = as_Register($dst$$reg);
2863 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2864 rscratch1, ldarb);
2865 __ sxtb(dst_reg, dst_reg);
2866 %}
2867
2868 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
2869 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2870 rscratch1, ldarb);
2871 %}
2872
2873 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
2874 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2875 rscratch1, ldarb);
2876 %}
2877
2878 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
2879 Register dst_reg = as_Register($dst$$reg);
2880 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2881 rscratch1, ldarh);
2882 __ sxthw(dst_reg, dst_reg);
2883 %}
2884
2885 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2888 rscratch1, ldarh);
2889 __ sxth(dst_reg, dst_reg);
2890 %}
2891
2892 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
2893 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2894 rscratch1, ldarh);
2895 %}
2896
2897 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
2898 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2899 rscratch1, ldarh);
2900 %}
2901
2902 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
2903 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2904 rscratch1, ldarw);
2905 %}
2906
2907 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
2908 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2909 rscratch1, ldarw);
2910 %}
2911
2912 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
2913 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2914 rscratch1, ldar);
2915 %}
2916
2917 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
2918 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2919 rscratch1, ldarw);
2920 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
2921 %}
2922
2923 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
2924 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2925 rscratch1, ldar);
2926 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
2927 %}
2928
2929 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
2930 Register src_reg = as_Register($src$$reg);
2931 // we sometimes get asked to store the stack pointer into the
2932 // current thread -- we cannot do that directly on AArch64
2933 if (src_reg == r31_sp) {
2934 C2_MacroAssembler _masm(&cbuf);
2935 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2936 __ mov(rscratch2, sp);
2937 src_reg = rscratch2;
2938 }
2939 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2940 rscratch1, stlr);
2941 %}
2942
2943 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
2944 {
2945 C2_MacroAssembler _masm(&cbuf);
2946 FloatRegister src_reg = as_FloatRegister($src$$reg);
2947 __ fmovs(rscratch2, src_reg);
2948 }
2949 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2950 rscratch1, stlrw);
2951 %}
2952
2953 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
2954 {
2955 C2_MacroAssembler _masm(&cbuf);
2956 FloatRegister src_reg = as_FloatRegister($src$$reg);
2957 __ fmovd(rscratch2, src_reg);
2958 }
2959 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2960 rscratch1, stlr);
2961 %}
2962
2963 // synchronized read/update encodings
2964
2965 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
2966 C2_MacroAssembler _masm(&cbuf);
2967 Register dst_reg = as_Register($dst$$reg);
2968 Register base = as_Register($mem$$base);
2969 int index = $mem$$index;
2970 int scale = $mem$$scale;
2971 int disp = $mem$$disp;
2972 if (index == -1) {
2973 if (disp != 0) {
2974 __ lea(rscratch1, Address(base, disp));
2975 __ ldaxr(dst_reg, rscratch1);
2976 } else {
2977 // TODO
2978 // should we ever get anything other than this case?
2979 __ ldaxr(dst_reg, base);
2980 }
2981 } else {
2982 Register index_reg = as_Register(index);
2983 if (disp == 0) {
2984 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
2985 __ ldaxr(dst_reg, rscratch1);
2986 } else {
2987 __ lea(rscratch1, Address(base, disp));
2988 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
2989 __ ldaxr(dst_reg, rscratch1);
2990 }
2991 }
2992 %}
2993
2994 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
2995 C2_MacroAssembler _masm(&cbuf);
2996 Register src_reg = as_Register($src$$reg);
2997 Register base = as_Register($mem$$base);
2998 int index = $mem$$index;
2999 int scale = $mem$$scale;
3000 int disp = $mem$$disp;
3001 if (index == -1) {
3002 if (disp != 0) {
3003 __ lea(rscratch2, Address(base, disp));
3004 __ stlxr(rscratch1, src_reg, rscratch2);
3005 } else {
3006 // TODO
3007 // should we ever get anything other than this case?
3008 __ stlxr(rscratch1, src_reg, base);
3009 }
3010 } else {
3011 Register index_reg = as_Register(index);
3012 if (disp == 0) {
3013 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3014 __ stlxr(rscratch1, src_reg, rscratch2);
3015 } else {
3016 __ lea(rscratch2, Address(base, disp));
3017 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3018 __ stlxr(rscratch1, src_reg, rscratch2);
3019 }
3020 }
3021 __ cmpw(rscratch1, zr);
3022 %}
3023
3024 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3025 C2_MacroAssembler _masm(&cbuf);
3026 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3027 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3028 Assembler::xword, /*acquire*/ false, /*release*/ true,
3029 /*weak*/ false, noreg);
3030 %}
3031
3032 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3033 C2_MacroAssembler _masm(&cbuf);
3034 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3035 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3036 Assembler::word, /*acquire*/ false, /*release*/ true,
3037 /*weak*/ false, noreg);
3038 %}
3039
3040 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3041 C2_MacroAssembler _masm(&cbuf);
3042 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3043 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3044 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3045 /*weak*/ false, noreg);
3046 %}
3047
3048 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3049 C2_MacroAssembler _masm(&cbuf);
3050 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3051 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3052 Assembler::byte, /*acquire*/ false, /*release*/ true,
3053 /*weak*/ false, noreg);
3054 %}
3055
3056
3057 // The only difference between aarch64_enc_cmpxchg and
3058 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3059 // CompareAndSwap sequence to serve as a barrier on acquiring a
3060 // lock.
3061 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3062 C2_MacroAssembler _masm(&cbuf);
3063 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3064 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3065 Assembler::xword, /*acquire*/ true, /*release*/ true,
3066 /*weak*/ false, noreg);
3067 %}
3068
3069 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3070 C2_MacroAssembler _masm(&cbuf);
3071 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3072 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3073 Assembler::word, /*acquire*/ true, /*release*/ true,
3074 /*weak*/ false, noreg);
3075 %}
3076
3077 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3078 C2_MacroAssembler _masm(&cbuf);
3079 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3080 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3081 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3082 /*weak*/ false, noreg);
3083 %}
3084
3085 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3086 C2_MacroAssembler _masm(&cbuf);
3087 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3088 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3089 Assembler::byte, /*acquire*/ true, /*release*/ true,
3090 /*weak*/ false, noreg);
3091 %}
3092
3093 // auxiliary used for CompareAndSwapX to set result register
3094 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3095 C2_MacroAssembler _masm(&cbuf);
3096 Register res_reg = as_Register($res$$reg);
3097 __ cset(res_reg, Assembler::EQ);
3098 %}
3099
3100 // prefetch encodings
3101
3102 enc_class aarch64_enc_prefetchw(memory mem) %{
3103 C2_MacroAssembler _masm(&cbuf);
3104 Register base = as_Register($mem$$base);
3105 int index = $mem$$index;
3106 int scale = $mem$$scale;
3107 int disp = $mem$$disp;
3108 if (index == -1) {
3109 __ prfm(Address(base, disp), PSTL1KEEP);
3110 } else {
3111 Register index_reg = as_Register(index);
3112 if (disp == 0) {
3113 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3114 } else {
3115 __ lea(rscratch1, Address(base, disp));
3116 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3117 }
3118 }
3119 %}
3120
3121 /// mov envcodings
3122
3123 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3124 C2_MacroAssembler _masm(&cbuf);
3125 uint32_t con = (uint32_t)$src$$constant;
3126 Register dst_reg = as_Register($dst$$reg);
3127 if (con == 0) {
3128 __ movw(dst_reg, zr);
3129 } else {
3130 __ movw(dst_reg, con);
3131 }
3132 %}
3133
3134 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3135 C2_MacroAssembler _masm(&cbuf);
3136 Register dst_reg = as_Register($dst$$reg);
3137 uint64_t con = (uint64_t)$src$$constant;
3138 if (con == 0) {
3139 __ mov(dst_reg, zr);
3140 } else {
3141 __ mov(dst_reg, con);
3142 }
3143 %}
3144
3145 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3146 C2_MacroAssembler _masm(&cbuf);
3147 Register dst_reg = as_Register($dst$$reg);
3148 address con = (address)$src$$constant;
3149 if (con == NULL || con == (address)1) {
3150 ShouldNotReachHere();
3151 } else {
3152 relocInfo::relocType rtype = $src->constant_reloc();
3153 if (rtype == relocInfo::oop_type) {
3154 __ movoop(dst_reg, (jobject)con, /*immediate*/true);
3155 } else if (rtype == relocInfo::metadata_type) {
3156 __ mov_metadata(dst_reg, (Metadata*)con);
3157 } else {
3158 assert(rtype == relocInfo::none, "unexpected reloc type");
3159 if (con < (address)(uintptr_t)os::vm_page_size()) {
3160 __ mov(dst_reg, con);
3161 } else {
3162 uintptr_t offset;
3163 __ adrp(dst_reg, con, offset);
3164 __ add(dst_reg, dst_reg, offset);
3165 }
3166 }
3167 }
3168 %}
3169
3170 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3171 C2_MacroAssembler _masm(&cbuf);
3172 Register dst_reg = as_Register($dst$$reg);
3173 __ mov(dst_reg, zr);
3174 %}
3175
3176 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3177 C2_MacroAssembler _masm(&cbuf);
3178 Register dst_reg = as_Register($dst$$reg);
3179 __ mov(dst_reg, (uint64_t)1);
3180 %}
3181
3182 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3183 C2_MacroAssembler _masm(&cbuf);
3184 __ load_byte_map_base($dst$$Register);
3185 %}
3186
3187 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3188 C2_MacroAssembler _masm(&cbuf);
3189 Register dst_reg = as_Register($dst$$reg);
3190 address con = (address)$src$$constant;
3191 if (con == NULL) {
3192 ShouldNotReachHere();
3193 } else {
3194 relocInfo::relocType rtype = $src->constant_reloc();
3195 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3196 __ set_narrow_oop(dst_reg, (jobject)con);
3197 }
3198 %}
3199
3200 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3201 C2_MacroAssembler _masm(&cbuf);
3202 Register dst_reg = as_Register($dst$$reg);
3203 __ mov(dst_reg, zr);
3204 %}
3205
3206 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3207 C2_MacroAssembler _masm(&cbuf);
3208 Register dst_reg = as_Register($dst$$reg);
3209 address con = (address)$src$$constant;
3210 if (con == NULL) {
3211 ShouldNotReachHere();
3212 } else {
3213 relocInfo::relocType rtype = $src->constant_reloc();
3214 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3215 __ set_narrow_klass(dst_reg, (Klass *)con);
3216 }
3217 %}
3218
3219 // arithmetic encodings
3220
3221 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3222 C2_MacroAssembler _masm(&cbuf);
3223 Register dst_reg = as_Register($dst$$reg);
3224 Register src_reg = as_Register($src1$$reg);
3225 int32_t con = (int32_t)$src2$$constant;
3226 // add has primary == 0, subtract has primary == 1
3227 if ($primary) { con = -con; }
3228 if (con < 0) {
3229 __ subw(dst_reg, src_reg, -con);
3230 } else {
3231 __ addw(dst_reg, src_reg, con);
3232 }
3233 %}
3234
3235 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3236 C2_MacroAssembler _masm(&cbuf);
3237 Register dst_reg = as_Register($dst$$reg);
3238 Register src_reg = as_Register($src1$$reg);
3239 int32_t con = (int32_t)$src2$$constant;
3240 // add has primary == 0, subtract has primary == 1
3241 if ($primary) { con = -con; }
3242 if (con < 0) {
3243 __ sub(dst_reg, src_reg, -con);
3244 } else {
3245 __ add(dst_reg, src_reg, con);
3246 }
3247 %}
3248
3249 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3250 C2_MacroAssembler _masm(&cbuf);
3251 Register dst_reg = as_Register($dst$$reg);
3252 Register src1_reg = as_Register($src1$$reg);
3253 Register src2_reg = as_Register($src2$$reg);
3254 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3255 %}
3256
3257 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3258 C2_MacroAssembler _masm(&cbuf);
3259 Register dst_reg = as_Register($dst$$reg);
3260 Register src1_reg = as_Register($src1$$reg);
3261 Register src2_reg = as_Register($src2$$reg);
3262 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3263 %}
3264
3265 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3266 C2_MacroAssembler _masm(&cbuf);
3267 Register dst_reg = as_Register($dst$$reg);
3268 Register src1_reg = as_Register($src1$$reg);
3269 Register src2_reg = as_Register($src2$$reg);
3270 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3271 %}
3272
3273 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3274 C2_MacroAssembler _masm(&cbuf);
3275 Register dst_reg = as_Register($dst$$reg);
3276 Register src1_reg = as_Register($src1$$reg);
3277 Register src2_reg = as_Register($src2$$reg);
3278 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3279 %}
3280
3281 // compare instruction encodings
3282
3283 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3284 C2_MacroAssembler _masm(&cbuf);
3285 Register reg1 = as_Register($src1$$reg);
3286 Register reg2 = as_Register($src2$$reg);
3287 __ cmpw(reg1, reg2);
3288 %}
3289
3290 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3291 C2_MacroAssembler _masm(&cbuf);
3292 Register reg = as_Register($src1$$reg);
3293 int32_t val = $src2$$constant;
3294 if (val >= 0) {
3295 __ subsw(zr, reg, val);
3296 } else {
3297 __ addsw(zr, reg, -val);
3298 }
3299 %}
3300
3301 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3302 C2_MacroAssembler _masm(&cbuf);
3303 Register reg1 = as_Register($src1$$reg);
3304 uint32_t val = (uint32_t)$src2$$constant;
3305 __ movw(rscratch1, val);
3306 __ cmpw(reg1, rscratch1);
3307 %}
3308
3309 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3310 C2_MacroAssembler _masm(&cbuf);
3311 Register reg1 = as_Register($src1$$reg);
3312 Register reg2 = as_Register($src2$$reg);
3313 __ cmp(reg1, reg2);
3314 %}
3315
3316 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3317 C2_MacroAssembler _masm(&cbuf);
3318 Register reg = as_Register($src1$$reg);
3319 int64_t val = $src2$$constant;
3320 if (val >= 0) {
3321 __ subs(zr, reg, val);
3322 } else if (val != -val) {
3323 __ adds(zr, reg, -val);
3324 } else {
3325 // aargh, Long.MIN_VALUE is a special case
3326 __ orr(rscratch1, zr, (uint64_t)val);
3327 __ subs(zr, reg, rscratch1);
3328 }
3329 %}
3330
3331 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3332 C2_MacroAssembler _masm(&cbuf);
3333 Register reg1 = as_Register($src1$$reg);
3334 uint64_t val = (uint64_t)$src2$$constant;
3335 __ mov(rscratch1, val);
3336 __ cmp(reg1, rscratch1);
3337 %}
3338
3339 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3340 C2_MacroAssembler _masm(&cbuf);
3341 Register reg1 = as_Register($src1$$reg);
3342 Register reg2 = as_Register($src2$$reg);
3343 __ cmp(reg1, reg2);
3344 %}
3345
3346 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3347 C2_MacroAssembler _masm(&cbuf);
3348 Register reg1 = as_Register($src1$$reg);
3349 Register reg2 = as_Register($src2$$reg);
3350 __ cmpw(reg1, reg2);
3351 %}
3352
3353 enc_class aarch64_enc_testp(iRegP src) %{
3354 C2_MacroAssembler _masm(&cbuf);
3355 Register reg = as_Register($src$$reg);
3356 __ cmp(reg, zr);
3357 %}
3358
3359 enc_class aarch64_enc_testn(iRegN src) %{
3360 C2_MacroAssembler _masm(&cbuf);
3361 Register reg = as_Register($src$$reg);
3362 __ cmpw(reg, zr);
3363 %}
3364
3365 enc_class aarch64_enc_b(label lbl) %{
3366 C2_MacroAssembler _masm(&cbuf);
3367 Label *L = $lbl$$label;
3368 __ b(*L);
3369 %}
3370
3371 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3372 C2_MacroAssembler _masm(&cbuf);
3373 Label *L = $lbl$$label;
3374 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3375 %}
3376
3377 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3378 C2_MacroAssembler _masm(&cbuf);
3379 Label *L = $lbl$$label;
3380 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3381 %}
3382
3383 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3384 %{
3385 Register sub_reg = as_Register($sub$$reg);
3386 Register super_reg = as_Register($super$$reg);
3387 Register temp_reg = as_Register($temp$$reg);
3388 Register result_reg = as_Register($result$$reg);
3389
3390 Label miss;
3391 C2_MacroAssembler _masm(&cbuf);
3392 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3393 NULL, &miss,
3394 /*set_cond_codes:*/ true);
3395 if ($primary) {
3396 __ mov(result_reg, zr);
3397 }
3398 __ bind(miss);
3399 %}
3400
3401 enc_class aarch64_enc_java_static_call(method meth) %{
3402 C2_MacroAssembler _masm(&cbuf);
3403
3404 address addr = (address)$meth$$method;
3405 address call;
3406 if (!_method) {
3407 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3408 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3409 } else {
3410 int method_index = resolved_method_index(cbuf);
3411 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3412 : static_call_Relocation::spec(method_index);
3413 call = __ trampoline_call(Address(addr, rspec), &cbuf);
3414
3415 // Emit stub for static call
3416 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
3417 if (stub == NULL) {
3418 ciEnv::current()->record_failure("CodeCache is full");
3419 return;
3420 }
3421 }
3422 if (call == NULL) {
3423 ciEnv::current()->record_failure("CodeCache is full");
3424 return;
3425 }
3426 %}
3427
3428 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3429 C2_MacroAssembler _masm(&cbuf);
3430 int method_index = resolved_method_index(cbuf);
3431 address call = __ ic_call((address)$meth$$method, method_index);
3432 if (call == NULL) {
3433 ciEnv::current()->record_failure("CodeCache is full");
3434 return;
3435 }
3436 %}
3437
3438 enc_class aarch64_enc_call_epilog() %{
3439 C2_MacroAssembler _masm(&cbuf);
3440 if (VerifyStackAtCalls) {
3441 // Check that stack depth is unchanged: find majik cookie on stack
3442 __ call_Unimplemented();
3443 }
3444 %}
3445
3446 enc_class aarch64_enc_java_to_runtime(method meth) %{
3447 C2_MacroAssembler _masm(&cbuf);
3448
3449 // some calls to generated routines (arraycopy code) are scheduled
3450 // by C2 as runtime calls. if so we can call them using a br (they
3451 // will be in a reachable segment) otherwise we have to use a blr
3452 // which loads the absolute address into a register.
3453 address entry = (address)$meth$$method;
3454 CodeBlob *cb = CodeCache::find_blob(entry);
3455 if (cb) {
3456 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3457 if (call == NULL) {
3458 ciEnv::current()->record_failure("CodeCache is full");
3459 return;
3460 }
3461 } else {
3462 Label retaddr;
3463 __ adr(rscratch2, retaddr);
3464 __ lea(rscratch1, RuntimeAddress(entry));
3465 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3466 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3467 __ blr(rscratch1);
3468 __ bind(retaddr);
3469 __ add(sp, sp, 2 * wordSize);
3470 }
3471 %}
3472
3473 enc_class aarch64_enc_rethrow() %{
3474 C2_MacroAssembler _masm(&cbuf);
3475 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3476 %}
3477
3478 enc_class aarch64_enc_ret() %{
3479 C2_MacroAssembler _masm(&cbuf);
3480 __ ret(lr);
3481 %}
3482
3483 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3484 C2_MacroAssembler _masm(&cbuf);
3485 Register target_reg = as_Register($jump_target$$reg);
3486 __ br(target_reg);
3487 %}
3488
3489 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3490 C2_MacroAssembler _masm(&cbuf);
3491 Register target_reg = as_Register($jump_target$$reg);
3492 // exception oop should be in r0
3493 // ret addr has been popped into lr
3494 // callee expects it in r3
3495 __ mov(r3, lr);
3496 __ br(target_reg);
3497 %}
3498
3499 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3500 C2_MacroAssembler _masm(&cbuf);
3501 Register oop = as_Register($object$$reg);
3502 Register box = as_Register($box$$reg);
3503 Register disp_hdr = as_Register($tmp$$reg);
3504 Register tmp = as_Register($tmp2$$reg);
3505 Label cont;
3506 Label object_has_monitor;
3507 Label cas_failed;
3508
3509 assert_different_registers(oop, box, tmp, disp_hdr);
3510
3511 // Load markWord from object into displaced_header.
3512 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3513
3514 if (UseBiasedLocking && !UseOptoBiasInlining) {
3515 __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
3516 }
3517
3518 // Check for existing monitor
3519 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3520
3521 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3522 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3523
3524 // Initialize the box. (Must happen before we update the object mark!)
3525 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3526
3527 // Compare object markWord with an unlocked value (tmp) and if
3528 // equal exchange the stack address of our box with object markWord.
3529 // On failure disp_hdr contains the possibly locked markWord.
3530 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3531 /*release*/ true, /*weak*/ false, disp_hdr);
3532 __ br(Assembler::EQ, cont);
3533
3534 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3535
3536 // If the compare-and-exchange succeeded, then we found an unlocked
3537 // object, will have now locked it will continue at label cont
3538
3539 __ bind(cas_failed);
3540 // We did not see an unlocked object so try the fast recursive case.
3541
3542 // Check if the owner is self by comparing the value in the
3543 // markWord of object (disp_hdr) with the stack pointer.
3544 __ mov(rscratch1, sp);
3545 __ sub(disp_hdr, disp_hdr, rscratch1);
3546 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3547 // If condition is true we are cont and hence we can store 0 as the
3548 // displaced header in the box, which indicates that it is a recursive lock.
3549 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3550 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3551
3552 __ b(cont);
3553
3554 // Handle existing monitor.
3555 __ bind(object_has_monitor);
3556
3557 // The object's monitor m is unlocked iff m->owner == NULL,
3558 // otherwise m->owner may contain a thread or a stack address.
3559 //
3560 // Try to CAS m->owner from NULL to current thread.
3561 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
3562 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3563 /*release*/ true, /*weak*/ false, noreg); // Sets flags for result
3564
3565 // Store a non-null value into the box to avoid looking like a re-entrant
3566 // lock. The fast-path monitor unlock code checks for
3567 // markWord::monitor_value so use markWord::unused_mark which has the
3568 // relevant bit set, and also matches ObjectSynchronizer::enter.
3569 __ mov(tmp, (address)markWord::unused_mark().value());
3570 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3571
3572 __ bind(cont);
3573 // flag == EQ indicates success
3574 // flag == NE indicates failure
3575 %}
3576
3577 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3578 C2_MacroAssembler _masm(&cbuf);
3579 Register oop = as_Register($object$$reg);
3580 Register box = as_Register($box$$reg);
3581 Register disp_hdr = as_Register($tmp$$reg);
3582 Register tmp = as_Register($tmp2$$reg);
3583 Label cont;
3584 Label object_has_monitor;
3585
3586 assert_different_registers(oop, box, tmp, disp_hdr);
3587
3588 if (UseBiasedLocking && !UseOptoBiasInlining) {
3589 __ biased_locking_exit(oop, tmp, cont);
3590 }
3591
3592 // Find the lock address and load the displaced header from the stack.
3593 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3594
3595 // If the displaced header is 0, we have a recursive unlock.
3596 __ cmp(disp_hdr, zr);
3597 __ br(Assembler::EQ, cont);
3598
3599 // Handle existing monitor.
3600 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3601 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3602
3603 // Check if it is still a light weight lock, this is is true if we
3604 // see the stack address of the basicLock in the markWord of the
3605 // object.
3606
3607 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3608 /*release*/ true, /*weak*/ false, tmp);
3609 __ b(cont);
3610
3611 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3612
3613 // Handle existing monitor.
3614 __ bind(object_has_monitor);
3615 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
3616 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
3617 __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3618 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3619 __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3620 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3621 __ cmp(rscratch1, zr); // Sets flags for result
3622 __ br(Assembler::NE, cont);
3623
3624 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3625 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3626 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3627 __ cmp(rscratch1, zr); // Sets flags for result
3628 __ cbnz(rscratch1, cont);
3629 // need a release store here
3630 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3631 __ stlr(zr, tmp); // set unowned
3632
3633 __ bind(cont);
3634 // flag == EQ indicates success
3635 // flag == NE indicates failure
3636 %}
3637
3638 %}
3639
3640 //----------FRAME--------------------------------------------------------------
3641 // Definition of frame structure and management information.
3642 //
3643 // S T A C K L A Y O U T Allocators stack-slot number
3644 // | (to get allocators register number
3645 // G Owned by | | v add OptoReg::stack0())
3646 // r CALLER | |
3647 // o | +--------+ pad to even-align allocators stack-slot
3648 // w V | pad0 | numbers; owned by CALLER
3649 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3650 // h ^ | in | 5
3651 // | | args | 4 Holes in incoming args owned by SELF
3652 // | | | | 3
3653 // | | +--------+
3654 // V | | old out| Empty on Intel, window on Sparc
3655 // | old |preserve| Must be even aligned.
3656 // | SP-+--------+----> Matcher::_old_SP, even aligned
3657 // | | in | 3 area for Intel ret address
3658 // Owned by |preserve| Empty on Sparc.
3659 // SELF +--------+
3660 // | | pad2 | 2 pad to align old SP
3661 // | +--------+ 1
3662 // | | locks | 0
3663 // | +--------+----> OptoReg::stack0(), even aligned
3664 // | | pad1 | 11 pad to align new SP
3665 // | +--------+
3666 // | | | 10
3667 // | | spills | 9 spills
3668 // V | | 8 (pad0 slot for callee)
3669 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3670 // ^ | out | 7
3671 // | | args | 6 Holes in outgoing args owned by CALLEE
3672 // Owned by +--------+
3673 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3674 // | new |preserve| Must be even-aligned.
3675 // | SP-+--------+----> Matcher::_new_SP, even aligned
3676 // | | |
3677 //
3678 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3679 // known from SELF's arguments and the Java calling convention.
3680 // Region 6-7 is determined per call site.
3681 // Note 2: If the calling convention leaves holes in the incoming argument
3682 // area, those holes are owned by SELF. Holes in the outgoing area
3683 // are owned by the CALLEE. Holes should not be nessecary in the
3684 // incoming area, as the Java calling convention is completely under
3685 // the control of the AD file. Doubles can be sorted and packed to
3686 // avoid holes. Holes in the outgoing arguments may be nessecary for
3687 // varargs C calling conventions.
3688 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3689 // even aligned with pad0 as needed.
3690 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3691 // (the latter is true on Intel but is it false on AArch64?)
3692 // region 6-11 is even aligned; it may be padded out more so that
3693 // the region from SP to FP meets the minimum stack alignment.
3694 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3695 // alignment. Region 11, pad1, may be dynamically extended so that
3696 // SP meets the minimum alignment.
3697
3698 frame %{
3699 // What direction does stack grow in (assumed to be same for C & Java)
3700 stack_direction(TOWARDS_LOW);
3701
3702 // These three registers define part of the calling convention
3703 // between compiled code and the interpreter.
3704
3705 // Inline Cache Register or Method for I2C.
3706 inline_cache_reg(R12);
3707
3708 // Method Oop Register when calling interpreter.
3709 interpreter_method_oop_reg(R12);
3710
3711 // Number of stack slots consumed by locking an object
3712 sync_stack_slots(2);
3713
3714 // Compiled code's Frame Pointer
3715 frame_pointer(R31);
3716
3717 // Interpreter stores its frame pointer in a register which is
3718 // stored to the stack by I2CAdaptors.
3719 // I2CAdaptors convert from interpreted java to compiled java.
3720 interpreter_frame_pointer(R29);
3721
3722 // Stack alignment requirement
3723 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3724
3725 // Number of stack slots between incoming argument block and the start of
3726 // a new frame. The PROLOG must add this many slots to the stack. The
3727 // EPILOG must remove this many slots. aarch64 needs two slots for
3728 // return address and fp.
3729 // TODO think this is correct but check
3730 in_preserve_stack_slots(4);
3731
3732 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3733 // for calls to C. Supports the var-args backing area for register parms.
3734 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3735
3736 // The after-PROLOG location of the return address. Location of
3737 // return address specifies a type (REG or STACK) and a number
3738 // representing the register number (i.e. - use a register name) or
3739 // stack slot.
3740 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3741 // Otherwise, it is above the locks and verification slot and alignment word
3742 // TODO this may well be correct but need to check why that - 2 is there
3743 // ppc port uses 0 but we definitely need to allow for fixed_slots
3744 // which folds in the space used for monitors
3745 return_addr(STACK - 2 +
3746 align_up((Compile::current()->in_preserve_stack_slots() +
3747 Compile::current()->fixed_slots()),
3748 stack_alignment_in_slots()));
3749
3750 // Body of function which returns an integer array locating
3751 // arguments either in registers or in stack slots. Passed an array
3752 // of ideal registers called "sig" and a "length" count. Stack-slot
3753 // offsets are based on outgoing arguments, i.e. a CALLER setting up
3754 // arguments for a CALLEE. Incoming stack arguments are
3755 // automatically biased by the preserve_stack_slots field above.
3756
3757 calling_convention
3758 %{
3759 // No difference between ingoing/outgoing just pass false
3760 SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3761 %}
3762
3763 c_calling_convention
3764 %{
3765 // This is obviously always outgoing
3766 (void) SharedRuntime::c_calling_convention(sig_bt, regs, NULL, length);
3767 %}
3768
3769 // Location of compiled Java return values. Same as C for now.
3770 return_value
3771 %{
3772 // TODO do we allow ideal_reg == Op_RegN???
3773 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3774 "only return normal values");
3775
3776 static const int lo[Op_RegL + 1] = { // enum name
3777 0, // Op_Node
3778 0, // Op_Set
3779 R0_num, // Op_RegN
3780 R0_num, // Op_RegI
3781 R0_num, // Op_RegP
3782 V0_num, // Op_RegF
3783 V0_num, // Op_RegD
3784 R0_num // Op_RegL
3785 };
3786
3787 static const int hi[Op_RegL + 1] = { // enum name
3788 0, // Op_Node
3789 0, // Op_Set
3790 OptoReg::Bad, // Op_RegN
3791 OptoReg::Bad, // Op_RegI
3792 R0_H_num, // Op_RegP
3793 OptoReg::Bad, // Op_RegF
3794 V0_H_num, // Op_RegD
3795 R0_H_num // Op_RegL
3796 };
3797
3798 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3799 %}
3800 %}
3801
3802 //----------ATTRIBUTES---------------------------------------------------------
3803 //----------Operand Attributes-------------------------------------------------
3804 op_attrib op_cost(1); // Required cost attribute
3805
3806 //----------Instruction Attributes---------------------------------------------
3807 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3808 ins_attrib ins_size(32); // Required size attribute (in bits)
3809 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3810 // a non-matching short branch variant
3811 // of some long branch?
3812 ins_attrib ins_alignment(4); // Required alignment attribute (must
3813 // be a power of 2) specifies the
3814 // alignment that some part of the
3815 // instruction (not necessarily the
3816 // start) requires. If > 1, a
3817 // compute_padding() function must be
3818 // provided for the instruction
3819
3820 //----------OPERANDS-----------------------------------------------------------
3821 // Operand definitions must precede instruction definitions for correct parsing
3822 // in the ADLC because operands constitute user defined types which are used in
3823 // instruction definitions.
3824
3825 //----------Simple Operands----------------------------------------------------
3826
3827 // Integer operands 32 bit
3828 // 32 bit immediate
3829 operand immI()
3830 %{
3831 match(ConI);
3832
3833 op_cost(0);
3834 format %{ %}
3835 interface(CONST_INTER);
3836 %}
3837
3838 // 32 bit zero
3839 operand immI0()
3840 %{
3841 predicate(n->get_int() == 0);
3842 match(ConI);
3843
3844 op_cost(0);
3845 format %{ %}
3846 interface(CONST_INTER);
3847 %}
3848
3849 // 32 bit unit increment
3850 operand immI_1()
3851 %{
3852 predicate(n->get_int() == 1);
3853 match(ConI);
3854
3855 op_cost(0);
3856 format %{ %}
3857 interface(CONST_INTER);
3858 %}
3859
3860 // 32 bit unit decrement
3861 operand immI_M1()
3862 %{
3863 predicate(n->get_int() == -1);
3864 match(ConI);
3865
3866 op_cost(0);
3867 format %{ %}
3868 interface(CONST_INTER);
3869 %}
3870
3871 // Shift values for add/sub extension shift
3872 operand immIExt()
3873 %{
3874 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3875 match(ConI);
3876
3877 op_cost(0);
3878 format %{ %}
3879 interface(CONST_INTER);
3880 %}
3881
3882 operand immI_le_4()
3883 %{
3884 predicate(n->get_int() <= 4);
3885 match(ConI);
3886
3887 op_cost(0);
3888 format %{ %}
3889 interface(CONST_INTER);
3890 %}
3891
3892 operand immI_31()
3893 %{
3894 predicate(n->get_int() == 31);
3895 match(ConI);
3896
3897 op_cost(0);
3898 format %{ %}
3899 interface(CONST_INTER);
3900 %}
3901
3902 operand immI_8()
3903 %{
3904 predicate(n->get_int() == 8);
3905 match(ConI);
3906
3907 op_cost(0);
3908 format %{ %}
3909 interface(CONST_INTER);
3910 %}
3911
3912 operand immI_16()
3913 %{
3914 predicate(n->get_int() == 16);
3915 match(ConI);
3916
3917 op_cost(0);
3918 format %{ %}
3919 interface(CONST_INTER);
3920 %}
3921
3922 operand immI_24()
3923 %{
3924 predicate(n->get_int() == 24);
3925 match(ConI);
3926
3927 op_cost(0);
3928 format %{ %}
3929 interface(CONST_INTER);
3930 %}
3931
3932 operand immI_32()
3933 %{
3934 predicate(n->get_int() == 32);
3935 match(ConI);
3936
3937 op_cost(0);
3938 format %{ %}
3939 interface(CONST_INTER);
3940 %}
3941
3942 operand immI_48()
3943 %{
3944 predicate(n->get_int() == 48);
3945 match(ConI);
3946
3947 op_cost(0);
3948 format %{ %}
3949 interface(CONST_INTER);
3950 %}
3951
3952 operand immI_56()
3953 %{
3954 predicate(n->get_int() == 56);
3955 match(ConI);
3956
3957 op_cost(0);
3958 format %{ %}
3959 interface(CONST_INTER);
3960 %}
3961
3962 operand immI_63()
3963 %{
3964 predicate(n->get_int() == 63);
3965 match(ConI);
3966
3967 op_cost(0);
3968 format %{ %}
3969 interface(CONST_INTER);
3970 %}
3971
3972 operand immI_64()
3973 %{
3974 predicate(n->get_int() == 64);
3975 match(ConI);
3976
3977 op_cost(0);
3978 format %{ %}
3979 interface(CONST_INTER);
3980 %}
3981
3982 operand immI_255()
3983 %{
3984 predicate(n->get_int() == 255);
3985 match(ConI);
3986
3987 op_cost(0);
3988 format %{ %}
3989 interface(CONST_INTER);
3990 %}
3991
3992 operand immI_65535()
3993 %{
3994 predicate(n->get_int() == 65535);
3995 match(ConI);
3996
3997 op_cost(0);
3998 format %{ %}
3999 interface(CONST_INTER);
4000 %}
4001
4002 operand immL_255()
4003 %{
4004 predicate(n->get_long() == 255L);
4005 match(ConL);
4006
4007 op_cost(0);
4008 format %{ %}
4009 interface(CONST_INTER);
4010 %}
4011
4012 operand immL_65535()
4013 %{
4014 predicate(n->get_long() == 65535L);
4015 match(ConL);
4016
4017 op_cost(0);
4018 format %{ %}
4019 interface(CONST_INTER);
4020 %}
4021
4022 operand immL_4294967295()
4023 %{
4024 predicate(n->get_long() == 4294967295L);
4025 match(ConL);
4026
4027 op_cost(0);
4028 format %{ %}
4029 interface(CONST_INTER);
4030 %}
4031
4032 operand immL_bitmask()
4033 %{
4034 predicate((n->get_long() != 0)
4035 && ((n->get_long() & 0xc000000000000000l) == 0)
4036 && is_power_of_2(n->get_long() + 1));
4037 match(ConL);
4038
4039 op_cost(0);
4040 format %{ %}
4041 interface(CONST_INTER);
4042 %}
4043
4044 operand immI_bitmask()
4045 %{
4046 predicate((n->get_int() != 0)
4047 && ((n->get_int() & 0xc0000000) == 0)
4048 && is_power_of_2(n->get_int() + 1));
4049 match(ConI);
4050
4051 op_cost(0);
4052 format %{ %}
4053 interface(CONST_INTER);
4054 %}
4055
4056 operand immL_positive_bitmaskI()
4057 %{
4058 predicate((n->get_long() != 0)
4059 && ((julong)n->get_long() < 0x80000000ULL)
4060 && is_power_of_2(n->get_long() + 1));
4061 match(ConL);
4062
4063 op_cost(0);
4064 format %{ %}
4065 interface(CONST_INTER);
4066 %}
4067
4068 // Scale values for scaled offset addressing modes (up to long but not quad)
4069 operand immIScale()
4070 %{
4071 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4072 match(ConI);
4073
4074 op_cost(0);
4075 format %{ %}
4076 interface(CONST_INTER);
4077 %}
4078
4079 // 26 bit signed offset -- for pc-relative branches
4080 operand immI26()
4081 %{
4082 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4083 match(ConI);
4084
4085 op_cost(0);
4086 format %{ %}
4087 interface(CONST_INTER);
4088 %}
4089
4090 // 19 bit signed offset -- for pc-relative loads
4091 operand immI19()
4092 %{
4093 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4094 match(ConI);
4095
4096 op_cost(0);
4097 format %{ %}
4098 interface(CONST_INTER);
4099 %}
4100
4101 // 12 bit unsigned offset -- for base plus immediate loads
4102 operand immIU12()
4103 %{
4104 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4105 match(ConI);
4106
4107 op_cost(0);
4108 format %{ %}
4109 interface(CONST_INTER);
4110 %}
4111
4112 operand immLU12()
4113 %{
4114 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4115 match(ConL);
4116
4117 op_cost(0);
4118 format %{ %}
4119 interface(CONST_INTER);
4120 %}
4121
4122 // Offset for scaled or unscaled immediate loads and stores
4123 operand immIOffset()
4124 %{
4125 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4126 match(ConI);
4127
4128 op_cost(0);
4129 format %{ %}
4130 interface(CONST_INTER);
4131 %}
4132
4133 operand immIOffset1()
4134 %{
4135 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4136 match(ConI);
4137
4138 op_cost(0);
4139 format %{ %}
4140 interface(CONST_INTER);
4141 %}
4142
4143 operand immIOffset2()
4144 %{
4145 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4146 match(ConI);
4147
4148 op_cost(0);
4149 format %{ %}
4150 interface(CONST_INTER);
4151 %}
4152
4153 operand immIOffset4()
4154 %{
4155 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4156 match(ConI);
4157
4158 op_cost(0);
4159 format %{ %}
4160 interface(CONST_INTER);
4161 %}
4162
4163 operand immIOffset8()
4164 %{
4165 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4166 match(ConI);
4167
4168 op_cost(0);
4169 format %{ %}
4170 interface(CONST_INTER);
4171 %}
4172
4173 operand immIOffset16()
4174 %{
4175 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4176 match(ConI);
4177
4178 op_cost(0);
4179 format %{ %}
4180 interface(CONST_INTER);
4181 %}
4182
4183 operand immLoffset()
4184 %{
4185 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4186 match(ConL);
4187
4188 op_cost(0);
4189 format %{ %}
4190 interface(CONST_INTER);
4191 %}
4192
4193 operand immLoffset1()
4194 %{
4195 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4196 match(ConL);
4197
4198 op_cost(0);
4199 format %{ %}
4200 interface(CONST_INTER);
4201 %}
4202
4203 operand immLoffset2()
4204 %{
4205 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4206 match(ConL);
4207
4208 op_cost(0);
4209 format %{ %}
4210 interface(CONST_INTER);
4211 %}
4212
4213 operand immLoffset4()
4214 %{
4215 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4216 match(ConL);
4217
4218 op_cost(0);
4219 format %{ %}
4220 interface(CONST_INTER);
4221 %}
4222
4223 operand immLoffset8()
4224 %{
4225 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4226 match(ConL);
4227
4228 op_cost(0);
4229 format %{ %}
4230 interface(CONST_INTER);
4231 %}
4232
4233 operand immLoffset16()
4234 %{
4235 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4236 match(ConL);
4237
4238 op_cost(0);
4239 format %{ %}
4240 interface(CONST_INTER);
4241 %}
4242
4243 // 32 bit integer valid for add sub immediate
4244 operand immIAddSub()
4245 %{
4246 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4247 match(ConI);
4248 op_cost(0);
4249 format %{ %}
4250 interface(CONST_INTER);
4251 %}
4252
4253 // 32 bit unsigned integer valid for logical immediate
4254 // TODO -- check this is right when e.g the mask is 0x80000000
4255 operand immILog()
4256 %{
4257 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4258 match(ConI);
4259
4260 op_cost(0);
4261 format %{ %}
4262 interface(CONST_INTER);
4263 %}
4264
4265 // Integer operands 64 bit
4266 // 64 bit immediate
4267 operand immL()
4268 %{
4269 match(ConL);
4270
4271 op_cost(0);
4272 format %{ %}
4273 interface(CONST_INTER);
4274 %}
4275
4276 // 64 bit zero
4277 operand immL0()
4278 %{
4279 predicate(n->get_long() == 0);
4280 match(ConL);
4281
4282 op_cost(0);
4283 format %{ %}
4284 interface(CONST_INTER);
4285 %}
4286
4287 // 64 bit unit increment
4288 operand immL_1()
4289 %{
4290 predicate(n->get_long() == 1);
4291 match(ConL);
4292
4293 op_cost(0);
4294 format %{ %}
4295 interface(CONST_INTER);
4296 %}
4297
4298 // 64 bit unit decrement
4299 operand immL_M1()
4300 %{
4301 predicate(n->get_long() == -1);
4302 match(ConL);
4303
4304 op_cost(0);
4305 format %{ %}
4306 interface(CONST_INTER);
4307 %}
4308
4309 // 32 bit offset of pc in thread anchor
4310
4311 operand immL_pc_off()
4312 %{
4313 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4314 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4315 match(ConL);
4316
4317 op_cost(0);
4318 format %{ %}
4319 interface(CONST_INTER);
4320 %}
4321
4322 // 64 bit integer valid for add sub immediate
4323 operand immLAddSub()
4324 %{
4325 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4326 match(ConL);
4327 op_cost(0);
4328 format %{ %}
4329 interface(CONST_INTER);
4330 %}
4331
4332 // 64 bit integer valid for logical immediate
4333 operand immLLog()
4334 %{
4335 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4336 match(ConL);
4337 op_cost(0);
4338 format %{ %}
4339 interface(CONST_INTER);
4340 %}
4341
4342 // Long Immediate: low 32-bit mask
4343 operand immL_32bits()
4344 %{
4345 predicate(n->get_long() == 0xFFFFFFFFL);
4346 match(ConL);
4347 op_cost(0);
4348 format %{ %}
4349 interface(CONST_INTER);
4350 %}
4351
4352 // Pointer operands
4353 // Pointer Immediate
4354 operand immP()
4355 %{
4356 match(ConP);
4357
4358 op_cost(0);
4359 format %{ %}
4360 interface(CONST_INTER);
4361 %}
4362
4363 // NULL Pointer Immediate
4364 operand immP0()
4365 %{
4366 predicate(n->get_ptr() == 0);
4367 match(ConP);
4368
4369 op_cost(0);
4370 format %{ %}
4371 interface(CONST_INTER);
4372 %}
4373
4374 // Pointer Immediate One
4375 // this is used in object initialization (initial object header)
4376 operand immP_1()
4377 %{
4378 predicate(n->get_ptr() == 1);
4379 match(ConP);
4380
4381 op_cost(0);
4382 format %{ %}
4383 interface(CONST_INTER);
4384 %}
4385
4386 // Card Table Byte Map Base
4387 operand immByteMapBase()
4388 %{
4389 // Get base of card map
4390 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4391 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4392 match(ConP);
4393
4394 op_cost(0);
4395 format %{ %}
4396 interface(CONST_INTER);
4397 %}
4398
4399 // Pointer Immediate Minus One
4400 // this is used when we want to write the current PC to the thread anchor
4401 operand immP_M1()
4402 %{
4403 predicate(n->get_ptr() == -1);
4404 match(ConP);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 // Pointer Immediate Minus Two
4412 // this is used when we want to write the current PC to the thread anchor
4413 operand immP_M2()
4414 %{
4415 predicate(n->get_ptr() == -2);
4416 match(ConP);
4417
4418 op_cost(0);
4419 format %{ %}
4420 interface(CONST_INTER);
4421 %}
4422
4423 // Float and Double operands
4424 // Double Immediate
4425 operand immD()
4426 %{
4427 match(ConD);
4428 op_cost(0);
4429 format %{ %}
4430 interface(CONST_INTER);
4431 %}
4432
4433 // Double Immediate: +0.0d
4434 operand immD0()
4435 %{
4436 predicate(jlong_cast(n->getd()) == 0);
4437 match(ConD);
4438
4439 op_cost(0);
4440 format %{ %}
4441 interface(CONST_INTER);
4442 %}
4443
4444 // constant 'double +0.0'.
4445 operand immDPacked()
4446 %{
4447 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4448 match(ConD);
4449 op_cost(0);
4450 format %{ %}
4451 interface(CONST_INTER);
4452 %}
4453
4454 // Float Immediate
4455 operand immF()
4456 %{
4457 match(ConF);
4458 op_cost(0);
4459 format %{ %}
4460 interface(CONST_INTER);
4461 %}
4462
4463 // Float Immediate: +0.0f.
4464 operand immF0()
4465 %{
4466 predicate(jint_cast(n->getf()) == 0);
4467 match(ConF);
4468
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 //
4475 operand immFPacked()
4476 %{
4477 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4478 match(ConF);
4479 op_cost(0);
4480 format %{ %}
4481 interface(CONST_INTER);
4482 %}
4483
4484 // Narrow pointer operands
4485 // Narrow Pointer Immediate
4486 operand immN()
4487 %{
4488 match(ConN);
4489
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 // Narrow NULL Pointer Immediate
4496 operand immN0()
4497 %{
4498 predicate(n->get_narrowcon() == 0);
4499 match(ConN);
4500
4501 op_cost(0);
4502 format %{ %}
4503 interface(CONST_INTER);
4504 %}
4505
4506 operand immNKlass()
4507 %{
4508 match(ConNKlass);
4509
4510 op_cost(0);
4511 format %{ %}
4512 interface(CONST_INTER);
4513 %}
4514
4515 // Integer 32 bit Register Operands
4516 // Integer 32 bitRegister (excludes SP)
4517 operand iRegI()
4518 %{
4519 constraint(ALLOC_IN_RC(any_reg32));
4520 match(RegI);
4521 match(iRegINoSp);
4522 op_cost(0);
4523 format %{ %}
4524 interface(REG_INTER);
4525 %}
4526
4527 // Integer 32 bit Register not Special
4528 operand iRegINoSp()
4529 %{
4530 constraint(ALLOC_IN_RC(no_special_reg32));
4531 match(RegI);
4532 op_cost(0);
4533 format %{ %}
4534 interface(REG_INTER);
4535 %}
4536
4537 // Integer 64 bit Register Operands
4538 // Integer 64 bit Register (includes SP)
4539 operand iRegL()
4540 %{
4541 constraint(ALLOC_IN_RC(any_reg));
4542 match(RegL);
4543 match(iRegLNoSp);
4544 op_cost(0);
4545 format %{ %}
4546 interface(REG_INTER);
4547 %}
4548
4549 // Integer 64 bit Register not Special
4550 operand iRegLNoSp()
4551 %{
4552 constraint(ALLOC_IN_RC(no_special_reg));
4553 match(RegL);
4554 match(iRegL_R0);
4555 format %{ %}
4556 interface(REG_INTER);
4557 %}
4558
4559 // Pointer Register Operands
4560 // Pointer Register
4561 operand iRegP()
4562 %{
4563 constraint(ALLOC_IN_RC(ptr_reg));
4564 match(RegP);
4565 match(iRegPNoSp);
4566 match(iRegP_R0);
4567 //match(iRegP_R2);
4568 //match(iRegP_R4);
4569 //match(iRegP_R5);
4570 match(thread_RegP);
4571 op_cost(0);
4572 format %{ %}
4573 interface(REG_INTER);
4574 %}
4575
4576 // Pointer 64 bit Register not Special
4577 operand iRegPNoSp()
4578 %{
4579 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4580 match(RegP);
4581 // match(iRegP);
4582 // match(iRegP_R0);
4583 // match(iRegP_R2);
4584 // match(iRegP_R4);
4585 // match(iRegP_R5);
4586 // match(thread_RegP);
4587 op_cost(0);
4588 format %{ %}
4589 interface(REG_INTER);
4590 %}
4591
4592 // Pointer 64 bit Register R0 only
4593 operand iRegP_R0()
4594 %{
4595 constraint(ALLOC_IN_RC(r0_reg));
4596 match(RegP);
4597 // match(iRegP);
4598 match(iRegPNoSp);
4599 op_cost(0);
4600 format %{ %}
4601 interface(REG_INTER);
4602 %}
4603
4604 // Pointer 64 bit Register R1 only
4605 operand iRegP_R1()
4606 %{
4607 constraint(ALLOC_IN_RC(r1_reg));
4608 match(RegP);
4609 // match(iRegP);
4610 match(iRegPNoSp);
4611 op_cost(0);
4612 format %{ %}
4613 interface(REG_INTER);
4614 %}
4615
4616 // Pointer 64 bit Register R2 only
4617 operand iRegP_R2()
4618 %{
4619 constraint(ALLOC_IN_RC(r2_reg));
4620 match(RegP);
4621 // match(iRegP);
4622 match(iRegPNoSp);
4623 op_cost(0);
4624 format %{ %}
4625 interface(REG_INTER);
4626 %}
4627
4628 // Pointer 64 bit Register R3 only
4629 operand iRegP_R3()
4630 %{
4631 constraint(ALLOC_IN_RC(r3_reg));
4632 match(RegP);
4633 // match(iRegP);
4634 match(iRegPNoSp);
4635 op_cost(0);
4636 format %{ %}
4637 interface(REG_INTER);
4638 %}
4639
4640 // Pointer 64 bit Register R4 only
4641 operand iRegP_R4()
4642 %{
4643 constraint(ALLOC_IN_RC(r4_reg));
4644 match(RegP);
4645 // match(iRegP);
4646 match(iRegPNoSp);
4647 op_cost(0);
4648 format %{ %}
4649 interface(REG_INTER);
4650 %}
4651
4652 // Pointer 64 bit Register R5 only
4653 operand iRegP_R5()
4654 %{
4655 constraint(ALLOC_IN_RC(r5_reg));
4656 match(RegP);
4657 // match(iRegP);
4658 match(iRegPNoSp);
4659 op_cost(0);
4660 format %{ %}
4661 interface(REG_INTER);
4662 %}
4663
4664 // Pointer 64 bit Register R10 only
4665 operand iRegP_R10()
4666 %{
4667 constraint(ALLOC_IN_RC(r10_reg));
4668 match(RegP);
4669 // match(iRegP);
4670 match(iRegPNoSp);
4671 op_cost(0);
4672 format %{ %}
4673 interface(REG_INTER);
4674 %}
4675
4676 // Long 64 bit Register R0 only
4677 operand iRegL_R0()
4678 %{
4679 constraint(ALLOC_IN_RC(r0_reg));
4680 match(RegL);
4681 match(iRegLNoSp);
4682 op_cost(0);
4683 format %{ %}
4684 interface(REG_INTER);
4685 %}
4686
4687 // Long 64 bit Register R2 only
4688 operand iRegL_R2()
4689 %{
4690 constraint(ALLOC_IN_RC(r2_reg));
4691 match(RegL);
4692 match(iRegLNoSp);
4693 op_cost(0);
4694 format %{ %}
4695 interface(REG_INTER);
4696 %}
4697
4698 // Long 64 bit Register R3 only
4699 operand iRegL_R3()
4700 %{
4701 constraint(ALLOC_IN_RC(r3_reg));
4702 match(RegL);
4703 match(iRegLNoSp);
4704 op_cost(0);
4705 format %{ %}
4706 interface(REG_INTER);
4707 %}
4708
4709 // Long 64 bit Register R11 only
4710 operand iRegL_R11()
4711 %{
4712 constraint(ALLOC_IN_RC(r11_reg));
4713 match(RegL);
4714 match(iRegLNoSp);
4715 op_cost(0);
4716 format %{ %}
4717 interface(REG_INTER);
4718 %}
4719
4720 // Pointer 64 bit Register FP only
4721 operand iRegP_FP()
4722 %{
4723 constraint(ALLOC_IN_RC(fp_reg));
4724 match(RegP);
4725 // match(iRegP);
4726 op_cost(0);
4727 format %{ %}
4728 interface(REG_INTER);
4729 %}
4730
4731 // Register R0 only
4732 operand iRegI_R0()
4733 %{
4734 constraint(ALLOC_IN_RC(int_r0_reg));
4735 match(RegI);
4736 match(iRegINoSp);
4737 op_cost(0);
4738 format %{ %}
4739 interface(REG_INTER);
4740 %}
4741
4742 // Register R2 only
4743 operand iRegI_R2()
4744 %{
4745 constraint(ALLOC_IN_RC(int_r2_reg));
4746 match(RegI);
4747 match(iRegINoSp);
4748 op_cost(0);
4749 format %{ %}
4750 interface(REG_INTER);
4751 %}
4752
4753 // Register R3 only
4754 operand iRegI_R3()
4755 %{
4756 constraint(ALLOC_IN_RC(int_r3_reg));
4757 match(RegI);
4758 match(iRegINoSp);
4759 op_cost(0);
4760 format %{ %}
4761 interface(REG_INTER);
4762 %}
4763
4764
4765 // Register R4 only
4766 operand iRegI_R4()
4767 %{
4768 constraint(ALLOC_IN_RC(int_r4_reg));
4769 match(RegI);
4770 match(iRegINoSp);
4771 op_cost(0);
4772 format %{ %}
4773 interface(REG_INTER);
4774 %}
4775
4776
4777 // Pointer Register Operands
4778 // Narrow Pointer Register
4779 operand iRegN()
4780 %{
4781 constraint(ALLOC_IN_RC(any_reg32));
4782 match(RegN);
4783 match(iRegNNoSp);
4784 op_cost(0);
4785 format %{ %}
4786 interface(REG_INTER);
4787 %}
4788
4789 operand iRegN_R0()
4790 %{
4791 constraint(ALLOC_IN_RC(r0_reg));
4792 match(iRegN);
4793 op_cost(0);
4794 format %{ %}
4795 interface(REG_INTER);
4796 %}
4797
4798 operand iRegN_R2()
4799 %{
4800 constraint(ALLOC_IN_RC(r2_reg));
4801 match(iRegN);
4802 op_cost(0);
4803 format %{ %}
4804 interface(REG_INTER);
4805 %}
4806
4807 operand iRegN_R3()
4808 %{
4809 constraint(ALLOC_IN_RC(r3_reg));
4810 match(iRegN);
4811 op_cost(0);
4812 format %{ %}
4813 interface(REG_INTER);
4814 %}
4815
4816 // Integer 64 bit Register not Special
4817 operand iRegNNoSp()
4818 %{
4819 constraint(ALLOC_IN_RC(no_special_reg32));
4820 match(RegN);
4821 op_cost(0);
4822 format %{ %}
4823 interface(REG_INTER);
4824 %}
4825
4826 // heap base register -- used for encoding immN0
4827
4828 operand iRegIHeapbase()
4829 %{
4830 constraint(ALLOC_IN_RC(heapbase_reg));
4831 match(RegI);
4832 op_cost(0);
4833 format %{ %}
4834 interface(REG_INTER);
4835 %}
4836
4837 // Float Register
4838 // Float register operands
4839 operand vRegF()
4840 %{
4841 constraint(ALLOC_IN_RC(float_reg));
4842 match(RegF);
4843
4844 op_cost(0);
4845 format %{ %}
4846 interface(REG_INTER);
4847 %}
4848
4849 // Double Register
4850 // Double register operands
4851 operand vRegD()
4852 %{
4853 constraint(ALLOC_IN_RC(double_reg));
4854 match(RegD);
4855
4856 op_cost(0);
4857 format %{ %}
4858 interface(REG_INTER);
4859 %}
4860
4861 operand vecD()
4862 %{
4863 constraint(ALLOC_IN_RC(vectord_reg));
4864 match(VecD);
4865
4866 op_cost(0);
4867 format %{ %}
4868 interface(REG_INTER);
4869 %}
4870
4871 operand vecX()
4872 %{
4873 constraint(ALLOC_IN_RC(vectorx_reg));
4874 match(VecX);
4875
4876 op_cost(0);
4877 format %{ %}
4878 interface(REG_INTER);
4879 %}
4880
4881 operand vRegD_V0()
4882 %{
4883 constraint(ALLOC_IN_RC(v0_reg));
4884 match(RegD);
4885 op_cost(0);
4886 format %{ %}
4887 interface(REG_INTER);
4888 %}
4889
4890 operand vRegD_V1()
4891 %{
4892 constraint(ALLOC_IN_RC(v1_reg));
4893 match(RegD);
4894 op_cost(0);
4895 format %{ %}
4896 interface(REG_INTER);
4897 %}
4898
4899 operand vRegD_V2()
4900 %{
4901 constraint(ALLOC_IN_RC(v2_reg));
4902 match(RegD);
4903 op_cost(0);
4904 format %{ %}
4905 interface(REG_INTER);
4906 %}
4907
4908 operand vRegD_V3()
4909 %{
4910 constraint(ALLOC_IN_RC(v3_reg));
4911 match(RegD);
4912 op_cost(0);
4913 format %{ %}
4914 interface(REG_INTER);
4915 %}
4916
4917 operand vRegD_V4()
4918 %{
4919 constraint(ALLOC_IN_RC(v4_reg));
4920 match(RegD);
4921 op_cost(0);
4922 format %{ %}
4923 interface(REG_INTER);
4924 %}
4925
4926 operand vRegD_V5()
4927 %{
4928 constraint(ALLOC_IN_RC(v5_reg));
4929 match(RegD);
4930 op_cost(0);
4931 format %{ %}
4932 interface(REG_INTER);
4933 %}
4934
4935 operand vRegD_V6()
4936 %{
4937 constraint(ALLOC_IN_RC(v6_reg));
4938 match(RegD);
4939 op_cost(0);
4940 format %{ %}
4941 interface(REG_INTER);
4942 %}
4943
4944 operand vRegD_V7()
4945 %{
4946 constraint(ALLOC_IN_RC(v7_reg));
4947 match(RegD);
4948 op_cost(0);
4949 format %{ %}
4950 interface(REG_INTER);
4951 %}
4952
4953 operand vRegD_V8()
4954 %{
4955 constraint(ALLOC_IN_RC(v8_reg));
4956 match(RegD);
4957 op_cost(0);
4958 format %{ %}
4959 interface(REG_INTER);
4960 %}
4961
4962 operand vRegD_V9()
4963 %{
4964 constraint(ALLOC_IN_RC(v9_reg));
4965 match(RegD);
4966 op_cost(0);
4967 format %{ %}
4968 interface(REG_INTER);
4969 %}
4970
4971 operand vRegD_V10()
4972 %{
4973 constraint(ALLOC_IN_RC(v10_reg));
4974 match(RegD);
4975 op_cost(0);
4976 format %{ %}
4977 interface(REG_INTER);
4978 %}
4979
4980 operand vRegD_V11()
4981 %{
4982 constraint(ALLOC_IN_RC(v11_reg));
4983 match(RegD);
4984 op_cost(0);
4985 format %{ %}
4986 interface(REG_INTER);
4987 %}
4988
4989 operand vRegD_V12()
4990 %{
4991 constraint(ALLOC_IN_RC(v12_reg));
4992 match(RegD);
4993 op_cost(0);
4994 format %{ %}
4995 interface(REG_INTER);
4996 %}
4997
4998 operand vRegD_V13()
4999 %{
5000 constraint(ALLOC_IN_RC(v13_reg));
5001 match(RegD);
5002 op_cost(0);
5003 format %{ %}
5004 interface(REG_INTER);
5005 %}
5006
5007 operand vRegD_V14()
5008 %{
5009 constraint(ALLOC_IN_RC(v14_reg));
5010 match(RegD);
5011 op_cost(0);
5012 format %{ %}
5013 interface(REG_INTER);
5014 %}
5015
5016 operand vRegD_V15()
5017 %{
5018 constraint(ALLOC_IN_RC(v15_reg));
5019 match(RegD);
5020 op_cost(0);
5021 format %{ %}
5022 interface(REG_INTER);
5023 %}
5024
5025 operand vRegD_V16()
5026 %{
5027 constraint(ALLOC_IN_RC(v16_reg));
5028 match(RegD);
5029 op_cost(0);
5030 format %{ %}
5031 interface(REG_INTER);
5032 %}
5033
5034 operand vRegD_V17()
5035 %{
5036 constraint(ALLOC_IN_RC(v17_reg));
5037 match(RegD);
5038 op_cost(0);
5039 format %{ %}
5040 interface(REG_INTER);
5041 %}
5042
5043 operand vRegD_V18()
5044 %{
5045 constraint(ALLOC_IN_RC(v18_reg));
5046 match(RegD);
5047 op_cost(0);
5048 format %{ %}
5049 interface(REG_INTER);
5050 %}
5051
5052 operand vRegD_V19()
5053 %{
5054 constraint(ALLOC_IN_RC(v19_reg));
5055 match(RegD);
5056 op_cost(0);
5057 format %{ %}
5058 interface(REG_INTER);
5059 %}
5060
5061 operand vRegD_V20()
5062 %{
5063 constraint(ALLOC_IN_RC(v20_reg));
5064 match(RegD);
5065 op_cost(0);
5066 format %{ %}
5067 interface(REG_INTER);
5068 %}
5069
5070 operand vRegD_V21()
5071 %{
5072 constraint(ALLOC_IN_RC(v21_reg));
5073 match(RegD);
5074 op_cost(0);
5075 format %{ %}
5076 interface(REG_INTER);
5077 %}
5078
5079 operand vRegD_V22()
5080 %{
5081 constraint(ALLOC_IN_RC(v22_reg));
5082 match(RegD);
5083 op_cost(0);
5084 format %{ %}
5085 interface(REG_INTER);
5086 %}
5087
5088 operand vRegD_V23()
5089 %{
5090 constraint(ALLOC_IN_RC(v23_reg));
5091 match(RegD);
5092 op_cost(0);
5093 format %{ %}
5094 interface(REG_INTER);
5095 %}
5096
5097 operand vRegD_V24()
5098 %{
5099 constraint(ALLOC_IN_RC(v24_reg));
5100 match(RegD);
5101 op_cost(0);
5102 format %{ %}
5103 interface(REG_INTER);
5104 %}
5105
5106 operand vRegD_V25()
5107 %{
5108 constraint(ALLOC_IN_RC(v25_reg));
5109 match(RegD);
5110 op_cost(0);
5111 format %{ %}
5112 interface(REG_INTER);
5113 %}
5114
5115 operand vRegD_V26()
5116 %{
5117 constraint(ALLOC_IN_RC(v26_reg));
5118 match(RegD);
5119 op_cost(0);
5120 format %{ %}
5121 interface(REG_INTER);
5122 %}
5123
5124 operand vRegD_V27()
5125 %{
5126 constraint(ALLOC_IN_RC(v27_reg));
5127 match(RegD);
5128 op_cost(0);
5129 format %{ %}
5130 interface(REG_INTER);
5131 %}
5132
5133 operand vRegD_V28()
5134 %{
5135 constraint(ALLOC_IN_RC(v28_reg));
5136 match(RegD);
5137 op_cost(0);
5138 format %{ %}
5139 interface(REG_INTER);
5140 %}
5141
5142 operand vRegD_V29()
5143 %{
5144 constraint(ALLOC_IN_RC(v29_reg));
5145 match(RegD);
5146 op_cost(0);
5147 format %{ %}
5148 interface(REG_INTER);
5149 %}
5150
5151 operand vRegD_V30()
5152 %{
5153 constraint(ALLOC_IN_RC(v30_reg));
5154 match(RegD);
5155 op_cost(0);
5156 format %{ %}
5157 interface(REG_INTER);
5158 %}
5159
5160 operand vRegD_V31()
5161 %{
5162 constraint(ALLOC_IN_RC(v31_reg));
5163 match(RegD);
5164 op_cost(0);
5165 format %{ %}
5166 interface(REG_INTER);
5167 %}
5168
5169 // Flags register, used as output of signed compare instructions
5170
5171 // note that on AArch64 we also use this register as the output for
5172 // for floating point compare instructions (CmpF CmpD). this ensures
5173 // that ordered inequality tests use GT, GE, LT or LE none of which
5174 // pass through cases where the result is unordered i.e. one or both
5175 // inputs to the compare is a NaN. this means that the ideal code can
5176 // replace e.g. a GT with an LE and not end up capturing the NaN case
5177 // (where the comparison should always fail). EQ and NE tests are
5178 // always generated in ideal code so that unordered folds into the NE
5179 // case, matching the behaviour of AArch64 NE.
5180 //
5181 // This differs from x86 where the outputs of FP compares use a
5182 // special FP flags registers and where compares based on this
5183 // register are distinguished into ordered inequalities (cmpOpUCF) and
5184 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5185 // to explicitly handle the unordered case in branches. x86 also has
5186 // to include extra CMoveX rules to accept a cmpOpUCF input.
5187
5188 operand rFlagsReg()
5189 %{
5190 constraint(ALLOC_IN_RC(int_flags));
5191 match(RegFlags);
5192
5193 op_cost(0);
5194 format %{ "RFLAGS" %}
5195 interface(REG_INTER);
5196 %}
5197
5198 // Flags register, used as output of unsigned compare instructions
5199 operand rFlagsRegU()
5200 %{
5201 constraint(ALLOC_IN_RC(int_flags));
5202 match(RegFlags);
5203
5204 op_cost(0);
5205 format %{ "RFLAGSU" %}
5206 interface(REG_INTER);
5207 %}
5208
5209 // Special Registers
5210
5211 // Method Register
5212 operand inline_cache_RegP(iRegP reg)
5213 %{
5214 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5215 match(reg);
5216 match(iRegPNoSp);
5217 op_cost(0);
5218 format %{ %}
5219 interface(REG_INTER);
5220 %}
5221
5222 operand interpreter_method_oop_RegP(iRegP reg)
5223 %{
5224 constraint(ALLOC_IN_RC(method_reg)); // interpreter_method_oop_reg
5225 match(reg);
5226 match(iRegPNoSp);
5227 op_cost(0);
5228 format %{ %}
5229 interface(REG_INTER);
5230 %}
5231
5232 // Thread Register
5233 operand thread_RegP(iRegP reg)
5234 %{
5235 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5236 match(reg);
5237 op_cost(0);
5238 format %{ %}
5239 interface(REG_INTER);
5240 %}
5241
5242 operand lr_RegP(iRegP reg)
5243 %{
5244 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5245 match(reg);
5246 op_cost(0);
5247 format %{ %}
5248 interface(REG_INTER);
5249 %}
5250
5251 //----------Memory Operands----------------------------------------------------
5252
5253 operand indirect(iRegP reg)
5254 %{
5255 constraint(ALLOC_IN_RC(ptr_reg));
5256 match(reg);
5257 op_cost(0);
5258 format %{ "[$reg]" %}
5259 interface(MEMORY_INTER) %{
5260 base($reg);
5261 index(0xffffffff);
5262 scale(0x0);
5263 disp(0x0);
5264 %}
5265 %}
5266
5267 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5268 %{
5269 constraint(ALLOC_IN_RC(ptr_reg));
5270 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5271 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5272 op_cost(0);
5273 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5274 interface(MEMORY_INTER) %{
5275 base($reg);
5276 index($ireg);
5277 scale($scale);
5278 disp(0x0);
5279 %}
5280 %}
5281
5282 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5283 %{
5284 constraint(ALLOC_IN_RC(ptr_reg));
5285 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5286 match(AddP reg (LShiftL lreg scale));
5287 op_cost(0);
5288 format %{ "$reg, $lreg lsl($scale)" %}
5289 interface(MEMORY_INTER) %{
5290 base($reg);
5291 index($lreg);
5292 scale($scale);
5293 disp(0x0);
5294 %}
5295 %}
5296
5297 operand indIndexI2L(iRegP reg, iRegI ireg)
5298 %{
5299 constraint(ALLOC_IN_RC(ptr_reg));
5300 match(AddP reg (ConvI2L ireg));
5301 op_cost(0);
5302 format %{ "$reg, $ireg, 0, I2L" %}
5303 interface(MEMORY_INTER) %{
5304 base($reg);
5305 index($ireg);
5306 scale(0x0);
5307 disp(0x0);
5308 %}
5309 %}
5310
5311 operand indIndex(iRegP reg, iRegL lreg)
5312 %{
5313 constraint(ALLOC_IN_RC(ptr_reg));
5314 match(AddP reg lreg);
5315 op_cost(0);
5316 format %{ "$reg, $lreg" %}
5317 interface(MEMORY_INTER) %{
5318 base($reg);
5319 index($lreg);
5320 scale(0x0);
5321 disp(0x0);
5322 %}
5323 %}
5324
5325 operand indOffI(iRegP reg, immIOffset off)
5326 %{
5327 constraint(ALLOC_IN_RC(ptr_reg));
5328 match(AddP reg off);
5329 op_cost(0);
5330 format %{ "[$reg, $off]" %}
5331 interface(MEMORY_INTER) %{
5332 base($reg);
5333 index(0xffffffff);
5334 scale(0x0);
5335 disp($off);
5336 %}
5337 %}
5338
5339 operand indOffI1(iRegP reg, immIOffset1 off)
5340 %{
5341 constraint(ALLOC_IN_RC(ptr_reg));
5342 match(AddP reg off);
5343 op_cost(0);
5344 format %{ "[$reg, $off]" %}
5345 interface(MEMORY_INTER) %{
5346 base($reg);
5347 index(0xffffffff);
5348 scale(0x0);
5349 disp($off);
5350 %}
5351 %}
5352
5353 operand indOffI2(iRegP reg, immIOffset2 off)
5354 %{
5355 constraint(ALLOC_IN_RC(ptr_reg));
5356 match(AddP reg off);
5357 op_cost(0);
5358 format %{ "[$reg, $off]" %}
5359 interface(MEMORY_INTER) %{
5360 base($reg);
5361 index(0xffffffff);
5362 scale(0x0);
5363 disp($off);
5364 %}
5365 %}
5366
5367 operand indOffI4(iRegP reg, immIOffset4 off)
5368 %{
5369 constraint(ALLOC_IN_RC(ptr_reg));
5370 match(AddP reg off);
5371 op_cost(0);
5372 format %{ "[$reg, $off]" %}
5373 interface(MEMORY_INTER) %{
5374 base($reg);
5375 index(0xffffffff);
5376 scale(0x0);
5377 disp($off);
5378 %}
5379 %}
5380
5381 operand indOffI8(iRegP reg, immIOffset8 off)
5382 %{
5383 constraint(ALLOC_IN_RC(ptr_reg));
5384 match(AddP reg off);
5385 op_cost(0);
5386 format %{ "[$reg, $off]" %}
5387 interface(MEMORY_INTER) %{
5388 base($reg);
5389 index(0xffffffff);
5390 scale(0x0);
5391 disp($off);
5392 %}
5393 %}
5394
5395 operand indOffI16(iRegP reg, immIOffset16 off)
5396 %{
5397 constraint(ALLOC_IN_RC(ptr_reg));
5398 match(AddP reg off);
5399 op_cost(0);
5400 format %{ "[$reg, $off]" %}
5401 interface(MEMORY_INTER) %{
5402 base($reg);
5403 index(0xffffffff);
5404 scale(0x0);
5405 disp($off);
5406 %}
5407 %}
5408
5409 operand indOffL(iRegP reg, immLoffset off)
5410 %{
5411 constraint(ALLOC_IN_RC(ptr_reg));
5412 match(AddP reg off);
5413 op_cost(0);
5414 format %{ "[$reg, $off]" %}
5415 interface(MEMORY_INTER) %{
5416 base($reg);
5417 index(0xffffffff);
5418 scale(0x0);
5419 disp($off);
5420 %}
5421 %}
5422
5423 operand indOffL1(iRegP reg, immLoffset1 off)
5424 %{
5425 constraint(ALLOC_IN_RC(ptr_reg));
5426 match(AddP reg off);
5427 op_cost(0);
5428 format %{ "[$reg, $off]" %}
5429 interface(MEMORY_INTER) %{
5430 base($reg);
5431 index(0xffffffff);
5432 scale(0x0);
5433 disp($off);
5434 %}
5435 %}
5436
5437 operand indOffL2(iRegP reg, immLoffset2 off)
5438 %{
5439 constraint(ALLOC_IN_RC(ptr_reg));
5440 match(AddP reg off);
5441 op_cost(0);
5442 format %{ "[$reg, $off]" %}
5443 interface(MEMORY_INTER) %{
5444 base($reg);
5445 index(0xffffffff);
5446 scale(0x0);
5447 disp($off);
5448 %}
5449 %}
5450
5451 operand indOffL4(iRegP reg, immLoffset4 off)
5452 %{
5453 constraint(ALLOC_IN_RC(ptr_reg));
5454 match(AddP reg off);
5455 op_cost(0);
5456 format %{ "[$reg, $off]" %}
5457 interface(MEMORY_INTER) %{
5458 base($reg);
5459 index(0xffffffff);
5460 scale(0x0);
5461 disp($off);
5462 %}
5463 %}
5464
5465 operand indOffL8(iRegP reg, immLoffset8 off)
5466 %{
5467 constraint(ALLOC_IN_RC(ptr_reg));
5468 match(AddP reg off);
5469 op_cost(0);
5470 format %{ "[$reg, $off]" %}
5471 interface(MEMORY_INTER) %{
5472 base($reg);
5473 index(0xffffffff);
5474 scale(0x0);
5475 disp($off);
5476 %}
5477 %}
5478
5479 operand indOffL16(iRegP reg, immLoffset16 off)
5480 %{
5481 constraint(ALLOC_IN_RC(ptr_reg));
5482 match(AddP reg off);
5483 op_cost(0);
5484 format %{ "[$reg, $off]" %}
5485 interface(MEMORY_INTER) %{
5486 base($reg);
5487 index(0xffffffff);
5488 scale(0x0);
5489 disp($off);
5490 %}
5491 %}
5492
5493 operand indirectN(iRegN reg)
5494 %{
5495 predicate(CompressedOops::shift() == 0);
5496 constraint(ALLOC_IN_RC(ptr_reg));
5497 match(DecodeN reg);
5498 op_cost(0);
5499 format %{ "[$reg]\t# narrow" %}
5500 interface(MEMORY_INTER) %{
5501 base($reg);
5502 index(0xffffffff);
5503 scale(0x0);
5504 disp(0x0);
5505 %}
5506 %}
5507
5508 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5509 %{
5510 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5511 constraint(ALLOC_IN_RC(ptr_reg));
5512 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5513 op_cost(0);
5514 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5515 interface(MEMORY_INTER) %{
5516 base($reg);
5517 index($ireg);
5518 scale($scale);
5519 disp(0x0);
5520 %}
5521 %}
5522
5523 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5524 %{
5525 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5526 constraint(ALLOC_IN_RC(ptr_reg));
5527 match(AddP (DecodeN reg) (LShiftL lreg scale));
5528 op_cost(0);
5529 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5530 interface(MEMORY_INTER) %{
5531 base($reg);
5532 index($lreg);
5533 scale($scale);
5534 disp(0x0);
5535 %}
5536 %}
5537
5538 operand indIndexI2LN(iRegN reg, iRegI ireg)
5539 %{
5540 predicate(CompressedOops::shift() == 0);
5541 constraint(ALLOC_IN_RC(ptr_reg));
5542 match(AddP (DecodeN reg) (ConvI2L ireg));
5543 op_cost(0);
5544 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5545 interface(MEMORY_INTER) %{
5546 base($reg);
5547 index($ireg);
5548 scale(0x0);
5549 disp(0x0);
5550 %}
5551 %}
5552
5553 operand indIndexN(iRegN reg, iRegL lreg)
5554 %{
5555 predicate(CompressedOops::shift() == 0);
5556 constraint(ALLOC_IN_RC(ptr_reg));
5557 match(AddP (DecodeN reg) lreg);
5558 op_cost(0);
5559 format %{ "$reg, $lreg\t# narrow" %}
5560 interface(MEMORY_INTER) %{
5561 base($reg);
5562 index($lreg);
5563 scale(0x0);
5564 disp(0x0);
5565 %}
5566 %}
5567
5568 operand indOffIN(iRegN reg, immIOffset off)
5569 %{
5570 predicate(CompressedOops::shift() == 0);
5571 constraint(ALLOC_IN_RC(ptr_reg));
5572 match(AddP (DecodeN reg) off);
5573 op_cost(0);
5574 format %{ "[$reg, $off]\t# narrow" %}
5575 interface(MEMORY_INTER) %{
5576 base($reg);
5577 index(0xffffffff);
5578 scale(0x0);
5579 disp($off);
5580 %}
5581 %}
5582
5583 operand indOffLN(iRegN reg, immLoffset off)
5584 %{
5585 predicate(CompressedOops::shift() == 0);
5586 constraint(ALLOC_IN_RC(ptr_reg));
5587 match(AddP (DecodeN reg) off);
5588 op_cost(0);
5589 format %{ "[$reg, $off]\t# narrow" %}
5590 interface(MEMORY_INTER) %{
5591 base($reg);
5592 index(0xffffffff);
5593 scale(0x0);
5594 disp($off);
5595 %}
5596 %}
5597
5598
5599
5600 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5601 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5602 %{
5603 constraint(ALLOC_IN_RC(ptr_reg));
5604 match(AddP reg off);
5605 op_cost(0);
5606 format %{ "[$reg, $off]" %}
5607 interface(MEMORY_INTER) %{
5608 base($reg);
5609 index(0xffffffff);
5610 scale(0x0);
5611 disp($off);
5612 %}
5613 %}
5614
5615 //----------Special Memory Operands--------------------------------------------
5616 // Stack Slot Operand - This operand is used for loading and storing temporary
5617 // values on the stack where a match requires a value to
5618 // flow through memory.
5619 operand stackSlotP(sRegP reg)
5620 %{
5621 constraint(ALLOC_IN_RC(stack_slots));
5622 op_cost(100);
5623 // No match rule because this operand is only generated in matching
5624 // match(RegP);
5625 format %{ "[$reg]" %}
5626 interface(MEMORY_INTER) %{
5627 base(0x1e); // RSP
5628 index(0x0); // No Index
5629 scale(0x0); // No Scale
5630 disp($reg); // Stack Offset
5631 %}
5632 %}
5633
5634 operand stackSlotI(sRegI reg)
5635 %{
5636 constraint(ALLOC_IN_RC(stack_slots));
5637 // No match rule because this operand is only generated in matching
5638 // match(RegI);
5639 format %{ "[$reg]" %}
5640 interface(MEMORY_INTER) %{
5641 base(0x1e); // RSP
5642 index(0x0); // No Index
5643 scale(0x0); // No Scale
5644 disp($reg); // Stack Offset
5645 %}
5646 %}
5647
5648 operand stackSlotF(sRegF reg)
5649 %{
5650 constraint(ALLOC_IN_RC(stack_slots));
5651 // No match rule because this operand is only generated in matching
5652 // match(RegF);
5653 format %{ "[$reg]" %}
5654 interface(MEMORY_INTER) %{
5655 base(0x1e); // RSP
5656 index(0x0); // No Index
5657 scale(0x0); // No Scale
5658 disp($reg); // Stack Offset
5659 %}
5660 %}
5661
5662 operand stackSlotD(sRegD reg)
5663 %{
5664 constraint(ALLOC_IN_RC(stack_slots));
5665 // No match rule because this operand is only generated in matching
5666 // match(RegD);
5667 format %{ "[$reg]" %}
5668 interface(MEMORY_INTER) %{
5669 base(0x1e); // RSP
5670 index(0x0); // No Index
5671 scale(0x0); // No Scale
5672 disp($reg); // Stack Offset
5673 %}
5674 %}
5675
5676 operand stackSlotL(sRegL reg)
5677 %{
5678 constraint(ALLOC_IN_RC(stack_slots));
5679 // No match rule because this operand is only generated in matching
5680 // match(RegL);
5681 format %{ "[$reg]" %}
5682 interface(MEMORY_INTER) %{
5683 base(0x1e); // RSP
5684 index(0x0); // No Index
5685 scale(0x0); // No Scale
5686 disp($reg); // Stack Offset
5687 %}
5688 %}
5689
5690 // Operands for expressing Control Flow
5691 // NOTE: Label is a predefined operand which should not be redefined in
5692 // the AD file. It is generically handled within the ADLC.
5693
5694 //----------Conditional Branch Operands----------------------------------------
5695 // Comparison Op - This is the operation of the comparison, and is limited to
5696 // the following set of codes:
5697 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5698 //
5699 // Other attributes of the comparison, such as unsignedness, are specified
5700 // by the comparison instruction that sets a condition code flags register.
5701 // That result is represented by a flags operand whose subtype is appropriate
5702 // to the unsignedness (etc.) of the comparison.
5703 //
5704 // Later, the instruction which matches both the Comparison Op (a Bool) and
5705 // the flags (produced by the Cmp) specifies the coding of the comparison op
5706 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5707
5708 // used for signed integral comparisons and fp comparisons
5709
5710 operand cmpOp()
5711 %{
5712 match(Bool);
5713
5714 format %{ "" %}
5715 interface(COND_INTER) %{
5716 equal(0x0, "eq");
5717 not_equal(0x1, "ne");
5718 less(0xb, "lt");
5719 greater_equal(0xa, "ge");
5720 less_equal(0xd, "le");
5721 greater(0xc, "gt");
5722 overflow(0x6, "vs");
5723 no_overflow(0x7, "vc");
5724 %}
5725 %}
5726
5727 // used for unsigned integral comparisons
5728
5729 operand cmpOpU()
5730 %{
5731 match(Bool);
5732
5733 format %{ "" %}
5734 interface(COND_INTER) %{
5735 equal(0x0, "eq");
5736 not_equal(0x1, "ne");
5737 less(0x3, "lo");
5738 greater_equal(0x2, "hs");
5739 less_equal(0x9, "ls");
5740 greater(0x8, "hi");
5741 overflow(0x6, "vs");
5742 no_overflow(0x7, "vc");
5743 %}
5744 %}
5745
5746 // used for certain integral comparisons which can be
5747 // converted to cbxx or tbxx instructions
5748
5749 operand cmpOpEqNe()
5750 %{
5751 match(Bool);
5752 op_cost(0);
5753 predicate(n->as_Bool()->_test._test == BoolTest::ne
5754 || n->as_Bool()->_test._test == BoolTest::eq);
5755
5756 format %{ "" %}
5757 interface(COND_INTER) %{
5758 equal(0x0, "eq");
5759 not_equal(0x1, "ne");
5760 less(0xb, "lt");
5761 greater_equal(0xa, "ge");
5762 less_equal(0xd, "le");
5763 greater(0xc, "gt");
5764 overflow(0x6, "vs");
5765 no_overflow(0x7, "vc");
5766 %}
5767 %}
5768
5769 // used for certain integral comparisons which can be
5770 // converted to cbxx or tbxx instructions
5771
5772 operand cmpOpLtGe()
5773 %{
5774 match(Bool);
5775 op_cost(0);
5776
5777 predicate(n->as_Bool()->_test._test == BoolTest::lt
5778 || n->as_Bool()->_test._test == BoolTest::ge);
5779
5780 format %{ "" %}
5781 interface(COND_INTER) %{
5782 equal(0x0, "eq");
5783 not_equal(0x1, "ne");
5784 less(0xb, "lt");
5785 greater_equal(0xa, "ge");
5786 less_equal(0xd, "le");
5787 greater(0xc, "gt");
5788 overflow(0x6, "vs");
5789 no_overflow(0x7, "vc");
5790 %}
5791 %}
5792
5793 // used for certain unsigned integral comparisons which can be
5794 // converted to cbxx or tbxx instructions
5795
5796 operand cmpOpUEqNeLtGe()
5797 %{
5798 match(Bool);
5799 op_cost(0);
5800
5801 predicate(n->as_Bool()->_test._test == BoolTest::eq
5802 || n->as_Bool()->_test._test == BoolTest::ne
5803 || n->as_Bool()->_test._test == BoolTest::lt
5804 || n->as_Bool()->_test._test == BoolTest::ge);
5805
5806 format %{ "" %}
5807 interface(COND_INTER) %{
5808 equal(0x0, "eq");
5809 not_equal(0x1, "ne");
5810 less(0xb, "lt");
5811 greater_equal(0xa, "ge");
5812 less_equal(0xd, "le");
5813 greater(0xc, "gt");
5814 overflow(0x6, "vs");
5815 no_overflow(0x7, "vc");
5816 %}
5817 %}
5818
5819 // Special operand allowing long args to int ops to be truncated for free
5820
5821 operand iRegL2I(iRegL reg) %{
5822
5823 op_cost(0);
5824
5825 match(ConvL2I reg);
5826
5827 format %{ "l2i($reg)" %}
5828
5829 interface(REG_INTER)
5830 %}
5831
5832 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5833 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5834 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5835
5836 //----------OPERAND CLASSES----------------------------------------------------
5837 // Operand Classes are groups of operands that are used as to simplify
5838 // instruction definitions by not requiring the AD writer to specify
5839 // separate instructions for every form of operand when the
5840 // instruction accepts multiple operand types with the same basic
5841 // encoding and format. The classic case of this is memory operands.
5842
5843 // memory is used to define read/write location for load/store
5844 // instruction defs. we can turn a memory op into an Address
5845
5846 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5847 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
5848
5849 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5850 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
5851
5852 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5853 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
5854
5855 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5856 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
5857
5858 // All of the memory operands. For the pipeline description.
5859 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5860 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5861 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
5862
5863
5864 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5865 // operations. it allows the src to be either an iRegI or a (ConvL2I
5866 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5867 // can be elided because the 32-bit instruction will just employ the
5868 // lower 32 bits anyway.
5869 //
5870 // n.b. this does not elide all L2I conversions. if the truncated
5871 // value is consumed by more than one operation then the ConvL2I
5872 // cannot be bundled into the consuming nodes so an l2i gets planted
5873 // (actually a movw $dst $src) and the downstream instructions consume
5874 // the result of the l2i as an iRegI input. That's a shame since the
5875 // movw is actually redundant but its not too costly.
5876
5877 opclass iRegIorL2I(iRegI, iRegL2I);
5878
5879 //----------PIPELINE-----------------------------------------------------------
5880 // Rules which define the behavior of the target architectures pipeline.
5881
5882 // For specific pipelines, eg A53, define the stages of that pipeline
5883 //pipe_desc(ISS, EX1, EX2, WR);
5884 #define ISS S0
5885 #define EX1 S1
5886 #define EX2 S2
5887 #define WR S3
5888
5889 // Integer ALU reg operation
5890 pipeline %{
5891
5892 attributes %{
5893 // ARM instructions are of fixed length
5894 fixed_size_instructions; // Fixed size instructions TODO does
5895 max_instructions_per_bundle = 2; // A53 = 2, A57 = 4
5896 // ARM instructions come in 32-bit word units
5897 instruction_unit_size = 4; // An instruction is 4 bytes long
5898 instruction_fetch_unit_size = 64; // The processor fetches one line
5899 instruction_fetch_units = 1; // of 64 bytes
5900
5901 // List of nop instructions
5902 nops( MachNop );
5903 %}
5904
5905 // We don't use an actual pipeline model so don't care about resources
5906 // or description. we do use pipeline classes to introduce fixed
5907 // latencies
5908
5909 //----------RESOURCES----------------------------------------------------------
5910 // Resources are the functional units available to the machine
5911
5912 resources( INS0, INS1, INS01 = INS0 | INS1,
5913 ALU0, ALU1, ALU = ALU0 | ALU1,
5914 MAC,
5915 DIV,
5916 BRANCH,
5917 LDST,
5918 NEON_FP);
5919
5920 //----------PIPELINE DESCRIPTION-----------------------------------------------
5921 // Pipeline Description specifies the stages in the machine's pipeline
5922
5923 // Define the pipeline as a generic 6 stage pipeline
5924 pipe_desc(S0, S1, S2, S3, S4, S5);
5925
5926 //----------PIPELINE CLASSES---------------------------------------------------
5927 // Pipeline Classes describe the stages in which input and output are
5928 // referenced by the hardware pipeline.
5929
5930 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5931 %{
5932 single_instruction;
5933 src1 : S1(read);
5934 src2 : S2(read);
5935 dst : S5(write);
5936 INS01 : ISS;
5937 NEON_FP : S5;
5938 %}
5939
5940 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5941 %{
5942 single_instruction;
5943 src1 : S1(read);
5944 src2 : S2(read);
5945 dst : S5(write);
5946 INS01 : ISS;
5947 NEON_FP : S5;
5948 %}
5949
5950 pipe_class fp_uop_s(vRegF dst, vRegF src)
5951 %{
5952 single_instruction;
5953 src : S1(read);
5954 dst : S5(write);
5955 INS01 : ISS;
5956 NEON_FP : S5;
5957 %}
5958
5959 pipe_class fp_uop_d(vRegD dst, vRegD src)
5960 %{
5961 single_instruction;
5962 src : S1(read);
5963 dst : S5(write);
5964 INS01 : ISS;
5965 NEON_FP : S5;
5966 %}
5967
5968 pipe_class fp_d2f(vRegF dst, vRegD src)
5969 %{
5970 single_instruction;
5971 src : S1(read);
5972 dst : S5(write);
5973 INS01 : ISS;
5974 NEON_FP : S5;
5975 %}
5976
5977 pipe_class fp_f2d(vRegD dst, vRegF src)
5978 %{
5979 single_instruction;
5980 src : S1(read);
5981 dst : S5(write);
5982 INS01 : ISS;
5983 NEON_FP : S5;
5984 %}
5985
5986 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5987 %{
5988 single_instruction;
5989 src : S1(read);
5990 dst : S5(write);
5991 INS01 : ISS;
5992 NEON_FP : S5;
5993 %}
5994
5995 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5996 %{
5997 single_instruction;
5998 src : S1(read);
5999 dst : S5(write);
6000 INS01 : ISS;
6001 NEON_FP : S5;
6002 %}
6003
6004 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6005 %{
6006 single_instruction;
6007 src : S1(read);
6008 dst : S5(write);
6009 INS01 : ISS;
6010 NEON_FP : S5;
6011 %}
6012
6013 pipe_class fp_l2f(vRegF dst, iRegL src)
6014 %{
6015 single_instruction;
6016 src : S1(read);
6017 dst : S5(write);
6018 INS01 : ISS;
6019 NEON_FP : S5;
6020 %}
6021
6022 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6023 %{
6024 single_instruction;
6025 src : S1(read);
6026 dst : S5(write);
6027 INS01 : ISS;
6028 NEON_FP : S5;
6029 %}
6030
6031 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6032 %{
6033 single_instruction;
6034 src : S1(read);
6035 dst : S5(write);
6036 INS01 : ISS;
6037 NEON_FP : S5;
6038 %}
6039
6040 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6041 %{
6042 single_instruction;
6043 src : S1(read);
6044 dst : S5(write);
6045 INS01 : ISS;
6046 NEON_FP : S5;
6047 %}
6048
6049 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6050 %{
6051 single_instruction;
6052 src : S1(read);
6053 dst : S5(write);
6054 INS01 : ISS;
6055 NEON_FP : S5;
6056 %}
6057
6058 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6059 %{
6060 single_instruction;
6061 src1 : S1(read);
6062 src2 : S2(read);
6063 dst : S5(write);
6064 INS0 : ISS;
6065 NEON_FP : S5;
6066 %}
6067
6068 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6069 %{
6070 single_instruction;
6071 src1 : S1(read);
6072 src2 : S2(read);
6073 dst : S5(write);
6074 INS0 : ISS;
6075 NEON_FP : S5;
6076 %}
6077
6078 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6079 %{
6080 single_instruction;
6081 cr : S1(read);
6082 src1 : S1(read);
6083 src2 : S1(read);
6084 dst : S3(write);
6085 INS01 : ISS;
6086 NEON_FP : S3;
6087 %}
6088
6089 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6090 %{
6091 single_instruction;
6092 cr : S1(read);
6093 src1 : S1(read);
6094 src2 : S1(read);
6095 dst : S3(write);
6096 INS01 : ISS;
6097 NEON_FP : S3;
6098 %}
6099
6100 pipe_class fp_imm_s(vRegF dst)
6101 %{
6102 single_instruction;
6103 dst : S3(write);
6104 INS01 : ISS;
6105 NEON_FP : S3;
6106 %}
6107
6108 pipe_class fp_imm_d(vRegD dst)
6109 %{
6110 single_instruction;
6111 dst : S3(write);
6112 INS01 : ISS;
6113 NEON_FP : S3;
6114 %}
6115
6116 pipe_class fp_load_constant_s(vRegF dst)
6117 %{
6118 single_instruction;
6119 dst : S4(write);
6120 INS01 : ISS;
6121 NEON_FP : S4;
6122 %}
6123
6124 pipe_class fp_load_constant_d(vRegD dst)
6125 %{
6126 single_instruction;
6127 dst : S4(write);
6128 INS01 : ISS;
6129 NEON_FP : S4;
6130 %}
6131
6132 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
6133 %{
6134 single_instruction;
6135 dst : S5(write);
6136 src1 : S1(read);
6137 src2 : S1(read);
6138 INS01 : ISS;
6139 NEON_FP : S5;
6140 %}
6141
6142 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
6143 %{
6144 single_instruction;
6145 dst : S5(write);
6146 src1 : S1(read);
6147 src2 : S1(read);
6148 INS0 : ISS;
6149 NEON_FP : S5;
6150 %}
6151
6152 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
6153 %{
6154 single_instruction;
6155 dst : S5(write);
6156 src1 : S1(read);
6157 src2 : S1(read);
6158 dst : S1(read);
6159 INS01 : ISS;
6160 NEON_FP : S5;
6161 %}
6162
6163 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
6164 %{
6165 single_instruction;
6166 dst : S5(write);
6167 src1 : S1(read);
6168 src2 : S1(read);
6169 dst : S1(read);
6170 INS0 : ISS;
6171 NEON_FP : S5;
6172 %}
6173
6174 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
6175 %{
6176 single_instruction;
6177 dst : S4(write);
6178 src1 : S2(read);
6179 src2 : S2(read);
6180 INS01 : ISS;
6181 NEON_FP : S4;
6182 %}
6183
6184 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
6185 %{
6186 single_instruction;
6187 dst : S4(write);
6188 src1 : S2(read);
6189 src2 : S2(read);
6190 INS0 : ISS;
6191 NEON_FP : S4;
6192 %}
6193
6194 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
6195 %{
6196 single_instruction;
6197 dst : S3(write);
6198 src1 : S2(read);
6199 src2 : S2(read);
6200 INS01 : ISS;
6201 NEON_FP : S3;
6202 %}
6203
6204 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
6205 %{
6206 single_instruction;
6207 dst : S3(write);
6208 src1 : S2(read);
6209 src2 : S2(read);
6210 INS0 : ISS;
6211 NEON_FP : S3;
6212 %}
6213
6214 pipe_class vshift64(vecD dst, vecD src, vecX shift)
6215 %{
6216 single_instruction;
6217 dst : S3(write);
6218 src : S1(read);
6219 shift : S1(read);
6220 INS01 : ISS;
6221 NEON_FP : S3;
6222 %}
6223
6224 pipe_class vshift128(vecX dst, vecX src, vecX shift)
6225 %{
6226 single_instruction;
6227 dst : S3(write);
6228 src : S1(read);
6229 shift : S1(read);
6230 INS0 : ISS;
6231 NEON_FP : S3;
6232 %}
6233
6234 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
6235 %{
6236 single_instruction;
6237 dst : S3(write);
6238 src : S1(read);
6239 INS01 : ISS;
6240 NEON_FP : S3;
6241 %}
6242
6243 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
6244 %{
6245 single_instruction;
6246 dst : S3(write);
6247 src : S1(read);
6248 INS0 : ISS;
6249 NEON_FP : S3;
6250 %}
6251
6252 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
6253 %{
6254 single_instruction;
6255 dst : S5(write);
6256 src1 : S1(read);
6257 src2 : S1(read);
6258 INS01 : ISS;
6259 NEON_FP : S5;
6260 %}
6261
6262 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
6263 %{
6264 single_instruction;
6265 dst : S5(write);
6266 src1 : S1(read);
6267 src2 : S1(read);
6268 INS0 : ISS;
6269 NEON_FP : S5;
6270 %}
6271
6272 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
6273 %{
6274 single_instruction;
6275 dst : S5(write);
6276 src1 : S1(read);
6277 src2 : S1(read);
6278 INS0 : ISS;
6279 NEON_FP : S5;
6280 %}
6281
6282 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
6283 %{
6284 single_instruction;
6285 dst : S5(write);
6286 src1 : S1(read);
6287 src2 : S1(read);
6288 INS0 : ISS;
6289 NEON_FP : S5;
6290 %}
6291
6292 pipe_class vsqrt_fp128(vecX dst, vecX src)
6293 %{
6294 single_instruction;
6295 dst : S5(write);
6296 src : S1(read);
6297 INS0 : ISS;
6298 NEON_FP : S5;
6299 %}
6300
6301 pipe_class vunop_fp64(vecD dst, vecD src)
6302 %{
6303 single_instruction;
6304 dst : S5(write);
6305 src : S1(read);
6306 INS01 : ISS;
6307 NEON_FP : S5;
6308 %}
6309
6310 pipe_class vunop_fp128(vecX dst, vecX src)
6311 %{
6312 single_instruction;
6313 dst : S5(write);
6314 src : S1(read);
6315 INS0 : ISS;
6316 NEON_FP : S5;
6317 %}
6318
6319 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
6320 %{
6321 single_instruction;
6322 dst : S3(write);
6323 src : S1(read);
6324 INS01 : ISS;
6325 NEON_FP : S3;
6326 %}
6327
6328 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
6329 %{
6330 single_instruction;
6331 dst : S3(write);
6332 src : S1(read);
6333 INS01 : ISS;
6334 NEON_FP : S3;
6335 %}
6336
6337 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
6338 %{
6339 single_instruction;
6340 dst : S3(write);
6341 src : S1(read);
6342 INS01 : ISS;
6343 NEON_FP : S3;
6344 %}
6345
6346 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
6347 %{
6348 single_instruction;
6349 dst : S3(write);
6350 src : S1(read);
6351 INS01 : ISS;
6352 NEON_FP : S3;
6353 %}
6354
6355 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
6356 %{
6357 single_instruction;
6358 dst : S3(write);
6359 src : S1(read);
6360 INS01 : ISS;
6361 NEON_FP : S3;
6362 %}
6363
6364 pipe_class vmovi_reg_imm64(vecD dst)
6365 %{
6366 single_instruction;
6367 dst : S3(write);
6368 INS01 : ISS;
6369 NEON_FP : S3;
6370 %}
6371
6372 pipe_class vmovi_reg_imm128(vecX dst)
6373 %{
6374 single_instruction;
6375 dst : S3(write);
6376 INS0 : ISS;
6377 NEON_FP : S3;
6378 %}
6379
6380 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
6381 %{
6382 single_instruction;
6383 dst : S5(write);
6384 mem : ISS(read);
6385 INS01 : ISS;
6386 NEON_FP : S3;
6387 %}
6388
6389 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
6390 %{
6391 single_instruction;
6392 dst : S5(write);
6393 mem : ISS(read);
6394 INS01 : ISS;
6395 NEON_FP : S3;
6396 %}
6397
6398 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
6399 %{
6400 single_instruction;
6401 mem : ISS(read);
6402 src : S2(read);
6403 INS01 : ISS;
6404 NEON_FP : S3;
6405 %}
6406
6407 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
6408 %{
6409 single_instruction;
6410 mem : ISS(read);
6411 src : S2(read);
6412 INS01 : ISS;
6413 NEON_FP : S3;
6414 %}
6415
6416 //------- Integer ALU operations --------------------------
6417
6418 // Integer ALU reg-reg operation
6419 // Operands needed in EX1, result generated in EX2
6420 // Eg. ADD x0, x1, x2
6421 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6422 %{
6423 single_instruction;
6424 dst : EX2(write);
6425 src1 : EX1(read);
6426 src2 : EX1(read);
6427 INS01 : ISS; // Dual issue as instruction 0 or 1
6428 ALU : EX2;
6429 %}
6430
6431 // Integer ALU reg-reg operation with constant shift
6432 // Shifted register must be available in LATE_ISS instead of EX1
6433 // Eg. ADD x0, x1, x2, LSL #2
6434 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6435 %{
6436 single_instruction;
6437 dst : EX2(write);
6438 src1 : EX1(read);
6439 src2 : ISS(read);
6440 INS01 : ISS;
6441 ALU : EX2;
6442 %}
6443
6444 // Integer ALU reg operation with constant shift
6445 // Eg. LSL x0, x1, #shift
6446 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6447 %{
6448 single_instruction;
6449 dst : EX2(write);
6450 src1 : ISS(read);
6451 INS01 : ISS;
6452 ALU : EX2;
6453 %}
6454
6455 // Integer ALU reg-reg operation with variable shift
6456 // Both operands must be available in LATE_ISS instead of EX1
6457 // Result is available in EX1 instead of EX2
6458 // Eg. LSLV x0, x1, x2
6459 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6460 %{
6461 single_instruction;
6462 dst : EX1(write);
6463 src1 : ISS(read);
6464 src2 : ISS(read);
6465 INS01 : ISS;
6466 ALU : EX1;
6467 %}
6468
6469 // Integer ALU reg-reg operation with extract
6470 // As for _vshift above, but result generated in EX2
6471 // Eg. EXTR x0, x1, x2, #N
6472 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6473 %{
6474 single_instruction;
6475 dst : EX2(write);
6476 src1 : ISS(read);
6477 src2 : ISS(read);
6478 INS1 : ISS; // Can only dual issue as Instruction 1
6479 ALU : EX1;
6480 %}
6481
6482 // Integer ALU reg operation
6483 // Eg. NEG x0, x1
6484 pipe_class ialu_reg(iRegI dst, iRegI src)
6485 %{
6486 single_instruction;
6487 dst : EX2(write);
6488 src : EX1(read);
6489 INS01 : ISS;
6490 ALU : EX2;
6491 %}
6492
6493 // Integer ALU reg mmediate operation
6494 // Eg. ADD x0, x1, #N
6495 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6496 %{
6497 single_instruction;
6498 dst : EX2(write);
6499 src1 : EX1(read);
6500 INS01 : ISS;
6501 ALU : EX2;
6502 %}
6503
6504 // Integer ALU immediate operation (no source operands)
6505 // Eg. MOV x0, #N
6506 pipe_class ialu_imm(iRegI dst)
6507 %{
6508 single_instruction;
6509 dst : EX1(write);
6510 INS01 : ISS;
6511 ALU : EX1;
6512 %}
6513
6514 //------- Compare operation -------------------------------
6515
6516 // Compare reg-reg
6517 // Eg. CMP x0, x1
6518 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6519 %{
6520 single_instruction;
6521 // fixed_latency(16);
6522 cr : EX2(write);
6523 op1 : EX1(read);
6524 op2 : EX1(read);
6525 INS01 : ISS;
6526 ALU : EX2;
6527 %}
6528
6529 // Compare reg-reg
6530 // Eg. CMP x0, #N
6531 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6532 %{
6533 single_instruction;
6534 // fixed_latency(16);
6535 cr : EX2(write);
6536 op1 : EX1(read);
6537 INS01 : ISS;
6538 ALU : EX2;
6539 %}
6540
6541 //------- Conditional instructions ------------------------
6542
6543 // Conditional no operands
6544 // Eg. CSINC x0, zr, zr, <cond>
6545 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6546 %{
6547 single_instruction;
6548 cr : EX1(read);
6549 dst : EX2(write);
6550 INS01 : ISS;
6551 ALU : EX2;
6552 %}
6553
6554 // Conditional 2 operand
6555 // EG. CSEL X0, X1, X2, <cond>
6556 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6557 %{
6558 single_instruction;
6559 cr : EX1(read);
6560 src1 : EX1(read);
6561 src2 : EX1(read);
6562 dst : EX2(write);
6563 INS01 : ISS;
6564 ALU : EX2;
6565 %}
6566
6567 // Conditional 2 operand
6568 // EG. CSEL X0, X1, X2, <cond>
6569 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6570 %{
6571 single_instruction;
6572 cr : EX1(read);
6573 src : EX1(read);
6574 dst : EX2(write);
6575 INS01 : ISS;
6576 ALU : EX2;
6577 %}
6578
6579 //------- Multiply pipeline operations --------------------
6580
6581 // Multiply reg-reg
6582 // Eg. MUL w0, w1, w2
6583 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6584 %{
6585 single_instruction;
6586 dst : WR(write);
6587 src1 : ISS(read);
6588 src2 : ISS(read);
6589 INS01 : ISS;
6590 MAC : WR;
6591 %}
6592
6593 // Multiply accumulate
6594 // Eg. MADD w0, w1, w2, w3
6595 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6596 %{
6597 single_instruction;
6598 dst : WR(write);
6599 src1 : ISS(read);
6600 src2 : ISS(read);
6601 src3 : ISS(read);
6602 INS01 : ISS;
6603 MAC : WR;
6604 %}
6605
6606 // Eg. MUL w0, w1, w2
6607 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6608 %{
6609 single_instruction;
6610 fixed_latency(3); // Maximum latency for 64 bit mul
6611 dst : WR(write);
6612 src1 : ISS(read);
6613 src2 : ISS(read);
6614 INS01 : ISS;
6615 MAC : WR;
6616 %}
6617
6618 // Multiply accumulate
6619 // Eg. MADD w0, w1, w2, w3
6620 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6621 %{
6622 single_instruction;
6623 fixed_latency(3); // Maximum latency for 64 bit mul
6624 dst : WR(write);
6625 src1 : ISS(read);
6626 src2 : ISS(read);
6627 src3 : ISS(read);
6628 INS01 : ISS;
6629 MAC : WR;
6630 %}
6631
6632 //------- Divide pipeline operations --------------------
6633
6634 // Eg. SDIV w0, w1, w2
6635 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6636 %{
6637 single_instruction;
6638 fixed_latency(8); // Maximum latency for 32 bit divide
6639 dst : WR(write);
6640 src1 : ISS(read);
6641 src2 : ISS(read);
6642 INS0 : ISS; // Can only dual issue as instruction 0
6643 DIV : WR;
6644 %}
6645
6646 // Eg. SDIV x0, x1, x2
6647 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6648 %{
6649 single_instruction;
6650 fixed_latency(16); // Maximum latency for 64 bit divide
6651 dst : WR(write);
6652 src1 : ISS(read);
6653 src2 : ISS(read);
6654 INS0 : ISS; // Can only dual issue as instruction 0
6655 DIV : WR;
6656 %}
6657
6658 //------- Load pipeline operations ------------------------
6659
6660 // Load - prefetch
6661 // Eg. PFRM <mem>
6662 pipe_class iload_prefetch(memory mem)
6663 %{
6664 single_instruction;
6665 mem : ISS(read);
6666 INS01 : ISS;
6667 LDST : WR;
6668 %}
6669
6670 // Load - reg, mem
6671 // Eg. LDR x0, <mem>
6672 pipe_class iload_reg_mem(iRegI dst, memory mem)
6673 %{
6674 single_instruction;
6675 dst : WR(write);
6676 mem : ISS(read);
6677 INS01 : ISS;
6678 LDST : WR;
6679 %}
6680
6681 // Load - reg, reg
6682 // Eg. LDR x0, [sp, x1]
6683 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6684 %{
6685 single_instruction;
6686 dst : WR(write);
6687 src : ISS(read);
6688 INS01 : ISS;
6689 LDST : WR;
6690 %}
6691
6692 //------- Store pipeline operations -----------------------
6693
6694 // Store - zr, mem
6695 // Eg. STR zr, <mem>
6696 pipe_class istore_mem(memory mem)
6697 %{
6698 single_instruction;
6699 mem : ISS(read);
6700 INS01 : ISS;
6701 LDST : WR;
6702 %}
6703
6704 // Store - reg, mem
6705 // Eg. STR x0, <mem>
6706 pipe_class istore_reg_mem(iRegI src, memory mem)
6707 %{
6708 single_instruction;
6709 mem : ISS(read);
6710 src : EX2(read);
6711 INS01 : ISS;
6712 LDST : WR;
6713 %}
6714
6715 // Store - reg, reg
6716 // Eg. STR x0, [sp, x1]
6717 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6718 %{
6719 single_instruction;
6720 dst : ISS(read);
6721 src : EX2(read);
6722 INS01 : ISS;
6723 LDST : WR;
6724 %}
6725
6726 //------- Store pipeline operations -----------------------
6727
6728 // Branch
6729 pipe_class pipe_branch()
6730 %{
6731 single_instruction;
6732 INS01 : ISS;
6733 BRANCH : EX1;
6734 %}
6735
6736 // Conditional branch
6737 pipe_class pipe_branch_cond(rFlagsReg cr)
6738 %{
6739 single_instruction;
6740 cr : EX1(read);
6741 INS01 : ISS;
6742 BRANCH : EX1;
6743 %}
6744
6745 // Compare & Branch
6746 // EG. CBZ/CBNZ
6747 pipe_class pipe_cmp_branch(iRegI op1)
6748 %{
6749 single_instruction;
6750 op1 : EX1(read);
6751 INS01 : ISS;
6752 BRANCH : EX1;
6753 %}
6754
6755 //------- Synchronisation operations ----------------------
6756
6757 // Any operation requiring serialization.
6758 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6759 pipe_class pipe_serial()
6760 %{
6761 single_instruction;
6762 force_serialization;
6763 fixed_latency(16);
6764 INS01 : ISS(2); // Cannot dual issue with any other instruction
6765 LDST : WR;
6766 %}
6767
6768 // Generic big/slow expanded idiom - also serialized
6769 pipe_class pipe_slow()
6770 %{
6771 instruction_count(10);
6772 multiple_bundles;
6773 force_serialization;
6774 fixed_latency(16);
6775 INS01 : ISS(2); // Cannot dual issue with any other instruction
6776 LDST : WR;
6777 %}
6778
6779 // Empty pipeline class
6780 pipe_class pipe_class_empty()
6781 %{
6782 single_instruction;
6783 fixed_latency(0);
6784 %}
6785
6786 // Default pipeline class.
6787 pipe_class pipe_class_default()
6788 %{
6789 single_instruction;
6790 fixed_latency(2);
6791 %}
6792
6793 // Pipeline class for compares.
6794 pipe_class pipe_class_compare()
6795 %{
6796 single_instruction;
6797 fixed_latency(16);
6798 %}
6799
6800 // Pipeline class for memory operations.
6801 pipe_class pipe_class_memory()
6802 %{
6803 single_instruction;
6804 fixed_latency(16);
6805 %}
6806
6807 // Pipeline class for call.
6808 pipe_class pipe_class_call()
6809 %{
6810 single_instruction;
6811 fixed_latency(100);
6812 %}
6813
6814 // Define the class for the Nop node.
6815 define %{
6816 MachNop = pipe_class_empty;
6817 %}
6818
6819 %}
6820 //----------INSTRUCTIONS-------------------------------------------------------
6821 //
6822 // match -- States which machine-independent subtree may be replaced
6823 // by this instruction.
6824 // ins_cost -- The estimated cost of this instruction is used by instruction
6825 // selection to identify a minimum cost tree of machine
6826 // instructions that matches a tree of machine-independent
6827 // instructions.
6828 // format -- A string providing the disassembly for this instruction.
6829 // The value of an instruction's operand may be inserted
6830 // by referring to it with a '$' prefix.
6831 // opcode -- Three instruction opcodes may be provided. These are referred
6832 // to within an encode class as $primary, $secondary, and $tertiary
6833 // rrspectively. The primary opcode is commonly used to
6834 // indicate the type of machine instruction, while secondary
6835 // and tertiary are often used for prefix options or addressing
6836 // modes.
6837 // ins_encode -- A list of encode classes with parameters. The encode class
6838 // name must have been defined in an 'enc_class' specification
6839 // in the encode section of the architecture description.
6840
6841 // ============================================================================
6842 // Memory (Load/Store) Instructions
6843
6844 // Load Instructions
6845
6846 // Load Byte (8 bit signed)
6847 instruct loadB(iRegINoSp dst, memory1 mem)
6848 %{
6849 match(Set dst (LoadB mem));
6850 predicate(!needs_acquiring_load(n));
6851
6852 ins_cost(4 * INSN_COST);
6853 format %{ "ldrsbw $dst, $mem\t# byte" %}
6854
6855 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6856
6857 ins_pipe(iload_reg_mem);
6858 %}
6859
6860 // Load Byte (8 bit signed) into long
6861 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6862 %{
6863 match(Set dst (ConvI2L (LoadB mem)));
6864 predicate(!needs_acquiring_load(n->in(1)));
6865
6866 ins_cost(4 * INSN_COST);
6867 format %{ "ldrsb $dst, $mem\t# byte" %}
6868
6869 ins_encode(aarch64_enc_ldrsb(dst, mem));
6870
6871 ins_pipe(iload_reg_mem);
6872 %}
6873
6874 // Load Byte (8 bit unsigned)
6875 instruct loadUB(iRegINoSp dst, memory1 mem)
6876 %{
6877 match(Set dst (LoadUB mem));
6878 predicate(!needs_acquiring_load(n));
6879
6880 ins_cost(4 * INSN_COST);
6881 format %{ "ldrbw $dst, $mem\t# byte" %}
6882
6883 ins_encode(aarch64_enc_ldrb(dst, mem));
6884
6885 ins_pipe(iload_reg_mem);
6886 %}
6887
6888 // Load Byte (8 bit unsigned) into long
6889 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6890 %{
6891 match(Set dst (ConvI2L (LoadUB mem)));
6892 predicate(!needs_acquiring_load(n->in(1)));
6893
6894 ins_cost(4 * INSN_COST);
6895 format %{ "ldrb $dst, $mem\t# byte" %}
6896
6897 ins_encode(aarch64_enc_ldrb(dst, mem));
6898
6899 ins_pipe(iload_reg_mem);
6900 %}
6901
6902 // Load Short (16 bit signed)
6903 instruct loadS(iRegINoSp dst, memory2 mem)
6904 %{
6905 match(Set dst (LoadS mem));
6906 predicate(!needs_acquiring_load(n));
6907
6908 ins_cost(4 * INSN_COST);
6909 format %{ "ldrshw $dst, $mem\t# short" %}
6910
6911 ins_encode(aarch64_enc_ldrshw(dst, mem));
6912
6913 ins_pipe(iload_reg_mem);
6914 %}
6915
6916 // Load Short (16 bit signed) into long
6917 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6918 %{
6919 match(Set dst (ConvI2L (LoadS mem)));
6920 predicate(!needs_acquiring_load(n->in(1)));
6921
6922 ins_cost(4 * INSN_COST);
6923 format %{ "ldrsh $dst, $mem\t# short" %}
6924
6925 ins_encode(aarch64_enc_ldrsh(dst, mem));
6926
6927 ins_pipe(iload_reg_mem);
6928 %}
6929
6930 // Load Char (16 bit unsigned)
6931 instruct loadUS(iRegINoSp dst, memory2 mem)
6932 %{
6933 match(Set dst (LoadUS mem));
6934 predicate(!needs_acquiring_load(n));
6935
6936 ins_cost(4 * INSN_COST);
6937 format %{ "ldrh $dst, $mem\t# short" %}
6938
6939 ins_encode(aarch64_enc_ldrh(dst, mem));
6940
6941 ins_pipe(iload_reg_mem);
6942 %}
6943
6944 // Load Short/Char (16 bit unsigned) into long
6945 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6946 %{
6947 match(Set dst (ConvI2L (LoadUS mem)));
6948 predicate(!needs_acquiring_load(n->in(1)));
6949
6950 ins_cost(4 * INSN_COST);
6951 format %{ "ldrh $dst, $mem\t# short" %}
6952
6953 ins_encode(aarch64_enc_ldrh(dst, mem));
6954
6955 ins_pipe(iload_reg_mem);
6956 %}
6957
6958 // Load Integer (32 bit signed)
6959 instruct loadI(iRegINoSp dst, memory4 mem)
6960 %{
6961 match(Set dst (LoadI mem));
6962 predicate(!needs_acquiring_load(n));
6963
6964 ins_cost(4 * INSN_COST);
6965 format %{ "ldrw $dst, $mem\t# int" %}
6966
6967 ins_encode(aarch64_enc_ldrw(dst, mem));
6968
6969 ins_pipe(iload_reg_mem);
6970 %}
6971
6972 // Load Integer (32 bit signed) into long
6973 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6974 %{
6975 match(Set dst (ConvI2L (LoadI mem)));
6976 predicate(!needs_acquiring_load(n->in(1)));
6977
6978 ins_cost(4 * INSN_COST);
6979 format %{ "ldrsw $dst, $mem\t# int" %}
6980
6981 ins_encode(aarch64_enc_ldrsw(dst, mem));
6982
6983 ins_pipe(iload_reg_mem);
6984 %}
6985
6986 // Load Integer (32 bit unsigned) into long
6987 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6988 %{
6989 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6990 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6991
6992 ins_cost(4 * INSN_COST);
6993 format %{ "ldrw $dst, $mem\t# int" %}
6994
6995 ins_encode(aarch64_enc_ldrw(dst, mem));
6996
6997 ins_pipe(iload_reg_mem);
6998 %}
6999
7000 // Load Long (64 bit signed)
7001 instruct loadL(iRegLNoSp dst, memory8 mem)
7002 %{
7003 match(Set dst (LoadL mem));
7004 predicate(!needs_acquiring_load(n));
7005
7006 ins_cost(4 * INSN_COST);
7007 format %{ "ldr $dst, $mem\t# int" %}
7008
7009 ins_encode(aarch64_enc_ldr(dst, mem));
7010
7011 ins_pipe(iload_reg_mem);
7012 %}
7013
7014 // Load Range
7015 instruct loadRange(iRegINoSp dst, memory4 mem)
7016 %{
7017 match(Set dst (LoadRange mem));
7018
7019 ins_cost(4 * INSN_COST);
7020 format %{ "ldrw $dst, $mem\t# range" %}
7021
7022 ins_encode(aarch64_enc_ldrw(dst, mem));
7023
7024 ins_pipe(iload_reg_mem);
7025 %}
7026
7027 // Load Pointer
7028 instruct loadP(iRegPNoSp dst, memory8 mem)
7029 %{
7030 match(Set dst (LoadP mem));
7031 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7032
7033 ins_cost(4 * INSN_COST);
7034 format %{ "ldr $dst, $mem\t# ptr" %}
7035
7036 ins_encode(aarch64_enc_ldr(dst, mem));
7037
7038 ins_pipe(iload_reg_mem);
7039 %}
7040
7041 // Load Compressed Pointer
7042 instruct loadN(iRegNNoSp dst, memory4 mem)
7043 %{
7044 match(Set dst (LoadN mem));
7045 predicate(!needs_acquiring_load(n));
7046
7047 ins_cost(4 * INSN_COST);
7048 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7049
7050 ins_encode(aarch64_enc_ldrw(dst, mem));
7051
7052 ins_pipe(iload_reg_mem);
7053 %}
7054
7055 // Load Klass Pointer
7056 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7057 %{
7058 match(Set dst (LoadKlass mem));
7059 predicate(!needs_acquiring_load(n));
7060
7061 ins_cost(4 * INSN_COST);
7062 format %{ "ldr $dst, $mem\t# class" %}
7063
7064 ins_encode(aarch64_enc_ldr(dst, mem));
7065
7066 ins_pipe(iload_reg_mem);
7067 %}
7068
7069 // Load Narrow Klass Pointer
7070 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7071 %{
7072 match(Set dst (LoadNKlass mem));
7073 predicate(!needs_acquiring_load(n));
7074
7075 ins_cost(4 * INSN_COST);
7076 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7077
7078 ins_encode(aarch64_enc_ldrw(dst, mem));
7079
7080 ins_pipe(iload_reg_mem);
7081 %}
7082
7083 // Load Float
7084 instruct loadF(vRegF dst, memory4 mem)
7085 %{
7086 match(Set dst (LoadF mem));
7087 predicate(!needs_acquiring_load(n));
7088
7089 ins_cost(4 * INSN_COST);
7090 format %{ "ldrs $dst, $mem\t# float" %}
7091
7092 ins_encode( aarch64_enc_ldrs(dst, mem) );
7093
7094 ins_pipe(pipe_class_memory);
7095 %}
7096
7097 // Load Double
7098 instruct loadD(vRegD dst, memory8 mem)
7099 %{
7100 match(Set dst (LoadD mem));
7101 predicate(!needs_acquiring_load(n));
7102
7103 ins_cost(4 * INSN_COST);
7104 format %{ "ldrd $dst, $mem\t# double" %}
7105
7106 ins_encode( aarch64_enc_ldrd(dst, mem) );
7107
7108 ins_pipe(pipe_class_memory);
7109 %}
7110
7111
7112 // Load Int Constant
7113 instruct loadConI(iRegINoSp dst, immI src)
7114 %{
7115 match(Set dst src);
7116
7117 ins_cost(INSN_COST);
7118 format %{ "mov $dst, $src\t# int" %}
7119
7120 ins_encode( aarch64_enc_movw_imm(dst, src) );
7121
7122 ins_pipe(ialu_imm);
7123 %}
7124
7125 // Load Long Constant
7126 instruct loadConL(iRegLNoSp dst, immL src)
7127 %{
7128 match(Set dst src);
7129
7130 ins_cost(INSN_COST);
7131 format %{ "mov $dst, $src\t# long" %}
7132
7133 ins_encode( aarch64_enc_mov_imm(dst, src) );
7134
7135 ins_pipe(ialu_imm);
7136 %}
7137
7138 // Load Pointer Constant
7139
7140 instruct loadConP(iRegPNoSp dst, immP con)
7141 %{
7142 match(Set dst con);
7143
7144 ins_cost(INSN_COST * 4);
7145 format %{
7146 "mov $dst, $con\t# ptr\n\t"
7147 %}
7148
7149 ins_encode(aarch64_enc_mov_p(dst, con));
7150
7151 ins_pipe(ialu_imm);
7152 %}
7153
7154 // Load Null Pointer Constant
7155
7156 instruct loadConP0(iRegPNoSp dst, immP0 con)
7157 %{
7158 match(Set dst con);
7159
7160 ins_cost(INSN_COST);
7161 format %{ "mov $dst, $con\t# NULL ptr" %}
7162
7163 ins_encode(aarch64_enc_mov_p0(dst, con));
7164
7165 ins_pipe(ialu_imm);
7166 %}
7167
7168 // Load Pointer Constant One
7169
7170 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7171 %{
7172 match(Set dst con);
7173
7174 ins_cost(INSN_COST);
7175 format %{ "mov $dst, $con\t# NULL ptr" %}
7176
7177 ins_encode(aarch64_enc_mov_p1(dst, con));
7178
7179 ins_pipe(ialu_imm);
7180 %}
7181
7182 // Load Byte Map Base Constant
7183
7184 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7185 %{
7186 match(Set dst con);
7187
7188 ins_cost(INSN_COST);
7189 format %{ "adr $dst, $con\t# Byte Map Base" %}
7190
7191 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7192
7193 ins_pipe(ialu_imm);
7194 %}
7195
7196 // Load Narrow Pointer Constant
7197
7198 instruct loadConN(iRegNNoSp dst, immN con)
7199 %{
7200 match(Set dst con);
7201
7202 ins_cost(INSN_COST * 4);
7203 format %{ "mov $dst, $con\t# compressed ptr" %}
7204
7205 ins_encode(aarch64_enc_mov_n(dst, con));
7206
7207 ins_pipe(ialu_imm);
7208 %}
7209
7210 // Load Narrow Null Pointer Constant
7211
7212 instruct loadConN0(iRegNNoSp dst, immN0 con)
7213 %{
7214 match(Set dst con);
7215
7216 ins_cost(INSN_COST);
7217 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7218
7219 ins_encode(aarch64_enc_mov_n0(dst, con));
7220
7221 ins_pipe(ialu_imm);
7222 %}
7223
7224 // Load Narrow Klass Constant
7225
7226 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7227 %{
7228 match(Set dst con);
7229
7230 ins_cost(INSN_COST);
7231 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7232
7233 ins_encode(aarch64_enc_mov_nk(dst, con));
7234
7235 ins_pipe(ialu_imm);
7236 %}
7237
7238 // Load Packed Float Constant
7239
7240 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7241 match(Set dst con);
7242 ins_cost(INSN_COST * 4);
7243 format %{ "fmovs $dst, $con"%}
7244 ins_encode %{
7245 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7246 %}
7247
7248 ins_pipe(fp_imm_s);
7249 %}
7250
7251 // Load Float Constant
7252
7253 instruct loadConF(vRegF dst, immF con) %{
7254 match(Set dst con);
7255
7256 ins_cost(INSN_COST * 4);
7257
7258 format %{
7259 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7260 %}
7261
7262 ins_encode %{
7263 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7264 %}
7265
7266 ins_pipe(fp_load_constant_s);
7267 %}
7268
7269 // Load Packed Double Constant
7270
7271 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7272 match(Set dst con);
7273 ins_cost(INSN_COST);
7274 format %{ "fmovd $dst, $con"%}
7275 ins_encode %{
7276 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7277 %}
7278
7279 ins_pipe(fp_imm_d);
7280 %}
7281
7282 // Load Double Constant
7283
7284 instruct loadConD(vRegD dst, immD con) %{
7285 match(Set dst con);
7286
7287 ins_cost(INSN_COST * 5);
7288 format %{
7289 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7290 %}
7291
7292 ins_encode %{
7293 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7294 %}
7295
7296 ins_pipe(fp_load_constant_d);
7297 %}
7298
7299 // Store Instructions
7300
7301 // Store CMS card-mark Immediate
7302 instruct storeimmCM0(immI0 zero, memory1 mem)
7303 %{
7304 match(Set mem (StoreCM mem zero));
7305
7306 ins_cost(INSN_COST);
7307 format %{ "storestore (elided)\n\t"
7308 "strb zr, $mem\t# byte" %}
7309
7310 ins_encode(aarch64_enc_strb0(mem));
7311
7312 ins_pipe(istore_mem);
7313 %}
7314
7315 // Store CMS card-mark Immediate with intervening StoreStore
7316 // needed when using CMS with no conditional card marking
7317 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7318 %{
7319 match(Set mem (StoreCM mem zero));
7320
7321 ins_cost(INSN_COST * 2);
7322 format %{ "storestore\n\t"
7323 "dmb ishst"
7324 "\n\tstrb zr, $mem\t# byte" %}
7325
7326 ins_encode(aarch64_enc_strb0_ordered(mem));
7327
7328 ins_pipe(istore_mem);
7329 %}
7330
7331 // Store Byte
7332 instruct storeB(iRegIorL2I src, memory1 mem)
7333 %{
7334 match(Set mem (StoreB mem src));
7335 predicate(!needs_releasing_store(n));
7336
7337 ins_cost(INSN_COST);
7338 format %{ "strb $src, $mem\t# byte" %}
7339
7340 ins_encode(aarch64_enc_strb(src, mem));
7341
7342 ins_pipe(istore_reg_mem);
7343 %}
7344
7345
7346 instruct storeimmB0(immI0 zero, memory1 mem)
7347 %{
7348 match(Set mem (StoreB mem zero));
7349 predicate(!needs_releasing_store(n));
7350
7351 ins_cost(INSN_COST);
7352 format %{ "strb rscractch2, $mem\t# byte" %}
7353
7354 ins_encode(aarch64_enc_strb0(mem));
7355
7356 ins_pipe(istore_mem);
7357 %}
7358
7359 // Store Char/Short
7360 instruct storeC(iRegIorL2I src, memory2 mem)
7361 %{
7362 match(Set mem (StoreC mem src));
7363 predicate(!needs_releasing_store(n));
7364
7365 ins_cost(INSN_COST);
7366 format %{ "strh $src, $mem\t# short" %}
7367
7368 ins_encode(aarch64_enc_strh(src, mem));
7369
7370 ins_pipe(istore_reg_mem);
7371 %}
7372
7373 instruct storeimmC0(immI0 zero, memory2 mem)
7374 %{
7375 match(Set mem (StoreC mem zero));
7376 predicate(!needs_releasing_store(n));
7377
7378 ins_cost(INSN_COST);
7379 format %{ "strh zr, $mem\t# short" %}
7380
7381 ins_encode(aarch64_enc_strh0(mem));
7382
7383 ins_pipe(istore_mem);
7384 %}
7385
7386 // Store Integer
7387
7388 instruct storeI(iRegIorL2I src, memory4 mem)
7389 %{
7390 match(Set mem(StoreI mem src));
7391 predicate(!needs_releasing_store(n));
7392
7393 ins_cost(INSN_COST);
7394 format %{ "strw $src, $mem\t# int" %}
7395
7396 ins_encode(aarch64_enc_strw(src, mem));
7397
7398 ins_pipe(istore_reg_mem);
7399 %}
7400
7401 instruct storeimmI0(immI0 zero, memory4 mem)
7402 %{
7403 match(Set mem(StoreI mem zero));
7404 predicate(!needs_releasing_store(n));
7405
7406 ins_cost(INSN_COST);
7407 format %{ "strw zr, $mem\t# int" %}
7408
7409 ins_encode(aarch64_enc_strw0(mem));
7410
7411 ins_pipe(istore_mem);
7412 %}
7413
7414 // Store Long (64 bit signed)
7415 instruct storeL(iRegL src, memory8 mem)
7416 %{
7417 match(Set mem (StoreL mem src));
7418 predicate(!needs_releasing_store(n));
7419
7420 ins_cost(INSN_COST);
7421 format %{ "str $src, $mem\t# int" %}
7422
7423 ins_encode(aarch64_enc_str(src, mem));
7424
7425 ins_pipe(istore_reg_mem);
7426 %}
7427
7428 // Store Long (64 bit signed)
7429 instruct storeimmL0(immL0 zero, memory8 mem)
7430 %{
7431 match(Set mem (StoreL mem zero));
7432 predicate(!needs_releasing_store(n));
7433
7434 ins_cost(INSN_COST);
7435 format %{ "str zr, $mem\t# int" %}
7436
7437 ins_encode(aarch64_enc_str0(mem));
7438
7439 ins_pipe(istore_mem);
7440 %}
7441
7442 // Store Pointer
7443 instruct storeP(iRegP src, memory8 mem)
7444 %{
7445 match(Set mem (StoreP mem src));
7446 predicate(!needs_releasing_store(n));
7447
7448 ins_cost(INSN_COST);
7449 format %{ "str $src, $mem\t# ptr" %}
7450
7451 ins_encode(aarch64_enc_str(src, mem));
7452
7453 ins_pipe(istore_reg_mem);
7454 %}
7455
7456 // Store Pointer
7457 instruct storeimmP0(immP0 zero, memory8 mem)
7458 %{
7459 match(Set mem (StoreP mem zero));
7460 predicate(!needs_releasing_store(n));
7461
7462 ins_cost(INSN_COST);
7463 format %{ "str zr, $mem\t# ptr" %}
7464
7465 ins_encode(aarch64_enc_str0(mem));
7466
7467 ins_pipe(istore_mem);
7468 %}
7469
7470 // Store Compressed Pointer
7471 instruct storeN(iRegN src, memory4 mem)
7472 %{
7473 match(Set mem (StoreN mem src));
7474 predicate(!needs_releasing_store(n));
7475
7476 ins_cost(INSN_COST);
7477 format %{ "strw $src, $mem\t# compressed ptr" %}
7478
7479 ins_encode(aarch64_enc_strw(src, mem));
7480
7481 ins_pipe(istore_reg_mem);
7482 %}
7483
7484 instruct storeImmN0(immN0 zero, memory4 mem)
7485 %{
7486 match(Set mem (StoreN mem zero));
7487 predicate(!needs_releasing_store(n));
7488
7489 ins_cost(INSN_COST);
7490 format %{ "strw zr, $mem\t# compressed ptr" %}
7491
7492 ins_encode(aarch64_enc_strw0(mem));
7493
7494 ins_pipe(istore_mem);
7495 %}
7496
7497 // Store Float
7498 instruct storeF(vRegF src, memory4 mem)
7499 %{
7500 match(Set mem (StoreF mem src));
7501 predicate(!needs_releasing_store(n));
7502
7503 ins_cost(INSN_COST);
7504 format %{ "strs $src, $mem\t# float" %}
7505
7506 ins_encode( aarch64_enc_strs(src, mem) );
7507
7508 ins_pipe(pipe_class_memory);
7509 %}
7510
7511 // TODO
7512 // implement storeImmF0 and storeFImmPacked
7513
7514 // Store Double
7515 instruct storeD(vRegD src, memory8 mem)
7516 %{
7517 match(Set mem (StoreD mem src));
7518 predicate(!needs_releasing_store(n));
7519
7520 ins_cost(INSN_COST);
7521 format %{ "strd $src, $mem\t# double" %}
7522
7523 ins_encode( aarch64_enc_strd(src, mem) );
7524
7525 ins_pipe(pipe_class_memory);
7526 %}
7527
7528 // Store Compressed Klass Pointer
7529 instruct storeNKlass(iRegN src, memory4 mem)
7530 %{
7531 predicate(!needs_releasing_store(n));
7532 match(Set mem (StoreNKlass mem src));
7533
7534 ins_cost(INSN_COST);
7535 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7536
7537 ins_encode(aarch64_enc_strw(src, mem));
7538
7539 ins_pipe(istore_reg_mem);
7540 %}
7541
7542 // TODO
7543 // implement storeImmD0 and storeDImmPacked
7544
7545 // prefetch instructions
7546 // Must be safe to execute with invalid address (cannot fault).
7547
7548 instruct prefetchalloc( memory8 mem ) %{
7549 match(PrefetchAllocation mem);
7550
7551 ins_cost(INSN_COST);
7552 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7553
7554 ins_encode( aarch64_enc_prefetchw(mem) );
7555
7556 ins_pipe(iload_prefetch);
7557 %}
7558
7559 // ---------------- volatile loads and stores ----------------
7560
7561 // Load Byte (8 bit signed)
7562 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7563 %{
7564 match(Set dst (LoadB mem));
7565
7566 ins_cost(VOLATILE_REF_COST);
7567 format %{ "ldarsb $dst, $mem\t# byte" %}
7568
7569 ins_encode(aarch64_enc_ldarsb(dst, mem));
7570
7571 ins_pipe(pipe_serial);
7572 %}
7573
7574 // Load Byte (8 bit signed) into long
7575 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7576 %{
7577 match(Set dst (ConvI2L (LoadB mem)));
7578
7579 ins_cost(VOLATILE_REF_COST);
7580 format %{ "ldarsb $dst, $mem\t# byte" %}
7581
7582 ins_encode(aarch64_enc_ldarsb(dst, mem));
7583
7584 ins_pipe(pipe_serial);
7585 %}
7586
7587 // Load Byte (8 bit unsigned)
7588 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7589 %{
7590 match(Set dst (LoadUB mem));
7591
7592 ins_cost(VOLATILE_REF_COST);
7593 format %{ "ldarb $dst, $mem\t# byte" %}
7594
7595 ins_encode(aarch64_enc_ldarb(dst, mem));
7596
7597 ins_pipe(pipe_serial);
7598 %}
7599
7600 // Load Byte (8 bit unsigned) into long
7601 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7602 %{
7603 match(Set dst (ConvI2L (LoadUB mem)));
7604
7605 ins_cost(VOLATILE_REF_COST);
7606 format %{ "ldarb $dst, $mem\t# byte" %}
7607
7608 ins_encode(aarch64_enc_ldarb(dst, mem));
7609
7610 ins_pipe(pipe_serial);
7611 %}
7612
7613 // Load Short (16 bit signed)
7614 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7615 %{
7616 match(Set dst (LoadS mem));
7617
7618 ins_cost(VOLATILE_REF_COST);
7619 format %{ "ldarshw $dst, $mem\t# short" %}
7620
7621 ins_encode(aarch64_enc_ldarshw(dst, mem));
7622
7623 ins_pipe(pipe_serial);
7624 %}
7625
7626 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7627 %{
7628 match(Set dst (LoadUS mem));
7629
7630 ins_cost(VOLATILE_REF_COST);
7631 format %{ "ldarhw $dst, $mem\t# short" %}
7632
7633 ins_encode(aarch64_enc_ldarhw(dst, mem));
7634
7635 ins_pipe(pipe_serial);
7636 %}
7637
7638 // Load Short/Char (16 bit unsigned) into long
7639 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7640 %{
7641 match(Set dst (ConvI2L (LoadUS mem)));
7642
7643 ins_cost(VOLATILE_REF_COST);
7644 format %{ "ldarh $dst, $mem\t# short" %}
7645
7646 ins_encode(aarch64_enc_ldarh(dst, mem));
7647
7648 ins_pipe(pipe_serial);
7649 %}
7650
7651 // Load Short/Char (16 bit signed) into long
7652 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7653 %{
7654 match(Set dst (ConvI2L (LoadS mem)));
7655
7656 ins_cost(VOLATILE_REF_COST);
7657 format %{ "ldarh $dst, $mem\t# short" %}
7658
7659 ins_encode(aarch64_enc_ldarsh(dst, mem));
7660
7661 ins_pipe(pipe_serial);
7662 %}
7663
7664 // Load Integer (32 bit signed)
7665 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7666 %{
7667 match(Set dst (LoadI mem));
7668
7669 ins_cost(VOLATILE_REF_COST);
7670 format %{ "ldarw $dst, $mem\t# int" %}
7671
7672 ins_encode(aarch64_enc_ldarw(dst, mem));
7673
7674 ins_pipe(pipe_serial);
7675 %}
7676
7677 // Load Integer (32 bit unsigned) into long
7678 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7679 %{
7680 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7681
7682 ins_cost(VOLATILE_REF_COST);
7683 format %{ "ldarw $dst, $mem\t# int" %}
7684
7685 ins_encode(aarch64_enc_ldarw(dst, mem));
7686
7687 ins_pipe(pipe_serial);
7688 %}
7689
7690 // Load Long (64 bit signed)
7691 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7692 %{
7693 match(Set dst (LoadL mem));
7694
7695 ins_cost(VOLATILE_REF_COST);
7696 format %{ "ldar $dst, $mem\t# int" %}
7697
7698 ins_encode(aarch64_enc_ldar(dst, mem));
7699
7700 ins_pipe(pipe_serial);
7701 %}
7702
7703 // Load Pointer
7704 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7705 %{
7706 match(Set dst (LoadP mem));
7707 predicate(n->as_Load()->barrier_data() == 0);
7708
7709 ins_cost(VOLATILE_REF_COST);
7710 format %{ "ldar $dst, $mem\t# ptr" %}
7711
7712 ins_encode(aarch64_enc_ldar(dst, mem));
7713
7714 ins_pipe(pipe_serial);
7715 %}
7716
7717 // Load Compressed Pointer
7718 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7719 %{
7720 match(Set dst (LoadN mem));
7721
7722 ins_cost(VOLATILE_REF_COST);
7723 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7724
7725 ins_encode(aarch64_enc_ldarw(dst, mem));
7726
7727 ins_pipe(pipe_serial);
7728 %}
7729
7730 // Load Float
7731 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7732 %{
7733 match(Set dst (LoadF mem));
7734
7735 ins_cost(VOLATILE_REF_COST);
7736 format %{ "ldars $dst, $mem\t# float" %}
7737
7738 ins_encode( aarch64_enc_fldars(dst, mem) );
7739
7740 ins_pipe(pipe_serial);
7741 %}
7742
7743 // Load Double
7744 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7745 %{
7746 match(Set dst (LoadD mem));
7747
7748 ins_cost(VOLATILE_REF_COST);
7749 format %{ "ldard $dst, $mem\t# double" %}
7750
7751 ins_encode( aarch64_enc_fldard(dst, mem) );
7752
7753 ins_pipe(pipe_serial);
7754 %}
7755
7756 // Store Byte
7757 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7758 %{
7759 match(Set mem (StoreB mem src));
7760
7761 ins_cost(VOLATILE_REF_COST);
7762 format %{ "stlrb $src, $mem\t# byte" %}
7763
7764 ins_encode(aarch64_enc_stlrb(src, mem));
7765
7766 ins_pipe(pipe_class_memory);
7767 %}
7768
7769 // Store Char/Short
7770 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7771 %{
7772 match(Set mem (StoreC mem src));
7773
7774 ins_cost(VOLATILE_REF_COST);
7775 format %{ "stlrh $src, $mem\t# short" %}
7776
7777 ins_encode(aarch64_enc_stlrh(src, mem));
7778
7779 ins_pipe(pipe_class_memory);
7780 %}
7781
7782 // Store Integer
7783
7784 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7785 %{
7786 match(Set mem(StoreI mem src));
7787
7788 ins_cost(VOLATILE_REF_COST);
7789 format %{ "stlrw $src, $mem\t# int" %}
7790
7791 ins_encode(aarch64_enc_stlrw(src, mem));
7792
7793 ins_pipe(pipe_class_memory);
7794 %}
7795
7796 // Store Long (64 bit signed)
7797 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7798 %{
7799 match(Set mem (StoreL mem src));
7800
7801 ins_cost(VOLATILE_REF_COST);
7802 format %{ "stlr $src, $mem\t# int" %}
7803
7804 ins_encode(aarch64_enc_stlr(src, mem));
7805
7806 ins_pipe(pipe_class_memory);
7807 %}
7808
7809 // Store Pointer
7810 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7811 %{
7812 match(Set mem (StoreP mem src));
7813
7814 ins_cost(VOLATILE_REF_COST);
7815 format %{ "stlr $src, $mem\t# ptr" %}
7816
7817 ins_encode(aarch64_enc_stlr(src, mem));
7818
7819 ins_pipe(pipe_class_memory);
7820 %}
7821
7822 // Store Compressed Pointer
7823 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7824 %{
7825 match(Set mem (StoreN mem src));
7826
7827 ins_cost(VOLATILE_REF_COST);
7828 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7829
7830 ins_encode(aarch64_enc_stlrw(src, mem));
7831
7832 ins_pipe(pipe_class_memory);
7833 %}
7834
7835 // Store Float
7836 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7837 %{
7838 match(Set mem (StoreF mem src));
7839
7840 ins_cost(VOLATILE_REF_COST);
7841 format %{ "stlrs $src, $mem\t# float" %}
7842
7843 ins_encode( aarch64_enc_fstlrs(src, mem) );
7844
7845 ins_pipe(pipe_class_memory);
7846 %}
7847
7848 // TODO
7849 // implement storeImmF0 and storeFImmPacked
7850
7851 // Store Double
7852 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7853 %{
7854 match(Set mem (StoreD mem src));
7855
7856 ins_cost(VOLATILE_REF_COST);
7857 format %{ "stlrd $src, $mem\t# double" %}
7858
7859 ins_encode( aarch64_enc_fstlrd(src, mem) );
7860
7861 ins_pipe(pipe_class_memory);
7862 %}
7863
7864 // ---------------- end of volatile loads and stores ----------------
7865
7866 instruct cacheWB(indirect addr)
7867 %{
7868 predicate(VM_Version::supports_data_cache_line_flush());
7869 match(CacheWB addr);
7870
7871 ins_cost(100);
7872 format %{"cache wb $addr" %}
7873 ins_encode %{
7874 assert($addr->index_position() < 0, "should be");
7875 assert($addr$$disp == 0, "should be");
7876 __ cache_wb(Address($addr$$base$$Register, 0));
7877 %}
7878 ins_pipe(pipe_slow); // XXX
7879 %}
7880
7881 instruct cacheWBPreSync()
7882 %{
7883 predicate(VM_Version::supports_data_cache_line_flush());
7884 match(CacheWBPreSync);
7885
7886 ins_cost(100);
7887 format %{"cache wb presync" %}
7888 ins_encode %{
7889 __ cache_wbsync(true);
7890 %}
7891 ins_pipe(pipe_slow); // XXX
7892 %}
7893
7894 instruct cacheWBPostSync()
7895 %{
7896 predicate(VM_Version::supports_data_cache_line_flush());
7897 match(CacheWBPostSync);
7898
7899 ins_cost(100);
7900 format %{"cache wb postsync" %}
7901 ins_encode %{
7902 __ cache_wbsync(false);
7903 %}
7904 ins_pipe(pipe_slow); // XXX
7905 %}
7906
7907 // ============================================================================
7908 // BSWAP Instructions
7909
7910 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7911 match(Set dst (ReverseBytesI src));
7912
7913 ins_cost(INSN_COST);
7914 format %{ "revw $dst, $src" %}
7915
7916 ins_encode %{
7917 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7918 %}
7919
7920 ins_pipe(ialu_reg);
7921 %}
7922
7923 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7924 match(Set dst (ReverseBytesL src));
7925
7926 ins_cost(INSN_COST);
7927 format %{ "rev $dst, $src" %}
7928
7929 ins_encode %{
7930 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7931 %}
7932
7933 ins_pipe(ialu_reg);
7934 %}
7935
7936 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7937 match(Set dst (ReverseBytesUS src));
7938
7939 ins_cost(INSN_COST);
7940 format %{ "rev16w $dst, $src" %}
7941
7942 ins_encode %{
7943 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7944 %}
7945
7946 ins_pipe(ialu_reg);
7947 %}
7948
7949 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7950 match(Set dst (ReverseBytesS src));
7951
7952 ins_cost(INSN_COST);
7953 format %{ "rev16w $dst, $src\n\t"
7954 "sbfmw $dst, $dst, #0, #15" %}
7955
7956 ins_encode %{
7957 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7958 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7959 %}
7960
7961 ins_pipe(ialu_reg);
7962 %}
7963
7964 // ============================================================================
7965 // Zero Count Instructions
7966
7967 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7968 match(Set dst (CountLeadingZerosI src));
7969
7970 ins_cost(INSN_COST);
7971 format %{ "clzw $dst, $src" %}
7972 ins_encode %{
7973 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7974 %}
7975
7976 ins_pipe(ialu_reg);
7977 %}
7978
7979 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7980 match(Set dst (CountLeadingZerosL src));
7981
7982 ins_cost(INSN_COST);
7983 format %{ "clz $dst, $src" %}
7984 ins_encode %{
7985 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7986 %}
7987
7988 ins_pipe(ialu_reg);
7989 %}
7990
7991 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7992 match(Set dst (CountTrailingZerosI src));
7993
7994 ins_cost(INSN_COST * 2);
7995 format %{ "rbitw $dst, $src\n\t"
7996 "clzw $dst, $dst" %}
7997 ins_encode %{
7998 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7999 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8000 %}
8001
8002 ins_pipe(ialu_reg);
8003 %}
8004
8005 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8006 match(Set dst (CountTrailingZerosL src));
8007
8008 ins_cost(INSN_COST * 2);
8009 format %{ "rbit $dst, $src\n\t"
8010 "clz $dst, $dst" %}
8011 ins_encode %{
8012 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8013 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8014 %}
8015
8016 ins_pipe(ialu_reg);
8017 %}
8018
8019 //---------- Population Count Instructions -------------------------------------
8020 //
8021
8022 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8023 predicate(UsePopCountInstruction);
8024 match(Set dst (PopCountI src));
8025 effect(TEMP tmp);
8026 ins_cost(INSN_COST * 13);
8027
8028 format %{ "movw $src, $src\n\t"
8029 "mov $tmp, $src\t# vector (1D)\n\t"
8030 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8031 "addv $tmp, $tmp\t# vector (8B)\n\t"
8032 "mov $dst, $tmp\t# vector (1D)" %}
8033 ins_encode %{
8034 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8035 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
8036 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8037 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8038 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8039 %}
8040
8041 ins_pipe(pipe_class_default);
8042 %}
8043
8044 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8045 predicate(UsePopCountInstruction);
8046 match(Set dst (PopCountI (LoadI mem)));
8047 effect(TEMP tmp);
8048 ins_cost(INSN_COST * 13);
8049
8050 format %{ "ldrs $tmp, $mem\n\t"
8051 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8052 "addv $tmp, $tmp\t# vector (8B)\n\t"
8053 "mov $dst, $tmp\t# vector (1D)" %}
8054 ins_encode %{
8055 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8056 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8057 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8058 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8059 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8060 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8061 %}
8062
8063 ins_pipe(pipe_class_default);
8064 %}
8065
8066 // Note: Long.bitCount(long) returns an int.
8067 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8068 predicate(UsePopCountInstruction);
8069 match(Set dst (PopCountL src));
8070 effect(TEMP tmp);
8071 ins_cost(INSN_COST * 13);
8072
8073 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8074 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8075 "addv $tmp, $tmp\t# vector (8B)\n\t"
8076 "mov $dst, $tmp\t# vector (1D)" %}
8077 ins_encode %{
8078 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
8079 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8080 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8081 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8082 %}
8083
8084 ins_pipe(pipe_class_default);
8085 %}
8086
8087 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8088 predicate(UsePopCountInstruction);
8089 match(Set dst (PopCountL (LoadL mem)));
8090 effect(TEMP tmp);
8091 ins_cost(INSN_COST * 13);
8092
8093 format %{ "ldrd $tmp, $mem\n\t"
8094 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8095 "addv $tmp, $tmp\t# vector (8B)\n\t"
8096 "mov $dst, $tmp\t# vector (1D)" %}
8097 ins_encode %{
8098 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8099 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8100 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8101 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8102 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8103 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8104 %}
8105
8106 ins_pipe(pipe_class_default);
8107 %}
8108
8109 // ============================================================================
8110 // MemBar Instruction
8111
8112 instruct load_fence() %{
8113 match(LoadFence);
8114 ins_cost(VOLATILE_REF_COST);
8115
8116 format %{ "load_fence" %}
8117
8118 ins_encode %{
8119 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8120 %}
8121 ins_pipe(pipe_serial);
8122 %}
8123
8124 instruct unnecessary_membar_acquire() %{
8125 predicate(unnecessary_acquire(n));
8126 match(MemBarAcquire);
8127 ins_cost(0);
8128
8129 format %{ "membar_acquire (elided)" %}
8130
8131 ins_encode %{
8132 __ block_comment("membar_acquire (elided)");
8133 %}
8134
8135 ins_pipe(pipe_class_empty);
8136 %}
8137
8138 instruct membar_acquire() %{
8139 match(MemBarAcquire);
8140 ins_cost(VOLATILE_REF_COST);
8141
8142 format %{ "membar_acquire\n\t"
8143 "dmb ish" %}
8144
8145 ins_encode %{
8146 __ block_comment("membar_acquire");
8147 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8148 %}
8149
8150 ins_pipe(pipe_serial);
8151 %}
8152
8153
8154 instruct membar_acquire_lock() %{
8155 match(MemBarAcquireLock);
8156 ins_cost(VOLATILE_REF_COST);
8157
8158 format %{ "membar_acquire_lock (elided)" %}
8159
8160 ins_encode %{
8161 __ block_comment("membar_acquire_lock (elided)");
8162 %}
8163
8164 ins_pipe(pipe_serial);
8165 %}
8166
8167 instruct store_fence() %{
8168 match(StoreFence);
8169 ins_cost(VOLATILE_REF_COST);
8170
8171 format %{ "store_fence" %}
8172
8173 ins_encode %{
8174 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8175 %}
8176 ins_pipe(pipe_serial);
8177 %}
8178
8179 instruct unnecessary_membar_release() %{
8180 predicate(unnecessary_release(n));
8181 match(MemBarRelease);
8182 ins_cost(0);
8183
8184 format %{ "membar_release (elided)" %}
8185
8186 ins_encode %{
8187 __ block_comment("membar_release (elided)");
8188 %}
8189 ins_pipe(pipe_serial);
8190 %}
8191
8192 instruct membar_release() %{
8193 match(MemBarRelease);
8194 ins_cost(VOLATILE_REF_COST);
8195
8196 format %{ "membar_release\n\t"
8197 "dmb ish" %}
8198
8199 ins_encode %{
8200 __ block_comment("membar_release");
8201 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8202 %}
8203 ins_pipe(pipe_serial);
8204 %}
8205
8206 instruct membar_storestore() %{
8207 match(MemBarStoreStore);
8208 ins_cost(VOLATILE_REF_COST);
8209
8210 format %{ "MEMBAR-store-store" %}
8211
8212 ins_encode %{
8213 __ membar(Assembler::StoreStore);
8214 %}
8215 ins_pipe(pipe_serial);
8216 %}
8217
8218 instruct membar_release_lock() %{
8219 match(MemBarReleaseLock);
8220 ins_cost(VOLATILE_REF_COST);
8221
8222 format %{ "membar_release_lock (elided)" %}
8223
8224 ins_encode %{
8225 __ block_comment("membar_release_lock (elided)");
8226 %}
8227
8228 ins_pipe(pipe_serial);
8229 %}
8230
8231 instruct unnecessary_membar_volatile() %{
8232 predicate(unnecessary_volatile(n));
8233 match(MemBarVolatile);
8234 ins_cost(0);
8235
8236 format %{ "membar_volatile (elided)" %}
8237
8238 ins_encode %{
8239 __ block_comment("membar_volatile (elided)");
8240 %}
8241
8242 ins_pipe(pipe_serial);
8243 %}
8244
8245 instruct membar_volatile() %{
8246 match(MemBarVolatile);
8247 ins_cost(VOLATILE_REF_COST*100);
8248
8249 format %{ "membar_volatile\n\t"
8250 "dmb ish"%}
8251
8252 ins_encode %{
8253 __ block_comment("membar_volatile");
8254 __ membar(Assembler::StoreLoad);
8255 %}
8256
8257 ins_pipe(pipe_serial);
8258 %}
8259
8260 // ============================================================================
8261 // Cast/Convert Instructions
8262
8263 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8264 match(Set dst (CastX2P src));
8265
8266 ins_cost(INSN_COST);
8267 format %{ "mov $dst, $src\t# long -> ptr" %}
8268
8269 ins_encode %{
8270 if ($dst$$reg != $src$$reg) {
8271 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8272 }
8273 %}
8274
8275 ins_pipe(ialu_reg);
8276 %}
8277
8278 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8279 match(Set dst (CastP2X src));
8280
8281 ins_cost(INSN_COST);
8282 format %{ "mov $dst, $src\t# ptr -> long" %}
8283
8284 ins_encode %{
8285 if ($dst$$reg != $src$$reg) {
8286 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8287 }
8288 %}
8289
8290 ins_pipe(ialu_reg);
8291 %}
8292
8293 // Convert oop into int for vectors alignment masking
8294 instruct convP2I(iRegINoSp dst, iRegP src) %{
8295 match(Set dst (ConvL2I (CastP2X src)));
8296
8297 ins_cost(INSN_COST);
8298 format %{ "movw $dst, $src\t# ptr -> int" %}
8299 ins_encode %{
8300 __ movw($dst$$Register, $src$$Register);
8301 %}
8302
8303 ins_pipe(ialu_reg);
8304 %}
8305
8306 // Convert compressed oop into int for vectors alignment masking
8307 // in case of 32bit oops (heap < 4Gb).
8308 instruct convN2I(iRegINoSp dst, iRegN src)
8309 %{
8310 predicate(CompressedOops::shift() == 0);
8311 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8312
8313 ins_cost(INSN_COST);
8314 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8315 ins_encode %{
8316 __ movw($dst$$Register, $src$$Register);
8317 %}
8318
8319 ins_pipe(ialu_reg);
8320 %}
8321
8322
8323 // Convert oop pointer into compressed form
8324 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8325 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8326 match(Set dst (EncodeP src));
8327 effect(KILL cr);
8328 ins_cost(INSN_COST * 3);
8329 format %{ "encode_heap_oop $dst, $src" %}
8330 ins_encode %{
8331 Register s = $src$$Register;
8332 Register d = $dst$$Register;
8333 __ encode_heap_oop(d, s);
8334 %}
8335 ins_pipe(ialu_reg);
8336 %}
8337
8338 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8339 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8340 match(Set dst (EncodeP src));
8341 ins_cost(INSN_COST * 3);
8342 format %{ "encode_heap_oop_not_null $dst, $src" %}
8343 ins_encode %{
8344 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8345 %}
8346 ins_pipe(ialu_reg);
8347 %}
8348
8349 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8350 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8351 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8352 match(Set dst (DecodeN src));
8353 ins_cost(INSN_COST * 3);
8354 format %{ "decode_heap_oop $dst, $src" %}
8355 ins_encode %{
8356 Register s = $src$$Register;
8357 Register d = $dst$$Register;
8358 __ decode_heap_oop(d, s);
8359 %}
8360 ins_pipe(ialu_reg);
8361 %}
8362
8363 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8364 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8365 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8366 match(Set dst (DecodeN src));
8367 ins_cost(INSN_COST * 3);
8368 format %{ "decode_heap_oop_not_null $dst, $src" %}
8369 ins_encode %{
8370 Register s = $src$$Register;
8371 Register d = $dst$$Register;
8372 __ decode_heap_oop_not_null(d, s);
8373 %}
8374 ins_pipe(ialu_reg);
8375 %}
8376
8377 // n.b. AArch64 implementations of encode_klass_not_null and
8378 // decode_klass_not_null do not modify the flags register so, unlike
8379 // Intel, we don't kill CR as a side effect here
8380
8381 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8382 match(Set dst (EncodePKlass src));
8383
8384 ins_cost(INSN_COST * 3);
8385 format %{ "encode_klass_not_null $dst,$src" %}
8386
8387 ins_encode %{
8388 Register src_reg = as_Register($src$$reg);
8389 Register dst_reg = as_Register($dst$$reg);
8390 __ encode_klass_not_null(dst_reg, src_reg);
8391 %}
8392
8393 ins_pipe(ialu_reg);
8394 %}
8395
8396 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8397 match(Set dst (DecodeNKlass src));
8398
8399 ins_cost(INSN_COST * 3);
8400 format %{ "decode_klass_not_null $dst,$src" %}
8401
8402 ins_encode %{
8403 Register src_reg = as_Register($src$$reg);
8404 Register dst_reg = as_Register($dst$$reg);
8405 if (dst_reg != src_reg) {
8406 __ decode_klass_not_null(dst_reg, src_reg);
8407 } else {
8408 __ decode_klass_not_null(dst_reg);
8409 }
8410 %}
8411
8412 ins_pipe(ialu_reg);
8413 %}
8414
8415 instruct checkCastPP(iRegPNoSp dst)
8416 %{
8417 match(Set dst (CheckCastPP dst));
8418
8419 size(0);
8420 format %{ "# checkcastPP of $dst" %}
8421 ins_encode(/* empty encoding */);
8422 ins_pipe(pipe_class_empty);
8423 %}
8424
8425 instruct castPP(iRegPNoSp dst)
8426 %{
8427 match(Set dst (CastPP dst));
8428
8429 size(0);
8430 format %{ "# castPP of $dst" %}
8431 ins_encode(/* empty encoding */);
8432 ins_pipe(pipe_class_empty);
8433 %}
8434
8435 instruct castII(iRegI dst)
8436 %{
8437 match(Set dst (CastII dst));
8438
8439 size(0);
8440 format %{ "# castII of $dst" %}
8441 ins_encode(/* empty encoding */);
8442 ins_cost(0);
8443 ins_pipe(pipe_class_empty);
8444 %}
8445
8446 // ============================================================================
8447 // Atomic operation instructions
8448 //
8449 // Intel and SPARC both implement Ideal Node LoadPLocked and
8450 // Store{PIL}Conditional instructions using a normal load for the
8451 // LoadPLocked and a CAS for the Store{PIL}Conditional.
8452 //
8453 // The ideal code appears only to use LoadPLocked/StorePLocked as a
8454 // pair to lock object allocations from Eden space when not using
8455 // TLABs.
8456 //
8457 // There does not appear to be a Load{IL}Locked Ideal Node and the
8458 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
8459 // and to use StoreIConditional only for 32-bit and StoreLConditional
8460 // only for 64-bit.
8461 //
8462 // We implement LoadPLocked and StorePLocked instructions using,
8463 // respectively the AArch64 hw load-exclusive and store-conditional
8464 // instructions. Whereas we must implement each of
8465 // Store{IL}Conditional using a CAS which employs a pair of
8466 // instructions comprising a load-exclusive followed by a
8467 // store-conditional.
8468
8469
8470 // Locked-load (linked load) of the current heap-top
8471 // used when updating the eden heap top
8472 // implemented using ldaxr on AArch64
8473
8474 instruct loadPLocked(iRegPNoSp dst, indirect mem)
8475 %{
8476 match(Set dst (LoadPLocked mem));
8477
8478 ins_cost(VOLATILE_REF_COST);
8479
8480 format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
8481
8482 ins_encode(aarch64_enc_ldaxr(dst, mem));
8483
8484 ins_pipe(pipe_serial);
8485 %}
8486
8487 // Conditional-store of the updated heap-top.
8488 // Used during allocation of the shared heap.
8489 // Sets flag (EQ) on success.
8490 // implemented using stlxr on AArch64.
8491
8492 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
8493 %{
8494 match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
8495
8496 ins_cost(VOLATILE_REF_COST);
8497
8498 // TODO
8499 // do we need to do a store-conditional release or can we just use a
8500 // plain store-conditional?
8501
8502 format %{
8503 "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
8504 "cmpw rscratch1, zr\t# EQ on successful write"
8505 %}
8506
8507 ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
8508
8509 ins_pipe(pipe_serial);
8510 %}
8511
8512
8513 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
8514 // when attempting to rebias a lock towards the current thread. We
8515 // must use the acquire form of cmpxchg in order to guarantee acquire
8516 // semantics in this case.
8517 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
8518 %{
8519 match(Set cr (StoreLConditional mem (Binary oldval newval)));
8520
8521 ins_cost(VOLATILE_REF_COST);
8522
8523 format %{
8524 "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8525 "cmpw rscratch1, zr\t# EQ on successful write"
8526 %}
8527
8528 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
8529
8530 ins_pipe(pipe_slow);
8531 %}
8532
8533 // storeIConditional also has acquire semantics, for no better reason
8534 // than matching storeLConditional. At the time of writing this
8535 // comment storeIConditional was not used anywhere by AArch64.
8536 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
8537 %{
8538 match(Set cr (StoreIConditional mem (Binary oldval newval)));
8539
8540 ins_cost(VOLATILE_REF_COST);
8541
8542 format %{
8543 "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8544 "cmpw rscratch1, zr\t# EQ on successful write"
8545 %}
8546
8547 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
8548
8549 ins_pipe(pipe_slow);
8550 %}
8551
8552 // standard CompareAndSwapX when we are using barriers
8553 // these have higher priority than the rules selected by a predicate
8554
8555 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8556 // can't match them
8557
8558 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8559
8560 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8561 ins_cost(2 * VOLATILE_REF_COST);
8562
8563 effect(KILL cr);
8564
8565 format %{
8566 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8567 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8568 %}
8569
8570 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8571 aarch64_enc_cset_eq(res));
8572
8573 ins_pipe(pipe_slow);
8574 %}
8575
8576 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8577
8578 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8579 ins_cost(2 * VOLATILE_REF_COST);
8580
8581 effect(KILL cr);
8582
8583 format %{
8584 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8585 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8586 %}
8587
8588 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8589 aarch64_enc_cset_eq(res));
8590
8591 ins_pipe(pipe_slow);
8592 %}
8593
8594 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8595
8596 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8597 ins_cost(2 * VOLATILE_REF_COST);
8598
8599 effect(KILL cr);
8600
8601 format %{
8602 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8603 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8604 %}
8605
8606 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8607 aarch64_enc_cset_eq(res));
8608
8609 ins_pipe(pipe_slow);
8610 %}
8611
8612 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8613
8614 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8615 ins_cost(2 * VOLATILE_REF_COST);
8616
8617 effect(KILL cr);
8618
8619 format %{
8620 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8621 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8622 %}
8623
8624 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8625 aarch64_enc_cset_eq(res));
8626
8627 ins_pipe(pipe_slow);
8628 %}
8629
8630 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8631
8632 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8633 predicate(n->as_LoadStore()->barrier_data() == 0);
8634 ins_cost(2 * VOLATILE_REF_COST);
8635
8636 effect(KILL cr);
8637
8638 format %{
8639 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8640 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8641 %}
8642
8643 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8644 aarch64_enc_cset_eq(res));
8645
8646 ins_pipe(pipe_slow);
8647 %}
8648
8649 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8650
8651 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8652 ins_cost(2 * VOLATILE_REF_COST);
8653
8654 effect(KILL cr);
8655
8656 format %{
8657 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8658 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8659 %}
8660
8661 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8662 aarch64_enc_cset_eq(res));
8663
8664 ins_pipe(pipe_slow);
8665 %}
8666
8667 // alternative CompareAndSwapX when we are eliding barriers
8668
8669 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8670
8671 predicate(needs_acquiring_load_exclusive(n));
8672 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8673 ins_cost(VOLATILE_REF_COST);
8674
8675 effect(KILL cr);
8676
8677 format %{
8678 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8679 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8680 %}
8681
8682 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8683 aarch64_enc_cset_eq(res));
8684
8685 ins_pipe(pipe_slow);
8686 %}
8687
8688 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8689
8690 predicate(needs_acquiring_load_exclusive(n));
8691 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8692 ins_cost(VOLATILE_REF_COST);
8693
8694 effect(KILL cr);
8695
8696 format %{
8697 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8698 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8699 %}
8700
8701 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8702 aarch64_enc_cset_eq(res));
8703
8704 ins_pipe(pipe_slow);
8705 %}
8706
8707 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8708
8709 predicate(needs_acquiring_load_exclusive(n));
8710 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8711 ins_cost(VOLATILE_REF_COST);
8712
8713 effect(KILL cr);
8714
8715 format %{
8716 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8717 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8718 %}
8719
8720 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8721 aarch64_enc_cset_eq(res));
8722
8723 ins_pipe(pipe_slow);
8724 %}
8725
8726 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8727
8728 predicate(needs_acquiring_load_exclusive(n));
8729 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8730 ins_cost(VOLATILE_REF_COST);
8731
8732 effect(KILL cr);
8733
8734 format %{
8735 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8736 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8737 %}
8738
8739 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8740 aarch64_enc_cset_eq(res));
8741
8742 ins_pipe(pipe_slow);
8743 %}
8744
8745 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8746
8747 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8748 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8749 ins_cost(VOLATILE_REF_COST);
8750
8751 effect(KILL cr);
8752
8753 format %{
8754 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8755 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8756 %}
8757
8758 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8759 aarch64_enc_cset_eq(res));
8760
8761 ins_pipe(pipe_slow);
8762 %}
8763
8764 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8765
8766 predicate(needs_acquiring_load_exclusive(n));
8767 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8768 ins_cost(VOLATILE_REF_COST);
8769
8770 effect(KILL cr);
8771
8772 format %{
8773 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8774 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8775 %}
8776
8777 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8778 aarch64_enc_cset_eq(res));
8779
8780 ins_pipe(pipe_slow);
8781 %}
8782
8783
8784 // ---------------------------------------------------------------------
8785
8786
8787 // BEGIN This section of the file is automatically generated. Do not edit --------------
8788
8789 // Sundry CAS operations. Note that release is always true,
8790 // regardless of the memory ordering of the CAS. This is because we
8791 // need the volatile case to be sequentially consistent but there is
8792 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8793 // can't check the type of memory ordering here, so we always emit a
8794 // STLXR.
8795
8796 // This section is generated from aarch64_ad_cas.m4
8797
8798
8799
8800 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8801 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8802 ins_cost(2 * VOLATILE_REF_COST);
8803 effect(TEMP_DEF res, KILL cr);
8804 format %{
8805 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8806 %}
8807 ins_encode %{
8808 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8809 Assembler::byte, /*acquire*/ false, /*release*/ true,
8810 /*weak*/ false, $res$$Register);
8811 __ sxtbw($res$$Register, $res$$Register);
8812 %}
8813 ins_pipe(pipe_slow);
8814 %}
8815
8816 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8817 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8818 ins_cost(2 * VOLATILE_REF_COST);
8819 effect(TEMP_DEF res, KILL cr);
8820 format %{
8821 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8822 %}
8823 ins_encode %{
8824 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8825 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8826 /*weak*/ false, $res$$Register);
8827 __ sxthw($res$$Register, $res$$Register);
8828 %}
8829 ins_pipe(pipe_slow);
8830 %}
8831
8832 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8833 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8834 ins_cost(2 * VOLATILE_REF_COST);
8835 effect(TEMP_DEF res, KILL cr);
8836 format %{
8837 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8838 %}
8839 ins_encode %{
8840 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8841 Assembler::word, /*acquire*/ false, /*release*/ true,
8842 /*weak*/ false, $res$$Register);
8843 %}
8844 ins_pipe(pipe_slow);
8845 %}
8846
8847 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8848 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8849 ins_cost(2 * VOLATILE_REF_COST);
8850 effect(TEMP_DEF res, KILL cr);
8851 format %{
8852 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8853 %}
8854 ins_encode %{
8855 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8856 Assembler::xword, /*acquire*/ false, /*release*/ true,
8857 /*weak*/ false, $res$$Register);
8858 %}
8859 ins_pipe(pipe_slow);
8860 %}
8861
8862 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8863 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8864 ins_cost(2 * VOLATILE_REF_COST);
8865 effect(TEMP_DEF res, KILL cr);
8866 format %{
8867 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8868 %}
8869 ins_encode %{
8870 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8871 Assembler::word, /*acquire*/ false, /*release*/ true,
8872 /*weak*/ false, $res$$Register);
8873 %}
8874 ins_pipe(pipe_slow);
8875 %}
8876
8877 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8878 predicate(n->as_LoadStore()->barrier_data() == 0);
8879 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8880 ins_cost(2 * VOLATILE_REF_COST);
8881 effect(TEMP_DEF res, KILL cr);
8882 format %{
8883 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8884 %}
8885 ins_encode %{
8886 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8887 Assembler::xword, /*acquire*/ false, /*release*/ true,
8888 /*weak*/ false, $res$$Register);
8889 %}
8890 ins_pipe(pipe_slow);
8891 %}
8892
8893 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8894 predicate(needs_acquiring_load_exclusive(n));
8895 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8896 ins_cost(VOLATILE_REF_COST);
8897 effect(TEMP_DEF res, KILL cr);
8898 format %{
8899 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8900 %}
8901 ins_encode %{
8902 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8903 Assembler::byte, /*acquire*/ true, /*release*/ true,
8904 /*weak*/ false, $res$$Register);
8905 __ sxtbw($res$$Register, $res$$Register);
8906 %}
8907 ins_pipe(pipe_slow);
8908 %}
8909
8910 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8911 predicate(needs_acquiring_load_exclusive(n));
8912 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8913 ins_cost(VOLATILE_REF_COST);
8914 effect(TEMP_DEF res, KILL cr);
8915 format %{
8916 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8917 %}
8918 ins_encode %{
8919 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8920 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8921 /*weak*/ false, $res$$Register);
8922 __ sxthw($res$$Register, $res$$Register);
8923 %}
8924 ins_pipe(pipe_slow);
8925 %}
8926
8927
8928 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8929 predicate(needs_acquiring_load_exclusive(n));
8930 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8931 ins_cost(VOLATILE_REF_COST);
8932 effect(TEMP_DEF res, KILL cr);
8933 format %{
8934 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8935 %}
8936 ins_encode %{
8937 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8938 Assembler::word, /*acquire*/ true, /*release*/ true,
8939 /*weak*/ false, $res$$Register);
8940 %}
8941 ins_pipe(pipe_slow);
8942 %}
8943
8944 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8945 predicate(needs_acquiring_load_exclusive(n));
8946 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8947 ins_cost(VOLATILE_REF_COST);
8948 effect(TEMP_DEF res, KILL cr);
8949 format %{
8950 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8951 %}
8952 ins_encode %{
8953 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8954 Assembler::xword, /*acquire*/ true, /*release*/ true,
8955 /*weak*/ false, $res$$Register);
8956 %}
8957 ins_pipe(pipe_slow);
8958 %}
8959
8960
8961 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8962 predicate(needs_acquiring_load_exclusive(n));
8963 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8964 ins_cost(VOLATILE_REF_COST);
8965 effect(TEMP_DEF res, KILL cr);
8966 format %{
8967 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8968 %}
8969 ins_encode %{
8970 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8971 Assembler::word, /*acquire*/ true, /*release*/ true,
8972 /*weak*/ false, $res$$Register);
8973 %}
8974 ins_pipe(pipe_slow);
8975 %}
8976
8977 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8978 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8979 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8980 ins_cost(VOLATILE_REF_COST);
8981 effect(TEMP_DEF res, KILL cr);
8982 format %{
8983 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8984 %}
8985 ins_encode %{
8986 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8987 Assembler::xword, /*acquire*/ true, /*release*/ true,
8988 /*weak*/ false, $res$$Register);
8989 %}
8990 ins_pipe(pipe_slow);
8991 %}
8992
8993 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8994 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8995 ins_cost(2 * VOLATILE_REF_COST);
8996 effect(KILL cr);
8997 format %{
8998 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8999 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9000 %}
9001 ins_encode %{
9002 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9003 Assembler::byte, /*acquire*/ false, /*release*/ true,
9004 /*weak*/ true, noreg);
9005 __ csetw($res$$Register, Assembler::EQ);
9006 %}
9007 ins_pipe(pipe_slow);
9008 %}
9009
9010 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9011 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9012 ins_cost(2 * VOLATILE_REF_COST);
9013 effect(KILL cr);
9014 format %{
9015 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9016 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9017 %}
9018 ins_encode %{
9019 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9020 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9021 /*weak*/ true, noreg);
9022 __ csetw($res$$Register, Assembler::EQ);
9023 %}
9024 ins_pipe(pipe_slow);
9025 %}
9026
9027 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9028 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9029 ins_cost(2 * VOLATILE_REF_COST);
9030 effect(KILL cr);
9031 format %{
9032 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9033 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9034 %}
9035 ins_encode %{
9036 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9037 Assembler::word, /*acquire*/ false, /*release*/ true,
9038 /*weak*/ true, noreg);
9039 __ csetw($res$$Register, Assembler::EQ);
9040 %}
9041 ins_pipe(pipe_slow);
9042 %}
9043
9044 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9045 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9046 ins_cost(2 * VOLATILE_REF_COST);
9047 effect(KILL cr);
9048 format %{
9049 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9050 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9051 %}
9052 ins_encode %{
9053 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9054 Assembler::xword, /*acquire*/ false, /*release*/ true,
9055 /*weak*/ true, noreg);
9056 __ csetw($res$$Register, Assembler::EQ);
9057 %}
9058 ins_pipe(pipe_slow);
9059 %}
9060
9061 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9062 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9063 ins_cost(2 * VOLATILE_REF_COST);
9064 effect(KILL cr);
9065 format %{
9066 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9067 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9068 %}
9069 ins_encode %{
9070 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9071 Assembler::word, /*acquire*/ false, /*release*/ true,
9072 /*weak*/ true, noreg);
9073 __ csetw($res$$Register, Assembler::EQ);
9074 %}
9075 ins_pipe(pipe_slow);
9076 %}
9077
9078 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9079 predicate(n->as_LoadStore()->barrier_data() == 0);
9080 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9081 ins_cost(2 * VOLATILE_REF_COST);
9082 effect(KILL cr);
9083 format %{
9084 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9085 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9086 %}
9087 ins_encode %{
9088 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9089 Assembler::xword, /*acquire*/ false, /*release*/ true,
9090 /*weak*/ true, noreg);
9091 __ csetw($res$$Register, Assembler::EQ);
9092 %}
9093 ins_pipe(pipe_slow);
9094 %}
9095
9096 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9097 predicate(needs_acquiring_load_exclusive(n));
9098 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9099 ins_cost(VOLATILE_REF_COST);
9100 effect(KILL cr);
9101 format %{
9102 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9103 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9104 %}
9105 ins_encode %{
9106 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9107 Assembler::byte, /*acquire*/ true, /*release*/ true,
9108 /*weak*/ true, noreg);
9109 __ csetw($res$$Register, Assembler::EQ);
9110 %}
9111 ins_pipe(pipe_slow);
9112 %}
9113
9114 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9115 predicate(needs_acquiring_load_exclusive(n));
9116 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9117 ins_cost(VOLATILE_REF_COST);
9118 effect(KILL cr);
9119 format %{
9120 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9121 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9122 %}
9123 ins_encode %{
9124 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9125 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9126 /*weak*/ true, noreg);
9127 __ csetw($res$$Register, Assembler::EQ);
9128 %}
9129 ins_pipe(pipe_slow);
9130 %}
9131
9132 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9133 predicate(needs_acquiring_load_exclusive(n));
9134 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9135 ins_cost(VOLATILE_REF_COST);
9136 effect(KILL cr);
9137 format %{
9138 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9139 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9140 %}
9141 ins_encode %{
9142 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9143 Assembler::word, /*acquire*/ true, /*release*/ true,
9144 /*weak*/ true, noreg);
9145 __ csetw($res$$Register, Assembler::EQ);
9146 %}
9147 ins_pipe(pipe_slow);
9148 %}
9149
9150 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9151 predicate(needs_acquiring_load_exclusive(n));
9152 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9153 ins_cost(VOLATILE_REF_COST);
9154 effect(KILL cr);
9155 format %{
9156 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9157 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9158 %}
9159 ins_encode %{
9160 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9161 Assembler::xword, /*acquire*/ true, /*release*/ true,
9162 /*weak*/ true, noreg);
9163 __ csetw($res$$Register, Assembler::EQ);
9164 %}
9165 ins_pipe(pipe_slow);
9166 %}
9167
9168 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9169 predicate(needs_acquiring_load_exclusive(n));
9170 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9171 ins_cost(VOLATILE_REF_COST);
9172 effect(KILL cr);
9173 format %{
9174 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9175 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9176 %}
9177 ins_encode %{
9178 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9179 Assembler::word, /*acquire*/ true, /*release*/ true,
9180 /*weak*/ true, noreg);
9181 __ csetw($res$$Register, Assembler::EQ);
9182 %}
9183 ins_pipe(pipe_slow);
9184 %}
9185
9186 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9187 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9188 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9189 ins_cost(VOLATILE_REF_COST);
9190 effect(KILL cr);
9191 format %{
9192 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9193 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9194 %}
9195 ins_encode %{
9196 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9197 Assembler::xword, /*acquire*/ true, /*release*/ true,
9198 /*weak*/ true, noreg);
9199 __ csetw($res$$Register, Assembler::EQ);
9200 %}
9201 ins_pipe(pipe_slow);
9202 %}
9203
9204 // END This section of the file is automatically generated. Do not edit --------------
9205 // ---------------------------------------------------------------------
9206
9207 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9208 match(Set prev (GetAndSetI mem newv));
9209 ins_cost(2 * VOLATILE_REF_COST);
9210 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9211 ins_encode %{
9212 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9213 %}
9214 ins_pipe(pipe_serial);
9215 %}
9216
9217 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9218 match(Set prev (GetAndSetL mem newv));
9219 ins_cost(2 * VOLATILE_REF_COST);
9220 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9221 ins_encode %{
9222 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9223 %}
9224 ins_pipe(pipe_serial);
9225 %}
9226
9227 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9228 match(Set prev (GetAndSetN mem newv));
9229 ins_cost(2 * VOLATILE_REF_COST);
9230 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9231 ins_encode %{
9232 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9233 %}
9234 ins_pipe(pipe_serial);
9235 %}
9236
9237 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9238 predicate(n->as_LoadStore()->barrier_data() == 0);
9239 match(Set prev (GetAndSetP mem newv));
9240 ins_cost(2 * VOLATILE_REF_COST);
9241 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9242 ins_encode %{
9243 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9244 %}
9245 ins_pipe(pipe_serial);
9246 %}
9247
9248 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9249 predicate(needs_acquiring_load_exclusive(n));
9250 match(Set prev (GetAndSetI mem newv));
9251 ins_cost(VOLATILE_REF_COST);
9252 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9253 ins_encode %{
9254 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9255 %}
9256 ins_pipe(pipe_serial);
9257 %}
9258
9259 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9260 predicate(needs_acquiring_load_exclusive(n));
9261 match(Set prev (GetAndSetL mem newv));
9262 ins_cost(VOLATILE_REF_COST);
9263 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9264 ins_encode %{
9265 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9266 %}
9267 ins_pipe(pipe_serial);
9268 %}
9269
9270 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9271 predicate(needs_acquiring_load_exclusive(n));
9272 match(Set prev (GetAndSetN mem newv));
9273 ins_cost(VOLATILE_REF_COST);
9274 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9275 ins_encode %{
9276 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9277 %}
9278 ins_pipe(pipe_serial);
9279 %}
9280
9281 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9282 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9283 match(Set prev (GetAndSetP mem newv));
9284 ins_cost(VOLATILE_REF_COST);
9285 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9286 ins_encode %{
9287 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9288 %}
9289 ins_pipe(pipe_serial);
9290 %}
9291
9292
9293 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9294 match(Set newval (GetAndAddL mem incr));
9295 ins_cost(2 * VOLATILE_REF_COST + 1);
9296 format %{ "get_and_addL $newval, [$mem], $incr" %}
9297 ins_encode %{
9298 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9299 %}
9300 ins_pipe(pipe_serial);
9301 %}
9302
9303 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9304 predicate(n->as_LoadStore()->result_not_used());
9305 match(Set dummy (GetAndAddL mem incr));
9306 ins_cost(2 * VOLATILE_REF_COST);
9307 format %{ "get_and_addL [$mem], $incr" %}
9308 ins_encode %{
9309 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9310 %}
9311 ins_pipe(pipe_serial);
9312 %}
9313
9314 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9315 match(Set newval (GetAndAddL mem incr));
9316 ins_cost(2 * VOLATILE_REF_COST + 1);
9317 format %{ "get_and_addL $newval, [$mem], $incr" %}
9318 ins_encode %{
9319 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9320 %}
9321 ins_pipe(pipe_serial);
9322 %}
9323
9324 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9325 predicate(n->as_LoadStore()->result_not_used());
9326 match(Set dummy (GetAndAddL mem incr));
9327 ins_cost(2 * VOLATILE_REF_COST);
9328 format %{ "get_and_addL [$mem], $incr" %}
9329 ins_encode %{
9330 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9331 %}
9332 ins_pipe(pipe_serial);
9333 %}
9334
9335 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9336 match(Set newval (GetAndAddI mem incr));
9337 ins_cost(2 * VOLATILE_REF_COST + 1);
9338 format %{ "get_and_addI $newval, [$mem], $incr" %}
9339 ins_encode %{
9340 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9341 %}
9342 ins_pipe(pipe_serial);
9343 %}
9344
9345 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9346 predicate(n->as_LoadStore()->result_not_used());
9347 match(Set dummy (GetAndAddI mem incr));
9348 ins_cost(2 * VOLATILE_REF_COST);
9349 format %{ "get_and_addI [$mem], $incr" %}
9350 ins_encode %{
9351 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9352 %}
9353 ins_pipe(pipe_serial);
9354 %}
9355
9356 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9357 match(Set newval (GetAndAddI mem incr));
9358 ins_cost(2 * VOLATILE_REF_COST + 1);
9359 format %{ "get_and_addI $newval, [$mem], $incr" %}
9360 ins_encode %{
9361 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9362 %}
9363 ins_pipe(pipe_serial);
9364 %}
9365
9366 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9367 predicate(n->as_LoadStore()->result_not_used());
9368 match(Set dummy (GetAndAddI mem incr));
9369 ins_cost(2 * VOLATILE_REF_COST);
9370 format %{ "get_and_addI [$mem], $incr" %}
9371 ins_encode %{
9372 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9373 %}
9374 ins_pipe(pipe_serial);
9375 %}
9376
9377 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9378 predicate(needs_acquiring_load_exclusive(n));
9379 match(Set newval (GetAndAddL mem incr));
9380 ins_cost(VOLATILE_REF_COST + 1);
9381 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9382 ins_encode %{
9383 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9384 %}
9385 ins_pipe(pipe_serial);
9386 %}
9387
9388 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9389 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9390 match(Set dummy (GetAndAddL mem incr));
9391 ins_cost(VOLATILE_REF_COST);
9392 format %{ "get_and_addL_acq [$mem], $incr" %}
9393 ins_encode %{
9394 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9395 %}
9396 ins_pipe(pipe_serial);
9397 %}
9398
9399 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9400 predicate(needs_acquiring_load_exclusive(n));
9401 match(Set newval (GetAndAddL mem incr));
9402 ins_cost(VOLATILE_REF_COST + 1);
9403 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9404 ins_encode %{
9405 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9406 %}
9407 ins_pipe(pipe_serial);
9408 %}
9409
9410 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9411 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9412 match(Set dummy (GetAndAddL mem incr));
9413 ins_cost(VOLATILE_REF_COST);
9414 format %{ "get_and_addL_acq [$mem], $incr" %}
9415 ins_encode %{
9416 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9417 %}
9418 ins_pipe(pipe_serial);
9419 %}
9420
9421 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9422 predicate(needs_acquiring_load_exclusive(n));
9423 match(Set newval (GetAndAddI mem incr));
9424 ins_cost(VOLATILE_REF_COST + 1);
9425 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9426 ins_encode %{
9427 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9428 %}
9429 ins_pipe(pipe_serial);
9430 %}
9431
9432 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9433 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9434 match(Set dummy (GetAndAddI mem incr));
9435 ins_cost(VOLATILE_REF_COST);
9436 format %{ "get_and_addI_acq [$mem], $incr" %}
9437 ins_encode %{
9438 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9439 %}
9440 ins_pipe(pipe_serial);
9441 %}
9442
9443 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9444 predicate(needs_acquiring_load_exclusive(n));
9445 match(Set newval (GetAndAddI mem incr));
9446 ins_cost(VOLATILE_REF_COST + 1);
9447 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9448 ins_encode %{
9449 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9450 %}
9451 ins_pipe(pipe_serial);
9452 %}
9453
9454 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9455 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9456 match(Set dummy (GetAndAddI mem incr));
9457 ins_cost(VOLATILE_REF_COST);
9458 format %{ "get_and_addI_acq [$mem], $incr" %}
9459 ins_encode %{
9460 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9461 %}
9462 ins_pipe(pipe_serial);
9463 %}
9464
9465 // Manifest a CmpL result in an integer register.
9466 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9467 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9468 %{
9469 match(Set dst (CmpL3 src1 src2));
9470 effect(KILL flags);
9471
9472 ins_cost(INSN_COST * 6);
9473 format %{
9474 "cmp $src1, $src2"
9475 "csetw $dst, ne"
9476 "cnegw $dst, lt"
9477 %}
9478 // format %{ "CmpL3 $dst, $src1, $src2" %}
9479 ins_encode %{
9480 __ cmp($src1$$Register, $src2$$Register);
9481 __ csetw($dst$$Register, Assembler::NE);
9482 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9483 %}
9484
9485 ins_pipe(pipe_class_default);
9486 %}
9487
9488 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9489 %{
9490 match(Set dst (CmpL3 src1 src2));
9491 effect(KILL flags);
9492
9493 ins_cost(INSN_COST * 6);
9494 format %{
9495 "cmp $src1, $src2"
9496 "csetw $dst, ne"
9497 "cnegw $dst, lt"
9498 %}
9499 ins_encode %{
9500 int32_t con = (int32_t)$src2$$constant;
9501 if (con < 0) {
9502 __ adds(zr, $src1$$Register, -con);
9503 } else {
9504 __ subs(zr, $src1$$Register, con);
9505 }
9506 __ csetw($dst$$Register, Assembler::NE);
9507 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9508 %}
9509
9510 ins_pipe(pipe_class_default);
9511 %}
9512
9513 // ============================================================================
9514 // Conditional Move Instructions
9515
9516 // n.b. we have identical rules for both a signed compare op (cmpOp)
9517 // and an unsigned compare op (cmpOpU). it would be nice if we could
9518 // define an op class which merged both inputs and use it to type the
9519 // argument to a single rule. unfortunatelyt his fails because the
9520 // opclass does not live up to the COND_INTER interface of its
9521 // component operands. When the generic code tries to negate the
9522 // operand it ends up running the generci Machoper::negate method
9523 // which throws a ShouldNotHappen. So, we have to provide two flavours
9524 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9525
9526 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9527 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9528
9529 ins_cost(INSN_COST * 2);
9530 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9531
9532 ins_encode %{
9533 __ cselw(as_Register($dst$$reg),
9534 as_Register($src2$$reg),
9535 as_Register($src1$$reg),
9536 (Assembler::Condition)$cmp$$cmpcode);
9537 %}
9538
9539 ins_pipe(icond_reg_reg);
9540 %}
9541
9542 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9543 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9544
9545 ins_cost(INSN_COST * 2);
9546 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9547
9548 ins_encode %{
9549 __ cselw(as_Register($dst$$reg),
9550 as_Register($src2$$reg),
9551 as_Register($src1$$reg),
9552 (Assembler::Condition)$cmp$$cmpcode);
9553 %}
9554
9555 ins_pipe(icond_reg_reg);
9556 %}
9557
9558 // special cases where one arg is zero
9559
9560 // n.b. this is selected in preference to the rule above because it
9561 // avoids loading constant 0 into a source register
9562
9563 // TODO
9564 // we ought only to be able to cull one of these variants as the ideal
9565 // transforms ought always to order the zero consistently (to left/right?)
9566
9567 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9568 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9569
9570 ins_cost(INSN_COST * 2);
9571 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9572
9573 ins_encode %{
9574 __ cselw(as_Register($dst$$reg),
9575 as_Register($src$$reg),
9576 zr,
9577 (Assembler::Condition)$cmp$$cmpcode);
9578 %}
9579
9580 ins_pipe(icond_reg);
9581 %}
9582
9583 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9584 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9585
9586 ins_cost(INSN_COST * 2);
9587 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9588
9589 ins_encode %{
9590 __ cselw(as_Register($dst$$reg),
9591 as_Register($src$$reg),
9592 zr,
9593 (Assembler::Condition)$cmp$$cmpcode);
9594 %}
9595
9596 ins_pipe(icond_reg);
9597 %}
9598
9599 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9600 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9601
9602 ins_cost(INSN_COST * 2);
9603 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9604
9605 ins_encode %{
9606 __ cselw(as_Register($dst$$reg),
9607 zr,
9608 as_Register($src$$reg),
9609 (Assembler::Condition)$cmp$$cmpcode);
9610 %}
9611
9612 ins_pipe(icond_reg);
9613 %}
9614
9615 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9616 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9617
9618 ins_cost(INSN_COST * 2);
9619 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9620
9621 ins_encode %{
9622 __ cselw(as_Register($dst$$reg),
9623 zr,
9624 as_Register($src$$reg),
9625 (Assembler::Condition)$cmp$$cmpcode);
9626 %}
9627
9628 ins_pipe(icond_reg);
9629 %}
9630
9631 // special case for creating a boolean 0 or 1
9632
9633 // n.b. this is selected in preference to the rule above because it
9634 // avoids loading constants 0 and 1 into a source register
9635
9636 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9637 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9638
9639 ins_cost(INSN_COST * 2);
9640 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9641
9642 ins_encode %{
9643 // equivalently
9644 // cset(as_Register($dst$$reg),
9645 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9646 __ csincw(as_Register($dst$$reg),
9647 zr,
9648 zr,
9649 (Assembler::Condition)$cmp$$cmpcode);
9650 %}
9651
9652 ins_pipe(icond_none);
9653 %}
9654
9655 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9656 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9657
9658 ins_cost(INSN_COST * 2);
9659 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9660
9661 ins_encode %{
9662 // equivalently
9663 // cset(as_Register($dst$$reg),
9664 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9665 __ csincw(as_Register($dst$$reg),
9666 zr,
9667 zr,
9668 (Assembler::Condition)$cmp$$cmpcode);
9669 %}
9670
9671 ins_pipe(icond_none);
9672 %}
9673
9674 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9675 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9676
9677 ins_cost(INSN_COST * 2);
9678 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9679
9680 ins_encode %{
9681 __ csel(as_Register($dst$$reg),
9682 as_Register($src2$$reg),
9683 as_Register($src1$$reg),
9684 (Assembler::Condition)$cmp$$cmpcode);
9685 %}
9686
9687 ins_pipe(icond_reg_reg);
9688 %}
9689
9690 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9691 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9692
9693 ins_cost(INSN_COST * 2);
9694 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9695
9696 ins_encode %{
9697 __ csel(as_Register($dst$$reg),
9698 as_Register($src2$$reg),
9699 as_Register($src1$$reg),
9700 (Assembler::Condition)$cmp$$cmpcode);
9701 %}
9702
9703 ins_pipe(icond_reg_reg);
9704 %}
9705
9706 // special cases where one arg is zero
9707
9708 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9709 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9710
9711 ins_cost(INSN_COST * 2);
9712 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9713
9714 ins_encode %{
9715 __ csel(as_Register($dst$$reg),
9716 zr,
9717 as_Register($src$$reg),
9718 (Assembler::Condition)$cmp$$cmpcode);
9719 %}
9720
9721 ins_pipe(icond_reg);
9722 %}
9723
9724 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9725 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9726
9727 ins_cost(INSN_COST * 2);
9728 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9729
9730 ins_encode %{
9731 __ csel(as_Register($dst$$reg),
9732 zr,
9733 as_Register($src$$reg),
9734 (Assembler::Condition)$cmp$$cmpcode);
9735 %}
9736
9737 ins_pipe(icond_reg);
9738 %}
9739
9740 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9741 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9742
9743 ins_cost(INSN_COST * 2);
9744 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9745
9746 ins_encode %{
9747 __ csel(as_Register($dst$$reg),
9748 as_Register($src$$reg),
9749 zr,
9750 (Assembler::Condition)$cmp$$cmpcode);
9751 %}
9752
9753 ins_pipe(icond_reg);
9754 %}
9755
9756 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9757 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9758
9759 ins_cost(INSN_COST * 2);
9760 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9761
9762 ins_encode %{
9763 __ csel(as_Register($dst$$reg),
9764 as_Register($src$$reg),
9765 zr,
9766 (Assembler::Condition)$cmp$$cmpcode);
9767 %}
9768
9769 ins_pipe(icond_reg);
9770 %}
9771
9772 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9773 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9774
9775 ins_cost(INSN_COST * 2);
9776 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9777
9778 ins_encode %{
9779 __ csel(as_Register($dst$$reg),
9780 as_Register($src2$$reg),
9781 as_Register($src1$$reg),
9782 (Assembler::Condition)$cmp$$cmpcode);
9783 %}
9784
9785 ins_pipe(icond_reg_reg);
9786 %}
9787
9788 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9789 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9790
9791 ins_cost(INSN_COST * 2);
9792 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9793
9794 ins_encode %{
9795 __ csel(as_Register($dst$$reg),
9796 as_Register($src2$$reg),
9797 as_Register($src1$$reg),
9798 (Assembler::Condition)$cmp$$cmpcode);
9799 %}
9800
9801 ins_pipe(icond_reg_reg);
9802 %}
9803
9804 // special cases where one arg is zero
9805
9806 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9807 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9808
9809 ins_cost(INSN_COST * 2);
9810 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9811
9812 ins_encode %{
9813 __ csel(as_Register($dst$$reg),
9814 zr,
9815 as_Register($src$$reg),
9816 (Assembler::Condition)$cmp$$cmpcode);
9817 %}
9818
9819 ins_pipe(icond_reg);
9820 %}
9821
9822 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9823 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9824
9825 ins_cost(INSN_COST * 2);
9826 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9827
9828 ins_encode %{
9829 __ csel(as_Register($dst$$reg),
9830 zr,
9831 as_Register($src$$reg),
9832 (Assembler::Condition)$cmp$$cmpcode);
9833 %}
9834
9835 ins_pipe(icond_reg);
9836 %}
9837
9838 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9839 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9840
9841 ins_cost(INSN_COST * 2);
9842 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9843
9844 ins_encode %{
9845 __ csel(as_Register($dst$$reg),
9846 as_Register($src$$reg),
9847 zr,
9848 (Assembler::Condition)$cmp$$cmpcode);
9849 %}
9850
9851 ins_pipe(icond_reg);
9852 %}
9853
9854 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9855 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9856
9857 ins_cost(INSN_COST * 2);
9858 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9859
9860 ins_encode %{
9861 __ csel(as_Register($dst$$reg),
9862 as_Register($src$$reg),
9863 zr,
9864 (Assembler::Condition)$cmp$$cmpcode);
9865 %}
9866
9867 ins_pipe(icond_reg);
9868 %}
9869
9870 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9871 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9872
9873 ins_cost(INSN_COST * 2);
9874 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9875
9876 ins_encode %{
9877 __ cselw(as_Register($dst$$reg),
9878 as_Register($src2$$reg),
9879 as_Register($src1$$reg),
9880 (Assembler::Condition)$cmp$$cmpcode);
9881 %}
9882
9883 ins_pipe(icond_reg_reg);
9884 %}
9885
9886 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9887 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9888
9889 ins_cost(INSN_COST * 2);
9890 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9891
9892 ins_encode %{
9893 __ cselw(as_Register($dst$$reg),
9894 as_Register($src2$$reg),
9895 as_Register($src1$$reg),
9896 (Assembler::Condition)$cmp$$cmpcode);
9897 %}
9898
9899 ins_pipe(icond_reg_reg);
9900 %}
9901
9902 // special cases where one arg is zero
9903
9904 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9905 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9906
9907 ins_cost(INSN_COST * 2);
9908 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9909
9910 ins_encode %{
9911 __ cselw(as_Register($dst$$reg),
9912 zr,
9913 as_Register($src$$reg),
9914 (Assembler::Condition)$cmp$$cmpcode);
9915 %}
9916
9917 ins_pipe(icond_reg);
9918 %}
9919
9920 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9921 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9922
9923 ins_cost(INSN_COST * 2);
9924 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9925
9926 ins_encode %{
9927 __ cselw(as_Register($dst$$reg),
9928 zr,
9929 as_Register($src$$reg),
9930 (Assembler::Condition)$cmp$$cmpcode);
9931 %}
9932
9933 ins_pipe(icond_reg);
9934 %}
9935
9936 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9937 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9938
9939 ins_cost(INSN_COST * 2);
9940 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9941
9942 ins_encode %{
9943 __ cselw(as_Register($dst$$reg),
9944 as_Register($src$$reg),
9945 zr,
9946 (Assembler::Condition)$cmp$$cmpcode);
9947 %}
9948
9949 ins_pipe(icond_reg);
9950 %}
9951
9952 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9953 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9954
9955 ins_cost(INSN_COST * 2);
9956 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9957
9958 ins_encode %{
9959 __ cselw(as_Register($dst$$reg),
9960 as_Register($src$$reg),
9961 zr,
9962 (Assembler::Condition)$cmp$$cmpcode);
9963 %}
9964
9965 ins_pipe(icond_reg);
9966 %}
9967
9968 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9969 %{
9970 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9971
9972 ins_cost(INSN_COST * 3);
9973
9974 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9975 ins_encode %{
9976 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9977 __ fcsels(as_FloatRegister($dst$$reg),
9978 as_FloatRegister($src2$$reg),
9979 as_FloatRegister($src1$$reg),
9980 cond);
9981 %}
9982
9983 ins_pipe(fp_cond_reg_reg_s);
9984 %}
9985
9986 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9987 %{
9988 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9989
9990 ins_cost(INSN_COST * 3);
9991
9992 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9993 ins_encode %{
9994 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9995 __ fcsels(as_FloatRegister($dst$$reg),
9996 as_FloatRegister($src2$$reg),
9997 as_FloatRegister($src1$$reg),
9998 cond);
9999 %}
10000
10001 ins_pipe(fp_cond_reg_reg_s);
10002 %}
10003
10004 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10005 %{
10006 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10007
10008 ins_cost(INSN_COST * 3);
10009
10010 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10011 ins_encode %{
10012 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10013 __ fcseld(as_FloatRegister($dst$$reg),
10014 as_FloatRegister($src2$$reg),
10015 as_FloatRegister($src1$$reg),
10016 cond);
10017 %}
10018
10019 ins_pipe(fp_cond_reg_reg_d);
10020 %}
10021
10022 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10023 %{
10024 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10025
10026 ins_cost(INSN_COST * 3);
10027
10028 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10029 ins_encode %{
10030 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10031 __ fcseld(as_FloatRegister($dst$$reg),
10032 as_FloatRegister($src2$$reg),
10033 as_FloatRegister($src1$$reg),
10034 cond);
10035 %}
10036
10037 ins_pipe(fp_cond_reg_reg_d);
10038 %}
10039
10040 // ============================================================================
10041 // Arithmetic Instructions
10042 //
10043
10044 // Integer Addition
10045
10046 // TODO
10047 // these currently employ operations which do not set CR and hence are
10048 // not flagged as killing CR but we would like to isolate the cases
10049 // where we want to set flags from those where we don't. need to work
10050 // out how to do that.
10051
10052 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10053 match(Set dst (AddI src1 src2));
10054
10055 ins_cost(INSN_COST);
10056 format %{ "addw $dst, $src1, $src2" %}
10057
10058 ins_encode %{
10059 __ addw(as_Register($dst$$reg),
10060 as_Register($src1$$reg),
10061 as_Register($src2$$reg));
10062 %}
10063
10064 ins_pipe(ialu_reg_reg);
10065 %}
10066
10067 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10068 match(Set dst (AddI src1 src2));
10069
10070 ins_cost(INSN_COST);
10071 format %{ "addw $dst, $src1, $src2" %}
10072
10073 // use opcode to indicate that this is an add not a sub
10074 opcode(0x0);
10075
10076 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10077
10078 ins_pipe(ialu_reg_imm);
10079 %}
10080
10081 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10082 match(Set dst (AddI (ConvL2I src1) src2));
10083
10084 ins_cost(INSN_COST);
10085 format %{ "addw $dst, $src1, $src2" %}
10086
10087 // use opcode to indicate that this is an add not a sub
10088 opcode(0x0);
10089
10090 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10091
10092 ins_pipe(ialu_reg_imm);
10093 %}
10094
10095 // Pointer Addition
10096 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10097 match(Set dst (AddP src1 src2));
10098
10099 ins_cost(INSN_COST);
10100 format %{ "add $dst, $src1, $src2\t# ptr" %}
10101
10102 ins_encode %{
10103 __ add(as_Register($dst$$reg),
10104 as_Register($src1$$reg),
10105 as_Register($src2$$reg));
10106 %}
10107
10108 ins_pipe(ialu_reg_reg);
10109 %}
10110
10111 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10112 match(Set dst (AddP src1 (ConvI2L src2)));
10113
10114 ins_cost(1.9 * INSN_COST);
10115 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10116
10117 ins_encode %{
10118 __ add(as_Register($dst$$reg),
10119 as_Register($src1$$reg),
10120 as_Register($src2$$reg), ext::sxtw);
10121 %}
10122
10123 ins_pipe(ialu_reg_reg);
10124 %}
10125
10126 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10127 match(Set dst (AddP src1 (LShiftL src2 scale)));
10128
10129 ins_cost(1.9 * INSN_COST);
10130 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10131
10132 ins_encode %{
10133 __ lea(as_Register($dst$$reg),
10134 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10135 Address::lsl($scale$$constant)));
10136 %}
10137
10138 ins_pipe(ialu_reg_reg_shift);
10139 %}
10140
10141 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10142 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10143
10144 ins_cost(1.9 * INSN_COST);
10145 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10146
10147 ins_encode %{
10148 __ lea(as_Register($dst$$reg),
10149 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10150 Address::sxtw($scale$$constant)));
10151 %}
10152
10153 ins_pipe(ialu_reg_reg_shift);
10154 %}
10155
10156 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10157 match(Set dst (LShiftL (ConvI2L src) scale));
10158
10159 ins_cost(INSN_COST);
10160 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10161
10162 ins_encode %{
10163 __ sbfiz(as_Register($dst$$reg),
10164 as_Register($src$$reg),
10165 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10166 %}
10167
10168 ins_pipe(ialu_reg_shift);
10169 %}
10170
10171 // Pointer Immediate Addition
10172 // n.b. this needs to be more expensive than using an indirect memory
10173 // operand
10174 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10175 match(Set dst (AddP src1 src2));
10176
10177 ins_cost(INSN_COST);
10178 format %{ "add $dst, $src1, $src2\t# ptr" %}
10179
10180 // use opcode to indicate that this is an add not a sub
10181 opcode(0x0);
10182
10183 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10184
10185 ins_pipe(ialu_reg_imm);
10186 %}
10187
10188 // Long Addition
10189 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10190
10191 match(Set dst (AddL src1 src2));
10192
10193 ins_cost(INSN_COST);
10194 format %{ "add $dst, $src1, $src2" %}
10195
10196 ins_encode %{
10197 __ add(as_Register($dst$$reg),
10198 as_Register($src1$$reg),
10199 as_Register($src2$$reg));
10200 %}
10201
10202 ins_pipe(ialu_reg_reg);
10203 %}
10204
10205 // No constant pool entries requiredLong Immediate Addition.
10206 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10207 match(Set dst (AddL src1 src2));
10208
10209 ins_cost(INSN_COST);
10210 format %{ "add $dst, $src1, $src2" %}
10211
10212 // use opcode to indicate that this is an add not a sub
10213 opcode(0x0);
10214
10215 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10216
10217 ins_pipe(ialu_reg_imm);
10218 %}
10219
10220 // Integer Subtraction
10221 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10222 match(Set dst (SubI src1 src2));
10223
10224 ins_cost(INSN_COST);
10225 format %{ "subw $dst, $src1, $src2" %}
10226
10227 ins_encode %{
10228 __ subw(as_Register($dst$$reg),
10229 as_Register($src1$$reg),
10230 as_Register($src2$$reg));
10231 %}
10232
10233 ins_pipe(ialu_reg_reg);
10234 %}
10235
10236 // Immediate Subtraction
10237 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10238 match(Set dst (SubI src1 src2));
10239
10240 ins_cost(INSN_COST);
10241 format %{ "subw $dst, $src1, $src2" %}
10242
10243 // use opcode to indicate that this is a sub not an add
10244 opcode(0x1);
10245
10246 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10247
10248 ins_pipe(ialu_reg_imm);
10249 %}
10250
10251 // Long Subtraction
10252 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10253
10254 match(Set dst (SubL src1 src2));
10255
10256 ins_cost(INSN_COST);
10257 format %{ "sub $dst, $src1, $src2" %}
10258
10259 ins_encode %{
10260 __ sub(as_Register($dst$$reg),
10261 as_Register($src1$$reg),
10262 as_Register($src2$$reg));
10263 %}
10264
10265 ins_pipe(ialu_reg_reg);
10266 %}
10267
10268 // No constant pool entries requiredLong Immediate Subtraction.
10269 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10270 match(Set dst (SubL src1 src2));
10271
10272 ins_cost(INSN_COST);
10273 format %{ "sub$dst, $src1, $src2" %}
10274
10275 // use opcode to indicate that this is a sub not an add
10276 opcode(0x1);
10277
10278 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10279
10280 ins_pipe(ialu_reg_imm);
10281 %}
10282
10283 // Integer Negation (special case for sub)
10284
10285 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10286 match(Set dst (SubI zero src));
10287
10288 ins_cost(INSN_COST);
10289 format %{ "negw $dst, $src\t# int" %}
10290
10291 ins_encode %{
10292 __ negw(as_Register($dst$$reg),
10293 as_Register($src$$reg));
10294 %}
10295
10296 ins_pipe(ialu_reg);
10297 %}
10298
10299 // Long Negation
10300
10301 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10302 match(Set dst (SubL zero src));
10303
10304 ins_cost(INSN_COST);
10305 format %{ "neg $dst, $src\t# long" %}
10306
10307 ins_encode %{
10308 __ neg(as_Register($dst$$reg),
10309 as_Register($src$$reg));
10310 %}
10311
10312 ins_pipe(ialu_reg);
10313 %}
10314
10315 // Integer Multiply
10316
10317 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10318 match(Set dst (MulI src1 src2));
10319
10320 ins_cost(INSN_COST * 3);
10321 format %{ "mulw $dst, $src1, $src2" %}
10322
10323 ins_encode %{
10324 __ mulw(as_Register($dst$$reg),
10325 as_Register($src1$$reg),
10326 as_Register($src2$$reg));
10327 %}
10328
10329 ins_pipe(imul_reg_reg);
10330 %}
10331
10332 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10333 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10334
10335 ins_cost(INSN_COST * 3);
10336 format %{ "smull $dst, $src1, $src2" %}
10337
10338 ins_encode %{
10339 __ smull(as_Register($dst$$reg),
10340 as_Register($src1$$reg),
10341 as_Register($src2$$reg));
10342 %}
10343
10344 ins_pipe(imul_reg_reg);
10345 %}
10346
10347 // Long Multiply
10348
10349 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10350 match(Set dst (MulL src1 src2));
10351
10352 ins_cost(INSN_COST * 5);
10353 format %{ "mul $dst, $src1, $src2" %}
10354
10355 ins_encode %{
10356 __ mul(as_Register($dst$$reg),
10357 as_Register($src1$$reg),
10358 as_Register($src2$$reg));
10359 %}
10360
10361 ins_pipe(lmul_reg_reg);
10362 %}
10363
10364 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10365 %{
10366 match(Set dst (MulHiL src1 src2));
10367
10368 ins_cost(INSN_COST * 7);
10369 format %{ "smulh $dst, $src1, $src2, \t# mulhi" %}
10370
10371 ins_encode %{
10372 __ smulh(as_Register($dst$$reg),
10373 as_Register($src1$$reg),
10374 as_Register($src2$$reg));
10375 %}
10376
10377 ins_pipe(lmul_reg_reg);
10378 %}
10379
10380 // Combined Integer Multiply & Add/Sub
10381
10382 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10383 match(Set dst (AddI src3 (MulI src1 src2)));
10384
10385 ins_cost(INSN_COST * 3);
10386 format %{ "madd $dst, $src1, $src2, $src3" %}
10387
10388 ins_encode %{
10389 __ maddw(as_Register($dst$$reg),
10390 as_Register($src1$$reg),
10391 as_Register($src2$$reg),
10392 as_Register($src3$$reg));
10393 %}
10394
10395 ins_pipe(imac_reg_reg);
10396 %}
10397
10398 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10399 match(Set dst (SubI src3 (MulI src1 src2)));
10400
10401 ins_cost(INSN_COST * 3);
10402 format %{ "msub $dst, $src1, $src2, $src3" %}
10403
10404 ins_encode %{
10405 __ msubw(as_Register($dst$$reg),
10406 as_Register($src1$$reg),
10407 as_Register($src2$$reg),
10408 as_Register($src3$$reg));
10409 %}
10410
10411 ins_pipe(imac_reg_reg);
10412 %}
10413
10414 // Combined Integer Multiply & Neg
10415
10416 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10417 match(Set dst (MulI (SubI zero src1) src2));
10418 match(Set dst (MulI src1 (SubI zero src2)));
10419
10420 ins_cost(INSN_COST * 3);
10421 format %{ "mneg $dst, $src1, $src2" %}
10422
10423 ins_encode %{
10424 __ mnegw(as_Register($dst$$reg),
10425 as_Register($src1$$reg),
10426 as_Register($src2$$reg));
10427 %}
10428
10429 ins_pipe(imac_reg_reg);
10430 %}
10431
10432 // Combined Long Multiply & Add/Sub
10433
10434 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10435 match(Set dst (AddL src3 (MulL src1 src2)));
10436
10437 ins_cost(INSN_COST * 5);
10438 format %{ "madd $dst, $src1, $src2, $src3" %}
10439
10440 ins_encode %{
10441 __ madd(as_Register($dst$$reg),
10442 as_Register($src1$$reg),
10443 as_Register($src2$$reg),
10444 as_Register($src3$$reg));
10445 %}
10446
10447 ins_pipe(lmac_reg_reg);
10448 %}
10449
10450 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10451 match(Set dst (SubL src3 (MulL src1 src2)));
10452
10453 ins_cost(INSN_COST * 5);
10454 format %{ "msub $dst, $src1, $src2, $src3" %}
10455
10456 ins_encode %{
10457 __ msub(as_Register($dst$$reg),
10458 as_Register($src1$$reg),
10459 as_Register($src2$$reg),
10460 as_Register($src3$$reg));
10461 %}
10462
10463 ins_pipe(lmac_reg_reg);
10464 %}
10465
10466 // Combined Long Multiply & Neg
10467
10468 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10469 match(Set dst (MulL (SubL zero src1) src2));
10470 match(Set dst (MulL src1 (SubL zero src2)));
10471
10472 ins_cost(INSN_COST * 5);
10473 format %{ "mneg $dst, $src1, $src2" %}
10474
10475 ins_encode %{
10476 __ mneg(as_Register($dst$$reg),
10477 as_Register($src1$$reg),
10478 as_Register($src2$$reg));
10479 %}
10480
10481 ins_pipe(lmac_reg_reg);
10482 %}
10483
10484 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10485
10486 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10487 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10488
10489 ins_cost(INSN_COST * 3);
10490 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10491
10492 ins_encode %{
10493 __ smaddl(as_Register($dst$$reg),
10494 as_Register($src1$$reg),
10495 as_Register($src2$$reg),
10496 as_Register($src3$$reg));
10497 %}
10498
10499 ins_pipe(imac_reg_reg);
10500 %}
10501
10502 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10503 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10504
10505 ins_cost(INSN_COST * 3);
10506 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10507
10508 ins_encode %{
10509 __ smsubl(as_Register($dst$$reg),
10510 as_Register($src1$$reg),
10511 as_Register($src2$$reg),
10512 as_Register($src3$$reg));
10513 %}
10514
10515 ins_pipe(imac_reg_reg);
10516 %}
10517
10518 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10519 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10520 match(Set dst (MulL (ConvI2L src1) (SubL zero (ConvI2L src2))));
10521
10522 ins_cost(INSN_COST * 3);
10523 format %{ "smnegl $dst, $src1, $src2" %}
10524
10525 ins_encode %{
10526 __ smnegl(as_Register($dst$$reg),
10527 as_Register($src1$$reg),
10528 as_Register($src2$$reg));
10529 %}
10530
10531 ins_pipe(imac_reg_reg);
10532 %}
10533
10534 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10535
10536 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10537 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10538
10539 ins_cost(INSN_COST * 5);
10540 format %{ "mulw rscratch1, $src1, $src2\n\t"
10541 "maddw $dst, $src3, $src4, rscratch1" %}
10542
10543 ins_encode %{
10544 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10545 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10546
10547 ins_pipe(imac_reg_reg);
10548 %}
10549
10550 // Integer Divide
10551
10552 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10553 match(Set dst (DivI src1 src2));
10554
10555 ins_cost(INSN_COST * 19);
10556 format %{ "sdivw $dst, $src1, $src2" %}
10557
10558 ins_encode(aarch64_enc_divw(dst, src1, src2));
10559 ins_pipe(idiv_reg_reg);
10560 %}
10561
10562 // Long Divide
10563
10564 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10565 match(Set dst (DivL src1 src2));
10566
10567 ins_cost(INSN_COST * 35);
10568 format %{ "sdiv $dst, $src1, $src2" %}
10569
10570 ins_encode(aarch64_enc_div(dst, src1, src2));
10571 ins_pipe(ldiv_reg_reg);
10572 %}
10573
10574 // Integer Remainder
10575
10576 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10577 match(Set dst (ModI src1 src2));
10578
10579 ins_cost(INSN_COST * 22);
10580 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10581 "msubw($dst, rscratch1, $src2, $src1" %}
10582
10583 ins_encode(aarch64_enc_modw(dst, src1, src2));
10584 ins_pipe(idiv_reg_reg);
10585 %}
10586
10587 // Long Remainder
10588
10589 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10590 match(Set dst (ModL src1 src2));
10591
10592 ins_cost(INSN_COST * 38);
10593 format %{ "sdiv rscratch1, $src1, $src2\n"
10594 "msub($dst, rscratch1, $src2, $src1" %}
10595
10596 ins_encode(aarch64_enc_mod(dst, src1, src2));
10597 ins_pipe(ldiv_reg_reg);
10598 %}
10599
10600 // Integer Shifts
10601
10602 // Shift Left Register
10603 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10604 match(Set dst (LShiftI src1 src2));
10605
10606 ins_cost(INSN_COST * 2);
10607 format %{ "lslvw $dst, $src1, $src2" %}
10608
10609 ins_encode %{
10610 __ lslvw(as_Register($dst$$reg),
10611 as_Register($src1$$reg),
10612 as_Register($src2$$reg));
10613 %}
10614
10615 ins_pipe(ialu_reg_reg_vshift);
10616 %}
10617
10618 // Shift Left Immediate
10619 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10620 match(Set dst (LShiftI src1 src2));
10621
10622 ins_cost(INSN_COST);
10623 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10624
10625 ins_encode %{
10626 __ lslw(as_Register($dst$$reg),
10627 as_Register($src1$$reg),
10628 $src2$$constant & 0x1f);
10629 %}
10630
10631 ins_pipe(ialu_reg_shift);
10632 %}
10633
10634 // Shift Right Logical Register
10635 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10636 match(Set dst (URShiftI src1 src2));
10637
10638 ins_cost(INSN_COST * 2);
10639 format %{ "lsrvw $dst, $src1, $src2" %}
10640
10641 ins_encode %{
10642 __ lsrvw(as_Register($dst$$reg),
10643 as_Register($src1$$reg),
10644 as_Register($src2$$reg));
10645 %}
10646
10647 ins_pipe(ialu_reg_reg_vshift);
10648 %}
10649
10650 // Shift Right Logical Immediate
10651 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10652 match(Set dst (URShiftI src1 src2));
10653
10654 ins_cost(INSN_COST);
10655 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10656
10657 ins_encode %{
10658 __ lsrw(as_Register($dst$$reg),
10659 as_Register($src1$$reg),
10660 $src2$$constant & 0x1f);
10661 %}
10662
10663 ins_pipe(ialu_reg_shift);
10664 %}
10665
10666 // Shift Right Arithmetic Register
10667 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10668 match(Set dst (RShiftI src1 src2));
10669
10670 ins_cost(INSN_COST * 2);
10671 format %{ "asrvw $dst, $src1, $src2" %}
10672
10673 ins_encode %{
10674 __ asrvw(as_Register($dst$$reg),
10675 as_Register($src1$$reg),
10676 as_Register($src2$$reg));
10677 %}
10678
10679 ins_pipe(ialu_reg_reg_vshift);
10680 %}
10681
10682 // Shift Right Arithmetic Immediate
10683 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10684 match(Set dst (RShiftI src1 src2));
10685
10686 ins_cost(INSN_COST);
10687 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10688
10689 ins_encode %{
10690 __ asrw(as_Register($dst$$reg),
10691 as_Register($src1$$reg),
10692 $src2$$constant & 0x1f);
10693 %}
10694
10695 ins_pipe(ialu_reg_shift);
10696 %}
10697
10698 // Combined Int Mask and Right Shift (using UBFM)
10699 // TODO
10700
10701 // Long Shifts
10702
10703 // Shift Left Register
10704 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10705 match(Set dst (LShiftL src1 src2));
10706
10707 ins_cost(INSN_COST * 2);
10708 format %{ "lslv $dst, $src1, $src2" %}
10709
10710 ins_encode %{
10711 __ lslv(as_Register($dst$$reg),
10712 as_Register($src1$$reg),
10713 as_Register($src2$$reg));
10714 %}
10715
10716 ins_pipe(ialu_reg_reg_vshift);
10717 %}
10718
10719 // Shift Left Immediate
10720 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10721 match(Set dst (LShiftL src1 src2));
10722
10723 ins_cost(INSN_COST);
10724 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10725
10726 ins_encode %{
10727 __ lsl(as_Register($dst$$reg),
10728 as_Register($src1$$reg),
10729 $src2$$constant & 0x3f);
10730 %}
10731
10732 ins_pipe(ialu_reg_shift);
10733 %}
10734
10735 // Shift Right Logical Register
10736 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10737 match(Set dst (URShiftL src1 src2));
10738
10739 ins_cost(INSN_COST * 2);
10740 format %{ "lsrv $dst, $src1, $src2" %}
10741
10742 ins_encode %{
10743 __ lsrv(as_Register($dst$$reg),
10744 as_Register($src1$$reg),
10745 as_Register($src2$$reg));
10746 %}
10747
10748 ins_pipe(ialu_reg_reg_vshift);
10749 %}
10750
10751 // Shift Right Logical Immediate
10752 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10753 match(Set dst (URShiftL src1 src2));
10754
10755 ins_cost(INSN_COST);
10756 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10757
10758 ins_encode %{
10759 __ lsr(as_Register($dst$$reg),
10760 as_Register($src1$$reg),
10761 $src2$$constant & 0x3f);
10762 %}
10763
10764 ins_pipe(ialu_reg_shift);
10765 %}
10766
10767 // A special-case pattern for card table stores.
10768 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10769 match(Set dst (URShiftL (CastP2X src1) src2));
10770
10771 ins_cost(INSN_COST);
10772 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10773
10774 ins_encode %{
10775 __ lsr(as_Register($dst$$reg),
10776 as_Register($src1$$reg),
10777 $src2$$constant & 0x3f);
10778 %}
10779
10780 ins_pipe(ialu_reg_shift);
10781 %}
10782
10783 // Shift Right Arithmetic Register
10784 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10785 match(Set dst (RShiftL src1 src2));
10786
10787 ins_cost(INSN_COST * 2);
10788 format %{ "asrv $dst, $src1, $src2" %}
10789
10790 ins_encode %{
10791 __ asrv(as_Register($dst$$reg),
10792 as_Register($src1$$reg),
10793 as_Register($src2$$reg));
10794 %}
10795
10796 ins_pipe(ialu_reg_reg_vshift);
10797 %}
10798
10799 // Shift Right Arithmetic Immediate
10800 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10801 match(Set dst (RShiftL src1 src2));
10802
10803 ins_cost(INSN_COST);
10804 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10805
10806 ins_encode %{
10807 __ asr(as_Register($dst$$reg),
10808 as_Register($src1$$reg),
10809 $src2$$constant & 0x3f);
10810 %}
10811
10812 ins_pipe(ialu_reg_shift);
10813 %}
10814
10815 // BEGIN This section of the file is automatically generated. Do not edit --------------
10816
10817
10818 // This pattern is automatically generated from aarch64_ad.m4
10819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10820 instruct regL_not_reg(iRegLNoSp dst,
10821 iRegL src1, immL_M1 m1,
10822 rFlagsReg cr) %{
10823 match(Set dst (XorL src1 m1));
10824 ins_cost(INSN_COST);
10825 format %{ "eon $dst, $src1, zr" %}
10826
10827 ins_encode %{
10828 __ eon(as_Register($dst$$reg),
10829 as_Register($src1$$reg),
10830 zr,
10831 Assembler::LSL, 0);
10832 %}
10833
10834 ins_pipe(ialu_reg);
10835 %}
10836
10837 // This pattern is automatically generated from aarch64_ad.m4
10838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10839 instruct regI_not_reg(iRegINoSp dst,
10840 iRegIorL2I src1, immI_M1 m1,
10841 rFlagsReg cr) %{
10842 match(Set dst (XorI src1 m1));
10843 ins_cost(INSN_COST);
10844 format %{ "eonw $dst, $src1, zr" %}
10845
10846 ins_encode %{
10847 __ eonw(as_Register($dst$$reg),
10848 as_Register($src1$$reg),
10849 zr,
10850 Assembler::LSL, 0);
10851 %}
10852
10853 ins_pipe(ialu_reg);
10854 %}
10855
10856 // This pattern is automatically generated from aarch64_ad.m4
10857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10858 instruct AndI_reg_not_reg(iRegINoSp dst,
10859 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10860 rFlagsReg cr) %{
10861 match(Set dst (AndI src1 (XorI src2 m1)));
10862 ins_cost(INSN_COST);
10863 format %{ "bicw $dst, $src1, $src2" %}
10864
10865 ins_encode %{
10866 __ bicw(as_Register($dst$$reg),
10867 as_Register($src1$$reg),
10868 as_Register($src2$$reg),
10869 Assembler::LSL, 0);
10870 %}
10871
10872 ins_pipe(ialu_reg_reg);
10873 %}
10874
10875 // This pattern is automatically generated from aarch64_ad.m4
10876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10877 instruct AndL_reg_not_reg(iRegLNoSp dst,
10878 iRegL src1, iRegL src2, immL_M1 m1,
10879 rFlagsReg cr) %{
10880 match(Set dst (AndL src1 (XorL src2 m1)));
10881 ins_cost(INSN_COST);
10882 format %{ "bic $dst, $src1, $src2" %}
10883
10884 ins_encode %{
10885 __ bic(as_Register($dst$$reg),
10886 as_Register($src1$$reg),
10887 as_Register($src2$$reg),
10888 Assembler::LSL, 0);
10889 %}
10890
10891 ins_pipe(ialu_reg_reg);
10892 %}
10893
10894 // This pattern is automatically generated from aarch64_ad.m4
10895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10896 instruct OrI_reg_not_reg(iRegINoSp dst,
10897 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10898 rFlagsReg cr) %{
10899 match(Set dst (OrI src1 (XorI src2 m1)));
10900 ins_cost(INSN_COST);
10901 format %{ "ornw $dst, $src1, $src2" %}
10902
10903 ins_encode %{
10904 __ ornw(as_Register($dst$$reg),
10905 as_Register($src1$$reg),
10906 as_Register($src2$$reg),
10907 Assembler::LSL, 0);
10908 %}
10909
10910 ins_pipe(ialu_reg_reg);
10911 %}
10912
10913 // This pattern is automatically generated from aarch64_ad.m4
10914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10915 instruct OrL_reg_not_reg(iRegLNoSp dst,
10916 iRegL src1, iRegL src2, immL_M1 m1,
10917 rFlagsReg cr) %{
10918 match(Set dst (OrL src1 (XorL src2 m1)));
10919 ins_cost(INSN_COST);
10920 format %{ "orn $dst, $src1, $src2" %}
10921
10922 ins_encode %{
10923 __ orn(as_Register($dst$$reg),
10924 as_Register($src1$$reg),
10925 as_Register($src2$$reg),
10926 Assembler::LSL, 0);
10927 %}
10928
10929 ins_pipe(ialu_reg_reg);
10930 %}
10931
10932 // This pattern is automatically generated from aarch64_ad.m4
10933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10934 instruct XorI_reg_not_reg(iRegINoSp dst,
10935 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10936 rFlagsReg cr) %{
10937 match(Set dst (XorI m1 (XorI src2 src1)));
10938 ins_cost(INSN_COST);
10939 format %{ "eonw $dst, $src1, $src2" %}
10940
10941 ins_encode %{
10942 __ eonw(as_Register($dst$$reg),
10943 as_Register($src1$$reg),
10944 as_Register($src2$$reg),
10945 Assembler::LSL, 0);
10946 %}
10947
10948 ins_pipe(ialu_reg_reg);
10949 %}
10950
10951 // This pattern is automatically generated from aarch64_ad.m4
10952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10953 instruct XorL_reg_not_reg(iRegLNoSp dst,
10954 iRegL src1, iRegL src2, immL_M1 m1,
10955 rFlagsReg cr) %{
10956 match(Set dst (XorL m1 (XorL src2 src1)));
10957 ins_cost(INSN_COST);
10958 format %{ "eon $dst, $src1, $src2" %}
10959
10960 ins_encode %{
10961 __ eon(as_Register($dst$$reg),
10962 as_Register($src1$$reg),
10963 as_Register($src2$$reg),
10964 Assembler::LSL, 0);
10965 %}
10966
10967 ins_pipe(ialu_reg_reg);
10968 %}
10969
10970 // This pattern is automatically generated from aarch64_ad.m4
10971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10972 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10973 iRegIorL2I src1, iRegIorL2I src2,
10974 immI src3, immI_M1 src4, rFlagsReg cr) %{
10975 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10976 ins_cost(1.9 * INSN_COST);
10977 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10978
10979 ins_encode %{
10980 __ bicw(as_Register($dst$$reg),
10981 as_Register($src1$$reg),
10982 as_Register($src2$$reg),
10983 Assembler::LSR,
10984 $src3$$constant & 0x1f);
10985 %}
10986
10987 ins_pipe(ialu_reg_reg_shift);
10988 %}
10989
10990 // This pattern is automatically generated from aarch64_ad.m4
10991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10992 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10993 iRegL src1, iRegL src2,
10994 immI src3, immL_M1 src4, rFlagsReg cr) %{
10995 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10996 ins_cost(1.9 * INSN_COST);
10997 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10998
10999 ins_encode %{
11000 __ bic(as_Register($dst$$reg),
11001 as_Register($src1$$reg),
11002 as_Register($src2$$reg),
11003 Assembler::LSR,
11004 $src3$$constant & 0x3f);
11005 %}
11006
11007 ins_pipe(ialu_reg_reg_shift);
11008 %}
11009
11010 // This pattern is automatically generated from aarch64_ad.m4
11011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11012 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11013 iRegIorL2I src1, iRegIorL2I src2,
11014 immI src3, immI_M1 src4, rFlagsReg cr) %{
11015 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11016 ins_cost(1.9 * INSN_COST);
11017 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11018
11019 ins_encode %{
11020 __ bicw(as_Register($dst$$reg),
11021 as_Register($src1$$reg),
11022 as_Register($src2$$reg),
11023 Assembler::ASR,
11024 $src3$$constant & 0x1f);
11025 %}
11026
11027 ins_pipe(ialu_reg_reg_shift);
11028 %}
11029
11030 // This pattern is automatically generated from aarch64_ad.m4
11031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11032 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11033 iRegL src1, iRegL src2,
11034 immI src3, immL_M1 src4, rFlagsReg cr) %{
11035 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11036 ins_cost(1.9 * INSN_COST);
11037 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11038
11039 ins_encode %{
11040 __ bic(as_Register($dst$$reg),
11041 as_Register($src1$$reg),
11042 as_Register($src2$$reg),
11043 Assembler::ASR,
11044 $src3$$constant & 0x3f);
11045 %}
11046
11047 ins_pipe(ialu_reg_reg_shift);
11048 %}
11049
11050 // This pattern is automatically generated from aarch64_ad.m4
11051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11052 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11053 iRegIorL2I src1, iRegIorL2I src2,
11054 immI src3, immI_M1 src4, rFlagsReg cr) %{
11055 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11056 ins_cost(1.9 * INSN_COST);
11057 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11058
11059 ins_encode %{
11060 __ bicw(as_Register($dst$$reg),
11061 as_Register($src1$$reg),
11062 as_Register($src2$$reg),
11063 Assembler::LSL,
11064 $src3$$constant & 0x1f);
11065 %}
11066
11067 ins_pipe(ialu_reg_reg_shift);
11068 %}
11069
11070 // This pattern is automatically generated from aarch64_ad.m4
11071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11072 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11073 iRegL src1, iRegL src2,
11074 immI src3, immL_M1 src4, rFlagsReg cr) %{
11075 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11076 ins_cost(1.9 * INSN_COST);
11077 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11078
11079 ins_encode %{
11080 __ bic(as_Register($dst$$reg),
11081 as_Register($src1$$reg),
11082 as_Register($src2$$reg),
11083 Assembler::LSL,
11084 $src3$$constant & 0x3f);
11085 %}
11086
11087 ins_pipe(ialu_reg_reg_shift);
11088 %}
11089
11090 // This pattern is automatically generated from aarch64_ad.m4
11091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11092 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11093 iRegIorL2I src1, iRegIorL2I src2,
11094 immI src3, immI_M1 src4, rFlagsReg cr) %{
11095 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11096 ins_cost(1.9 * INSN_COST);
11097 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11098
11099 ins_encode %{
11100 __ eonw(as_Register($dst$$reg),
11101 as_Register($src1$$reg),
11102 as_Register($src2$$reg),
11103 Assembler::LSR,
11104 $src3$$constant & 0x1f);
11105 %}
11106
11107 ins_pipe(ialu_reg_reg_shift);
11108 %}
11109
11110 // This pattern is automatically generated from aarch64_ad.m4
11111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11112 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11113 iRegL src1, iRegL src2,
11114 immI src3, immL_M1 src4, rFlagsReg cr) %{
11115 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11116 ins_cost(1.9 * INSN_COST);
11117 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11118
11119 ins_encode %{
11120 __ eon(as_Register($dst$$reg),
11121 as_Register($src1$$reg),
11122 as_Register($src2$$reg),
11123 Assembler::LSR,
11124 $src3$$constant & 0x3f);
11125 %}
11126
11127 ins_pipe(ialu_reg_reg_shift);
11128 %}
11129
11130 // This pattern is automatically generated from aarch64_ad.m4
11131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11132 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11133 iRegIorL2I src1, iRegIorL2I src2,
11134 immI src3, immI_M1 src4, rFlagsReg cr) %{
11135 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11136 ins_cost(1.9 * INSN_COST);
11137 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11138
11139 ins_encode %{
11140 __ eonw(as_Register($dst$$reg),
11141 as_Register($src1$$reg),
11142 as_Register($src2$$reg),
11143 Assembler::ASR,
11144 $src3$$constant & 0x1f);
11145 %}
11146
11147 ins_pipe(ialu_reg_reg_shift);
11148 %}
11149
11150 // This pattern is automatically generated from aarch64_ad.m4
11151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11152 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11153 iRegL src1, iRegL src2,
11154 immI src3, immL_M1 src4, rFlagsReg cr) %{
11155 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11156 ins_cost(1.9 * INSN_COST);
11157 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11158
11159 ins_encode %{
11160 __ eon(as_Register($dst$$reg),
11161 as_Register($src1$$reg),
11162 as_Register($src2$$reg),
11163 Assembler::ASR,
11164 $src3$$constant & 0x3f);
11165 %}
11166
11167 ins_pipe(ialu_reg_reg_shift);
11168 %}
11169
11170 // This pattern is automatically generated from aarch64_ad.m4
11171 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11172 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11173 iRegIorL2I src1, iRegIorL2I src2,
11174 immI src3, immI_M1 src4, rFlagsReg cr) %{
11175 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11176 ins_cost(1.9 * INSN_COST);
11177 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11178
11179 ins_encode %{
11180 __ eonw(as_Register($dst$$reg),
11181 as_Register($src1$$reg),
11182 as_Register($src2$$reg),
11183 Assembler::LSL,
11184 $src3$$constant & 0x1f);
11185 %}
11186
11187 ins_pipe(ialu_reg_reg_shift);
11188 %}
11189
11190 // This pattern is automatically generated from aarch64_ad.m4
11191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11192 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11193 iRegL src1, iRegL src2,
11194 immI src3, immL_M1 src4, rFlagsReg cr) %{
11195 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11196 ins_cost(1.9 * INSN_COST);
11197 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11198
11199 ins_encode %{
11200 __ eon(as_Register($dst$$reg),
11201 as_Register($src1$$reg),
11202 as_Register($src2$$reg),
11203 Assembler::LSL,
11204 $src3$$constant & 0x3f);
11205 %}
11206
11207 ins_pipe(ialu_reg_reg_shift);
11208 %}
11209
11210 // This pattern is automatically generated from aarch64_ad.m4
11211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11212 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11213 iRegIorL2I src1, iRegIorL2I src2,
11214 immI src3, immI_M1 src4, rFlagsReg cr) %{
11215 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11216 ins_cost(1.9 * INSN_COST);
11217 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11218
11219 ins_encode %{
11220 __ ornw(as_Register($dst$$reg),
11221 as_Register($src1$$reg),
11222 as_Register($src2$$reg),
11223 Assembler::LSR,
11224 $src3$$constant & 0x1f);
11225 %}
11226
11227 ins_pipe(ialu_reg_reg_shift);
11228 %}
11229
11230 // This pattern is automatically generated from aarch64_ad.m4
11231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11232 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11233 iRegL src1, iRegL src2,
11234 immI src3, immL_M1 src4, rFlagsReg cr) %{
11235 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11236 ins_cost(1.9 * INSN_COST);
11237 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11238
11239 ins_encode %{
11240 __ orn(as_Register($dst$$reg),
11241 as_Register($src1$$reg),
11242 as_Register($src2$$reg),
11243 Assembler::LSR,
11244 $src3$$constant & 0x3f);
11245 %}
11246
11247 ins_pipe(ialu_reg_reg_shift);
11248 %}
11249
11250 // This pattern is automatically generated from aarch64_ad.m4
11251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11252 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11253 iRegIorL2I src1, iRegIorL2I src2,
11254 immI src3, immI_M1 src4, rFlagsReg cr) %{
11255 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11256 ins_cost(1.9 * INSN_COST);
11257 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11258
11259 ins_encode %{
11260 __ ornw(as_Register($dst$$reg),
11261 as_Register($src1$$reg),
11262 as_Register($src2$$reg),
11263 Assembler::ASR,
11264 $src3$$constant & 0x1f);
11265 %}
11266
11267 ins_pipe(ialu_reg_reg_shift);
11268 %}
11269
11270 // This pattern is automatically generated from aarch64_ad.m4
11271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11272 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11273 iRegL src1, iRegL src2,
11274 immI src3, immL_M1 src4, rFlagsReg cr) %{
11275 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11276 ins_cost(1.9 * INSN_COST);
11277 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11278
11279 ins_encode %{
11280 __ orn(as_Register($dst$$reg),
11281 as_Register($src1$$reg),
11282 as_Register($src2$$reg),
11283 Assembler::ASR,
11284 $src3$$constant & 0x3f);
11285 %}
11286
11287 ins_pipe(ialu_reg_reg_shift);
11288 %}
11289
11290 // This pattern is automatically generated from aarch64_ad.m4
11291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11292 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11293 iRegIorL2I src1, iRegIorL2I src2,
11294 immI src3, immI_M1 src4, rFlagsReg cr) %{
11295 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11296 ins_cost(1.9 * INSN_COST);
11297 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11298
11299 ins_encode %{
11300 __ ornw(as_Register($dst$$reg),
11301 as_Register($src1$$reg),
11302 as_Register($src2$$reg),
11303 Assembler::LSL,
11304 $src3$$constant & 0x1f);
11305 %}
11306
11307 ins_pipe(ialu_reg_reg_shift);
11308 %}
11309
11310 // This pattern is automatically generated from aarch64_ad.m4
11311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11312 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11313 iRegL src1, iRegL src2,
11314 immI src3, immL_M1 src4, rFlagsReg cr) %{
11315 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11316 ins_cost(1.9 * INSN_COST);
11317 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11318
11319 ins_encode %{
11320 __ orn(as_Register($dst$$reg),
11321 as_Register($src1$$reg),
11322 as_Register($src2$$reg),
11323 Assembler::LSL,
11324 $src3$$constant & 0x3f);
11325 %}
11326
11327 ins_pipe(ialu_reg_reg_shift);
11328 %}
11329
11330 // This pattern is automatically generated from aarch64_ad.m4
11331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11332 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11333 iRegIorL2I src1, iRegIorL2I src2,
11334 immI src3, rFlagsReg cr) %{
11335 match(Set dst (AndI src1 (URShiftI src2 src3)));
11336
11337 ins_cost(1.9 * INSN_COST);
11338 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11339
11340 ins_encode %{
11341 __ andw(as_Register($dst$$reg),
11342 as_Register($src1$$reg),
11343 as_Register($src2$$reg),
11344 Assembler::LSR,
11345 $src3$$constant & 0x1f);
11346 %}
11347
11348 ins_pipe(ialu_reg_reg_shift);
11349 %}
11350
11351 // This pattern is automatically generated from aarch64_ad.m4
11352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11353 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11354 iRegL src1, iRegL src2,
11355 immI src3, rFlagsReg cr) %{
11356 match(Set dst (AndL src1 (URShiftL src2 src3)));
11357
11358 ins_cost(1.9 * INSN_COST);
11359 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11360
11361 ins_encode %{
11362 __ andr(as_Register($dst$$reg),
11363 as_Register($src1$$reg),
11364 as_Register($src2$$reg),
11365 Assembler::LSR,
11366 $src3$$constant & 0x3f);
11367 %}
11368
11369 ins_pipe(ialu_reg_reg_shift);
11370 %}
11371
11372 // This pattern is automatically generated from aarch64_ad.m4
11373 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11374 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11375 iRegIorL2I src1, iRegIorL2I src2,
11376 immI src3, rFlagsReg cr) %{
11377 match(Set dst (AndI src1 (RShiftI src2 src3)));
11378
11379 ins_cost(1.9 * INSN_COST);
11380 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11381
11382 ins_encode %{
11383 __ andw(as_Register($dst$$reg),
11384 as_Register($src1$$reg),
11385 as_Register($src2$$reg),
11386 Assembler::ASR,
11387 $src3$$constant & 0x1f);
11388 %}
11389
11390 ins_pipe(ialu_reg_reg_shift);
11391 %}
11392
11393 // This pattern is automatically generated from aarch64_ad.m4
11394 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11395 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11396 iRegL src1, iRegL src2,
11397 immI src3, rFlagsReg cr) %{
11398 match(Set dst (AndL src1 (RShiftL src2 src3)));
11399
11400 ins_cost(1.9 * INSN_COST);
11401 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11402
11403 ins_encode %{
11404 __ andr(as_Register($dst$$reg),
11405 as_Register($src1$$reg),
11406 as_Register($src2$$reg),
11407 Assembler::ASR,
11408 $src3$$constant & 0x3f);
11409 %}
11410
11411 ins_pipe(ialu_reg_reg_shift);
11412 %}
11413
11414 // This pattern is automatically generated from aarch64_ad.m4
11415 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11416 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11417 iRegIorL2I src1, iRegIorL2I src2,
11418 immI src3, rFlagsReg cr) %{
11419 match(Set dst (AndI src1 (LShiftI src2 src3)));
11420
11421 ins_cost(1.9 * INSN_COST);
11422 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11423
11424 ins_encode %{
11425 __ andw(as_Register($dst$$reg),
11426 as_Register($src1$$reg),
11427 as_Register($src2$$reg),
11428 Assembler::LSL,
11429 $src3$$constant & 0x1f);
11430 %}
11431
11432 ins_pipe(ialu_reg_reg_shift);
11433 %}
11434
11435 // This pattern is automatically generated from aarch64_ad.m4
11436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11437 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11438 iRegL src1, iRegL src2,
11439 immI src3, rFlagsReg cr) %{
11440 match(Set dst (AndL src1 (LShiftL src2 src3)));
11441
11442 ins_cost(1.9 * INSN_COST);
11443 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11444
11445 ins_encode %{
11446 __ andr(as_Register($dst$$reg),
11447 as_Register($src1$$reg),
11448 as_Register($src2$$reg),
11449 Assembler::LSL,
11450 $src3$$constant & 0x3f);
11451 %}
11452
11453 ins_pipe(ialu_reg_reg_shift);
11454 %}
11455
11456 // This pattern is automatically generated from aarch64_ad.m4
11457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11458 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11459 iRegIorL2I src1, iRegIorL2I src2,
11460 immI src3, rFlagsReg cr) %{
11461 match(Set dst (XorI src1 (URShiftI src2 src3)));
11462
11463 ins_cost(1.9 * INSN_COST);
11464 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11465
11466 ins_encode %{
11467 __ eorw(as_Register($dst$$reg),
11468 as_Register($src1$$reg),
11469 as_Register($src2$$reg),
11470 Assembler::LSR,
11471 $src3$$constant & 0x1f);
11472 %}
11473
11474 ins_pipe(ialu_reg_reg_shift);
11475 %}
11476
11477 // This pattern is automatically generated from aarch64_ad.m4
11478 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11479 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11480 iRegL src1, iRegL src2,
11481 immI src3, rFlagsReg cr) %{
11482 match(Set dst (XorL src1 (URShiftL src2 src3)));
11483
11484 ins_cost(1.9 * INSN_COST);
11485 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11486
11487 ins_encode %{
11488 __ eor(as_Register($dst$$reg),
11489 as_Register($src1$$reg),
11490 as_Register($src2$$reg),
11491 Assembler::LSR,
11492 $src3$$constant & 0x3f);
11493 %}
11494
11495 ins_pipe(ialu_reg_reg_shift);
11496 %}
11497
11498 // This pattern is automatically generated from aarch64_ad.m4
11499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11500 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11501 iRegIorL2I src1, iRegIorL2I src2,
11502 immI src3, rFlagsReg cr) %{
11503 match(Set dst (XorI src1 (RShiftI src2 src3)));
11504
11505 ins_cost(1.9 * INSN_COST);
11506 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11507
11508 ins_encode %{
11509 __ eorw(as_Register($dst$$reg),
11510 as_Register($src1$$reg),
11511 as_Register($src2$$reg),
11512 Assembler::ASR,
11513 $src3$$constant & 0x1f);
11514 %}
11515
11516 ins_pipe(ialu_reg_reg_shift);
11517 %}
11518
11519 // This pattern is automatically generated from aarch64_ad.m4
11520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11521 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11522 iRegL src1, iRegL src2,
11523 immI src3, rFlagsReg cr) %{
11524 match(Set dst (XorL src1 (RShiftL src2 src3)));
11525
11526 ins_cost(1.9 * INSN_COST);
11527 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11528
11529 ins_encode %{
11530 __ eor(as_Register($dst$$reg),
11531 as_Register($src1$$reg),
11532 as_Register($src2$$reg),
11533 Assembler::ASR,
11534 $src3$$constant & 0x3f);
11535 %}
11536
11537 ins_pipe(ialu_reg_reg_shift);
11538 %}
11539
11540 // This pattern is automatically generated from aarch64_ad.m4
11541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11542 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11543 iRegIorL2I src1, iRegIorL2I src2,
11544 immI src3, rFlagsReg cr) %{
11545 match(Set dst (XorI src1 (LShiftI src2 src3)));
11546
11547 ins_cost(1.9 * INSN_COST);
11548 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11549
11550 ins_encode %{
11551 __ eorw(as_Register($dst$$reg),
11552 as_Register($src1$$reg),
11553 as_Register($src2$$reg),
11554 Assembler::LSL,
11555 $src3$$constant & 0x1f);
11556 %}
11557
11558 ins_pipe(ialu_reg_reg_shift);
11559 %}
11560
11561 // This pattern is automatically generated from aarch64_ad.m4
11562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11563 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11564 iRegL src1, iRegL src2,
11565 immI src3, rFlagsReg cr) %{
11566 match(Set dst (XorL src1 (LShiftL src2 src3)));
11567
11568 ins_cost(1.9 * INSN_COST);
11569 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11570
11571 ins_encode %{
11572 __ eor(as_Register($dst$$reg),
11573 as_Register($src1$$reg),
11574 as_Register($src2$$reg),
11575 Assembler::LSL,
11576 $src3$$constant & 0x3f);
11577 %}
11578
11579 ins_pipe(ialu_reg_reg_shift);
11580 %}
11581
11582 // This pattern is automatically generated from aarch64_ad.m4
11583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11584 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11585 iRegIorL2I src1, iRegIorL2I src2,
11586 immI src3, rFlagsReg cr) %{
11587 match(Set dst (OrI src1 (URShiftI src2 src3)));
11588
11589 ins_cost(1.9 * INSN_COST);
11590 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11591
11592 ins_encode %{
11593 __ orrw(as_Register($dst$$reg),
11594 as_Register($src1$$reg),
11595 as_Register($src2$$reg),
11596 Assembler::LSR,
11597 $src3$$constant & 0x1f);
11598 %}
11599
11600 ins_pipe(ialu_reg_reg_shift);
11601 %}
11602
11603 // This pattern is automatically generated from aarch64_ad.m4
11604 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11605 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11606 iRegL src1, iRegL src2,
11607 immI src3, rFlagsReg cr) %{
11608 match(Set dst (OrL src1 (URShiftL src2 src3)));
11609
11610 ins_cost(1.9 * INSN_COST);
11611 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11612
11613 ins_encode %{
11614 __ orr(as_Register($dst$$reg),
11615 as_Register($src1$$reg),
11616 as_Register($src2$$reg),
11617 Assembler::LSR,
11618 $src3$$constant & 0x3f);
11619 %}
11620
11621 ins_pipe(ialu_reg_reg_shift);
11622 %}
11623
11624 // This pattern is automatically generated from aarch64_ad.m4
11625 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11626 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11627 iRegIorL2I src1, iRegIorL2I src2,
11628 immI src3, rFlagsReg cr) %{
11629 match(Set dst (OrI src1 (RShiftI src2 src3)));
11630
11631 ins_cost(1.9 * INSN_COST);
11632 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11633
11634 ins_encode %{
11635 __ orrw(as_Register($dst$$reg),
11636 as_Register($src1$$reg),
11637 as_Register($src2$$reg),
11638 Assembler::ASR,
11639 $src3$$constant & 0x1f);
11640 %}
11641
11642 ins_pipe(ialu_reg_reg_shift);
11643 %}
11644
11645 // This pattern is automatically generated from aarch64_ad.m4
11646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11647 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11648 iRegL src1, iRegL src2,
11649 immI src3, rFlagsReg cr) %{
11650 match(Set dst (OrL src1 (RShiftL src2 src3)));
11651
11652 ins_cost(1.9 * INSN_COST);
11653 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11654
11655 ins_encode %{
11656 __ orr(as_Register($dst$$reg),
11657 as_Register($src1$$reg),
11658 as_Register($src2$$reg),
11659 Assembler::ASR,
11660 $src3$$constant & 0x3f);
11661 %}
11662
11663 ins_pipe(ialu_reg_reg_shift);
11664 %}
11665
11666 // This pattern is automatically generated from aarch64_ad.m4
11667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11668 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11669 iRegIorL2I src1, iRegIorL2I src2,
11670 immI src3, rFlagsReg cr) %{
11671 match(Set dst (OrI src1 (LShiftI src2 src3)));
11672
11673 ins_cost(1.9 * INSN_COST);
11674 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11675
11676 ins_encode %{
11677 __ orrw(as_Register($dst$$reg),
11678 as_Register($src1$$reg),
11679 as_Register($src2$$reg),
11680 Assembler::LSL,
11681 $src3$$constant & 0x1f);
11682 %}
11683
11684 ins_pipe(ialu_reg_reg_shift);
11685 %}
11686
11687 // This pattern is automatically generated from aarch64_ad.m4
11688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11689 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11690 iRegL src1, iRegL src2,
11691 immI src3, rFlagsReg cr) %{
11692 match(Set dst (OrL src1 (LShiftL src2 src3)));
11693
11694 ins_cost(1.9 * INSN_COST);
11695 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11696
11697 ins_encode %{
11698 __ orr(as_Register($dst$$reg),
11699 as_Register($src1$$reg),
11700 as_Register($src2$$reg),
11701 Assembler::LSL,
11702 $src3$$constant & 0x3f);
11703 %}
11704
11705 ins_pipe(ialu_reg_reg_shift);
11706 %}
11707
11708 // This pattern is automatically generated from aarch64_ad.m4
11709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11710 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11711 iRegIorL2I src1, iRegIorL2I src2,
11712 immI src3, rFlagsReg cr) %{
11713 match(Set dst (AddI src1 (URShiftI src2 src3)));
11714
11715 ins_cost(1.9 * INSN_COST);
11716 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11717
11718 ins_encode %{
11719 __ addw(as_Register($dst$$reg),
11720 as_Register($src1$$reg),
11721 as_Register($src2$$reg),
11722 Assembler::LSR,
11723 $src3$$constant & 0x1f);
11724 %}
11725
11726 ins_pipe(ialu_reg_reg_shift);
11727 %}
11728
11729 // This pattern is automatically generated from aarch64_ad.m4
11730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11731 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11732 iRegL src1, iRegL src2,
11733 immI src3, rFlagsReg cr) %{
11734 match(Set dst (AddL src1 (URShiftL src2 src3)));
11735
11736 ins_cost(1.9 * INSN_COST);
11737 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11738
11739 ins_encode %{
11740 __ add(as_Register($dst$$reg),
11741 as_Register($src1$$reg),
11742 as_Register($src2$$reg),
11743 Assembler::LSR,
11744 $src3$$constant & 0x3f);
11745 %}
11746
11747 ins_pipe(ialu_reg_reg_shift);
11748 %}
11749
11750 // This pattern is automatically generated from aarch64_ad.m4
11751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11752 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11753 iRegIorL2I src1, iRegIorL2I src2,
11754 immI src3, rFlagsReg cr) %{
11755 match(Set dst (AddI src1 (RShiftI src2 src3)));
11756
11757 ins_cost(1.9 * INSN_COST);
11758 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11759
11760 ins_encode %{
11761 __ addw(as_Register($dst$$reg),
11762 as_Register($src1$$reg),
11763 as_Register($src2$$reg),
11764 Assembler::ASR,
11765 $src3$$constant & 0x1f);
11766 %}
11767
11768 ins_pipe(ialu_reg_reg_shift);
11769 %}
11770
11771 // This pattern is automatically generated from aarch64_ad.m4
11772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11773 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11774 iRegL src1, iRegL src2,
11775 immI src3, rFlagsReg cr) %{
11776 match(Set dst (AddL src1 (RShiftL src2 src3)));
11777
11778 ins_cost(1.9 * INSN_COST);
11779 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11780
11781 ins_encode %{
11782 __ add(as_Register($dst$$reg),
11783 as_Register($src1$$reg),
11784 as_Register($src2$$reg),
11785 Assembler::ASR,
11786 $src3$$constant & 0x3f);
11787 %}
11788
11789 ins_pipe(ialu_reg_reg_shift);
11790 %}
11791
11792 // This pattern is automatically generated from aarch64_ad.m4
11793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11794 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11795 iRegIorL2I src1, iRegIorL2I src2,
11796 immI src3, rFlagsReg cr) %{
11797 match(Set dst (AddI src1 (LShiftI src2 src3)));
11798
11799 ins_cost(1.9 * INSN_COST);
11800 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11801
11802 ins_encode %{
11803 __ addw(as_Register($dst$$reg),
11804 as_Register($src1$$reg),
11805 as_Register($src2$$reg),
11806 Assembler::LSL,
11807 $src3$$constant & 0x1f);
11808 %}
11809
11810 ins_pipe(ialu_reg_reg_shift);
11811 %}
11812
11813 // This pattern is automatically generated from aarch64_ad.m4
11814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11815 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11816 iRegL src1, iRegL src2,
11817 immI src3, rFlagsReg cr) %{
11818 match(Set dst (AddL src1 (LShiftL src2 src3)));
11819
11820 ins_cost(1.9 * INSN_COST);
11821 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11822
11823 ins_encode %{
11824 __ add(as_Register($dst$$reg),
11825 as_Register($src1$$reg),
11826 as_Register($src2$$reg),
11827 Assembler::LSL,
11828 $src3$$constant & 0x3f);
11829 %}
11830
11831 ins_pipe(ialu_reg_reg_shift);
11832 %}
11833
11834 // This pattern is automatically generated from aarch64_ad.m4
11835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11836 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11837 iRegIorL2I src1, iRegIorL2I src2,
11838 immI src3, rFlagsReg cr) %{
11839 match(Set dst (SubI src1 (URShiftI src2 src3)));
11840
11841 ins_cost(1.9 * INSN_COST);
11842 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11843
11844 ins_encode %{
11845 __ subw(as_Register($dst$$reg),
11846 as_Register($src1$$reg),
11847 as_Register($src2$$reg),
11848 Assembler::LSR,
11849 $src3$$constant & 0x1f);
11850 %}
11851
11852 ins_pipe(ialu_reg_reg_shift);
11853 %}
11854
11855 // This pattern is automatically generated from aarch64_ad.m4
11856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11857 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11858 iRegL src1, iRegL src2,
11859 immI src3, rFlagsReg cr) %{
11860 match(Set dst (SubL src1 (URShiftL src2 src3)));
11861
11862 ins_cost(1.9 * INSN_COST);
11863 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11864
11865 ins_encode %{
11866 __ sub(as_Register($dst$$reg),
11867 as_Register($src1$$reg),
11868 as_Register($src2$$reg),
11869 Assembler::LSR,
11870 $src3$$constant & 0x3f);
11871 %}
11872
11873 ins_pipe(ialu_reg_reg_shift);
11874 %}
11875
11876 // This pattern is automatically generated from aarch64_ad.m4
11877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11878 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11879 iRegIorL2I src1, iRegIorL2I src2,
11880 immI src3, rFlagsReg cr) %{
11881 match(Set dst (SubI src1 (RShiftI src2 src3)));
11882
11883 ins_cost(1.9 * INSN_COST);
11884 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11885
11886 ins_encode %{
11887 __ subw(as_Register($dst$$reg),
11888 as_Register($src1$$reg),
11889 as_Register($src2$$reg),
11890 Assembler::ASR,
11891 $src3$$constant & 0x1f);
11892 %}
11893
11894 ins_pipe(ialu_reg_reg_shift);
11895 %}
11896
11897 // This pattern is automatically generated from aarch64_ad.m4
11898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11899 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11900 iRegL src1, iRegL src2,
11901 immI src3, rFlagsReg cr) %{
11902 match(Set dst (SubL src1 (RShiftL src2 src3)));
11903
11904 ins_cost(1.9 * INSN_COST);
11905 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11906
11907 ins_encode %{
11908 __ sub(as_Register($dst$$reg),
11909 as_Register($src1$$reg),
11910 as_Register($src2$$reg),
11911 Assembler::ASR,
11912 $src3$$constant & 0x3f);
11913 %}
11914
11915 ins_pipe(ialu_reg_reg_shift);
11916 %}
11917
11918 // This pattern is automatically generated from aarch64_ad.m4
11919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11920 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11921 iRegIorL2I src1, iRegIorL2I src2,
11922 immI src3, rFlagsReg cr) %{
11923 match(Set dst (SubI src1 (LShiftI src2 src3)));
11924
11925 ins_cost(1.9 * INSN_COST);
11926 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11927
11928 ins_encode %{
11929 __ subw(as_Register($dst$$reg),
11930 as_Register($src1$$reg),
11931 as_Register($src2$$reg),
11932 Assembler::LSL,
11933 $src3$$constant & 0x1f);
11934 %}
11935
11936 ins_pipe(ialu_reg_reg_shift);
11937 %}
11938
11939 // This pattern is automatically generated from aarch64_ad.m4
11940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11941 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11942 iRegL src1, iRegL src2,
11943 immI src3, rFlagsReg cr) %{
11944 match(Set dst (SubL src1 (LShiftL src2 src3)));
11945
11946 ins_cost(1.9 * INSN_COST);
11947 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11948
11949 ins_encode %{
11950 __ sub(as_Register($dst$$reg),
11951 as_Register($src1$$reg),
11952 as_Register($src2$$reg),
11953 Assembler::LSL,
11954 $src3$$constant & 0x3f);
11955 %}
11956
11957 ins_pipe(ialu_reg_reg_shift);
11958 %}
11959
11960
11961 // This pattern is automatically generated from aarch64_ad.m4
11962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11963
11964 // Shift Left followed by Shift Right.
11965 // This idiom is used by the compiler for the i2b bytecode etc.
11966 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11967 %{
11968 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11969 ins_cost(INSN_COST * 2);
11970 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11971 ins_encode %{
11972 int lshift = $lshift_count$$constant & 63;
11973 int rshift = $rshift_count$$constant & 63;
11974 int s = 63 - lshift;
11975 int r = (rshift - lshift) & 63;
11976 __ sbfm(as_Register($dst$$reg),
11977 as_Register($src$$reg),
11978 r, s);
11979 %}
11980
11981 ins_pipe(ialu_reg_shift);
11982 %}
11983
11984 // This pattern is automatically generated from aarch64_ad.m4
11985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11986
11987 // Shift Left followed by Shift Right.
11988 // This idiom is used by the compiler for the i2b bytecode etc.
11989 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11990 %{
11991 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11992 ins_cost(INSN_COST * 2);
11993 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11994 ins_encode %{
11995 int lshift = $lshift_count$$constant & 31;
11996 int rshift = $rshift_count$$constant & 31;
11997 int s = 31 - lshift;
11998 int r = (rshift - lshift) & 31;
11999 __ sbfmw(as_Register($dst$$reg),
12000 as_Register($src$$reg),
12001 r, s);
12002 %}
12003
12004 ins_pipe(ialu_reg_shift);
12005 %}
12006
12007 // This pattern is automatically generated from aarch64_ad.m4
12008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12009
12010 // Shift Left followed by Shift Right.
12011 // This idiom is used by the compiler for the i2b bytecode etc.
12012 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12013 %{
12014 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12015 ins_cost(INSN_COST * 2);
12016 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12017 ins_encode %{
12018 int lshift = $lshift_count$$constant & 63;
12019 int rshift = $rshift_count$$constant & 63;
12020 int s = 63 - lshift;
12021 int r = (rshift - lshift) & 63;
12022 __ ubfm(as_Register($dst$$reg),
12023 as_Register($src$$reg),
12024 r, s);
12025 %}
12026
12027 ins_pipe(ialu_reg_shift);
12028 %}
12029
12030 // This pattern is automatically generated from aarch64_ad.m4
12031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12032
12033 // Shift Left followed by Shift Right.
12034 // This idiom is used by the compiler for the i2b bytecode etc.
12035 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12036 %{
12037 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12038 ins_cost(INSN_COST * 2);
12039 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12040 ins_encode %{
12041 int lshift = $lshift_count$$constant & 31;
12042 int rshift = $rshift_count$$constant & 31;
12043 int s = 31 - lshift;
12044 int r = (rshift - lshift) & 31;
12045 __ ubfmw(as_Register($dst$$reg),
12046 as_Register($src$$reg),
12047 r, s);
12048 %}
12049
12050 ins_pipe(ialu_reg_shift);
12051 %}
12052
12053 // Bitfield extract with shift & mask
12054
12055 // This pattern is automatically generated from aarch64_ad.m4
12056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12057 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12058 %{
12059 match(Set dst (AndI (URShiftI src rshift) mask));
12060 // Make sure we are not going to exceed what ubfxw can do.
12061 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12062
12063 ins_cost(INSN_COST);
12064 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12065 ins_encode %{
12066 int rshift = $rshift$$constant & 31;
12067 intptr_t mask = $mask$$constant;
12068 int width = exact_log2(mask+1);
12069 __ ubfxw(as_Register($dst$$reg),
12070 as_Register($src$$reg), rshift, width);
12071 %}
12072 ins_pipe(ialu_reg_shift);
12073 %}
12074
12075 // This pattern is automatically generated from aarch64_ad.m4
12076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12077 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12078 %{
12079 match(Set dst (AndL (URShiftL src rshift) mask));
12080 // Make sure we are not going to exceed what ubfx can do.
12081 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12082
12083 ins_cost(INSN_COST);
12084 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12085 ins_encode %{
12086 int rshift = $rshift$$constant & 63;
12087 intptr_t mask = $mask$$constant;
12088 int width = exact_log2_long(mask+1);
12089 __ ubfx(as_Register($dst$$reg),
12090 as_Register($src$$reg), rshift, width);
12091 %}
12092 ins_pipe(ialu_reg_shift);
12093 %}
12094
12095
12096 // This pattern is automatically generated from aarch64_ad.m4
12097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12098
12099 // We can use ubfx when extending an And with a mask when we know mask
12100 // is positive. We know that because immI_bitmask guarantees it.
12101 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12102 %{
12103 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12104 // Make sure we are not going to exceed what ubfxw can do.
12105 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12106
12107 ins_cost(INSN_COST * 2);
12108 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12109 ins_encode %{
12110 int rshift = $rshift$$constant & 31;
12111 intptr_t mask = $mask$$constant;
12112 int width = exact_log2(mask+1);
12113 __ ubfx(as_Register($dst$$reg),
12114 as_Register($src$$reg), rshift, width);
12115 %}
12116 ins_pipe(ialu_reg_shift);
12117 %}
12118
12119
12120 // This pattern is automatically generated from aarch64_ad.m4
12121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12122
12123 // We can use ubfiz when masking by a positive number and then left shifting the result.
12124 // We know that the mask is positive because immI_bitmask guarantees it.
12125 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12126 %{
12127 match(Set dst (LShiftI (AndI src mask) lshift));
12128 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12129
12130 ins_cost(INSN_COST);
12131 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12132 ins_encode %{
12133 int lshift = $lshift$$constant & 31;
12134 intptr_t mask = $mask$$constant;
12135 int width = exact_log2(mask+1);
12136 __ ubfizw(as_Register($dst$$reg),
12137 as_Register($src$$reg), lshift, width);
12138 %}
12139 ins_pipe(ialu_reg_shift);
12140 %}
12141
12142 // This pattern is automatically generated from aarch64_ad.m4
12143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12144
12145 // We can use ubfiz when masking by a positive number and then left shifting the result.
12146 // We know that the mask is positive because immL_bitmask guarantees it.
12147 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12148 %{
12149 match(Set dst (LShiftL (AndL src mask) lshift));
12150 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12151
12152 ins_cost(INSN_COST);
12153 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12154 ins_encode %{
12155 int lshift = $lshift$$constant & 63;
12156 intptr_t mask = $mask$$constant;
12157 int width = exact_log2_long(mask+1);
12158 __ ubfiz(as_Register($dst$$reg),
12159 as_Register($src$$reg), lshift, width);
12160 %}
12161 ins_pipe(ialu_reg_shift);
12162 %}
12163
12164 // This pattern is automatically generated from aarch64_ad.m4
12165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12166
12167 // We can use ubfiz when masking by a positive number and then left shifting the result.
12168 // We know that the mask is positive because immI_bitmask guarantees it.
12169 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12170 %{
12171 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12172 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12173
12174 ins_cost(INSN_COST);
12175 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12176 ins_encode %{
12177 int lshift = $lshift$$constant & 31;
12178 intptr_t mask = $mask$$constant;
12179 int width = exact_log2(mask+1);
12180 __ ubfizw(as_Register($dst$$reg),
12181 as_Register($src$$reg), lshift, width);
12182 %}
12183 ins_pipe(ialu_reg_shift);
12184 %}
12185
12186 // This pattern is automatically generated from aarch64_ad.m4
12187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12188
12189 // We can use ubfiz when masking by a positive number and then left shifting the result.
12190 // We know that the mask is positive because immL_bitmask guarantees it.
12191 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12192 %{
12193 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12194 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12195
12196 ins_cost(INSN_COST);
12197 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12198 ins_encode %{
12199 int lshift = $lshift$$constant & 63;
12200 intptr_t mask = $mask$$constant;
12201 int width = exact_log2_long(mask+1);
12202 __ ubfiz(as_Register($dst$$reg),
12203 as_Register($src$$reg), lshift, width);
12204 %}
12205 ins_pipe(ialu_reg_shift);
12206 %}
12207
12208
12209 // This pattern is automatically generated from aarch64_ad.m4
12210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12211
12212 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12213 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12214 %{
12215 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12216 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12217
12218 ins_cost(INSN_COST);
12219 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12220 ins_encode %{
12221 int lshift = $lshift$$constant & 63;
12222 intptr_t mask = $mask$$constant;
12223 int width = exact_log2(mask+1);
12224 __ ubfiz(as_Register($dst$$reg),
12225 as_Register($src$$reg), lshift, width);
12226 %}
12227 ins_pipe(ialu_reg_shift);
12228 %}
12229
12230 // This pattern is automatically generated from aarch64_ad.m4
12231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12232
12233 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12234 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12235 %{
12236 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12237 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12238
12239 ins_cost(INSN_COST);
12240 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12241 ins_encode %{
12242 int lshift = $lshift$$constant & 31;
12243 intptr_t mask = $mask$$constant;
12244 int width = exact_log2(mask+1);
12245 __ ubfiz(as_Register($dst$$reg),
12246 as_Register($src$$reg), lshift, width);
12247 %}
12248 ins_pipe(ialu_reg_shift);
12249 %}
12250
12251 // This pattern is automatically generated from aarch64_ad.m4
12252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12253
12254 // Can skip int2long conversions after AND with small bitmask
12255 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12256 %{
12257 match(Set dst (ConvI2L (AndI src msk)));
12258 ins_cost(INSN_COST);
12259 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12260 ins_encode %{
12261 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12262 %}
12263 ins_pipe(ialu_reg_shift);
12264 %}
12265
12266
12267 // Rotations
12268 // This pattern is automatically generated from aarch64_ad.m4
12269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12270 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12271 %{
12272 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12273 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12274
12275 ins_cost(INSN_COST);
12276 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12277
12278 ins_encode %{
12279 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12280 $rshift$$constant & 63);
12281 %}
12282 ins_pipe(ialu_reg_reg_extr);
12283 %}
12284
12285
12286 // This pattern is automatically generated from aarch64_ad.m4
12287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12288 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12289 %{
12290 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12291 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12292
12293 ins_cost(INSN_COST);
12294 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12295
12296 ins_encode %{
12297 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12298 $rshift$$constant & 31);
12299 %}
12300 ins_pipe(ialu_reg_reg_extr);
12301 %}
12302
12303
12304 // This pattern is automatically generated from aarch64_ad.m4
12305 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12306 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12307 %{
12308 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12309 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12310
12311 ins_cost(INSN_COST);
12312 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12313
12314 ins_encode %{
12315 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12316 $rshift$$constant & 63);
12317 %}
12318 ins_pipe(ialu_reg_reg_extr);
12319 %}
12320
12321
12322 // This pattern is automatically generated from aarch64_ad.m4
12323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12324 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12325 %{
12326 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12327 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12328
12329 ins_cost(INSN_COST);
12330 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12331
12332 ins_encode %{
12333 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12334 $rshift$$constant & 31);
12335 %}
12336 ins_pipe(ialu_reg_reg_extr);
12337 %}
12338
12339
12340 // This pattern is automatically generated from aarch64_ad.m4
12341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12342
12343 // rol expander
12344 instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
12345 %{
12346 effect(DEF dst, USE src, USE shift);
12347
12348 format %{ "rol $dst, $src, $shift" %}
12349 ins_cost(INSN_COST * 3);
12350 ins_encode %{
12351 __ subw(rscratch1, zr, as_Register($shift$$reg));
12352 __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
12353 rscratch1);
12354 %}
12355 ins_pipe(ialu_reg_reg_vshift);
12356 %}
12357
12358 // This pattern is automatically generated from aarch64_ad.m4
12359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12360
12361 // rol expander
12362 instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
12363 %{
12364 effect(DEF dst, USE src, USE shift);
12365
12366 format %{ "rol $dst, $src, $shift" %}
12367 ins_cost(INSN_COST * 3);
12368 ins_encode %{
12369 __ subw(rscratch1, zr, as_Register($shift$$reg));
12370 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
12371 rscratch1);
12372 %}
12373 ins_pipe(ialu_reg_reg_vshift);
12374 %}
12375
12376 // This pattern is automatically generated from aarch64_ad.m4
12377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12378 instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
12379 %{
12380 match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
12381
12382 expand %{
12383 rolL_rReg(dst, src, shift, cr);
12384 %}
12385 %}
12386
12387 // This pattern is automatically generated from aarch64_ad.m4
12388 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12389 instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
12390 %{
12391 match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
12392
12393 expand %{
12394 rolL_rReg(dst, src, shift, cr);
12395 %}
12396 %}
12397
12398 // This pattern is automatically generated from aarch64_ad.m4
12399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12400 instruct rolI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
12401 %{
12402 match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
12403
12404 expand %{
12405 rolI_rReg(dst, src, shift, cr);
12406 %}
12407 %}
12408
12409 // This pattern is automatically generated from aarch64_ad.m4
12410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12411 instruct rolI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
12412 %{
12413 match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
12414
12415 expand %{
12416 rolI_rReg(dst, src, shift, cr);
12417 %}
12418 %}
12419
12420 // This pattern is automatically generated from aarch64_ad.m4
12421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12422
12423 // ror expander
12424 instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
12425 %{
12426 effect(DEF dst, USE src, USE shift);
12427
12428 format %{ "ror $dst, $src, $shift" %}
12429 ins_cost(INSN_COST);
12430 ins_encode %{
12431 __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
12432 as_Register($shift$$reg));
12433 %}
12434 ins_pipe(ialu_reg_reg_vshift);
12435 %}
12436
12437 // This pattern is automatically generated from aarch64_ad.m4
12438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12439
12440 // ror expander
12441 instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
12442 %{
12443 effect(DEF dst, USE src, USE shift);
12444
12445 format %{ "ror $dst, $src, $shift" %}
12446 ins_cost(INSN_COST);
12447 ins_encode %{
12448 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
12449 as_Register($shift$$reg));
12450 %}
12451 ins_pipe(ialu_reg_reg_vshift);
12452 %}
12453
12454 // This pattern is automatically generated from aarch64_ad.m4
12455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12456 instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
12457 %{
12458 match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
12459
12460 expand %{
12461 rorL_rReg(dst, src, shift, cr);
12462 %}
12463 %}
12464
12465 // This pattern is automatically generated from aarch64_ad.m4
12466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12467 instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
12468 %{
12469 match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
12470
12471 expand %{
12472 rorL_rReg(dst, src, shift, cr);
12473 %}
12474 %}
12475
12476 // This pattern is automatically generated from aarch64_ad.m4
12477 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12478 instruct rorI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
12479 %{
12480 match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
12481
12482 expand %{
12483 rorI_rReg(dst, src, shift, cr);
12484 %}
12485 %}
12486
12487 // This pattern is automatically generated from aarch64_ad.m4
12488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12489 instruct rorI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
12490 %{
12491 match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
12492
12493 expand %{
12494 rorI_rReg(dst, src, shift, cr);
12495 %}
12496 %}
12497
12498
12499 // Add/subtract (extended)
12500
12501 // This pattern is automatically generated from aarch64_ad.m4
12502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12503 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12504 %{
12505 match(Set dst (AddL src1 (ConvI2L src2)));
12506 ins_cost(INSN_COST);
12507 format %{ "add $dst, $src1, $src2, sxtw" %}
12508
12509 ins_encode %{
12510 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12511 as_Register($src2$$reg), ext::sxtw);
12512 %}
12513 ins_pipe(ialu_reg_reg);
12514 %}
12515
12516 // This pattern is automatically generated from aarch64_ad.m4
12517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12518 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12519 %{
12520 match(Set dst (SubL src1 (ConvI2L src2)));
12521 ins_cost(INSN_COST);
12522 format %{ "sub $dst, $src1, $src2, sxtw" %}
12523
12524 ins_encode %{
12525 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12526 as_Register($src2$$reg), ext::sxtw);
12527 %}
12528 ins_pipe(ialu_reg_reg);
12529 %}
12530
12531 // This pattern is automatically generated from aarch64_ad.m4
12532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12533 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12534 %{
12535 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12536 ins_cost(INSN_COST);
12537 format %{ "add $dst, $src1, $src2, sxth" %}
12538
12539 ins_encode %{
12540 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12541 as_Register($src2$$reg), ext::sxth);
12542 %}
12543 ins_pipe(ialu_reg_reg);
12544 %}
12545
12546 // This pattern is automatically generated from aarch64_ad.m4
12547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12548 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12549 %{
12550 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12551 ins_cost(INSN_COST);
12552 format %{ "add $dst, $src1, $src2, sxtb" %}
12553
12554 ins_encode %{
12555 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12556 as_Register($src2$$reg), ext::sxtb);
12557 %}
12558 ins_pipe(ialu_reg_reg);
12559 %}
12560
12561 // This pattern is automatically generated from aarch64_ad.m4
12562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12563 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12564 %{
12565 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12566 ins_cost(INSN_COST);
12567 format %{ "add $dst, $src1, $src2, uxtb" %}
12568
12569 ins_encode %{
12570 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12571 as_Register($src2$$reg), ext::uxtb);
12572 %}
12573 ins_pipe(ialu_reg_reg);
12574 %}
12575
12576 // This pattern is automatically generated from aarch64_ad.m4
12577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12578 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12579 %{
12580 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12581 ins_cost(INSN_COST);
12582 format %{ "add $dst, $src1, $src2, sxth" %}
12583
12584 ins_encode %{
12585 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12586 as_Register($src2$$reg), ext::sxth);
12587 %}
12588 ins_pipe(ialu_reg_reg);
12589 %}
12590
12591 // This pattern is automatically generated from aarch64_ad.m4
12592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12593 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12594 %{
12595 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12596 ins_cost(INSN_COST);
12597 format %{ "add $dst, $src1, $src2, sxtw" %}
12598
12599 ins_encode %{
12600 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12601 as_Register($src2$$reg), ext::sxtw);
12602 %}
12603 ins_pipe(ialu_reg_reg);
12604 %}
12605
12606 // This pattern is automatically generated from aarch64_ad.m4
12607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12608 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12609 %{
12610 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12611 ins_cost(INSN_COST);
12612 format %{ "add $dst, $src1, $src2, sxtb" %}
12613
12614 ins_encode %{
12615 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12616 as_Register($src2$$reg), ext::sxtb);
12617 %}
12618 ins_pipe(ialu_reg_reg);
12619 %}
12620
12621 // This pattern is automatically generated from aarch64_ad.m4
12622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12623 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12624 %{
12625 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12626 ins_cost(INSN_COST);
12627 format %{ "add $dst, $src1, $src2, uxtb" %}
12628
12629 ins_encode %{
12630 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12631 as_Register($src2$$reg), ext::uxtb);
12632 %}
12633 ins_pipe(ialu_reg_reg);
12634 %}
12635
12636 // This pattern is automatically generated from aarch64_ad.m4
12637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12638 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12639 %{
12640 match(Set dst (AddI src1 (AndI src2 mask)));
12641 ins_cost(INSN_COST);
12642 format %{ "addw $dst, $src1, $src2, uxtb" %}
12643
12644 ins_encode %{
12645 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12646 as_Register($src2$$reg), ext::uxtb);
12647 %}
12648 ins_pipe(ialu_reg_reg);
12649 %}
12650
12651 // This pattern is automatically generated from aarch64_ad.m4
12652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12653 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12654 %{
12655 match(Set dst (AddI src1 (AndI src2 mask)));
12656 ins_cost(INSN_COST);
12657 format %{ "addw $dst, $src1, $src2, uxth" %}
12658
12659 ins_encode %{
12660 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12661 as_Register($src2$$reg), ext::uxth);
12662 %}
12663 ins_pipe(ialu_reg_reg);
12664 %}
12665
12666 // This pattern is automatically generated from aarch64_ad.m4
12667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12668 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12669 %{
12670 match(Set dst (AddL src1 (AndL src2 mask)));
12671 ins_cost(INSN_COST);
12672 format %{ "add $dst, $src1, $src2, uxtb" %}
12673
12674 ins_encode %{
12675 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12676 as_Register($src2$$reg), ext::uxtb);
12677 %}
12678 ins_pipe(ialu_reg_reg);
12679 %}
12680
12681 // This pattern is automatically generated from aarch64_ad.m4
12682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12683 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12684 %{
12685 match(Set dst (AddL src1 (AndL src2 mask)));
12686 ins_cost(INSN_COST);
12687 format %{ "add $dst, $src1, $src2, uxth" %}
12688
12689 ins_encode %{
12690 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12691 as_Register($src2$$reg), ext::uxth);
12692 %}
12693 ins_pipe(ialu_reg_reg);
12694 %}
12695
12696 // This pattern is automatically generated from aarch64_ad.m4
12697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12698 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12699 %{
12700 match(Set dst (AddL src1 (AndL src2 mask)));
12701 ins_cost(INSN_COST);
12702 format %{ "add $dst, $src1, $src2, uxtw" %}
12703
12704 ins_encode %{
12705 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12706 as_Register($src2$$reg), ext::uxtw);
12707 %}
12708 ins_pipe(ialu_reg_reg);
12709 %}
12710
12711 // This pattern is automatically generated from aarch64_ad.m4
12712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12713 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12714 %{
12715 match(Set dst (SubI src1 (AndI src2 mask)));
12716 ins_cost(INSN_COST);
12717 format %{ "subw $dst, $src1, $src2, uxtb" %}
12718
12719 ins_encode %{
12720 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12721 as_Register($src2$$reg), ext::uxtb);
12722 %}
12723 ins_pipe(ialu_reg_reg);
12724 %}
12725
12726 // This pattern is automatically generated from aarch64_ad.m4
12727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12728 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12729 %{
12730 match(Set dst (SubI src1 (AndI src2 mask)));
12731 ins_cost(INSN_COST);
12732 format %{ "subw $dst, $src1, $src2, uxth" %}
12733
12734 ins_encode %{
12735 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12736 as_Register($src2$$reg), ext::uxth);
12737 %}
12738 ins_pipe(ialu_reg_reg);
12739 %}
12740
12741 // This pattern is automatically generated from aarch64_ad.m4
12742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12743 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12744 %{
12745 match(Set dst (SubL src1 (AndL src2 mask)));
12746 ins_cost(INSN_COST);
12747 format %{ "sub $dst, $src1, $src2, uxtb" %}
12748
12749 ins_encode %{
12750 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12751 as_Register($src2$$reg), ext::uxtb);
12752 %}
12753 ins_pipe(ialu_reg_reg);
12754 %}
12755
12756 // This pattern is automatically generated from aarch64_ad.m4
12757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12758 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12759 %{
12760 match(Set dst (SubL src1 (AndL src2 mask)));
12761 ins_cost(INSN_COST);
12762 format %{ "sub $dst, $src1, $src2, uxth" %}
12763
12764 ins_encode %{
12765 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12766 as_Register($src2$$reg), ext::uxth);
12767 %}
12768 ins_pipe(ialu_reg_reg);
12769 %}
12770
12771 // This pattern is automatically generated from aarch64_ad.m4
12772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12773 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12774 %{
12775 match(Set dst (SubL src1 (AndL src2 mask)));
12776 ins_cost(INSN_COST);
12777 format %{ "sub $dst, $src1, $src2, uxtw" %}
12778
12779 ins_encode %{
12780 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12781 as_Register($src2$$reg), ext::uxtw);
12782 %}
12783 ins_pipe(ialu_reg_reg);
12784 %}
12785
12786
12787 // This pattern is automatically generated from aarch64_ad.m4
12788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12789 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12790 %{
12791 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12792 ins_cost(1.9 * INSN_COST);
12793 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12794
12795 ins_encode %{
12796 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12797 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12798 %}
12799 ins_pipe(ialu_reg_reg_shift);
12800 %}
12801
12802 // This pattern is automatically generated from aarch64_ad.m4
12803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12804 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12805 %{
12806 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12807 ins_cost(1.9 * INSN_COST);
12808 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12809
12810 ins_encode %{
12811 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12812 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12813 %}
12814 ins_pipe(ialu_reg_reg_shift);
12815 %}
12816
12817 // This pattern is automatically generated from aarch64_ad.m4
12818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12819 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12820 %{
12821 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12822 ins_cost(1.9 * INSN_COST);
12823 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12824
12825 ins_encode %{
12826 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12827 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12828 %}
12829 ins_pipe(ialu_reg_reg_shift);
12830 %}
12831
12832 // This pattern is automatically generated from aarch64_ad.m4
12833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12834 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12835 %{
12836 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12837 ins_cost(1.9 * INSN_COST);
12838 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12839
12840 ins_encode %{
12841 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12842 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12843 %}
12844 ins_pipe(ialu_reg_reg_shift);
12845 %}
12846
12847 // This pattern is automatically generated from aarch64_ad.m4
12848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12849 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12850 %{
12851 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12852 ins_cost(1.9 * INSN_COST);
12853 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12854
12855 ins_encode %{
12856 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12857 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12858 %}
12859 ins_pipe(ialu_reg_reg_shift);
12860 %}
12861
12862 // This pattern is automatically generated from aarch64_ad.m4
12863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12864 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12865 %{
12866 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12867 ins_cost(1.9 * INSN_COST);
12868 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12869
12870 ins_encode %{
12871 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12872 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12873 %}
12874 ins_pipe(ialu_reg_reg_shift);
12875 %}
12876
12877 // This pattern is automatically generated from aarch64_ad.m4
12878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12879 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12880 %{
12881 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12882 ins_cost(1.9 * INSN_COST);
12883 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12884
12885 ins_encode %{
12886 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12887 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12888 %}
12889 ins_pipe(ialu_reg_reg_shift);
12890 %}
12891
12892 // This pattern is automatically generated from aarch64_ad.m4
12893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12894 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12895 %{
12896 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12897 ins_cost(1.9 * INSN_COST);
12898 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12899
12900 ins_encode %{
12901 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12902 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12903 %}
12904 ins_pipe(ialu_reg_reg_shift);
12905 %}
12906
12907 // This pattern is automatically generated from aarch64_ad.m4
12908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12909 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12910 %{
12911 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12912 ins_cost(1.9 * INSN_COST);
12913 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12914
12915 ins_encode %{
12916 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12917 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12918 %}
12919 ins_pipe(ialu_reg_reg_shift);
12920 %}
12921
12922 // This pattern is automatically generated from aarch64_ad.m4
12923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12924 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12925 %{
12926 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12927 ins_cost(1.9 * INSN_COST);
12928 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12929
12930 ins_encode %{
12931 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12932 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12933 %}
12934 ins_pipe(ialu_reg_reg_shift);
12935 %}
12936
12937 // This pattern is automatically generated from aarch64_ad.m4
12938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12939 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12940 %{
12941 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12942 ins_cost(1.9 * INSN_COST);
12943 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12944
12945 ins_encode %{
12946 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12947 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12948 %}
12949 ins_pipe(ialu_reg_reg_shift);
12950 %}
12951
12952 // This pattern is automatically generated from aarch64_ad.m4
12953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12954 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12955 %{
12956 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12957 ins_cost(1.9 * INSN_COST);
12958 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12959
12960 ins_encode %{
12961 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12962 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12963 %}
12964 ins_pipe(ialu_reg_reg_shift);
12965 %}
12966
12967 // This pattern is automatically generated from aarch64_ad.m4
12968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12969 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12970 %{
12971 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12972 ins_cost(1.9 * INSN_COST);
12973 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12974
12975 ins_encode %{
12976 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12977 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12978 %}
12979 ins_pipe(ialu_reg_reg_shift);
12980 %}
12981
12982 // This pattern is automatically generated from aarch64_ad.m4
12983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12984 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12985 %{
12986 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12987 ins_cost(1.9 * INSN_COST);
12988 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12989
12990 ins_encode %{
12991 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12992 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12993 %}
12994 ins_pipe(ialu_reg_reg_shift);
12995 %}
12996
12997 // This pattern is automatically generated from aarch64_ad.m4
12998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12999 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13000 %{
13001 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13002 ins_cost(1.9 * INSN_COST);
13003 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13004
13005 ins_encode %{
13006 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13007 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13008 %}
13009 ins_pipe(ialu_reg_reg_shift);
13010 %}
13011
13012 // This pattern is automatically generated from aarch64_ad.m4
13013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13014 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13015 %{
13016 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13017 ins_cost(1.9 * INSN_COST);
13018 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13019
13020 ins_encode %{
13021 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13022 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13023 %}
13024 ins_pipe(ialu_reg_reg_shift);
13025 %}
13026
13027 // This pattern is automatically generated from aarch64_ad.m4
13028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13029 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13030 %{
13031 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13032 ins_cost(1.9 * INSN_COST);
13033 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13034
13035 ins_encode %{
13036 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13037 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13038 %}
13039 ins_pipe(ialu_reg_reg_shift);
13040 %}
13041
13042 // This pattern is automatically generated from aarch64_ad.m4
13043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13044 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13045 %{
13046 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13047 ins_cost(1.9 * INSN_COST);
13048 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13049
13050 ins_encode %{
13051 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13052 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13053 %}
13054 ins_pipe(ialu_reg_reg_shift);
13055 %}
13056
13057 // This pattern is automatically generated from aarch64_ad.m4
13058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13059 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13060 %{
13061 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13062 ins_cost(1.9 * INSN_COST);
13063 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13064
13065 ins_encode %{
13066 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13067 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13068 %}
13069 ins_pipe(ialu_reg_reg_shift);
13070 %}
13071
13072 // This pattern is automatically generated from aarch64_ad.m4
13073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13074 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13075 %{
13076 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13077 ins_cost(1.9 * INSN_COST);
13078 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13079
13080 ins_encode %{
13081 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13082 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13083 %}
13084 ins_pipe(ialu_reg_reg_shift);
13085 %}
13086
13087 // This pattern is automatically generated from aarch64_ad.m4
13088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13089 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13090 %{
13091 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13092 ins_cost(1.9 * INSN_COST);
13093 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13094
13095 ins_encode %{
13096 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13097 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13098 %}
13099 ins_pipe(ialu_reg_reg_shift);
13100 %}
13101
13102 // This pattern is automatically generated from aarch64_ad.m4
13103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13104 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13105 %{
13106 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13107 ins_cost(1.9 * INSN_COST);
13108 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13109
13110 ins_encode %{
13111 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13112 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13113 %}
13114 ins_pipe(ialu_reg_reg_shift);
13115 %}
13116
13117
13118
13119 // END This section of the file is automatically generated. Do not edit --------------
13120
13121
13122 // ============================================================================
13123 // Floating Point Arithmetic Instructions
13124
13125 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13126 match(Set dst (AddF src1 src2));
13127
13128 ins_cost(INSN_COST * 5);
13129 format %{ "fadds $dst, $src1, $src2" %}
13130
13131 ins_encode %{
13132 __ fadds(as_FloatRegister($dst$$reg),
13133 as_FloatRegister($src1$$reg),
13134 as_FloatRegister($src2$$reg));
13135 %}
13136
13137 ins_pipe(fp_dop_reg_reg_s);
13138 %}
13139
13140 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13141 match(Set dst (AddD src1 src2));
13142
13143 ins_cost(INSN_COST * 5);
13144 format %{ "faddd $dst, $src1, $src2" %}
13145
13146 ins_encode %{
13147 __ faddd(as_FloatRegister($dst$$reg),
13148 as_FloatRegister($src1$$reg),
13149 as_FloatRegister($src2$$reg));
13150 %}
13151
13152 ins_pipe(fp_dop_reg_reg_d);
13153 %}
13154
13155 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13156 match(Set dst (SubF src1 src2));
13157
13158 ins_cost(INSN_COST * 5);
13159 format %{ "fsubs $dst, $src1, $src2" %}
13160
13161 ins_encode %{
13162 __ fsubs(as_FloatRegister($dst$$reg),
13163 as_FloatRegister($src1$$reg),
13164 as_FloatRegister($src2$$reg));
13165 %}
13166
13167 ins_pipe(fp_dop_reg_reg_s);
13168 %}
13169
13170 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13171 match(Set dst (SubD src1 src2));
13172
13173 ins_cost(INSN_COST * 5);
13174 format %{ "fsubd $dst, $src1, $src2" %}
13175
13176 ins_encode %{
13177 __ fsubd(as_FloatRegister($dst$$reg),
13178 as_FloatRegister($src1$$reg),
13179 as_FloatRegister($src2$$reg));
13180 %}
13181
13182 ins_pipe(fp_dop_reg_reg_d);
13183 %}
13184
13185 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13186 match(Set dst (MulF src1 src2));
13187
13188 ins_cost(INSN_COST * 6);
13189 format %{ "fmuls $dst, $src1, $src2" %}
13190
13191 ins_encode %{
13192 __ fmuls(as_FloatRegister($dst$$reg),
13193 as_FloatRegister($src1$$reg),
13194 as_FloatRegister($src2$$reg));
13195 %}
13196
13197 ins_pipe(fp_dop_reg_reg_s);
13198 %}
13199
13200 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13201 match(Set dst (MulD src1 src2));
13202
13203 ins_cost(INSN_COST * 6);
13204 format %{ "fmuld $dst, $src1, $src2" %}
13205
13206 ins_encode %{
13207 __ fmuld(as_FloatRegister($dst$$reg),
13208 as_FloatRegister($src1$$reg),
13209 as_FloatRegister($src2$$reg));
13210 %}
13211
13212 ins_pipe(fp_dop_reg_reg_d);
13213 %}
13214
13215 // src1 * src2 + src3
13216 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13217 predicate(UseFMA);
13218 match(Set dst (FmaF src3 (Binary src1 src2)));
13219
13220 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13221
13222 ins_encode %{
13223 __ fmadds(as_FloatRegister($dst$$reg),
13224 as_FloatRegister($src1$$reg),
13225 as_FloatRegister($src2$$reg),
13226 as_FloatRegister($src3$$reg));
13227 %}
13228
13229 ins_pipe(pipe_class_default);
13230 %}
13231
13232 // src1 * src2 + src3
13233 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13234 predicate(UseFMA);
13235 match(Set dst (FmaD src3 (Binary src1 src2)));
13236
13237 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13238
13239 ins_encode %{
13240 __ fmaddd(as_FloatRegister($dst$$reg),
13241 as_FloatRegister($src1$$reg),
13242 as_FloatRegister($src2$$reg),
13243 as_FloatRegister($src3$$reg));
13244 %}
13245
13246 ins_pipe(pipe_class_default);
13247 %}
13248
13249 // -src1 * src2 + src3
13250 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13251 predicate(UseFMA);
13252 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13253 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13254
13255 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13256
13257 ins_encode %{
13258 __ fmsubs(as_FloatRegister($dst$$reg),
13259 as_FloatRegister($src1$$reg),
13260 as_FloatRegister($src2$$reg),
13261 as_FloatRegister($src3$$reg));
13262 %}
13263
13264 ins_pipe(pipe_class_default);
13265 %}
13266
13267 // -src1 * src2 + src3
13268 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13269 predicate(UseFMA);
13270 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13271 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13272
13273 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13274
13275 ins_encode %{
13276 __ fmsubd(as_FloatRegister($dst$$reg),
13277 as_FloatRegister($src1$$reg),
13278 as_FloatRegister($src2$$reg),
13279 as_FloatRegister($src3$$reg));
13280 %}
13281
13282 ins_pipe(pipe_class_default);
13283 %}
13284
13285 // -src1 * src2 - src3
13286 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13287 predicate(UseFMA);
13288 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13289 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13290
13291 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13292
13293 ins_encode %{
13294 __ fnmadds(as_FloatRegister($dst$$reg),
13295 as_FloatRegister($src1$$reg),
13296 as_FloatRegister($src2$$reg),
13297 as_FloatRegister($src3$$reg));
13298 %}
13299
13300 ins_pipe(pipe_class_default);
13301 %}
13302
13303 // -src1 * src2 - src3
13304 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13305 predicate(UseFMA);
13306 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
13307 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13308
13309 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13310
13311 ins_encode %{
13312 __ fnmaddd(as_FloatRegister($dst$$reg),
13313 as_FloatRegister($src1$$reg),
13314 as_FloatRegister($src2$$reg),
13315 as_FloatRegister($src3$$reg));
13316 %}
13317
13318 ins_pipe(pipe_class_default);
13319 %}
13320
13321 // src1 * src2 - src3
13322 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13323 predicate(UseFMA);
13324 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13325
13326 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13327
13328 ins_encode %{
13329 __ fnmsubs(as_FloatRegister($dst$$reg),
13330 as_FloatRegister($src1$$reg),
13331 as_FloatRegister($src2$$reg),
13332 as_FloatRegister($src3$$reg));
13333 %}
13334
13335 ins_pipe(pipe_class_default);
13336 %}
13337
13338 // src1 * src2 - src3
13339 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13340 predicate(UseFMA);
13341 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13342
13343 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13344
13345 ins_encode %{
13346 // n.b. insn name should be fnmsubd
13347 __ fnmsub(as_FloatRegister($dst$$reg),
13348 as_FloatRegister($src1$$reg),
13349 as_FloatRegister($src2$$reg),
13350 as_FloatRegister($src3$$reg));
13351 %}
13352
13353 ins_pipe(pipe_class_default);
13354 %}
13355
13356
13357 // Math.max(FF)F
13358 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13359 match(Set dst (MaxF src1 src2));
13360
13361 format %{ "fmaxs $dst, $src1, $src2" %}
13362 ins_encode %{
13363 __ fmaxs(as_FloatRegister($dst$$reg),
13364 as_FloatRegister($src1$$reg),
13365 as_FloatRegister($src2$$reg));
13366 %}
13367
13368 ins_pipe(fp_dop_reg_reg_s);
13369 %}
13370
13371 // Math.min(FF)F
13372 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13373 match(Set dst (MinF src1 src2));
13374
13375 format %{ "fmins $dst, $src1, $src2" %}
13376 ins_encode %{
13377 __ fmins(as_FloatRegister($dst$$reg),
13378 as_FloatRegister($src1$$reg),
13379 as_FloatRegister($src2$$reg));
13380 %}
13381
13382 ins_pipe(fp_dop_reg_reg_s);
13383 %}
13384
13385 // Math.max(DD)D
13386 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13387 match(Set dst (MaxD src1 src2));
13388
13389 format %{ "fmaxd $dst, $src1, $src2" %}
13390 ins_encode %{
13391 __ fmaxd(as_FloatRegister($dst$$reg),
13392 as_FloatRegister($src1$$reg),
13393 as_FloatRegister($src2$$reg));
13394 %}
13395
13396 ins_pipe(fp_dop_reg_reg_d);
13397 %}
13398
13399 // Math.min(DD)D
13400 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13401 match(Set dst (MinD src1 src2));
13402
13403 format %{ "fmind $dst, $src1, $src2" %}
13404 ins_encode %{
13405 __ fmind(as_FloatRegister($dst$$reg),
13406 as_FloatRegister($src1$$reg),
13407 as_FloatRegister($src2$$reg));
13408 %}
13409
13410 ins_pipe(fp_dop_reg_reg_d);
13411 %}
13412
13413
13414 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13415 match(Set dst (DivF src1 src2));
13416
13417 ins_cost(INSN_COST * 18);
13418 format %{ "fdivs $dst, $src1, $src2" %}
13419
13420 ins_encode %{
13421 __ fdivs(as_FloatRegister($dst$$reg),
13422 as_FloatRegister($src1$$reg),
13423 as_FloatRegister($src2$$reg));
13424 %}
13425
13426 ins_pipe(fp_div_s);
13427 %}
13428
13429 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13430 match(Set dst (DivD src1 src2));
13431
13432 ins_cost(INSN_COST * 32);
13433 format %{ "fdivd $dst, $src1, $src2" %}
13434
13435 ins_encode %{
13436 __ fdivd(as_FloatRegister($dst$$reg),
13437 as_FloatRegister($src1$$reg),
13438 as_FloatRegister($src2$$reg));
13439 %}
13440
13441 ins_pipe(fp_div_d);
13442 %}
13443
13444 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13445 match(Set dst (NegF src));
13446
13447 ins_cost(INSN_COST * 3);
13448 format %{ "fneg $dst, $src" %}
13449
13450 ins_encode %{
13451 __ fnegs(as_FloatRegister($dst$$reg),
13452 as_FloatRegister($src$$reg));
13453 %}
13454
13455 ins_pipe(fp_uop_s);
13456 %}
13457
13458 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13459 match(Set dst (NegD src));
13460
13461 ins_cost(INSN_COST * 3);
13462 format %{ "fnegd $dst, $src" %}
13463
13464 ins_encode %{
13465 __ fnegd(as_FloatRegister($dst$$reg),
13466 as_FloatRegister($src$$reg));
13467 %}
13468
13469 ins_pipe(fp_uop_d);
13470 %}
13471
13472 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13473 %{
13474 match(Set dst (AbsI src));
13475
13476 effect(KILL cr);
13477 ins_cost(INSN_COST * 2);
13478 format %{ "cmpw $src, zr\n\t"
13479 "cnegw $dst, $src, Assembler::LT\t# int abs"
13480 %}
13481
13482 ins_encode %{
13483 __ cmpw(as_Register($src$$reg), zr);
13484 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13485 %}
13486 ins_pipe(pipe_class_default);
13487 %}
13488
13489 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13490 %{
13491 match(Set dst (AbsL src));
13492
13493 effect(KILL cr);
13494 ins_cost(INSN_COST * 2);
13495 format %{ "cmp $src, zr\n\t"
13496 "cneg $dst, $src, Assembler::LT\t# long abs"
13497 %}
13498
13499 ins_encode %{
13500 __ cmp(as_Register($src$$reg), zr);
13501 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13502 %}
13503 ins_pipe(pipe_class_default);
13504 %}
13505
13506 instruct absF_reg(vRegF dst, vRegF src) %{
13507 match(Set dst (AbsF src));
13508
13509 ins_cost(INSN_COST * 3);
13510 format %{ "fabss $dst, $src" %}
13511 ins_encode %{
13512 __ fabss(as_FloatRegister($dst$$reg),
13513 as_FloatRegister($src$$reg));
13514 %}
13515
13516 ins_pipe(fp_uop_s);
13517 %}
13518
13519 instruct absD_reg(vRegD dst, vRegD src) %{
13520 match(Set dst (AbsD src));
13521
13522 ins_cost(INSN_COST * 3);
13523 format %{ "fabsd $dst, $src" %}
13524 ins_encode %{
13525 __ fabsd(as_FloatRegister($dst$$reg),
13526 as_FloatRegister($src$$reg));
13527 %}
13528
13529 ins_pipe(fp_uop_d);
13530 %}
13531
13532 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13533 match(Set dst (SqrtD src));
13534
13535 ins_cost(INSN_COST * 50);
13536 format %{ "fsqrtd $dst, $src" %}
13537 ins_encode %{
13538 __ fsqrtd(as_FloatRegister($dst$$reg),
13539 as_FloatRegister($src$$reg));
13540 %}
13541
13542 ins_pipe(fp_div_s);
13543 %}
13544
13545 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13546 match(Set dst (SqrtF src));
13547
13548 ins_cost(INSN_COST * 50);
13549 format %{ "fsqrts $dst, $src" %}
13550 ins_encode %{
13551 __ fsqrts(as_FloatRegister($dst$$reg),
13552 as_FloatRegister($src$$reg));
13553 %}
13554
13555 ins_pipe(fp_div_d);
13556 %}
13557
13558 // Math.rint, floor, ceil
13559 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13560 match(Set dst (RoundDoubleMode src rmode));
13561 format %{ "frint $dst, $src, $rmode" %}
13562 ins_encode %{
13563 switch ($rmode$$constant) {
13564 case RoundDoubleModeNode::rmode_rint:
13565 __ frintnd(as_FloatRegister($dst$$reg),
13566 as_FloatRegister($src$$reg));
13567 break;
13568 case RoundDoubleModeNode::rmode_floor:
13569 __ frintmd(as_FloatRegister($dst$$reg),
13570 as_FloatRegister($src$$reg));
13571 break;
13572 case RoundDoubleModeNode::rmode_ceil:
13573 __ frintpd(as_FloatRegister($dst$$reg),
13574 as_FloatRegister($src$$reg));
13575 break;
13576 }
13577 %}
13578 ins_pipe(fp_uop_d);
13579 %}
13580
13581 // ============================================================================
13582 // Logical Instructions
13583
13584 // Integer Logical Instructions
13585
13586 // And Instructions
13587
13588
13589 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13590 match(Set dst (AndI src1 src2));
13591
13592 format %{ "andw $dst, $src1, $src2\t# int" %}
13593
13594 ins_cost(INSN_COST);
13595 ins_encode %{
13596 __ andw(as_Register($dst$$reg),
13597 as_Register($src1$$reg),
13598 as_Register($src2$$reg));
13599 %}
13600
13601 ins_pipe(ialu_reg_reg);
13602 %}
13603
13604 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13605 match(Set dst (AndI src1 src2));
13606
13607 format %{ "andsw $dst, $src1, $src2\t# int" %}
13608
13609 ins_cost(INSN_COST);
13610 ins_encode %{
13611 __ andw(as_Register($dst$$reg),
13612 as_Register($src1$$reg),
13613 (uint64_t)($src2$$constant));
13614 %}
13615
13616 ins_pipe(ialu_reg_imm);
13617 %}
13618
13619 // Or Instructions
13620
13621 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13622 match(Set dst (OrI src1 src2));
13623
13624 format %{ "orrw $dst, $src1, $src2\t# int" %}
13625
13626 ins_cost(INSN_COST);
13627 ins_encode %{
13628 __ orrw(as_Register($dst$$reg),
13629 as_Register($src1$$reg),
13630 as_Register($src2$$reg));
13631 %}
13632
13633 ins_pipe(ialu_reg_reg);
13634 %}
13635
13636 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13637 match(Set dst (OrI src1 src2));
13638
13639 format %{ "orrw $dst, $src1, $src2\t# int" %}
13640
13641 ins_cost(INSN_COST);
13642 ins_encode %{
13643 __ orrw(as_Register($dst$$reg),
13644 as_Register($src1$$reg),
13645 (uint64_t)($src2$$constant));
13646 %}
13647
13648 ins_pipe(ialu_reg_imm);
13649 %}
13650
13651 // Xor Instructions
13652
13653 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13654 match(Set dst (XorI src1 src2));
13655
13656 format %{ "eorw $dst, $src1, $src2\t# int" %}
13657
13658 ins_cost(INSN_COST);
13659 ins_encode %{
13660 __ eorw(as_Register($dst$$reg),
13661 as_Register($src1$$reg),
13662 as_Register($src2$$reg));
13663 %}
13664
13665 ins_pipe(ialu_reg_reg);
13666 %}
13667
13668 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13669 match(Set dst (XorI src1 src2));
13670
13671 format %{ "eorw $dst, $src1, $src2\t# int" %}
13672
13673 ins_cost(INSN_COST);
13674 ins_encode %{
13675 __ eorw(as_Register($dst$$reg),
13676 as_Register($src1$$reg),
13677 (uint64_t)($src2$$constant));
13678 %}
13679
13680 ins_pipe(ialu_reg_imm);
13681 %}
13682
13683 // Long Logical Instructions
13684 // TODO
13685
13686 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13687 match(Set dst (AndL src1 src2));
13688
13689 format %{ "and $dst, $src1, $src2\t# int" %}
13690
13691 ins_cost(INSN_COST);
13692 ins_encode %{
13693 __ andr(as_Register($dst$$reg),
13694 as_Register($src1$$reg),
13695 as_Register($src2$$reg));
13696 %}
13697
13698 ins_pipe(ialu_reg_reg);
13699 %}
13700
13701 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13702 match(Set dst (AndL src1 src2));
13703
13704 format %{ "and $dst, $src1, $src2\t# int" %}
13705
13706 ins_cost(INSN_COST);
13707 ins_encode %{
13708 __ andr(as_Register($dst$$reg),
13709 as_Register($src1$$reg),
13710 (uint64_t)($src2$$constant));
13711 %}
13712
13713 ins_pipe(ialu_reg_imm);
13714 %}
13715
13716 // Or Instructions
13717
13718 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13719 match(Set dst (OrL src1 src2));
13720
13721 format %{ "orr $dst, $src1, $src2\t# int" %}
13722
13723 ins_cost(INSN_COST);
13724 ins_encode %{
13725 __ orr(as_Register($dst$$reg),
13726 as_Register($src1$$reg),
13727 as_Register($src2$$reg));
13728 %}
13729
13730 ins_pipe(ialu_reg_reg);
13731 %}
13732
13733 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13734 match(Set dst (OrL src1 src2));
13735
13736 format %{ "orr $dst, $src1, $src2\t# int" %}
13737
13738 ins_cost(INSN_COST);
13739 ins_encode %{
13740 __ orr(as_Register($dst$$reg),
13741 as_Register($src1$$reg),
13742 (uint64_t)($src2$$constant));
13743 %}
13744
13745 ins_pipe(ialu_reg_imm);
13746 %}
13747
13748 // Xor Instructions
13749
13750 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13751 match(Set dst (XorL src1 src2));
13752
13753 format %{ "eor $dst, $src1, $src2\t# int" %}
13754
13755 ins_cost(INSN_COST);
13756 ins_encode %{
13757 __ eor(as_Register($dst$$reg),
13758 as_Register($src1$$reg),
13759 as_Register($src2$$reg));
13760 %}
13761
13762 ins_pipe(ialu_reg_reg);
13763 %}
13764
13765 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13766 match(Set dst (XorL src1 src2));
13767
13768 ins_cost(INSN_COST);
13769 format %{ "eor $dst, $src1, $src2\t# int" %}
13770
13771 ins_encode %{
13772 __ eor(as_Register($dst$$reg),
13773 as_Register($src1$$reg),
13774 (uint64_t)($src2$$constant));
13775 %}
13776
13777 ins_pipe(ialu_reg_imm);
13778 %}
13779
13780 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13781 %{
13782 match(Set dst (ConvI2L src));
13783
13784 ins_cost(INSN_COST);
13785 format %{ "sxtw $dst, $src\t# i2l" %}
13786 ins_encode %{
13787 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13788 %}
13789 ins_pipe(ialu_reg_shift);
13790 %}
13791
13792 // this pattern occurs in bigmath arithmetic
13793 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13794 %{
13795 match(Set dst (AndL (ConvI2L src) mask));
13796
13797 ins_cost(INSN_COST);
13798 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13799 ins_encode %{
13800 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13801 %}
13802
13803 ins_pipe(ialu_reg_shift);
13804 %}
13805
13806 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13807 match(Set dst (ConvL2I src));
13808
13809 ins_cost(INSN_COST);
13810 format %{ "movw $dst, $src \t// l2i" %}
13811
13812 ins_encode %{
13813 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13814 %}
13815
13816 ins_pipe(ialu_reg);
13817 %}
13818
13819 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13820 %{
13821 match(Set dst (Conv2B src));
13822 effect(KILL cr);
13823
13824 format %{
13825 "cmpw $src, zr\n\t"
13826 "cset $dst, ne"
13827 %}
13828
13829 ins_encode %{
13830 __ cmpw(as_Register($src$$reg), zr);
13831 __ cset(as_Register($dst$$reg), Assembler::NE);
13832 %}
13833
13834 ins_pipe(ialu_reg);
13835 %}
13836
13837 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
13838 %{
13839 match(Set dst (Conv2B src));
13840 effect(KILL cr);
13841
13842 format %{
13843 "cmp $src, zr\n\t"
13844 "cset $dst, ne"
13845 %}
13846
13847 ins_encode %{
13848 __ cmp(as_Register($src$$reg), zr);
13849 __ cset(as_Register($dst$$reg), Assembler::NE);
13850 %}
13851
13852 ins_pipe(ialu_reg);
13853 %}
13854
13855 instruct convD2F_reg(vRegF dst, vRegD src) %{
13856 match(Set dst (ConvD2F src));
13857
13858 ins_cost(INSN_COST * 5);
13859 format %{ "fcvtd $dst, $src \t// d2f" %}
13860
13861 ins_encode %{
13862 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13863 %}
13864
13865 ins_pipe(fp_d2f);
13866 %}
13867
13868 instruct convF2D_reg(vRegD dst, vRegF src) %{
13869 match(Set dst (ConvF2D src));
13870
13871 ins_cost(INSN_COST * 5);
13872 format %{ "fcvts $dst, $src \t// f2d" %}
13873
13874 ins_encode %{
13875 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13876 %}
13877
13878 ins_pipe(fp_f2d);
13879 %}
13880
13881 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13882 match(Set dst (ConvF2I src));
13883
13884 ins_cost(INSN_COST * 5);
13885 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13886
13887 ins_encode %{
13888 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13889 %}
13890
13891 ins_pipe(fp_f2i);
13892 %}
13893
13894 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13895 match(Set dst (ConvF2L src));
13896
13897 ins_cost(INSN_COST * 5);
13898 format %{ "fcvtzs $dst, $src \t// f2l" %}
13899
13900 ins_encode %{
13901 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13902 %}
13903
13904 ins_pipe(fp_f2l);
13905 %}
13906
13907 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13908 match(Set dst (ConvI2F src));
13909
13910 ins_cost(INSN_COST * 5);
13911 format %{ "scvtfws $dst, $src \t// i2f" %}
13912
13913 ins_encode %{
13914 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13915 %}
13916
13917 ins_pipe(fp_i2f);
13918 %}
13919
13920 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13921 match(Set dst (ConvL2F src));
13922
13923 ins_cost(INSN_COST * 5);
13924 format %{ "scvtfs $dst, $src \t// l2f" %}
13925
13926 ins_encode %{
13927 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13928 %}
13929
13930 ins_pipe(fp_l2f);
13931 %}
13932
13933 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13934 match(Set dst (ConvD2I src));
13935
13936 ins_cost(INSN_COST * 5);
13937 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13938
13939 ins_encode %{
13940 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13941 %}
13942
13943 ins_pipe(fp_d2i);
13944 %}
13945
13946 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13947 match(Set dst (ConvD2L src));
13948
13949 ins_cost(INSN_COST * 5);
13950 format %{ "fcvtzd $dst, $src \t// d2l" %}
13951
13952 ins_encode %{
13953 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13954 %}
13955
13956 ins_pipe(fp_d2l);
13957 %}
13958
13959 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13960 match(Set dst (ConvI2D src));
13961
13962 ins_cost(INSN_COST * 5);
13963 format %{ "scvtfwd $dst, $src \t// i2d" %}
13964
13965 ins_encode %{
13966 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13967 %}
13968
13969 ins_pipe(fp_i2d);
13970 %}
13971
13972 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13973 match(Set dst (ConvL2D src));
13974
13975 ins_cost(INSN_COST * 5);
13976 format %{ "scvtfd $dst, $src \t// l2d" %}
13977
13978 ins_encode %{
13979 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13980 %}
13981
13982 ins_pipe(fp_l2d);
13983 %}
13984
13985 // stack <-> reg and reg <-> reg shuffles with no conversion
13986
13987 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13988
13989 match(Set dst (MoveF2I src));
13990
13991 effect(DEF dst, USE src);
13992
13993 ins_cost(4 * INSN_COST);
13994
13995 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13996
13997 ins_encode %{
13998 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13999 %}
14000
14001 ins_pipe(iload_reg_reg);
14002
14003 %}
14004
14005 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14006
14007 match(Set dst (MoveI2F src));
14008
14009 effect(DEF dst, USE src);
14010
14011 ins_cost(4 * INSN_COST);
14012
14013 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14014
14015 ins_encode %{
14016 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14017 %}
14018
14019 ins_pipe(pipe_class_memory);
14020
14021 %}
14022
14023 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14024
14025 match(Set dst (MoveD2L src));
14026
14027 effect(DEF dst, USE src);
14028
14029 ins_cost(4 * INSN_COST);
14030
14031 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14032
14033 ins_encode %{
14034 __ ldr($dst$$Register, Address(sp, $src$$disp));
14035 %}
14036
14037 ins_pipe(iload_reg_reg);
14038
14039 %}
14040
14041 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14042
14043 match(Set dst (MoveL2D src));
14044
14045 effect(DEF dst, USE src);
14046
14047 ins_cost(4 * INSN_COST);
14048
14049 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14050
14051 ins_encode %{
14052 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14053 %}
14054
14055 ins_pipe(pipe_class_memory);
14056
14057 %}
14058
14059 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14060
14061 match(Set dst (MoveF2I src));
14062
14063 effect(DEF dst, USE src);
14064
14065 ins_cost(INSN_COST);
14066
14067 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14068
14069 ins_encode %{
14070 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14071 %}
14072
14073 ins_pipe(pipe_class_memory);
14074
14075 %}
14076
14077 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14078
14079 match(Set dst (MoveI2F src));
14080
14081 effect(DEF dst, USE src);
14082
14083 ins_cost(INSN_COST);
14084
14085 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14086
14087 ins_encode %{
14088 __ strw($src$$Register, Address(sp, $dst$$disp));
14089 %}
14090
14091 ins_pipe(istore_reg_reg);
14092
14093 %}
14094
14095 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14096
14097 match(Set dst (MoveD2L src));
14098
14099 effect(DEF dst, USE src);
14100
14101 ins_cost(INSN_COST);
14102
14103 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14104
14105 ins_encode %{
14106 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14107 %}
14108
14109 ins_pipe(pipe_class_memory);
14110
14111 %}
14112
14113 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14114
14115 match(Set dst (MoveL2D src));
14116
14117 effect(DEF dst, USE src);
14118
14119 ins_cost(INSN_COST);
14120
14121 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14122
14123 ins_encode %{
14124 __ str($src$$Register, Address(sp, $dst$$disp));
14125 %}
14126
14127 ins_pipe(istore_reg_reg);
14128
14129 %}
14130
14131 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14132
14133 match(Set dst (MoveF2I src));
14134
14135 effect(DEF dst, USE src);
14136
14137 ins_cost(INSN_COST);
14138
14139 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14140
14141 ins_encode %{
14142 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14143 %}
14144
14145 ins_pipe(fp_f2i);
14146
14147 %}
14148
14149 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14150
14151 match(Set dst (MoveI2F src));
14152
14153 effect(DEF dst, USE src);
14154
14155 ins_cost(INSN_COST);
14156
14157 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14158
14159 ins_encode %{
14160 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14161 %}
14162
14163 ins_pipe(fp_i2f);
14164
14165 %}
14166
14167 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14168
14169 match(Set dst (MoveD2L src));
14170
14171 effect(DEF dst, USE src);
14172
14173 ins_cost(INSN_COST);
14174
14175 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14176
14177 ins_encode %{
14178 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14179 %}
14180
14181 ins_pipe(fp_d2l);
14182
14183 %}
14184
14185 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14186
14187 match(Set dst (MoveL2D src));
14188
14189 effect(DEF dst, USE src);
14190
14191 ins_cost(INSN_COST);
14192
14193 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14194
14195 ins_encode %{
14196 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14197 %}
14198
14199 ins_pipe(fp_l2d);
14200
14201 %}
14202
14203 // ============================================================================
14204 // clearing of an array
14205
14206 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14207 %{
14208 match(Set dummy (ClearArray cnt base));
14209 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14210
14211 ins_cost(4 * INSN_COST);
14212 format %{ "ClearArray $cnt, $base" %}
14213
14214 ins_encode %{
14215 __ zero_words($base$$Register, $cnt$$Register);
14216 %}
14217
14218 ins_pipe(pipe_class_memory);
14219 %}
14220
14221 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14222 %{
14223 predicate((uint64_t)n->in(2)->get_long()
14224 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14225 match(Set dummy (ClearArray cnt base));
14226 effect(USE_KILL base);
14227
14228 ins_cost(4 * INSN_COST);
14229 format %{ "ClearArray $cnt, $base" %}
14230
14231 ins_encode %{
14232 __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14233 %}
14234
14235 ins_pipe(pipe_class_memory);
14236 %}
14237
14238 // ============================================================================
14239 // Overflow Math Instructions
14240
14241 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14242 %{
14243 match(Set cr (OverflowAddI op1 op2));
14244
14245 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14246 ins_cost(INSN_COST);
14247 ins_encode %{
14248 __ cmnw($op1$$Register, $op2$$Register);
14249 %}
14250
14251 ins_pipe(icmp_reg_reg);
14252 %}
14253
14254 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14255 %{
14256 match(Set cr (OverflowAddI op1 op2));
14257
14258 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14259 ins_cost(INSN_COST);
14260 ins_encode %{
14261 __ cmnw($op1$$Register, $op2$$constant);
14262 %}
14263
14264 ins_pipe(icmp_reg_imm);
14265 %}
14266
14267 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14268 %{
14269 match(Set cr (OverflowAddL op1 op2));
14270
14271 format %{ "cmn $op1, $op2\t# overflow check long" %}
14272 ins_cost(INSN_COST);
14273 ins_encode %{
14274 __ cmn($op1$$Register, $op2$$Register);
14275 %}
14276
14277 ins_pipe(icmp_reg_reg);
14278 %}
14279
14280 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14281 %{
14282 match(Set cr (OverflowAddL op1 op2));
14283
14284 format %{ "cmn $op1, $op2\t# overflow check long" %}
14285 ins_cost(INSN_COST);
14286 ins_encode %{
14287 __ cmn($op1$$Register, $op2$$constant);
14288 %}
14289
14290 ins_pipe(icmp_reg_imm);
14291 %}
14292
14293 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14294 %{
14295 match(Set cr (OverflowSubI op1 op2));
14296
14297 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14298 ins_cost(INSN_COST);
14299 ins_encode %{
14300 __ cmpw($op1$$Register, $op2$$Register);
14301 %}
14302
14303 ins_pipe(icmp_reg_reg);
14304 %}
14305
14306 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14307 %{
14308 match(Set cr (OverflowSubI op1 op2));
14309
14310 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14311 ins_cost(INSN_COST);
14312 ins_encode %{
14313 __ cmpw($op1$$Register, $op2$$constant);
14314 %}
14315
14316 ins_pipe(icmp_reg_imm);
14317 %}
14318
14319 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14320 %{
14321 match(Set cr (OverflowSubL op1 op2));
14322
14323 format %{ "cmp $op1, $op2\t# overflow check long" %}
14324 ins_cost(INSN_COST);
14325 ins_encode %{
14326 __ cmp($op1$$Register, $op2$$Register);
14327 %}
14328
14329 ins_pipe(icmp_reg_reg);
14330 %}
14331
14332 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14333 %{
14334 match(Set cr (OverflowSubL op1 op2));
14335
14336 format %{ "cmp $op1, $op2\t# overflow check long" %}
14337 ins_cost(INSN_COST);
14338 ins_encode %{
14339 __ subs(zr, $op1$$Register, $op2$$constant);
14340 %}
14341
14342 ins_pipe(icmp_reg_imm);
14343 %}
14344
14345 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14346 %{
14347 match(Set cr (OverflowSubI zero op1));
14348
14349 format %{ "cmpw zr, $op1\t# overflow check int" %}
14350 ins_cost(INSN_COST);
14351 ins_encode %{
14352 __ cmpw(zr, $op1$$Register);
14353 %}
14354
14355 ins_pipe(icmp_reg_imm);
14356 %}
14357
14358 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14359 %{
14360 match(Set cr (OverflowSubL zero op1));
14361
14362 format %{ "cmp zr, $op1\t# overflow check long" %}
14363 ins_cost(INSN_COST);
14364 ins_encode %{
14365 __ cmp(zr, $op1$$Register);
14366 %}
14367
14368 ins_pipe(icmp_reg_imm);
14369 %}
14370
14371 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14372 %{
14373 match(Set cr (OverflowMulI op1 op2));
14374
14375 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14376 "cmp rscratch1, rscratch1, sxtw\n\t"
14377 "movw rscratch1, #0x80000000\n\t"
14378 "cselw rscratch1, rscratch1, zr, NE\n\t"
14379 "cmpw rscratch1, #1" %}
14380 ins_cost(5 * INSN_COST);
14381 ins_encode %{
14382 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14383 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14384 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14385 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14386 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14387 %}
14388
14389 ins_pipe(pipe_slow);
14390 %}
14391
14392 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14393 %{
14394 match(If cmp (OverflowMulI op1 op2));
14395 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14396 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14397 effect(USE labl, KILL cr);
14398
14399 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14400 "cmp rscratch1, rscratch1, sxtw\n\t"
14401 "b$cmp $labl" %}
14402 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14403 ins_encode %{
14404 Label* L = $labl$$label;
14405 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14406 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14407 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14408 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14409 %}
14410
14411 ins_pipe(pipe_serial);
14412 %}
14413
14414 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14415 %{
14416 match(Set cr (OverflowMulL op1 op2));
14417
14418 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14419 "smulh rscratch2, $op1, $op2\n\t"
14420 "cmp rscratch2, rscratch1, ASR #63\n\t"
14421 "movw rscratch1, #0x80000000\n\t"
14422 "cselw rscratch1, rscratch1, zr, NE\n\t"
14423 "cmpw rscratch1, #1" %}
14424 ins_cost(6 * INSN_COST);
14425 ins_encode %{
14426 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14427 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14428 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14429 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14430 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14431 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14432 %}
14433
14434 ins_pipe(pipe_slow);
14435 %}
14436
14437 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14438 %{
14439 match(If cmp (OverflowMulL op1 op2));
14440 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14441 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14442 effect(USE labl, KILL cr);
14443
14444 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14445 "smulh rscratch2, $op1, $op2\n\t"
14446 "cmp rscratch2, rscratch1, ASR #63\n\t"
14447 "b$cmp $labl" %}
14448 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14449 ins_encode %{
14450 Label* L = $labl$$label;
14451 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14452 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14453 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14454 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14455 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14456 %}
14457
14458 ins_pipe(pipe_serial);
14459 %}
14460
14461 // ============================================================================
14462 // Compare Instructions
14463
14464 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14465 %{
14466 match(Set cr (CmpI op1 op2));
14467
14468 effect(DEF cr, USE op1, USE op2);
14469
14470 ins_cost(INSN_COST);
14471 format %{ "cmpw $op1, $op2" %}
14472
14473 ins_encode(aarch64_enc_cmpw(op1, op2));
14474
14475 ins_pipe(icmp_reg_reg);
14476 %}
14477
14478 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14479 %{
14480 match(Set cr (CmpI op1 zero));
14481
14482 effect(DEF cr, USE op1);
14483
14484 ins_cost(INSN_COST);
14485 format %{ "cmpw $op1, 0" %}
14486
14487 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14488
14489 ins_pipe(icmp_reg_imm);
14490 %}
14491
14492 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14493 %{
14494 match(Set cr (CmpI op1 op2));
14495
14496 effect(DEF cr, USE op1);
14497
14498 ins_cost(INSN_COST);
14499 format %{ "cmpw $op1, $op2" %}
14500
14501 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14502
14503 ins_pipe(icmp_reg_imm);
14504 %}
14505
14506 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14507 %{
14508 match(Set cr (CmpI op1 op2));
14509
14510 effect(DEF cr, USE op1);
14511
14512 ins_cost(INSN_COST * 2);
14513 format %{ "cmpw $op1, $op2" %}
14514
14515 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14516
14517 ins_pipe(icmp_reg_imm);
14518 %}
14519
14520 // Unsigned compare Instructions; really, same as signed compare
14521 // except it should only be used to feed an If or a CMovI which takes a
14522 // cmpOpU.
14523
14524 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14525 %{
14526 match(Set cr (CmpU op1 op2));
14527
14528 effect(DEF cr, USE op1, USE op2);
14529
14530 ins_cost(INSN_COST);
14531 format %{ "cmpw $op1, $op2\t# unsigned" %}
14532
14533 ins_encode(aarch64_enc_cmpw(op1, op2));
14534
14535 ins_pipe(icmp_reg_reg);
14536 %}
14537
14538 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14539 %{
14540 match(Set cr (CmpU op1 zero));
14541
14542 effect(DEF cr, USE op1);
14543
14544 ins_cost(INSN_COST);
14545 format %{ "cmpw $op1, #0\t# unsigned" %}
14546
14547 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14548
14549 ins_pipe(icmp_reg_imm);
14550 %}
14551
14552 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14553 %{
14554 match(Set cr (CmpU op1 op2));
14555
14556 effect(DEF cr, USE op1);
14557
14558 ins_cost(INSN_COST);
14559 format %{ "cmpw $op1, $op2\t# unsigned" %}
14560
14561 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14562
14563 ins_pipe(icmp_reg_imm);
14564 %}
14565
14566 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14567 %{
14568 match(Set cr (CmpU op1 op2));
14569
14570 effect(DEF cr, USE op1);
14571
14572 ins_cost(INSN_COST * 2);
14573 format %{ "cmpw $op1, $op2\t# unsigned" %}
14574
14575 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14576
14577 ins_pipe(icmp_reg_imm);
14578 %}
14579
14580 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14581 %{
14582 match(Set cr (CmpL op1 op2));
14583
14584 effect(DEF cr, USE op1, USE op2);
14585
14586 ins_cost(INSN_COST);
14587 format %{ "cmp $op1, $op2" %}
14588
14589 ins_encode(aarch64_enc_cmp(op1, op2));
14590
14591 ins_pipe(icmp_reg_reg);
14592 %}
14593
14594 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14595 %{
14596 match(Set cr (CmpL op1 zero));
14597
14598 effect(DEF cr, USE op1);
14599
14600 ins_cost(INSN_COST);
14601 format %{ "tst $op1" %}
14602
14603 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14604
14605 ins_pipe(icmp_reg_imm);
14606 %}
14607
14608 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14609 %{
14610 match(Set cr (CmpL op1 op2));
14611
14612 effect(DEF cr, USE op1);
14613
14614 ins_cost(INSN_COST);
14615 format %{ "cmp $op1, $op2" %}
14616
14617 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14618
14619 ins_pipe(icmp_reg_imm);
14620 %}
14621
14622 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14623 %{
14624 match(Set cr (CmpL op1 op2));
14625
14626 effect(DEF cr, USE op1);
14627
14628 ins_cost(INSN_COST * 2);
14629 format %{ "cmp $op1, $op2" %}
14630
14631 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14632
14633 ins_pipe(icmp_reg_imm);
14634 %}
14635
14636 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14637 %{
14638 match(Set cr (CmpUL op1 op2));
14639
14640 effect(DEF cr, USE op1, USE op2);
14641
14642 ins_cost(INSN_COST);
14643 format %{ "cmp $op1, $op2" %}
14644
14645 ins_encode(aarch64_enc_cmp(op1, op2));
14646
14647 ins_pipe(icmp_reg_reg);
14648 %}
14649
14650 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14651 %{
14652 match(Set cr (CmpUL op1 zero));
14653
14654 effect(DEF cr, USE op1);
14655
14656 ins_cost(INSN_COST);
14657 format %{ "tst $op1" %}
14658
14659 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14660
14661 ins_pipe(icmp_reg_imm);
14662 %}
14663
14664 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14665 %{
14666 match(Set cr (CmpUL op1 op2));
14667
14668 effect(DEF cr, USE op1);
14669
14670 ins_cost(INSN_COST);
14671 format %{ "cmp $op1, $op2" %}
14672
14673 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14674
14675 ins_pipe(icmp_reg_imm);
14676 %}
14677
14678 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14679 %{
14680 match(Set cr (CmpUL op1 op2));
14681
14682 effect(DEF cr, USE op1);
14683
14684 ins_cost(INSN_COST * 2);
14685 format %{ "cmp $op1, $op2" %}
14686
14687 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14688
14689 ins_pipe(icmp_reg_imm);
14690 %}
14691
14692 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14693 %{
14694 match(Set cr (CmpP op1 op2));
14695
14696 effect(DEF cr, USE op1, USE op2);
14697
14698 ins_cost(INSN_COST);
14699 format %{ "cmp $op1, $op2\t // ptr" %}
14700
14701 ins_encode(aarch64_enc_cmpp(op1, op2));
14702
14703 ins_pipe(icmp_reg_reg);
14704 %}
14705
14706 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14707 %{
14708 match(Set cr (CmpN op1 op2));
14709
14710 effect(DEF cr, USE op1, USE op2);
14711
14712 ins_cost(INSN_COST);
14713 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14714
14715 ins_encode(aarch64_enc_cmpn(op1, op2));
14716
14717 ins_pipe(icmp_reg_reg);
14718 %}
14719
14720 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14721 %{
14722 match(Set cr (CmpP op1 zero));
14723
14724 effect(DEF cr, USE op1, USE zero);
14725
14726 ins_cost(INSN_COST);
14727 format %{ "cmp $op1, 0\t // ptr" %}
14728
14729 ins_encode(aarch64_enc_testp(op1));
14730
14731 ins_pipe(icmp_reg_imm);
14732 %}
14733
14734 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14735 %{
14736 match(Set cr (CmpN op1 zero));
14737
14738 effect(DEF cr, USE op1, USE zero);
14739
14740 ins_cost(INSN_COST);
14741 format %{ "cmp $op1, 0\t // compressed ptr" %}
14742
14743 ins_encode(aarch64_enc_testn(op1));
14744
14745 ins_pipe(icmp_reg_imm);
14746 %}
14747
14748 // FP comparisons
14749 //
14750 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14751 // using normal cmpOp. See declaration of rFlagsReg for details.
14752
14753 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14754 %{
14755 match(Set cr (CmpF src1 src2));
14756
14757 ins_cost(3 * INSN_COST);
14758 format %{ "fcmps $src1, $src2" %}
14759
14760 ins_encode %{
14761 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14762 %}
14763
14764 ins_pipe(pipe_class_compare);
14765 %}
14766
14767 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14768 %{
14769 match(Set cr (CmpF src1 src2));
14770
14771 ins_cost(3 * INSN_COST);
14772 format %{ "fcmps $src1, 0.0" %}
14773
14774 ins_encode %{
14775 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14776 %}
14777
14778 ins_pipe(pipe_class_compare);
14779 %}
14780 // FROM HERE
14781
14782 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14783 %{
14784 match(Set cr (CmpD src1 src2));
14785
14786 ins_cost(3 * INSN_COST);
14787 format %{ "fcmpd $src1, $src2" %}
14788
14789 ins_encode %{
14790 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14791 %}
14792
14793 ins_pipe(pipe_class_compare);
14794 %}
14795
14796 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14797 %{
14798 match(Set cr (CmpD src1 src2));
14799
14800 ins_cost(3 * INSN_COST);
14801 format %{ "fcmpd $src1, 0.0" %}
14802
14803 ins_encode %{
14804 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14805 %}
14806
14807 ins_pipe(pipe_class_compare);
14808 %}
14809
14810 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14811 %{
14812 match(Set dst (CmpF3 src1 src2));
14813 effect(KILL cr);
14814
14815 ins_cost(5 * INSN_COST);
14816 format %{ "fcmps $src1, $src2\n\t"
14817 "csinvw($dst, zr, zr, eq\n\t"
14818 "csnegw($dst, $dst, $dst, lt)"
14819 %}
14820
14821 ins_encode %{
14822 Label done;
14823 FloatRegister s1 = as_FloatRegister($src1$$reg);
14824 FloatRegister s2 = as_FloatRegister($src2$$reg);
14825 Register d = as_Register($dst$$reg);
14826 __ fcmps(s1, s2);
14827 // installs 0 if EQ else -1
14828 __ csinvw(d, zr, zr, Assembler::EQ);
14829 // keeps -1 if less or unordered else installs 1
14830 __ csnegw(d, d, d, Assembler::LT);
14831 __ bind(done);
14832 %}
14833
14834 ins_pipe(pipe_class_default);
14835
14836 %}
14837
14838 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14839 %{
14840 match(Set dst (CmpD3 src1 src2));
14841 effect(KILL cr);
14842
14843 ins_cost(5 * INSN_COST);
14844 format %{ "fcmpd $src1, $src2\n\t"
14845 "csinvw($dst, zr, zr, eq\n\t"
14846 "csnegw($dst, $dst, $dst, lt)"
14847 %}
14848
14849 ins_encode %{
14850 Label done;
14851 FloatRegister s1 = as_FloatRegister($src1$$reg);
14852 FloatRegister s2 = as_FloatRegister($src2$$reg);
14853 Register d = as_Register($dst$$reg);
14854 __ fcmpd(s1, s2);
14855 // installs 0 if EQ else -1
14856 __ csinvw(d, zr, zr, Assembler::EQ);
14857 // keeps -1 if less or unordered else installs 1
14858 __ csnegw(d, d, d, Assembler::LT);
14859 __ bind(done);
14860 %}
14861 ins_pipe(pipe_class_default);
14862
14863 %}
14864
14865 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14866 %{
14867 match(Set dst (CmpF3 src1 zero));
14868 effect(KILL cr);
14869
14870 ins_cost(5 * INSN_COST);
14871 format %{ "fcmps $src1, 0.0\n\t"
14872 "csinvw($dst, zr, zr, eq\n\t"
14873 "csnegw($dst, $dst, $dst, lt)"
14874 %}
14875
14876 ins_encode %{
14877 Label done;
14878 FloatRegister s1 = as_FloatRegister($src1$$reg);
14879 Register d = as_Register($dst$$reg);
14880 __ fcmps(s1, 0.0);
14881 // installs 0 if EQ else -1
14882 __ csinvw(d, zr, zr, Assembler::EQ);
14883 // keeps -1 if less or unordered else installs 1
14884 __ csnegw(d, d, d, Assembler::LT);
14885 __ bind(done);
14886 %}
14887
14888 ins_pipe(pipe_class_default);
14889
14890 %}
14891
14892 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14893 %{
14894 match(Set dst (CmpD3 src1 zero));
14895 effect(KILL cr);
14896
14897 ins_cost(5 * INSN_COST);
14898 format %{ "fcmpd $src1, 0.0\n\t"
14899 "csinvw($dst, zr, zr, eq\n\t"
14900 "csnegw($dst, $dst, $dst, lt)"
14901 %}
14902
14903 ins_encode %{
14904 Label done;
14905 FloatRegister s1 = as_FloatRegister($src1$$reg);
14906 Register d = as_Register($dst$$reg);
14907 __ fcmpd(s1, 0.0);
14908 // installs 0 if EQ else -1
14909 __ csinvw(d, zr, zr, Assembler::EQ);
14910 // keeps -1 if less or unordered else installs 1
14911 __ csnegw(d, d, d, Assembler::LT);
14912 __ bind(done);
14913 %}
14914 ins_pipe(pipe_class_default);
14915
14916 %}
14917
14918 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14919 %{
14920 match(Set dst (CmpLTMask p q));
14921 effect(KILL cr);
14922
14923 ins_cost(3 * INSN_COST);
14924
14925 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14926 "csetw $dst, lt\n\t"
14927 "subw $dst, zr, $dst"
14928 %}
14929
14930 ins_encode %{
14931 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14932 __ csetw(as_Register($dst$$reg), Assembler::LT);
14933 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14934 %}
14935
14936 ins_pipe(ialu_reg_reg);
14937 %}
14938
14939 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14940 %{
14941 match(Set dst (CmpLTMask src zero));
14942 effect(KILL cr);
14943
14944 ins_cost(INSN_COST);
14945
14946 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14947
14948 ins_encode %{
14949 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14950 %}
14951
14952 ins_pipe(ialu_reg_shift);
14953 %}
14954
14955 // ============================================================================
14956 // Max and Min
14957
14958 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14959 %{
14960 effect( DEF dst, USE src1, USE src2, USE cr );
14961
14962 ins_cost(INSN_COST * 2);
14963 format %{ "cselw $dst, $src1, $src2 lt\t" %}
14964
14965 ins_encode %{
14966 __ cselw(as_Register($dst$$reg),
14967 as_Register($src1$$reg),
14968 as_Register($src2$$reg),
14969 Assembler::LT);
14970 %}
14971
14972 ins_pipe(icond_reg_reg);
14973 %}
14974
14975 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
14976 %{
14977 match(Set dst (MinI src1 src2));
14978 ins_cost(INSN_COST * 3);
14979
14980 expand %{
14981 rFlagsReg cr;
14982 compI_reg_reg(cr, src1, src2);
14983 cmovI_reg_reg_lt(dst, src1, src2, cr);
14984 %}
14985
14986 %}
14987 // FROM HERE
14988
14989 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14990 %{
14991 effect( DEF dst, USE src1, USE src2, USE cr );
14992
14993 ins_cost(INSN_COST * 2);
14994 format %{ "cselw $dst, $src1, $src2 gt\t" %}
14995
14996 ins_encode %{
14997 __ cselw(as_Register($dst$$reg),
14998 as_Register($src1$$reg),
14999 as_Register($src2$$reg),
15000 Assembler::GT);
15001 %}
15002
15003 ins_pipe(icond_reg_reg);
15004 %}
15005
15006 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15007 %{
15008 match(Set dst (MaxI src1 src2));
15009 ins_cost(INSN_COST * 3);
15010 expand %{
15011 rFlagsReg cr;
15012 compI_reg_reg(cr, src1, src2);
15013 cmovI_reg_reg_gt(dst, src1, src2, cr);
15014 %}
15015 %}
15016
15017 // ============================================================================
15018 // Branch Instructions
15019
15020 // Direct Branch.
15021 instruct branch(label lbl)
15022 %{
15023 match(Goto);
15024
15025 effect(USE lbl);
15026
15027 ins_cost(BRANCH_COST);
15028 format %{ "b $lbl" %}
15029
15030 ins_encode(aarch64_enc_b(lbl));
15031
15032 ins_pipe(pipe_branch);
15033 %}
15034
15035 // Conditional Near Branch
15036 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15037 %{
15038 // Same match rule as `branchConFar'.
15039 match(If cmp cr);
15040
15041 effect(USE lbl);
15042
15043 ins_cost(BRANCH_COST);
15044 // If set to 1 this indicates that the current instruction is a
15045 // short variant of a long branch. This avoids using this
15046 // instruction in first-pass matching. It will then only be used in
15047 // the `Shorten_branches' pass.
15048 // ins_short_branch(1);
15049 format %{ "b$cmp $lbl" %}
15050
15051 ins_encode(aarch64_enc_br_con(cmp, lbl));
15052
15053 ins_pipe(pipe_branch_cond);
15054 %}
15055
15056 // Conditional Near Branch Unsigned
15057 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15058 %{
15059 // Same match rule as `branchConFar'.
15060 match(If cmp cr);
15061
15062 effect(USE lbl);
15063
15064 ins_cost(BRANCH_COST);
15065 // If set to 1 this indicates that the current instruction is a
15066 // short variant of a long branch. This avoids using this
15067 // instruction in first-pass matching. It will then only be used in
15068 // the `Shorten_branches' pass.
15069 // ins_short_branch(1);
15070 format %{ "b$cmp $lbl\t# unsigned" %}
15071
15072 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15073
15074 ins_pipe(pipe_branch_cond);
15075 %}
15076
15077 // Make use of CBZ and CBNZ. These instructions, as well as being
15078 // shorter than (cmp; branch), have the additional benefit of not
15079 // killing the flags.
15080
15081 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15082 match(If cmp (CmpI op1 op2));
15083 effect(USE labl);
15084
15085 ins_cost(BRANCH_COST);
15086 format %{ "cbw$cmp $op1, $labl" %}
15087 ins_encode %{
15088 Label* L = $labl$$label;
15089 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15090 if (cond == Assembler::EQ)
15091 __ cbzw($op1$$Register, *L);
15092 else
15093 __ cbnzw($op1$$Register, *L);
15094 %}
15095 ins_pipe(pipe_cmp_branch);
15096 %}
15097
15098 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15099 match(If cmp (CmpL op1 op2));
15100 effect(USE labl);
15101
15102 ins_cost(BRANCH_COST);
15103 format %{ "cb$cmp $op1, $labl" %}
15104 ins_encode %{
15105 Label* L = $labl$$label;
15106 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15107 if (cond == Assembler::EQ)
15108 __ cbz($op1$$Register, *L);
15109 else
15110 __ cbnz($op1$$Register, *L);
15111 %}
15112 ins_pipe(pipe_cmp_branch);
15113 %}
15114
15115 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15116 match(If cmp (CmpP op1 op2));
15117 effect(USE labl);
15118
15119 ins_cost(BRANCH_COST);
15120 format %{ "cb$cmp $op1, $labl" %}
15121 ins_encode %{
15122 Label* L = $labl$$label;
15123 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15124 if (cond == Assembler::EQ)
15125 __ cbz($op1$$Register, *L);
15126 else
15127 __ cbnz($op1$$Register, *L);
15128 %}
15129 ins_pipe(pipe_cmp_branch);
15130 %}
15131
15132 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15133 match(If cmp (CmpN op1 op2));
15134 effect(USE labl);
15135
15136 ins_cost(BRANCH_COST);
15137 format %{ "cbw$cmp $op1, $labl" %}
15138 ins_encode %{
15139 Label* L = $labl$$label;
15140 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15141 if (cond == Assembler::EQ)
15142 __ cbzw($op1$$Register, *L);
15143 else
15144 __ cbnzw($op1$$Register, *L);
15145 %}
15146 ins_pipe(pipe_cmp_branch);
15147 %}
15148
15149 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15150 match(If cmp (CmpP (DecodeN oop) zero));
15151 effect(USE labl);
15152
15153 ins_cost(BRANCH_COST);
15154 format %{ "cb$cmp $oop, $labl" %}
15155 ins_encode %{
15156 Label* L = $labl$$label;
15157 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15158 if (cond == Assembler::EQ)
15159 __ cbzw($oop$$Register, *L);
15160 else
15161 __ cbnzw($oop$$Register, *L);
15162 %}
15163 ins_pipe(pipe_cmp_branch);
15164 %}
15165
15166 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
15167 match(If cmp (CmpU op1 op2));
15168 effect(USE labl);
15169
15170 ins_cost(BRANCH_COST);
15171 format %{ "cbw$cmp $op1, $labl" %}
15172 ins_encode %{
15173 Label* L = $labl$$label;
15174 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15175 if (cond == Assembler::EQ || cond == Assembler::LS)
15176 __ cbzw($op1$$Register, *L);
15177 else
15178 __ cbnzw($op1$$Register, *L);
15179 %}
15180 ins_pipe(pipe_cmp_branch);
15181 %}
15182
15183 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
15184 match(If cmp (CmpUL op1 op2));
15185 effect(USE labl);
15186
15187 ins_cost(BRANCH_COST);
15188 format %{ "cb$cmp $op1, $labl" %}
15189 ins_encode %{
15190 Label* L = $labl$$label;
15191 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15192 if (cond == Assembler::EQ || cond == Assembler::LS)
15193 __ cbz($op1$$Register, *L);
15194 else
15195 __ cbnz($op1$$Register, *L);
15196 %}
15197 ins_pipe(pipe_cmp_branch);
15198 %}
15199
15200 // Test bit and Branch
15201
15202 // Patterns for short (< 32KiB) variants
15203 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15204 match(If cmp (CmpL op1 op2));
15205 effect(USE labl);
15206
15207 ins_cost(BRANCH_COST);
15208 format %{ "cb$cmp $op1, $labl # long" %}
15209 ins_encode %{
15210 Label* L = $labl$$label;
15211 Assembler::Condition cond =
15212 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15213 __ tbr(cond, $op1$$Register, 63, *L);
15214 %}
15215 ins_pipe(pipe_cmp_branch);
15216 ins_short_branch(1);
15217 %}
15218
15219 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15220 match(If cmp (CmpI op1 op2));
15221 effect(USE labl);
15222
15223 ins_cost(BRANCH_COST);
15224 format %{ "cb$cmp $op1, $labl # int" %}
15225 ins_encode %{
15226 Label* L = $labl$$label;
15227 Assembler::Condition cond =
15228 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15229 __ tbr(cond, $op1$$Register, 31, *L);
15230 %}
15231 ins_pipe(pipe_cmp_branch);
15232 ins_short_branch(1);
15233 %}
15234
15235 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15236 match(If cmp (CmpL (AndL op1 op2) op3));
15237 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15238 effect(USE labl);
15239
15240 ins_cost(BRANCH_COST);
15241 format %{ "tb$cmp $op1, $op2, $labl" %}
15242 ins_encode %{
15243 Label* L = $labl$$label;
15244 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15245 int bit = exact_log2_long($op2$$constant);
15246 __ tbr(cond, $op1$$Register, bit, *L);
15247 %}
15248 ins_pipe(pipe_cmp_branch);
15249 ins_short_branch(1);
15250 %}
15251
15252 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15253 match(If cmp (CmpI (AndI op1 op2) op3));
15254 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15255 effect(USE labl);
15256
15257 ins_cost(BRANCH_COST);
15258 format %{ "tb$cmp $op1, $op2, $labl" %}
15259 ins_encode %{
15260 Label* L = $labl$$label;
15261 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15262 int bit = exact_log2((juint)$op2$$constant);
15263 __ tbr(cond, $op1$$Register, bit, *L);
15264 %}
15265 ins_pipe(pipe_cmp_branch);
15266 ins_short_branch(1);
15267 %}
15268
15269 // And far variants
15270 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15271 match(If cmp (CmpL op1 op2));
15272 effect(USE labl);
15273
15274 ins_cost(BRANCH_COST);
15275 format %{ "cb$cmp $op1, $labl # long" %}
15276 ins_encode %{
15277 Label* L = $labl$$label;
15278 Assembler::Condition cond =
15279 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15280 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15281 %}
15282 ins_pipe(pipe_cmp_branch);
15283 %}
15284
15285 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15286 match(If cmp (CmpI op1 op2));
15287 effect(USE labl);
15288
15289 ins_cost(BRANCH_COST);
15290 format %{ "cb$cmp $op1, $labl # int" %}
15291 ins_encode %{
15292 Label* L = $labl$$label;
15293 Assembler::Condition cond =
15294 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15295 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15296 %}
15297 ins_pipe(pipe_cmp_branch);
15298 %}
15299
15300 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15301 match(If cmp (CmpL (AndL op1 op2) op3));
15302 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15303 effect(USE labl);
15304
15305 ins_cost(BRANCH_COST);
15306 format %{ "tb$cmp $op1, $op2, $labl" %}
15307 ins_encode %{
15308 Label* L = $labl$$label;
15309 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15310 int bit = exact_log2_long($op2$$constant);
15311 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15312 %}
15313 ins_pipe(pipe_cmp_branch);
15314 %}
15315
15316 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15317 match(If cmp (CmpI (AndI op1 op2) op3));
15318 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15319 effect(USE labl);
15320
15321 ins_cost(BRANCH_COST);
15322 format %{ "tb$cmp $op1, $op2, $labl" %}
15323 ins_encode %{
15324 Label* L = $labl$$label;
15325 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15326 int bit = exact_log2((juint)$op2$$constant);
15327 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15328 %}
15329 ins_pipe(pipe_cmp_branch);
15330 %}
15331
15332 // Test bits
15333
15334 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15335 match(Set cr (CmpL (AndL op1 op2) op3));
15336 predicate(Assembler::operand_valid_for_logical_immediate
15337 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15338
15339 ins_cost(INSN_COST);
15340 format %{ "tst $op1, $op2 # long" %}
15341 ins_encode %{
15342 __ tst($op1$$Register, $op2$$constant);
15343 %}
15344 ins_pipe(ialu_reg_reg);
15345 %}
15346
15347 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15348 match(Set cr (CmpI (AndI op1 op2) op3));
15349 predicate(Assembler::operand_valid_for_logical_immediate
15350 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15351
15352 ins_cost(INSN_COST);
15353 format %{ "tst $op1, $op2 # int" %}
15354 ins_encode %{
15355 __ tstw($op1$$Register, $op2$$constant);
15356 %}
15357 ins_pipe(ialu_reg_reg);
15358 %}
15359
15360 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15361 match(Set cr (CmpL (AndL op1 op2) op3));
15362
15363 ins_cost(INSN_COST);
15364 format %{ "tst $op1, $op2 # long" %}
15365 ins_encode %{
15366 __ tst($op1$$Register, $op2$$Register);
15367 %}
15368 ins_pipe(ialu_reg_reg);
15369 %}
15370
15371 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15372 match(Set cr (CmpI (AndI op1 op2) op3));
15373
15374 ins_cost(INSN_COST);
15375 format %{ "tstw $op1, $op2 # int" %}
15376 ins_encode %{
15377 __ tstw($op1$$Register, $op2$$Register);
15378 %}
15379 ins_pipe(ialu_reg_reg);
15380 %}
15381
15382
15383 // Conditional Far Branch
15384 // Conditional Far Branch Unsigned
15385 // TODO: fixme
15386
15387 // counted loop end branch near
15388 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15389 %{
15390 match(CountedLoopEnd cmp cr);
15391
15392 effect(USE lbl);
15393
15394 ins_cost(BRANCH_COST);
15395 // short variant.
15396 // ins_short_branch(1);
15397 format %{ "b$cmp $lbl \t// counted loop end" %}
15398
15399 ins_encode(aarch64_enc_br_con(cmp, lbl));
15400
15401 ins_pipe(pipe_branch);
15402 %}
15403
15404 // counted loop end branch near Unsigned
15405 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15406 %{
15407 match(CountedLoopEnd cmp cr);
15408
15409 effect(USE lbl);
15410
15411 ins_cost(BRANCH_COST);
15412 // short variant.
15413 // ins_short_branch(1);
15414 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
15415
15416 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15417
15418 ins_pipe(pipe_branch);
15419 %}
15420
15421 // counted loop end branch far
15422 // counted loop end branch far unsigned
15423 // TODO: fixme
15424
15425 // ============================================================================
15426 // inlined locking and unlocking
15427
15428 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
15429 %{
15430 match(Set cr (FastLock object box));
15431 effect(TEMP tmp, TEMP tmp2);
15432
15433 // TODO
15434 // identify correct cost
15435 ins_cost(5 * INSN_COST);
15436 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
15437
15438 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
15439
15440 ins_pipe(pipe_serial);
15441 %}
15442
15443 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
15444 %{
15445 match(Set cr (FastUnlock object box));
15446 effect(TEMP tmp, TEMP tmp2);
15447
15448 ins_cost(5 * INSN_COST);
15449 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
15450
15451 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
15452
15453 ins_pipe(pipe_serial);
15454 %}
15455
15456
15457 // ============================================================================
15458 // Safepoint Instructions
15459
15460 // TODO
15461 // provide a near and far version of this code
15462
15463 instruct safePoint(rFlagsReg cr, iRegP poll)
15464 %{
15465 match(SafePoint poll);
15466 effect(KILL cr);
15467
15468 format %{
15469 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15470 %}
15471 ins_encode %{
15472 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15473 %}
15474 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15475 %}
15476
15477
15478 // ============================================================================
15479 // Procedure Call/Return Instructions
15480
15481 // Call Java Static Instruction
15482
15483 instruct CallStaticJavaDirect(method meth)
15484 %{
15485 match(CallStaticJava);
15486
15487 effect(USE meth);
15488
15489 ins_cost(CALL_COST);
15490
15491 format %{ "call,static $meth \t// ==> " %}
15492
15493 ins_encode( aarch64_enc_java_static_call(meth),
15494 aarch64_enc_call_epilog );
15495
15496 ins_pipe(pipe_class_call);
15497 %}
15498
15499 // TO HERE
15500
15501 // Call Java Dynamic Instruction
15502 instruct CallDynamicJavaDirect(method meth)
15503 %{
15504 match(CallDynamicJava);
15505
15506 effect(USE meth);
15507
15508 ins_cost(CALL_COST);
15509
15510 format %{ "CALL,dynamic $meth \t// ==> " %}
15511
15512 ins_encode( aarch64_enc_java_dynamic_call(meth),
15513 aarch64_enc_call_epilog );
15514
15515 ins_pipe(pipe_class_call);
15516 %}
15517
15518 // Call Runtime Instruction
15519
15520 instruct CallRuntimeDirect(method meth)
15521 %{
15522 match(CallRuntime);
15523
15524 effect(USE meth);
15525
15526 ins_cost(CALL_COST);
15527
15528 format %{ "CALL, runtime $meth" %}
15529
15530 ins_encode( aarch64_enc_java_to_runtime(meth) );
15531
15532 ins_pipe(pipe_class_call);
15533 %}
15534
15535 // Call Runtime Instruction
15536
15537 instruct CallLeafDirect(method meth)
15538 %{
15539 match(CallLeaf);
15540
15541 effect(USE meth);
15542
15543 ins_cost(CALL_COST);
15544
15545 format %{ "CALL, runtime leaf $meth" %}
15546
15547 ins_encode( aarch64_enc_java_to_runtime(meth) );
15548
15549 ins_pipe(pipe_class_call);
15550 %}
15551
15552 // Call Runtime Instruction
15553
15554 instruct CallLeafNoFPDirect(method meth)
15555 %{
15556 match(CallLeafNoFP);
15557
15558 effect(USE meth);
15559
15560 ins_cost(CALL_COST);
15561
15562 format %{ "CALL, runtime leaf nofp $meth" %}
15563
15564 ins_encode( aarch64_enc_java_to_runtime(meth) );
15565
15566 ins_pipe(pipe_class_call);
15567 %}
15568
15569 // Tail Call; Jump from runtime stub to Java code.
15570 // Also known as an 'interprocedural jump'.
15571 // Target of jump will eventually return to caller.
15572 // TailJump below removes the return address.
15573 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
15574 %{
15575 match(TailCall jump_target method_ptr);
15576
15577 ins_cost(CALL_COST);
15578
15579 format %{ "br $jump_target\t# $method_ptr holds method" %}
15580
15581 ins_encode(aarch64_enc_tail_call(jump_target));
15582
15583 ins_pipe(pipe_class_call);
15584 %}
15585
15586 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
15587 %{
15588 match(TailJump jump_target ex_oop);
15589
15590 ins_cost(CALL_COST);
15591
15592 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15593
15594 ins_encode(aarch64_enc_tail_jmp(jump_target));
15595
15596 ins_pipe(pipe_class_call);
15597 %}
15598
15599 // Create exception oop: created by stack-crawling runtime code.
15600 // Created exception is now available to this handler, and is setup
15601 // just prior to jumping to this handler. No code emitted.
15602 // TODO check
15603 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15604 instruct CreateException(iRegP_R0 ex_oop)
15605 %{
15606 match(Set ex_oop (CreateEx));
15607
15608 format %{ " -- \t// exception oop; no code emitted" %}
15609
15610 size(0);
15611
15612 ins_encode( /*empty*/ );
15613
15614 ins_pipe(pipe_class_empty);
15615 %}
15616
15617 // Rethrow exception: The exception oop will come in the first
15618 // argument position. Then JUMP (not call) to the rethrow stub code.
15619 instruct RethrowException() %{
15620 match(Rethrow);
15621 ins_cost(CALL_COST);
15622
15623 format %{ "b rethrow_stub" %}
15624
15625 ins_encode( aarch64_enc_rethrow() );
15626
15627 ins_pipe(pipe_class_call);
15628 %}
15629
15630
15631 // Return Instruction
15632 // epilog node loads ret address into lr as part of frame pop
15633 instruct Ret()
15634 %{
15635 match(Return);
15636
15637 format %{ "ret\t// return register" %}
15638
15639 ins_encode( aarch64_enc_ret() );
15640
15641 ins_pipe(pipe_branch);
15642 %}
15643
15644 // Die now.
15645 instruct ShouldNotReachHere() %{
15646 match(Halt);
15647
15648 ins_cost(CALL_COST);
15649 format %{ "ShouldNotReachHere" %}
15650
15651 ins_encode %{
15652 if (is_reachable()) {
15653 __ stop(_halt_reason);
15654 }
15655 %}
15656
15657 ins_pipe(pipe_class_default);
15658 %}
15659
15660 // ============================================================================
15661 // Partial Subtype Check
15662 //
15663 // superklass array for an instance of the superklass. Set a hidden
15664 // internal cache on a hit (cache is checked with exposed code in
15665 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15666 // encoding ALSO sets flags.
15667
15668 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15669 %{
15670 match(Set result (PartialSubtypeCheck sub super));
15671 effect(KILL cr, KILL temp);
15672
15673 ins_cost(1100); // slightly larger than the next version
15674 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15675
15676 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15677
15678 opcode(0x1); // Force zero of result reg on hit
15679
15680 ins_pipe(pipe_class_memory);
15681 %}
15682
15683 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
15684 %{
15685 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
15686 effect(KILL temp, KILL result);
15687
15688 ins_cost(1100); // slightly larger than the next version
15689 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
15690
15691 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15692
15693 opcode(0x0); // Don't zero result reg on hit
15694
15695 ins_pipe(pipe_class_memory);
15696 %}
15697
15698 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15699 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15700 %{
15701 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
15702 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15703 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15704
15705 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15706 ins_encode %{
15707 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15708 __ string_compare($str1$$Register, $str2$$Register,
15709 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15710 $tmp1$$Register, $tmp2$$Register,
15711 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
15712 %}
15713 ins_pipe(pipe_class_memory);
15714 %}
15715
15716 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15717 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15718 %{
15719 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
15720 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15721 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15722
15723 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15724 ins_encode %{
15725 __ string_compare($str1$$Register, $str2$$Register,
15726 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15727 $tmp1$$Register, $tmp2$$Register,
15728 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
15729 %}
15730 ins_pipe(pipe_class_memory);
15731 %}
15732
15733 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15734 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15735 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15736 %{
15737 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15738 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15739 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15740 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15741
15742 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15743 ins_encode %{
15744 __ string_compare($str1$$Register, $str2$$Register,
15745 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15746 $tmp1$$Register, $tmp2$$Register,
15747 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15748 $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
15749 %}
15750 ins_pipe(pipe_class_memory);
15751 %}
15752
15753 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15754 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15755 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15756 %{
15757 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
15758 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15759 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15760 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15761
15762 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15763 ins_encode %{
15764 __ string_compare($str1$$Register, $str2$$Register,
15765 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15766 $tmp1$$Register, $tmp2$$Register,
15767 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15768 $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
15769 %}
15770 ins_pipe(pipe_class_memory);
15771 %}
15772
15773 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15774 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15775 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
15776 %{
15777 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15778 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15779 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15780 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
15781 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
15782
15783 ins_encode %{
15784 __ string_indexof($str1$$Register, $str2$$Register,
15785 $cnt1$$Register, $cnt2$$Register,
15786 $tmp1$$Register, $tmp2$$Register,
15787 $tmp3$$Register, $tmp4$$Register,
15788 $tmp5$$Register, $tmp6$$Register,
15789 -1, $result$$Register, StrIntrinsicNode::UU);
15790 %}
15791 ins_pipe(pipe_class_memory);
15792 %}
15793
15794 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15795 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15796 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
15797 %{
15798 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15799 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15800 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15801 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
15802 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
15803
15804 ins_encode %{
15805 __ string_indexof($str1$$Register, $str2$$Register,
15806 $cnt1$$Register, $cnt2$$Register,
15807 $tmp1$$Register, $tmp2$$Register,
15808 $tmp3$$Register, $tmp4$$Register,
15809 $tmp5$$Register, $tmp6$$Register,
15810 -1, $result$$Register, StrIntrinsicNode::LL);
15811 %}
15812 ins_pipe(pipe_class_memory);
15813 %}
15814
15815 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15816 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15817 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
15818 %{
15819 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15820 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15821 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15822 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
15823 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
15824
15825 ins_encode %{
15826 __ string_indexof($str1$$Register, $str2$$Register,
15827 $cnt1$$Register, $cnt2$$Register,
15828 $tmp1$$Register, $tmp2$$Register,
15829 $tmp3$$Register, $tmp4$$Register,
15830 $tmp5$$Register, $tmp6$$Register,
15831 -1, $result$$Register, StrIntrinsicNode::UL);
15832 %}
15833 ins_pipe(pipe_class_memory);
15834 %}
15835
15836 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15837 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15838 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15839 %{
15840 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15841 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15842 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15843 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15844 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
15845
15846 ins_encode %{
15847 int icnt2 = (int)$int_cnt2$$constant;
15848 __ string_indexof($str1$$Register, $str2$$Register,
15849 $cnt1$$Register, zr,
15850 $tmp1$$Register, $tmp2$$Register,
15851 $tmp3$$Register, $tmp4$$Register, zr, zr,
15852 icnt2, $result$$Register, StrIntrinsicNode::UU);
15853 %}
15854 ins_pipe(pipe_class_memory);
15855 %}
15856
15857 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15858 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15859 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15860 %{
15861 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15862 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15863 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15864 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15865 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
15866
15867 ins_encode %{
15868 int icnt2 = (int)$int_cnt2$$constant;
15869 __ string_indexof($str1$$Register, $str2$$Register,
15870 $cnt1$$Register, zr,
15871 $tmp1$$Register, $tmp2$$Register,
15872 $tmp3$$Register, $tmp4$$Register, zr, zr,
15873 icnt2, $result$$Register, StrIntrinsicNode::LL);
15874 %}
15875 ins_pipe(pipe_class_memory);
15876 %}
15877
15878 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15879 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15880 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15881 %{
15882 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15883 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15884 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15885 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15886 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
15887
15888 ins_encode %{
15889 int icnt2 = (int)$int_cnt2$$constant;
15890 __ string_indexof($str1$$Register, $str2$$Register,
15891 $cnt1$$Register, zr,
15892 $tmp1$$Register, $tmp2$$Register,
15893 $tmp3$$Register, $tmp4$$Register, zr, zr,
15894 icnt2, $result$$Register, StrIntrinsicNode::UL);
15895 %}
15896 ins_pipe(pipe_class_memory);
15897 %}
15898
15899 instruct string_indexofU_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15900 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15901 iRegINoSp tmp3, rFlagsReg cr)
15902 %{
15903 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15904 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15905 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15906
15907 format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15908
15909 ins_encode %{
15910 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15911 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15912 $tmp3$$Register);
15913 %}
15914 ins_pipe(pipe_class_memory);
15915 %}
15916
15917 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15918 iRegI_R0 result, rFlagsReg cr)
15919 %{
15920 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
15921 match(Set result (StrEquals (Binary str1 str2) cnt));
15922 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15923
15924 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15925 ins_encode %{
15926 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15927 __ string_equals($str1$$Register, $str2$$Register,
15928 $result$$Register, $cnt$$Register, 1);
15929 %}
15930 ins_pipe(pipe_class_memory);
15931 %}
15932
15933 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15934 iRegI_R0 result, rFlagsReg cr)
15935 %{
15936 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
15937 match(Set result (StrEquals (Binary str1 str2) cnt));
15938 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15939
15940 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15941 ins_encode %{
15942 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15943 __ string_equals($str1$$Register, $str2$$Register,
15944 $result$$Register, $cnt$$Register, 2);
15945 %}
15946 ins_pipe(pipe_class_memory);
15947 %}
15948
15949 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
15950 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
15951 iRegP_R10 tmp, rFlagsReg cr)
15952 %{
15953 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15954 match(Set result (AryEq ary1 ary2));
15955 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15956
15957 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
15958 ins_encode %{
15959 __ arrays_equals($ary1$$Register, $ary2$$Register,
15960 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15961 $result$$Register, $tmp$$Register, 1);
15962 %}
15963 ins_pipe(pipe_class_memory);
15964 %}
15965
15966 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
15967 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
15968 iRegP_R10 tmp, rFlagsReg cr)
15969 %{
15970 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15971 match(Set result (AryEq ary1 ary2));
15972 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15973
15974 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
15975 ins_encode %{
15976 __ arrays_equals($ary1$$Register, $ary2$$Register,
15977 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15978 $result$$Register, $tmp$$Register, 2);
15979 %}
15980 ins_pipe(pipe_class_memory);
15981 %}
15982
15983 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
15984 %{
15985 match(Set result (HasNegatives ary1 len));
15986 effect(USE_KILL ary1, USE_KILL len, KILL cr);
15987 format %{ "has negatives byte[] $ary1,$len -> $result" %}
15988 ins_encode %{
15989 __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
15990 %}
15991 ins_pipe( pipe_slow );
15992 %}
15993
15994 // fast char[] to byte[] compression
15995 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
15996 vRegD_V0 tmp1, vRegD_V1 tmp2,
15997 vRegD_V2 tmp3, vRegD_V3 tmp4,
15998 iRegI_R0 result, rFlagsReg cr)
15999 %{
16000 match(Set result (StrCompressedCopy src (Binary dst len)));
16001 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16002
16003 format %{ "String Compress $src,$dst -> $result // KILL R1, R2, R3, R4" %}
16004 ins_encode %{
16005 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16006 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16007 $tmp3$$FloatRegister, $tmp4$$FloatRegister,
16008 $result$$Register);
16009 %}
16010 ins_pipe( pipe_slow );
16011 %}
16012
16013 // fast byte[] to char[] inflation
16014 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16015 vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16016 %{
16017 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16018 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16019
16020 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
16021 ins_encode %{
16022 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16023 $tmp1$$FloatRegister, $tmp2$$FloatRegister, $tmp3$$FloatRegister, $tmp4$$Register);
16024 %}
16025 ins_pipe(pipe_class_memory);
16026 %}
16027
16028 // encode char[] to byte[] in ISO_8859_1
16029 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16030 vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
16031 vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
16032 iRegI_R0 result, rFlagsReg cr)
16033 %{
16034 match(Set result (EncodeISOArray src (Binary dst len)));
16035 effect(USE_KILL src, USE_KILL dst, USE_KILL len,
16036 KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
16037
16038 format %{ "Encode array $src,$dst,$len -> $result" %}
16039 ins_encode %{
16040 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16041 $result$$Register, $Vtmp1$$FloatRegister, $Vtmp2$$FloatRegister,
16042 $Vtmp3$$FloatRegister, $Vtmp4$$FloatRegister);
16043 %}
16044 ins_pipe( pipe_class_memory );
16045 %}
16046
16047 // ============================================================================
16048 // This name is KNOWN by the ADLC and cannot be changed.
16049 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16050 // for this guy.
16051 instruct tlsLoadP(thread_RegP dst)
16052 %{
16053 match(Set dst (ThreadLocal));
16054
16055 ins_cost(0);
16056
16057 format %{ " -- \t// $dst=Thread::current(), empty" %}
16058
16059 size(0);
16060
16061 ins_encode( /*empty*/ );
16062
16063 ins_pipe(pipe_class_empty);
16064 %}
16065
16066 // ====================VECTOR INSTRUCTIONS=====================================
16067
16068 // Load vector (32 bits)
16069 instruct loadV4(vecD dst, vmem4 mem)
16070 %{
16071 predicate(n->as_LoadVector()->memory_size() == 4);
16072 match(Set dst (LoadVector mem));
16073 ins_cost(4 * INSN_COST);
16074 format %{ "ldrs $dst,$mem\t# vector (32 bits)" %}
16075 ins_encode( aarch64_enc_ldrvS(dst, mem) );
16076 ins_pipe(vload_reg_mem64);
16077 %}
16078
16079 // Load vector (64 bits)
16080 instruct loadV8(vecD dst, vmem8 mem)
16081 %{
16082 predicate(n->as_LoadVector()->memory_size() == 8);
16083 match(Set dst (LoadVector mem));
16084 ins_cost(4 * INSN_COST);
16085 format %{ "ldrd $dst,$mem\t# vector (64 bits)" %}
16086 ins_encode( aarch64_enc_ldrvD(dst, mem) );
16087 ins_pipe(vload_reg_mem64);
16088 %}
16089
16090 // Load Vector (128 bits)
16091 instruct loadV16(vecX dst, vmem16 mem)
16092 %{
16093 predicate(n->as_LoadVector()->memory_size() == 16);
16094 match(Set dst (LoadVector mem));
16095 ins_cost(4 * INSN_COST);
16096 format %{ "ldrq $dst,$mem\t# vector (128 bits)" %}
16097 ins_encode( aarch64_enc_ldrvQ(dst, mem) );
16098 ins_pipe(vload_reg_mem128);
16099 %}
16100
16101 // Store Vector (32 bits)
16102 instruct storeV4(vecD src, vmem4 mem)
16103 %{
16104 predicate(n->as_StoreVector()->memory_size() == 4);
16105 match(Set mem (StoreVector mem src));
16106 ins_cost(4 * INSN_COST);
16107 format %{ "strs $mem,$src\t# vector (32 bits)" %}
16108 ins_encode( aarch64_enc_strvS(src, mem) );
16109 ins_pipe(vstore_reg_mem64);
16110 %}
16111
16112 // Store Vector (64 bits)
16113 instruct storeV8(vecD src, vmem8 mem)
16114 %{
16115 predicate(n->as_StoreVector()->memory_size() == 8);
16116 match(Set mem (StoreVector mem src));
16117 ins_cost(4 * INSN_COST);
16118 format %{ "strd $mem,$src\t# vector (64 bits)" %}
16119 ins_encode( aarch64_enc_strvD(src, mem) );
16120 ins_pipe(vstore_reg_mem64);
16121 %}
16122
16123 // Store Vector (128 bits)
16124 instruct storeV16(vecX src, vmem16 mem)
16125 %{
16126 predicate(n->as_StoreVector()->memory_size() == 16);
16127 match(Set mem (StoreVector mem src));
16128 ins_cost(4 * INSN_COST);
16129 format %{ "strq $mem,$src\t# vector (128 bits)" %}
16130 ins_encode( aarch64_enc_strvQ(src, mem) );
16131 ins_pipe(vstore_reg_mem128);
16132 %}
16133
16134 instruct replicate8B(vecD dst, iRegIorL2I src)
16135 %{
16136 predicate(n->as_Vector()->length() == 4 ||
16137 n->as_Vector()->length() == 8);
16138 match(Set dst (ReplicateB src));
16139 ins_cost(INSN_COST);
16140 format %{ "dup $dst, $src\t# vector (8B)" %}
16141 ins_encode %{
16142 __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($src$$reg));
16143 %}
16144 ins_pipe(vdup_reg_reg64);
16145 %}
16146
16147 instruct replicate16B(vecX dst, iRegIorL2I src)
16148 %{
16149 predicate(n->as_Vector()->length() == 16);
16150 match(Set dst (ReplicateB src));
16151 ins_cost(INSN_COST);
16152 format %{ "dup $dst, $src\t# vector (16B)" %}
16153 ins_encode %{
16154 __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($src$$reg));
16155 %}
16156 ins_pipe(vdup_reg_reg128);
16157 %}
16158
16159 instruct replicate8B_imm(vecD dst, immI con)
16160 %{
16161 predicate(n->as_Vector()->length() == 4 ||
16162 n->as_Vector()->length() == 8);
16163 match(Set dst (ReplicateB con));
16164 ins_cost(INSN_COST);
16165 format %{ "movi $dst, $con\t# vector(8B)" %}
16166 ins_encode %{
16167 __ mov(as_FloatRegister($dst$$reg), __ T8B, $con$$constant & 0xff);
16168 %}
16169 ins_pipe(vmovi_reg_imm64);
16170 %}
16171
16172 instruct replicate16B_imm(vecX dst, immI con)
16173 %{
16174 predicate(n->as_Vector()->length() == 16);
16175 match(Set dst (ReplicateB con));
16176 ins_cost(INSN_COST);
16177 format %{ "movi $dst, $con\t# vector(16B)" %}
16178 ins_encode %{
16179 __ mov(as_FloatRegister($dst$$reg), __ T16B, $con$$constant & 0xff);
16180 %}
16181 ins_pipe(vmovi_reg_imm128);
16182 %}
16183
16184 instruct replicate4S(vecD dst, iRegIorL2I src)
16185 %{
16186 predicate(n->as_Vector()->length() == 2 ||
16187 n->as_Vector()->length() == 4);
16188 match(Set dst (ReplicateS src));
16189 ins_cost(INSN_COST);
16190 format %{ "dup $dst, $src\t# vector (4S)" %}
16191 ins_encode %{
16192 __ dup(as_FloatRegister($dst$$reg), __ T4H, as_Register($src$$reg));
16193 %}
16194 ins_pipe(vdup_reg_reg64);
16195 %}
16196
16197 instruct replicate8S(vecX dst, iRegIorL2I src)
16198 %{
16199 predicate(n->as_Vector()->length() == 8);
16200 match(Set dst (ReplicateS src));
16201 ins_cost(INSN_COST);
16202 format %{ "dup $dst, $src\t# vector (8S)" %}
16203 ins_encode %{
16204 __ dup(as_FloatRegister($dst$$reg), __ T8H, as_Register($src$$reg));
16205 %}
16206 ins_pipe(vdup_reg_reg128);
16207 %}
16208
16209 instruct replicate4S_imm(vecD dst, immI con)
16210 %{
16211 predicate(n->as_Vector()->length() == 2 ||
16212 n->as_Vector()->length() == 4);
16213 match(Set dst (ReplicateS con));
16214 ins_cost(INSN_COST);
16215 format %{ "movi $dst, $con\t# vector(4H)" %}
16216 ins_encode %{
16217 __ mov(as_FloatRegister($dst$$reg), __ T4H, $con$$constant & 0xffff);
16218 %}
16219 ins_pipe(vmovi_reg_imm64);
16220 %}
16221
16222 instruct replicate8S_imm(vecX dst, immI con)
16223 %{
16224 predicate(n->as_Vector()->length() == 8);
16225 match(Set dst (ReplicateS con));
16226 ins_cost(INSN_COST);
16227 format %{ "movi $dst, $con\t# vector(8H)" %}
16228 ins_encode %{
16229 __ mov(as_FloatRegister($dst$$reg), __ T8H, $con$$constant & 0xffff);
16230 %}
16231 ins_pipe(vmovi_reg_imm128);
16232 %}
16233
16234 instruct replicate2I(vecD dst, iRegIorL2I src)
16235 %{
16236 predicate(n->as_Vector()->length() == 2);
16237 match(Set dst (ReplicateI src));
16238 ins_cost(INSN_COST);
16239 format %{ "dup $dst, $src\t# vector (2I)" %}
16240 ins_encode %{
16241 __ dup(as_FloatRegister($dst$$reg), __ T2S, as_Register($src$$reg));
16242 %}
16243 ins_pipe(vdup_reg_reg64);
16244 %}
16245
16246 instruct replicate4I(vecX dst, iRegIorL2I src)
16247 %{
16248 predicate(n->as_Vector()->length() == 4);
16249 match(Set dst (ReplicateI src));
16250 ins_cost(INSN_COST);
16251 format %{ "dup $dst, $src\t# vector (4I)" %}
16252 ins_encode %{
16253 __ dup(as_FloatRegister($dst$$reg), __ T4S, as_Register($src$$reg));
16254 %}
16255 ins_pipe(vdup_reg_reg128);
16256 %}
16257
16258 instruct replicate2I_imm(vecD dst, immI con)
16259 %{
16260 predicate(n->as_Vector()->length() == 2);
16261 match(Set dst (ReplicateI con));
16262 ins_cost(INSN_COST);
16263 format %{ "movi $dst, $con\t# vector(2I)" %}
16264 ins_encode %{
16265 __ mov(as_FloatRegister($dst$$reg), __ T2S, $con$$constant);
16266 %}
16267 ins_pipe(vmovi_reg_imm64);
16268 %}
16269
16270 instruct replicate4I_imm(vecX dst, immI con)
16271 %{
16272 predicate(n->as_Vector()->length() == 4);
16273 match(Set dst (ReplicateI con));
16274 ins_cost(INSN_COST);
16275 format %{ "movi $dst, $con\t# vector(4I)" %}
16276 ins_encode %{
16277 __ mov(as_FloatRegister($dst$$reg), __ T4S, $con$$constant);
16278 %}
16279 ins_pipe(vmovi_reg_imm128);
16280 %}
16281
16282 instruct replicate2L(vecX dst, iRegL src)
16283 %{
16284 predicate(n->as_Vector()->length() == 2);
16285 match(Set dst (ReplicateL src));
16286 ins_cost(INSN_COST);
16287 format %{ "dup $dst, $src\t# vector (2L)" %}
16288 ins_encode %{
16289 __ dup(as_FloatRegister($dst$$reg), __ T2D, as_Register($src$$reg));
16290 %}
16291 ins_pipe(vdup_reg_reg128);
16292 %}
16293
16294 instruct replicate2L_zero(vecX dst, immI0 zero)
16295 %{
16296 predicate(n->as_Vector()->length() == 2);
16297 match(Set dst (ReplicateI zero));
16298 ins_cost(INSN_COST);
16299 format %{ "movi $dst, $zero\t# vector(4I)" %}
16300 ins_encode %{
16301 __ eor(as_FloatRegister($dst$$reg), __ T16B,
16302 as_FloatRegister($dst$$reg),
16303 as_FloatRegister($dst$$reg));
16304 %}
16305 ins_pipe(vmovi_reg_imm128);
16306 %}
16307
16308 instruct replicate2F(vecD dst, vRegF src)
16309 %{
16310 predicate(n->as_Vector()->length() == 2);
16311 match(Set dst (ReplicateF src));
16312 ins_cost(INSN_COST);
16313 format %{ "dup $dst, $src\t# vector (2F)" %}
16314 ins_encode %{
16315 __ dup(as_FloatRegister($dst$$reg), __ T2S,
16316 as_FloatRegister($src$$reg));
16317 %}
16318 ins_pipe(vdup_reg_freg64);
16319 %}
16320
16321 instruct replicate4F(vecX dst, vRegF src)
16322 %{
16323 predicate(n->as_Vector()->length() == 4);
16324 match(Set dst (ReplicateF src));
16325 ins_cost(INSN_COST);
16326 format %{ "dup $dst, $src\t# vector (4F)" %}
16327 ins_encode %{
16328 __ dup(as_FloatRegister($dst$$reg), __ T4S,
16329 as_FloatRegister($src$$reg));
16330 %}
16331 ins_pipe(vdup_reg_freg128);
16332 %}
16333
16334 instruct replicate2D(vecX dst, vRegD src)
16335 %{
16336 predicate(n->as_Vector()->length() == 2);
16337 match(Set dst (ReplicateD src));
16338 ins_cost(INSN_COST);
16339 format %{ "dup $dst, $src\t# vector (2D)" %}
16340 ins_encode %{
16341 __ dup(as_FloatRegister($dst$$reg), __ T2D,
16342 as_FloatRegister($src$$reg));
16343 %}
16344 ins_pipe(vdup_reg_dreg128);
16345 %}
16346
16347 // ====================REDUCTION ARITHMETIC====================================
16348
16349 instruct reduce_add2I(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, iRegINoSp tmp, iRegINoSp tmp2)
16350 %{
16351 match(Set dst (AddReductionVI isrc vsrc));
16352 ins_cost(INSN_COST);
16353 effect(TEMP tmp, TEMP tmp2);
16354 format %{ "umov $tmp, $vsrc, S, 0\n\t"
16355 "umov $tmp2, $vsrc, S, 1\n\t"
16356 "addw $tmp, $isrc, $tmp\n\t"
16357 "addw $dst, $tmp, $tmp2\t# add reduction2I"
16358 %}
16359 ins_encode %{
16360 __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 0);
16361 __ umov($tmp2$$Register, as_FloatRegister($vsrc$$reg), __ S, 1);
16362 __ addw($tmp$$Register, $isrc$$Register, $tmp$$Register);
16363 __ addw($dst$$Register, $tmp$$Register, $tmp2$$Register);
16364 %}
16365 ins_pipe(pipe_class_default);
16366 %}
16367
16368 instruct reduce_add4I(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX vtmp, iRegINoSp itmp)
16369 %{
16370 match(Set dst (AddReductionVI isrc vsrc));
16371 ins_cost(INSN_COST);
16372 effect(TEMP vtmp, TEMP itmp);
16373 format %{ "addv $vtmp, T4S, $vsrc\n\t"
16374 "umov $itmp, $vtmp, S, 0\n\t"
16375 "addw $dst, $itmp, $isrc\t# add reduction4I"
16376 %}
16377 ins_encode %{
16378 __ addv(as_FloatRegister($vtmp$$reg), __ T4S,
16379 as_FloatRegister($vsrc$$reg));
16380 __ umov($itmp$$Register, as_FloatRegister($vtmp$$reg), __ S, 0);
16381 __ addw($dst$$Register, $itmp$$Register, $isrc$$Register);
16382 %}
16383 ins_pipe(pipe_class_default);
16384 %}
16385
16386 instruct reduce_mul2I(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, iRegINoSp tmp)
16387 %{
16388 match(Set dst (MulReductionVI isrc vsrc));
16389 ins_cost(INSN_COST);
16390 effect(TEMP tmp, TEMP dst);
16391 format %{ "umov $tmp, $vsrc, S, 0\n\t"
16392 "mul $dst, $tmp, $isrc\n\t"
16393 "umov $tmp, $vsrc, S, 1\n\t"
16394 "mul $dst, $tmp, $dst\t# mul reduction2I"
16395 %}
16396 ins_encode %{
16397 __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 0);
16398 __ mul($dst$$Register, $tmp$$Register, $isrc$$Register);
16399 __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 1);
16400 __ mul($dst$$Register, $tmp$$Register, $dst$$Register);
16401 %}
16402 ins_pipe(pipe_class_default);
16403 %}
16404
16405 instruct reduce_mul4I(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX vtmp, iRegINoSp itmp)
16406 %{
16407 match(Set dst (MulReductionVI isrc vsrc));
16408 ins_cost(INSN_COST);
16409 effect(TEMP vtmp, TEMP itmp, TEMP dst);
16410 format %{ "ins $vtmp, D, $vsrc, 0, 1\n\t"
16411 "mulv $vtmp, T2S, $vtmp, $vsrc\n\t"
16412 "umov $itmp, $vtmp, S, 0\n\t"
16413 "mul $dst, $itmp, $isrc\n\t"
16414 "umov $itmp, $vtmp, S, 1\n\t"
16415 "mul $dst, $itmp, $dst\t# mul reduction4I"
16416 %}
16417 ins_encode %{
16418 __ ins(as_FloatRegister($vtmp$$reg), __ D,
16419 as_FloatRegister($vsrc$$reg), 0, 1);
16420 __ mulv(as_FloatRegister($vtmp$$reg), __ T2S,
16421 as_FloatRegister($vtmp$$reg), as_FloatRegister($vsrc$$reg));
16422 __ umov($itmp$$Register, as_FloatRegister($vtmp$$reg), __ S, 0);
16423 __ mul($dst$$Register, $itmp$$Register, $isrc$$Register);
16424 __ umov($itmp$$Register, as_FloatRegister($vtmp$$reg), __ S, 1);
16425 __ mul($dst$$Register, $itmp$$Register, $dst$$Register);
16426 %}
16427 ins_pipe(pipe_class_default);
16428 %}
16429
16430 instruct reduce_add2F(vRegF dst, vRegF fsrc, vecD vsrc, vecD tmp)
16431 %{
16432 match(Set dst (AddReductionVF fsrc vsrc));
16433 ins_cost(INSN_COST);
16434 effect(TEMP tmp, TEMP dst);
16435 format %{ "fadds $dst, $fsrc, $vsrc\n\t"
16436 "ins $tmp, S, $vsrc, 0, 1\n\t"
16437 "fadds $dst, $dst, $tmp\t# add reduction2F"
16438 %}
16439 ins_encode %{
16440 __ fadds(as_FloatRegister($dst$$reg),
16441 as_FloatRegister($fsrc$$reg), as_FloatRegister($vsrc$$reg));
16442 __ ins(as_FloatRegister($tmp$$reg), __ S,
16443 as_FloatRegister($vsrc$$reg), 0, 1);
16444 __ fadds(as_FloatRegister($dst$$reg),
16445 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16446 %}
16447 ins_pipe(pipe_class_default);
16448 %}
16449
16450 instruct reduce_add4F(vRegF dst, vRegF fsrc, vecX vsrc, vecX tmp)
16451 %{
16452 match(Set dst (AddReductionVF fsrc vsrc));
16453 ins_cost(INSN_COST);
16454 effect(TEMP tmp, TEMP dst);
16455 format %{ "fadds $dst, $fsrc, $vsrc\n\t"
16456 "ins $tmp, S, $vsrc, 0, 1\n\t"
16457 "fadds $dst, $dst, $tmp\n\t"
16458 "ins $tmp, S, $vsrc, 0, 2\n\t"
16459 "fadds $dst, $dst, $tmp\n\t"
16460 "ins $tmp, S, $vsrc, 0, 3\n\t"
16461 "fadds $dst, $dst, $tmp\t# add reduction4F"
16462 %}
16463 ins_encode %{
16464 __ fadds(as_FloatRegister($dst$$reg),
16465 as_FloatRegister($fsrc$$reg), as_FloatRegister($vsrc$$reg));
16466 __ ins(as_FloatRegister($tmp$$reg), __ S,
16467 as_FloatRegister($vsrc$$reg), 0, 1);
16468 __ fadds(as_FloatRegister($dst$$reg),
16469 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16470 __ ins(as_FloatRegister($tmp$$reg), __ S,
16471 as_FloatRegister($vsrc$$reg), 0, 2);
16472 __ fadds(as_FloatRegister($dst$$reg),
16473 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16474 __ ins(as_FloatRegister($tmp$$reg), __ S,
16475 as_FloatRegister($vsrc$$reg), 0, 3);
16476 __ fadds(as_FloatRegister($dst$$reg),
16477 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16478 %}
16479 ins_pipe(pipe_class_default);
16480 %}
16481
16482 instruct reduce_mul2F(vRegF dst, vRegF fsrc, vecD vsrc, vecD tmp)
16483 %{
16484 match(Set dst (MulReductionVF fsrc vsrc));
16485 ins_cost(INSN_COST);
16486 effect(TEMP tmp, TEMP dst);
16487 format %{ "fmuls $dst, $fsrc, $vsrc\n\t"
16488 "ins $tmp, S, $vsrc, 0, 1\n\t"
16489 "fmuls $dst, $dst, $tmp\t# mul reduction2F"
16490 %}
16491 ins_encode %{
16492 __ fmuls(as_FloatRegister($dst$$reg),
16493 as_FloatRegister($fsrc$$reg), as_FloatRegister($vsrc$$reg));
16494 __ ins(as_FloatRegister($tmp$$reg), __ S,
16495 as_FloatRegister($vsrc$$reg), 0, 1);
16496 __ fmuls(as_FloatRegister($dst$$reg),
16497 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16498 %}
16499 ins_pipe(pipe_class_default);
16500 %}
16501
16502 instruct reduce_mul4F(vRegF dst, vRegF fsrc, vecX vsrc, vecX tmp)
16503 %{
16504 match(Set dst (MulReductionVF fsrc vsrc));
16505 ins_cost(INSN_COST);
16506 effect(TEMP tmp, TEMP dst);
16507 format %{ "fmuls $dst, $fsrc, $vsrc\n\t"
16508 "ins $tmp, S, $vsrc, 0, 1\n\t"
16509 "fmuls $dst, $dst, $tmp\n\t"
16510 "ins $tmp, S, $vsrc, 0, 2\n\t"
16511 "fmuls $dst, $dst, $tmp\n\t"
16512 "ins $tmp, S, $vsrc, 0, 3\n\t"
16513 "fmuls $dst, $dst, $tmp\t# mul reduction4F"
16514 %}
16515 ins_encode %{
16516 __ fmuls(as_FloatRegister($dst$$reg),
16517 as_FloatRegister($fsrc$$reg), as_FloatRegister($vsrc$$reg));
16518 __ ins(as_FloatRegister($tmp$$reg), __ S,
16519 as_FloatRegister($vsrc$$reg), 0, 1);
16520 __ fmuls(as_FloatRegister($dst$$reg),
16521 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16522 __ ins(as_FloatRegister($tmp$$reg), __ S,
16523 as_FloatRegister($vsrc$$reg), 0, 2);
16524 __ fmuls(as_FloatRegister($dst$$reg),
16525 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16526 __ ins(as_FloatRegister($tmp$$reg), __ S,
16527 as_FloatRegister($vsrc$$reg), 0, 3);
16528 __ fmuls(as_FloatRegister($dst$$reg),
16529 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16530 %}
16531 ins_pipe(pipe_class_default);
16532 %}
16533
16534 instruct reduce_add2D(vRegD dst, vRegD dsrc, vecX vsrc, vecX tmp)
16535 %{
16536 match(Set dst (AddReductionVD dsrc vsrc));
16537 ins_cost(INSN_COST);
16538 effect(TEMP tmp, TEMP dst);
16539 format %{ "faddd $dst, $dsrc, $vsrc\n\t"
16540 "ins $tmp, D, $vsrc, 0, 1\n\t"
16541 "faddd $dst, $dst, $tmp\t# add reduction2D"
16542 %}
16543 ins_encode %{
16544 __ faddd(as_FloatRegister($dst$$reg),
16545 as_FloatRegister($dsrc$$reg), as_FloatRegister($vsrc$$reg));
16546 __ ins(as_FloatRegister($tmp$$reg), __ D,
16547 as_FloatRegister($vsrc$$reg), 0, 1);
16548 __ faddd(as_FloatRegister($dst$$reg),
16549 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16550 %}
16551 ins_pipe(pipe_class_default);
16552 %}
16553
16554 instruct reduce_mul2D(vRegD dst, vRegD dsrc, vecX vsrc, vecX tmp)
16555 %{
16556 match(Set dst (MulReductionVD dsrc vsrc));
16557 ins_cost(INSN_COST);
16558 effect(TEMP tmp, TEMP dst);
16559 format %{ "fmuld $dst, $dsrc, $vsrc\n\t"
16560 "ins $tmp, D, $vsrc, 0, 1\n\t"
16561 "fmuld $dst, $dst, $tmp\t# mul reduction2D"
16562 %}
16563 ins_encode %{
16564 __ fmuld(as_FloatRegister($dst$$reg),
16565 as_FloatRegister($dsrc$$reg), as_FloatRegister($vsrc$$reg));
16566 __ ins(as_FloatRegister($tmp$$reg), __ D,
16567 as_FloatRegister($vsrc$$reg), 0, 1);
16568 __ fmuld(as_FloatRegister($dst$$reg),
16569 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16570 %}
16571 ins_pipe(pipe_class_default);
16572 %}
16573
16574 instruct reduce_max2F(vRegF dst, vRegF fsrc, vecD vsrc, vecD tmp) %{
16575 predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
16576 match(Set dst (MaxReductionV fsrc vsrc));
16577 ins_cost(INSN_COST);
16578 effect(TEMP_DEF dst, TEMP tmp);
16579 format %{ "fmaxs $dst, $fsrc, $vsrc\n\t"
16580 "ins $tmp, S, $vsrc, 0, 1\n\t"
16581 "fmaxs $dst, $dst, $tmp\t# max reduction2F" %}
16582 ins_encode %{
16583 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($fsrc$$reg), as_FloatRegister($vsrc$$reg));
16584 __ ins(as_FloatRegister($tmp$$reg), __ S, as_FloatRegister($vsrc$$reg), 0, 1);
16585 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16586 %}
16587 ins_pipe(pipe_class_default);
16588 %}
16589
16590 instruct reduce_max4F(vRegF dst, vRegF fsrc, vecX vsrc) %{
16591 predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
16592 match(Set dst (MaxReductionV fsrc vsrc));
16593 ins_cost(INSN_COST);
16594 effect(TEMP_DEF dst);
16595 format %{ "fmaxv $dst, T4S, $vsrc\n\t"
16596 "fmaxs $dst, $dst, $fsrc\t# max reduction4F" %}
16597 ins_encode %{
16598 __ fmaxv(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($vsrc$$reg));
16599 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($fsrc$$reg));
16600 %}
16601 ins_pipe(pipe_class_default);
16602 %}
16603
16604 instruct reduce_max2D(vRegD dst, vRegD dsrc, vecX vsrc, vecX tmp) %{
16605 predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
16606 match(Set dst (MaxReductionV dsrc vsrc));
16607 ins_cost(INSN_COST);
16608 effect(TEMP_DEF dst, TEMP tmp);
16609 format %{ "fmaxd $dst, $dsrc, $vsrc\n\t"
16610 "ins $tmp, D, $vsrc, 0, 1\n\t"
16611 "fmaxd $dst, $dst, $tmp\t# max reduction2D" %}
16612 ins_encode %{
16613 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dsrc$$reg), as_FloatRegister($vsrc$$reg));
16614 __ ins(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($vsrc$$reg), 0, 1);
16615 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16616 %}
16617 ins_pipe(pipe_class_default);
16618 %}
16619
16620 instruct reduce_min2F(vRegF dst, vRegF fsrc, vecD vsrc, vecD tmp) %{
16621 predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
16622 match(Set dst (MinReductionV fsrc vsrc));
16623 ins_cost(INSN_COST);
16624 effect(TEMP_DEF dst, TEMP tmp);
16625 format %{ "fmins $dst, $fsrc, $vsrc\n\t"
16626 "ins $tmp, S, $vsrc, 0, 1\n\t"
16627 "fmins $dst, $dst, $tmp\t# min reduction2F" %}
16628 ins_encode %{
16629 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($fsrc$$reg), as_FloatRegister($vsrc$$reg));
16630 __ ins(as_FloatRegister($tmp$$reg), __ S, as_FloatRegister($vsrc$$reg), 0, 1);
16631 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16632 %}
16633 ins_pipe(pipe_class_default);
16634 %}
16635
16636 instruct reduce_min4F(vRegF dst, vRegF fsrc, vecX vsrc) %{
16637 predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
16638 match(Set dst (MinReductionV fsrc vsrc));
16639 ins_cost(INSN_COST);
16640 effect(TEMP_DEF dst);
16641 format %{ "fminv $dst, T4S, $vsrc\n\t"
16642 "fmins $dst, $dst, $fsrc\t# min reduction4F" %}
16643 ins_encode %{
16644 __ fminv(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($vsrc$$reg));
16645 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($fsrc$$reg));
16646 %}
16647 ins_pipe(pipe_class_default);
16648 %}
16649
16650 instruct reduce_min2D(vRegD dst, vRegD dsrc, vecX vsrc, vecX tmp) %{
16651 predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
16652 match(Set dst (MinReductionV dsrc vsrc));
16653 ins_cost(INSN_COST);
16654 effect(TEMP_DEF dst, TEMP tmp);
16655 format %{ "fmind $dst, $dsrc, $vsrc\n\t"
16656 "ins $tmp, D, $vsrc, 0, 1\n\t"
16657 "fmind $dst, $dst, $tmp\t# min reduction2D" %}
16658 ins_encode %{
16659 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dsrc$$reg), as_FloatRegister($vsrc$$reg));
16660 __ ins(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($vsrc$$reg), 0, 1);
16661 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
16662 %}
16663 ins_pipe(pipe_class_default);
16664 %}
16665
16666 // ====================VECTOR ARITHMETIC=======================================
16667
16668 // --------------------------------- ADD --------------------------------------
16669
16670 instruct vadd8B(vecD dst, vecD src1, vecD src2)
16671 %{
16672 predicate(n->as_Vector()->length() == 4 ||
16673 n->as_Vector()->length() == 8);
16674 match(Set dst (AddVB src1 src2));
16675 ins_cost(INSN_COST);
16676 format %{ "addv $dst,$src1,$src2\t# vector (8B)" %}
16677 ins_encode %{
16678 __ addv(as_FloatRegister($dst$$reg), __ T8B,
16679 as_FloatRegister($src1$$reg),
16680 as_FloatRegister($src2$$reg));
16681 %}
16682 ins_pipe(vdop64);
16683 %}
16684
16685 instruct vadd16B(vecX dst, vecX src1, vecX src2)
16686 %{
16687 predicate(n->as_Vector()->length() == 16);
16688 match(Set dst (AddVB src1 src2));
16689 ins_cost(INSN_COST);
16690 format %{ "addv $dst,$src1,$src2\t# vector (16B)" %}
16691 ins_encode %{
16692 __ addv(as_FloatRegister($dst$$reg), __ T16B,
16693 as_FloatRegister($src1$$reg),
16694 as_FloatRegister($src2$$reg));
16695 %}
16696 ins_pipe(vdop128);
16697 %}
16698
16699 instruct vadd4S(vecD dst, vecD src1, vecD src2)
16700 %{
16701 predicate(n->as_Vector()->length() == 2 ||
16702 n->as_Vector()->length() == 4);
16703 match(Set dst (AddVS src1 src2));
16704 ins_cost(INSN_COST);
16705 format %{ "addv $dst,$src1,$src2\t# vector (4H)" %}
16706 ins_encode %{
16707 __ addv(as_FloatRegister($dst$$reg), __ T4H,
16708 as_FloatRegister($src1$$reg),
16709 as_FloatRegister($src2$$reg));
16710 %}
16711 ins_pipe(vdop64);
16712 %}
16713
16714 instruct vadd8S(vecX dst, vecX src1, vecX src2)
16715 %{
16716 predicate(n->as_Vector()->length() == 8);
16717 match(Set dst (AddVS src1 src2));
16718 ins_cost(INSN_COST);
16719 format %{ "addv $dst,$src1,$src2\t# vector (8H)" %}
16720 ins_encode %{
16721 __ addv(as_FloatRegister($dst$$reg), __ T8H,
16722 as_FloatRegister($src1$$reg),
16723 as_FloatRegister($src2$$reg));
16724 %}
16725 ins_pipe(vdop128);
16726 %}
16727
16728 instruct vadd2I(vecD dst, vecD src1, vecD src2)
16729 %{
16730 predicate(n->as_Vector()->length() == 2);
16731 match(Set dst (AddVI src1 src2));
16732 ins_cost(INSN_COST);
16733 format %{ "addv $dst,$src1,$src2\t# vector (2S)" %}
16734 ins_encode %{
16735 __ addv(as_FloatRegister($dst$$reg), __ T2S,
16736 as_FloatRegister($src1$$reg),
16737 as_FloatRegister($src2$$reg));
16738 %}
16739 ins_pipe(vdop64);
16740 %}
16741
16742 instruct vadd4I(vecX dst, vecX src1, vecX src2)
16743 %{
16744 predicate(n->as_Vector()->length() == 4);
16745 match(Set dst (AddVI src1 src2));
16746 ins_cost(INSN_COST);
16747 format %{ "addv $dst,$src1,$src2\t# vector (4S)" %}
16748 ins_encode %{
16749 __ addv(as_FloatRegister($dst$$reg), __ T4S,
16750 as_FloatRegister($src1$$reg),
16751 as_FloatRegister($src2$$reg));
16752 %}
16753 ins_pipe(vdop128);
16754 %}
16755
16756 instruct vadd2L(vecX dst, vecX src1, vecX src2)
16757 %{
16758 predicate(n->as_Vector()->length() == 2);
16759 match(Set dst (AddVL src1 src2));
16760 ins_cost(INSN_COST);
16761 format %{ "addv $dst,$src1,$src2\t# vector (2L)" %}
16762 ins_encode %{
16763 __ addv(as_FloatRegister($dst$$reg), __ T2D,
16764 as_FloatRegister($src1$$reg),
16765 as_FloatRegister($src2$$reg));
16766 %}
16767 ins_pipe(vdop128);
16768 %}
16769
16770 instruct vadd2F(vecD dst, vecD src1, vecD src2)
16771 %{
16772 predicate(n->as_Vector()->length() == 2);
16773 match(Set dst (AddVF src1 src2));
16774 ins_cost(INSN_COST);
16775 format %{ "fadd $dst,$src1,$src2\t# vector (2S)" %}
16776 ins_encode %{
16777 __ fadd(as_FloatRegister($dst$$reg), __ T2S,
16778 as_FloatRegister($src1$$reg),
16779 as_FloatRegister($src2$$reg));
16780 %}
16781 ins_pipe(vdop_fp64);
16782 %}
16783
16784 instruct vadd4F(vecX dst, vecX src1, vecX src2)
16785 %{
16786 predicate(n->as_Vector()->length() == 4);
16787 match(Set dst (AddVF src1 src2));
16788 ins_cost(INSN_COST);
16789 format %{ "fadd $dst,$src1,$src2\t# vector (4S)" %}
16790 ins_encode %{
16791 __ fadd(as_FloatRegister($dst$$reg), __ T4S,
16792 as_FloatRegister($src1$$reg),
16793 as_FloatRegister($src2$$reg));
16794 %}
16795 ins_pipe(vdop_fp128);
16796 %}
16797
16798 instruct vadd2D(vecX dst, vecX src1, vecX src2)
16799 %{
16800 match(Set dst (AddVD src1 src2));
16801 ins_cost(INSN_COST);
16802 format %{ "fadd $dst,$src1,$src2\t# vector (2D)" %}
16803 ins_encode %{
16804 __ fadd(as_FloatRegister($dst$$reg), __ T2D,
16805 as_FloatRegister($src1$$reg),
16806 as_FloatRegister($src2$$reg));
16807 %}
16808 ins_pipe(vdop_fp128);
16809 %}
16810
16811 // --------------------------------- SUB --------------------------------------
16812
16813 instruct vsub8B(vecD dst, vecD src1, vecD src2)
16814 %{
16815 predicate(n->as_Vector()->length() == 4 ||
16816 n->as_Vector()->length() == 8);
16817 match(Set dst (SubVB src1 src2));
16818 ins_cost(INSN_COST);
16819 format %{ "subv $dst,$src1,$src2\t# vector (8B)" %}
16820 ins_encode %{
16821 __ subv(as_FloatRegister($dst$$reg), __ T8B,
16822 as_FloatRegister($src1$$reg),
16823 as_FloatRegister($src2$$reg));
16824 %}
16825 ins_pipe(vdop64);
16826 %}
16827
16828 instruct vsub16B(vecX dst, vecX src1, vecX src2)
16829 %{
16830 predicate(n->as_Vector()->length() == 16);
16831 match(Set dst (SubVB src1 src2));
16832 ins_cost(INSN_COST);
16833 format %{ "subv $dst,$src1,$src2\t# vector (16B)" %}
16834 ins_encode %{
16835 __ subv(as_FloatRegister($dst$$reg), __ T16B,
16836 as_FloatRegister($src1$$reg),
16837 as_FloatRegister($src2$$reg));
16838 %}
16839 ins_pipe(vdop128);
16840 %}
16841
16842 instruct vsub4S(vecD dst, vecD src1, vecD src2)
16843 %{
16844 predicate(n->as_Vector()->length() == 2 ||
16845 n->as_Vector()->length() == 4);
16846 match(Set dst (SubVS src1 src2));
16847 ins_cost(INSN_COST);
16848 format %{ "subv $dst,$src1,$src2\t# vector (4H)" %}
16849 ins_encode %{
16850 __ subv(as_FloatRegister($dst$$reg), __ T4H,
16851 as_FloatRegister($src1$$reg),
16852 as_FloatRegister($src2$$reg));
16853 %}
16854 ins_pipe(vdop64);
16855 %}
16856
16857 instruct vsub8S(vecX dst, vecX src1, vecX src2)
16858 %{
16859 predicate(n->as_Vector()->length() == 8);
16860 match(Set dst (SubVS src1 src2));
16861 ins_cost(INSN_COST);
16862 format %{ "subv $dst,$src1,$src2\t# vector (8H)" %}
16863 ins_encode %{
16864 __ subv(as_FloatRegister($dst$$reg), __ T8H,
16865 as_FloatRegister($src1$$reg),
16866 as_FloatRegister($src2$$reg));
16867 %}
16868 ins_pipe(vdop128);
16869 %}
16870
16871 instruct vsub2I(vecD dst, vecD src1, vecD src2)
16872 %{
16873 predicate(n->as_Vector()->length() == 2);
16874 match(Set dst (SubVI src1 src2));
16875 ins_cost(INSN_COST);
16876 format %{ "subv $dst,$src1,$src2\t# vector (2S)" %}
16877 ins_encode %{
16878 __ subv(as_FloatRegister($dst$$reg), __ T2S,
16879 as_FloatRegister($src1$$reg),
16880 as_FloatRegister($src2$$reg));
16881 %}
16882 ins_pipe(vdop64);
16883 %}
16884
16885 instruct vsub4I(vecX dst, vecX src1, vecX src2)
16886 %{
16887 predicate(n->as_Vector()->length() == 4);
16888 match(Set dst (SubVI src1 src2));
16889 ins_cost(INSN_COST);
16890 format %{ "subv $dst,$src1,$src2\t# vector (4S)" %}
16891 ins_encode %{
16892 __ subv(as_FloatRegister($dst$$reg), __ T4S,
16893 as_FloatRegister($src1$$reg),
16894 as_FloatRegister($src2$$reg));
16895 %}
16896 ins_pipe(vdop128);
16897 %}
16898
16899 instruct vsub2L(vecX dst, vecX src1, vecX src2)
16900 %{
16901 predicate(n->as_Vector()->length() == 2);
16902 match(Set dst (SubVL src1 src2));
16903 ins_cost(INSN_COST);
16904 format %{ "subv $dst,$src1,$src2\t# vector (2L)" %}
16905 ins_encode %{
16906 __ subv(as_FloatRegister($dst$$reg), __ T2D,
16907 as_FloatRegister($src1$$reg),
16908 as_FloatRegister($src2$$reg));
16909 %}
16910 ins_pipe(vdop128);
16911 %}
16912
16913 instruct vsub2F(vecD dst, vecD src1, vecD src2)
16914 %{
16915 predicate(n->as_Vector()->length() == 2);
16916 match(Set dst (SubVF src1 src2));
16917 ins_cost(INSN_COST);
16918 format %{ "fsub $dst,$src1,$src2\t# vector (2S)" %}
16919 ins_encode %{
16920 __ fsub(as_FloatRegister($dst$$reg), __ T2S,
16921 as_FloatRegister($src1$$reg),
16922 as_FloatRegister($src2$$reg));
16923 %}
16924 ins_pipe(vdop_fp64);
16925 %}
16926
16927 instruct vsub4F(vecX dst, vecX src1, vecX src2)
16928 %{
16929 predicate(n->as_Vector()->length() == 4);
16930 match(Set dst (SubVF src1 src2));
16931 ins_cost(INSN_COST);
16932 format %{ "fsub $dst,$src1,$src2\t# vector (4S)" %}
16933 ins_encode %{
16934 __ fsub(as_FloatRegister($dst$$reg), __ T4S,
16935 as_FloatRegister($src1$$reg),
16936 as_FloatRegister($src2$$reg));
16937 %}
16938 ins_pipe(vdop_fp128);
16939 %}
16940
16941 instruct vsub2D(vecX dst, vecX src1, vecX src2)
16942 %{
16943 predicate(n->as_Vector()->length() == 2);
16944 match(Set dst (SubVD src1 src2));
16945 ins_cost(INSN_COST);
16946 format %{ "fsub $dst,$src1,$src2\t# vector (2D)" %}
16947 ins_encode %{
16948 __ fsub(as_FloatRegister($dst$$reg), __ T2D,
16949 as_FloatRegister($src1$$reg),
16950 as_FloatRegister($src2$$reg));
16951 %}
16952 ins_pipe(vdop_fp128);
16953 %}
16954
16955 // --------------------------------- MUL --------------------------------------
16956
16957 instruct vmul8B(vecD dst, vecD src1, vecD src2)
16958 %{
16959 predicate(n->as_Vector()->length() == 4 ||
16960 n->as_Vector()->length() == 8);
16961 match(Set dst (MulVB src1 src2));
16962 ins_cost(INSN_COST);
16963 format %{ "mulv $dst,$src1,$src2\t# vector (8B)" %}
16964 ins_encode %{
16965 __ mulv(as_FloatRegister($dst$$reg), __ T8B,
16966 as_FloatRegister($src1$$reg),
16967 as_FloatRegister($src2$$reg));
16968 %}
16969 ins_pipe(vmul64);
16970 %}
16971
16972 instruct vmul16B(vecX dst, vecX src1, vecX src2)
16973 %{
16974 predicate(n->as_Vector()->length() == 16);
16975 match(Set dst (MulVB src1 src2));
16976 ins_cost(INSN_COST);
16977 format %{ "mulv $dst,$src1,$src2\t# vector (16B)" %}
16978 ins_encode %{
16979 __ mulv(as_FloatRegister($dst$$reg), __ T16B,
16980 as_FloatRegister($src1$$reg),
16981 as_FloatRegister($src2$$reg));
16982 %}
16983 ins_pipe(vmul128);
16984 %}
16985
16986 instruct vmul4S(vecD dst, vecD src1, vecD src2)
16987 %{
16988 predicate(n->as_Vector()->length() == 2 ||
16989 n->as_Vector()->length() == 4);
16990 match(Set dst (MulVS src1 src2));
16991 ins_cost(INSN_COST);
16992 format %{ "mulv $dst,$src1,$src2\t# vector (4H)" %}
16993 ins_encode %{
16994 __ mulv(as_FloatRegister($dst$$reg), __ T4H,
16995 as_FloatRegister($src1$$reg),
16996 as_FloatRegister($src2$$reg));
16997 %}
16998 ins_pipe(vmul64);
16999 %}
17000
17001 instruct vmul8S(vecX dst, vecX src1, vecX src2)
17002 %{
17003 predicate(n->as_Vector()->length() == 8);
17004 match(Set dst (MulVS src1 src2));
17005 ins_cost(INSN_COST);
17006 format %{ "mulv $dst,$src1,$src2\t# vector (8H)" %}
17007 ins_encode %{
17008 __ mulv(as_FloatRegister($dst$$reg), __ T8H,
17009 as_FloatRegister($src1$$reg),
17010 as_FloatRegister($src2$$reg));
17011 %}
17012 ins_pipe(vmul128);
17013 %}
17014
17015 instruct vmul2I(vecD dst, vecD src1, vecD src2)
17016 %{
17017 predicate(n->as_Vector()->length() == 2);
17018 match(Set dst (MulVI src1 src2));
17019 ins_cost(INSN_COST);
17020 format %{ "mulv $dst,$src1,$src2\t# vector (2S)" %}
17021 ins_encode %{
17022 __ mulv(as_FloatRegister($dst$$reg), __ T2S,
17023 as_FloatRegister($src1$$reg),
17024 as_FloatRegister($src2$$reg));
17025 %}
17026 ins_pipe(vmul64);
17027 %}
17028
17029 instruct vmul4I(vecX dst, vecX src1, vecX src2)
17030 %{
17031 predicate(n->as_Vector()->length() == 4);
17032 match(Set dst (MulVI src1 src2));
17033 ins_cost(INSN_COST);
17034 format %{ "mulv $dst,$src1,$src2\t# vector (4S)" %}
17035 ins_encode %{
17036 __ mulv(as_FloatRegister($dst$$reg), __ T4S,
17037 as_FloatRegister($src1$$reg),
17038 as_FloatRegister($src2$$reg));
17039 %}
17040 ins_pipe(vmul128);
17041 %}
17042
17043 instruct vmul2F(vecD dst, vecD src1, vecD src2)
17044 %{
17045 predicate(n->as_Vector()->length() == 2);
17046 match(Set dst (MulVF src1 src2));
17047 ins_cost(INSN_COST);
17048 format %{ "fmul $dst,$src1,$src2\t# vector (2S)" %}
17049 ins_encode %{
17050 __ fmul(as_FloatRegister($dst$$reg), __ T2S,
17051 as_FloatRegister($src1$$reg),
17052 as_FloatRegister($src2$$reg));
17053 %}
17054 ins_pipe(vmuldiv_fp64);
17055 %}
17056
17057 instruct vmul4F(vecX dst, vecX src1, vecX src2)
17058 %{
17059 predicate(n->as_Vector()->length() == 4);
17060 match(Set dst (MulVF src1 src2));
17061 ins_cost(INSN_COST);
17062 format %{ "fmul $dst,$src1,$src2\t# vector (4S)" %}
17063 ins_encode %{
17064 __ fmul(as_FloatRegister($dst$$reg), __ T4S,
17065 as_FloatRegister($src1$$reg),
17066 as_FloatRegister($src2$$reg));
17067 %}
17068 ins_pipe(vmuldiv_fp128);
17069 %}
17070
17071 instruct vmul2D(vecX dst, vecX src1, vecX src2)
17072 %{
17073 predicate(n->as_Vector()->length() == 2);
17074 match(Set dst (MulVD src1 src2));
17075 ins_cost(INSN_COST);
17076 format %{ "fmul $dst,$src1,$src2\t# vector (2D)" %}
17077 ins_encode %{
17078 __ fmul(as_FloatRegister($dst$$reg), __ T2D,
17079 as_FloatRegister($src1$$reg),
17080 as_FloatRegister($src2$$reg));
17081 %}
17082 ins_pipe(vmuldiv_fp128);
17083 %}
17084
17085 // --------------------------------- MLA --------------------------------------
17086
17087 instruct vmla4S(vecD dst, vecD src1, vecD src2)
17088 %{
17089 predicate(n->as_Vector()->length() == 2 ||
17090 n->as_Vector()->length() == 4);
17091 match(Set dst (AddVS dst (MulVS src1 src2)));
17092 ins_cost(INSN_COST);
17093 format %{ "mlav $dst,$src1,$src2\t# vector (4H)" %}
17094 ins_encode %{
17095 __ mlav(as_FloatRegister($dst$$reg), __ T4H,
17096 as_FloatRegister($src1$$reg),
17097 as_FloatRegister($src2$$reg));
17098 %}
17099 ins_pipe(vmla64);
17100 %}
17101
17102 instruct vmla8S(vecX dst, vecX src1, vecX src2)
17103 %{
17104 predicate(n->as_Vector()->length() == 8);
17105 match(Set dst (AddVS dst (MulVS src1 src2)));
17106 ins_cost(INSN_COST);
17107 format %{ "mlav $dst,$src1,$src2\t# vector (8H)" %}
17108 ins_encode %{
17109 __ mlav(as_FloatRegister($dst$$reg), __ T8H,
17110 as_FloatRegister($src1$$reg),
17111 as_FloatRegister($src2$$reg));
17112 %}
17113 ins_pipe(vmla128);
17114 %}
17115
17116 instruct vmla2I(vecD dst, vecD src1, vecD src2)
17117 %{
17118 predicate(n->as_Vector()->length() == 2);
17119 match(Set dst (AddVI dst (MulVI src1 src2)));
17120 ins_cost(INSN_COST);
17121 format %{ "mlav $dst,$src1,$src2\t# vector (2S)" %}
17122 ins_encode %{
17123 __ mlav(as_FloatRegister($dst$$reg), __ T2S,
17124 as_FloatRegister($src1$$reg),
17125 as_FloatRegister($src2$$reg));
17126 %}
17127 ins_pipe(vmla64);
17128 %}
17129
17130 instruct vmla4I(vecX dst, vecX src1, vecX src2)
17131 %{
17132 predicate(n->as_Vector()->length() == 4);
17133 match(Set dst (AddVI dst (MulVI src1 src2)));
17134 ins_cost(INSN_COST);
17135 format %{ "mlav $dst,$src1,$src2\t# vector (4S)" %}
17136 ins_encode %{
17137 __ mlav(as_FloatRegister($dst$$reg), __ T4S,
17138 as_FloatRegister($src1$$reg),
17139 as_FloatRegister($src2$$reg));
17140 %}
17141 ins_pipe(vmla128);
17142 %}
17143
17144 // dst + src1 * src2
17145 instruct vmla2F(vecD dst, vecD src1, vecD src2) %{
17146 predicate(UseFMA && n->as_Vector()->length() == 2);
17147 match(Set dst (FmaVF dst (Binary src1 src2)));
17148 format %{ "fmla $dst,$src1,$src2\t# vector (2S)" %}
17149 ins_cost(INSN_COST);
17150 ins_encode %{
17151 __ fmla(as_FloatRegister($dst$$reg), __ T2S,
17152 as_FloatRegister($src1$$reg),
17153 as_FloatRegister($src2$$reg));
17154 %}
17155 ins_pipe(vmuldiv_fp64);
17156 %}
17157
17158 // dst + src1 * src2
17159 instruct vmla4F(vecX dst, vecX src1, vecX src2) %{
17160 predicate(UseFMA && n->as_Vector()->length() == 4);
17161 match(Set dst (FmaVF dst (Binary src1 src2)));
17162 format %{ "fmla $dst,$src1,$src2\t# vector (4S)" %}
17163 ins_cost(INSN_COST);
17164 ins_encode %{
17165 __ fmla(as_FloatRegister($dst$$reg), __ T4S,
17166 as_FloatRegister($src1$$reg),
17167 as_FloatRegister($src2$$reg));
17168 %}
17169 ins_pipe(vmuldiv_fp128);
17170 %}
17171
17172 // dst + src1 * src2
17173 instruct vmla2D(vecX dst, vecX src1, vecX src2) %{
17174 predicate(UseFMA && n->as_Vector()->length() == 2);
17175 match(Set dst (FmaVD dst (Binary src1 src2)));
17176 format %{ "fmla $dst,$src1,$src2\t# vector (2D)" %}
17177 ins_cost(INSN_COST);
17178 ins_encode %{
17179 __ fmla(as_FloatRegister($dst$$reg), __ T2D,
17180 as_FloatRegister($src1$$reg),
17181 as_FloatRegister($src2$$reg));
17182 %}
17183 ins_pipe(vmuldiv_fp128);
17184 %}
17185
17186 // --------------------------------- MLS --------------------------------------
17187
17188 instruct vmls4S(vecD dst, vecD src1, vecD src2)
17189 %{
17190 predicate(n->as_Vector()->length() == 2 ||
17191 n->as_Vector()->length() == 4);
17192 match(Set dst (SubVS dst (MulVS src1 src2)));
17193 ins_cost(INSN_COST);
17194 format %{ "mlsv $dst,$src1,$src2\t# vector (4H)" %}
17195 ins_encode %{
17196 __ mlsv(as_FloatRegister($dst$$reg), __ T4H,
17197 as_FloatRegister($src1$$reg),
17198 as_FloatRegister($src2$$reg));
17199 %}
17200 ins_pipe(vmla64);
17201 %}
17202
17203 instruct vmls8S(vecX dst, vecX src1, vecX src2)
17204 %{
17205 predicate(n->as_Vector()->length() == 8);
17206 match(Set dst (SubVS dst (MulVS src1 src2)));
17207 ins_cost(INSN_COST);
17208 format %{ "mlsv $dst,$src1,$src2\t# vector (8H)" %}
17209 ins_encode %{
17210 __ mlsv(as_FloatRegister($dst$$reg), __ T8H,
17211 as_FloatRegister($src1$$reg),
17212 as_FloatRegister($src2$$reg));
17213 %}
17214 ins_pipe(vmla128);
17215 %}
17216
17217 instruct vmls2I(vecD dst, vecD src1, vecD src2)
17218 %{
17219 predicate(n->as_Vector()->length() == 2);
17220 match(Set dst (SubVI dst (MulVI src1 src2)));
17221 ins_cost(INSN_COST);
17222 format %{ "mlsv $dst,$src1,$src2\t# vector (2S)" %}
17223 ins_encode %{
17224 __ mlsv(as_FloatRegister($dst$$reg), __ T2S,
17225 as_FloatRegister($src1$$reg),
17226 as_FloatRegister($src2$$reg));
17227 %}
17228 ins_pipe(vmla64);
17229 %}
17230
17231 instruct vmls4I(vecX dst, vecX src1, vecX src2)
17232 %{
17233 predicate(n->as_Vector()->length() == 4);
17234 match(Set dst (SubVI dst (MulVI src1 src2)));
17235 ins_cost(INSN_COST);
17236 format %{ "mlsv $dst,$src1,$src2\t# vector (4S)" %}
17237 ins_encode %{
17238 __ mlsv(as_FloatRegister($dst$$reg), __ T4S,
17239 as_FloatRegister($src1$$reg),
17240 as_FloatRegister($src2$$reg));
17241 %}
17242 ins_pipe(vmla128);
17243 %}
17244
17245 // dst - src1 * src2
17246 instruct vmls2F(vecD dst, vecD src1, vecD src2) %{
17247 predicate(UseFMA && n->as_Vector()->length() == 2);
17248 match(Set dst (FmaVF dst (Binary (NegVF src1) src2)));
17249 match(Set dst (FmaVF dst (Binary src1 (NegVF src2))));
17250 format %{ "fmls $dst,$src1,$src2\t# vector (2S)" %}
17251 ins_cost(INSN_COST);
17252 ins_encode %{
17253 __ fmls(as_FloatRegister($dst$$reg), __ T2S,
17254 as_FloatRegister($src1$$reg),
17255 as_FloatRegister($src2$$reg));
17256 %}
17257 ins_pipe(vmuldiv_fp64);
17258 %}
17259
17260 // dst - src1 * src2
17261 instruct vmls4F(vecX dst, vecX src1, vecX src2) %{
17262 predicate(UseFMA && n->as_Vector()->length() == 4);
17263 match(Set dst (FmaVF dst (Binary (NegVF src1) src2)));
17264 match(Set dst (FmaVF dst (Binary src1 (NegVF src2))));
17265 format %{ "fmls $dst,$src1,$src2\t# vector (4S)" %}
17266 ins_cost(INSN_COST);
17267 ins_encode %{
17268 __ fmls(as_FloatRegister($dst$$reg), __ T4S,
17269 as_FloatRegister($src1$$reg),
17270 as_FloatRegister($src2$$reg));
17271 %}
17272 ins_pipe(vmuldiv_fp128);
17273 %}
17274
17275 // dst - src1 * src2
17276 instruct vmls2D(vecX dst, vecX src1, vecX src2) %{
17277 predicate(UseFMA && n->as_Vector()->length() == 2);
17278 match(Set dst (FmaVD dst (Binary (NegVD src1) src2)));
17279 match(Set dst (FmaVD dst (Binary src1 (NegVD src2))));
17280 format %{ "fmls $dst,$src1,$src2\t# vector (2D)" %}
17281 ins_cost(INSN_COST);
17282 ins_encode %{
17283 __ fmls(as_FloatRegister($dst$$reg), __ T2D,
17284 as_FloatRegister($src1$$reg),
17285 as_FloatRegister($src2$$reg));
17286 %}
17287 ins_pipe(vmuldiv_fp128);
17288 %}
17289
17290 // --------------- Vector Multiply-Add Shorts into Integer --------------------
17291
17292 instruct vmuladdS2I(vecX dst, vecX src1, vecX src2, vecX tmp) %{
17293 predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
17294 match(Set dst (MulAddVS2VI src1 src2));
17295 ins_cost(INSN_COST);
17296 effect(TEMP_DEF dst, TEMP tmp);
17297 format %{ "smullv $tmp, $src1, $src2\t# vector (4H)\n\t"
17298 "smullv $dst, $src1, $src2\t# vector (8H)\n\t"
17299 "addpv $dst, $tmp, $dst\t# vector (4S)\n\t" %}
17300 ins_encode %{
17301 __ smullv(as_FloatRegister($tmp$$reg), __ T4H,
17302 as_FloatRegister($src1$$reg),
17303 as_FloatRegister($src2$$reg));
17304 __ smullv(as_FloatRegister($dst$$reg), __ T8H,
17305 as_FloatRegister($src1$$reg),
17306 as_FloatRegister($src2$$reg));
17307 __ addpv(as_FloatRegister($dst$$reg), __ T4S,
17308 as_FloatRegister($tmp$$reg),
17309 as_FloatRegister($dst$$reg));
17310 %}
17311 ins_pipe(vmuldiv_fp128);
17312 %}
17313
17314 // --------------------------------- DIV --------------------------------------
17315
17316 instruct vdiv2F(vecD dst, vecD src1, vecD src2)
17317 %{
17318 predicate(n->as_Vector()->length() == 2);
17319 match(Set dst (DivVF src1 src2));
17320 ins_cost(INSN_COST);
17321 format %{ "fdiv $dst,$src1,$src2\t# vector (2S)" %}
17322 ins_encode %{
17323 __ fdiv(as_FloatRegister($dst$$reg), __ T2S,
17324 as_FloatRegister($src1$$reg),
17325 as_FloatRegister($src2$$reg));
17326 %}
17327 ins_pipe(vmuldiv_fp64);
17328 %}
17329
17330 instruct vdiv4F(vecX dst, vecX src1, vecX src2)
17331 %{
17332 predicate(n->as_Vector()->length() == 4);
17333 match(Set dst (DivVF src1 src2));
17334 ins_cost(INSN_COST);
17335 format %{ "fdiv $dst,$src1,$src2\t# vector (4S)" %}
17336 ins_encode %{
17337 __ fdiv(as_FloatRegister($dst$$reg), __ T4S,
17338 as_FloatRegister($src1$$reg),
17339 as_FloatRegister($src2$$reg));
17340 %}
17341 ins_pipe(vmuldiv_fp128);
17342 %}
17343
17344 instruct vdiv2D(vecX dst, vecX src1, vecX src2)
17345 %{
17346 predicate(n->as_Vector()->length() == 2);
17347 match(Set dst (DivVD src1 src2));
17348 ins_cost(INSN_COST);
17349 format %{ "fdiv $dst,$src1,$src2\t# vector (2D)" %}
17350 ins_encode %{
17351 __ fdiv(as_FloatRegister($dst$$reg), __ T2D,
17352 as_FloatRegister($src1$$reg),
17353 as_FloatRegister($src2$$reg));
17354 %}
17355 ins_pipe(vmuldiv_fp128);
17356 %}
17357
17358 // --------------------------------- SQRT -------------------------------------
17359
17360 instruct vsqrt2F(vecD dst, vecD src)
17361 %{
17362 predicate(n->as_Vector()->length() == 2);
17363 match(Set dst (SqrtVF src));
17364 format %{ "fsqrt $dst, $src\t# vector (2F)" %}
17365 ins_encode %{
17366 __ fsqrt(as_FloatRegister($dst$$reg), __ T2S, as_FloatRegister($src$$reg));
17367 %}
17368 ins_pipe(vunop_fp64);
17369 %}
17370
17371 instruct vsqrt4F(vecX dst, vecX src)
17372 %{
17373 predicate(n->as_Vector()->length() == 4);
17374 match(Set dst (SqrtVF src));
17375 format %{ "fsqrt $dst, $src\t# vector (4F)" %}
17376 ins_encode %{
17377 __ fsqrt(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src$$reg));
17378 %}
17379 ins_pipe(vsqrt_fp128);
17380 %}
17381
17382 instruct vsqrt2D(vecX dst, vecX src)
17383 %{
17384 predicate(n->as_Vector()->length() == 2);
17385 match(Set dst (SqrtVD src));
17386 format %{ "fsqrt $dst, $src\t# vector (2D)" %}
17387 ins_encode %{
17388 __ fsqrt(as_FloatRegister($dst$$reg), __ T2D,
17389 as_FloatRegister($src$$reg));
17390 %}
17391 ins_pipe(vsqrt_fp128);
17392 %}
17393
17394 // --------------------------------- ABS --------------------------------------
17395
17396 instruct vabs8B(vecD dst, vecD src)
17397 %{
17398 predicate(n->as_Vector()->length() == 4 ||
17399 n->as_Vector()->length() == 8);
17400 match(Set dst (AbsVB src));
17401 ins_cost(INSN_COST);
17402 format %{ "abs $dst, $src\t# vector (8B)" %}
17403 ins_encode %{
17404 __ absr(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($src$$reg));
17405 %}
17406 ins_pipe(vlogical64);
17407 %}
17408
17409 instruct vabs16B(vecX dst, vecX src)
17410 %{
17411 predicate(n->as_Vector()->length() == 16);
17412 match(Set dst (AbsVB src));
17413 ins_cost(INSN_COST);
17414 format %{ "abs $dst, $src\t# vector (16B)" %}
17415 ins_encode %{
17416 __ absr(as_FloatRegister($dst$$reg), __ T16B, as_FloatRegister($src$$reg));
17417 %}
17418 ins_pipe(vlogical128);
17419 %}
17420
17421 instruct vabs4S(vecD dst, vecD src)
17422 %{
17423 predicate(n->as_Vector()->length() == 4);
17424 match(Set dst (AbsVS src));
17425 ins_cost(INSN_COST);
17426 format %{ "abs $dst, $src\t# vector (4H)" %}
17427 ins_encode %{
17428 __ absr(as_FloatRegister($dst$$reg), __ T4H, as_FloatRegister($src$$reg));
17429 %}
17430 ins_pipe(vlogical64);
17431 %}
17432
17433 instruct vabs8S(vecX dst, vecX src)
17434 %{
17435 predicate(n->as_Vector()->length() == 8);
17436 match(Set dst (AbsVS src));
17437 ins_cost(INSN_COST);
17438 format %{ "abs $dst, $src\t# vector (8H)" %}
17439 ins_encode %{
17440 __ absr(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg));
17441 %}
17442 ins_pipe(vlogical128);
17443 %}
17444
17445 instruct vabs2I(vecD dst, vecD src)
17446 %{
17447 predicate(n->as_Vector()->length() == 2);
17448 match(Set dst (AbsVI src));
17449 ins_cost(INSN_COST);
17450 format %{ "abs $dst, $src\t# vector (2S)" %}
17451 ins_encode %{
17452 __ absr(as_FloatRegister($dst$$reg), __ T2S, as_FloatRegister($src$$reg));
17453 %}
17454 ins_pipe(vlogical64);
17455 %}
17456
17457 instruct vabs4I(vecX dst, vecX src)
17458 %{
17459 predicate(n->as_Vector()->length() == 4);
17460 match(Set dst (AbsVI src));
17461 ins_cost(INSN_COST);
17462 format %{ "abs $dst, $src\t# vector (4S)" %}
17463 ins_encode %{
17464 __ absr(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src$$reg));
17465 %}
17466 ins_pipe(vlogical128);
17467 %}
17468
17469 instruct vabs2L(vecX dst, vecX src)
17470 %{
17471 predicate(n->as_Vector()->length() == 2);
17472 match(Set dst (AbsVL src));
17473 ins_cost(INSN_COST);
17474 format %{ "abs $dst, $src\t# vector (2D)" %}
17475 ins_encode %{
17476 __ absr(as_FloatRegister($dst$$reg), __ T2D, as_FloatRegister($src$$reg));
17477 %}
17478 ins_pipe(vlogical128);
17479 %}
17480
17481 instruct vabs2F(vecD dst, vecD src)
17482 %{
17483 predicate(n->as_Vector()->length() == 2);
17484 match(Set dst (AbsVF src));
17485 ins_cost(INSN_COST * 3);
17486 format %{ "fabs $dst,$src\t# vector (2S)" %}
17487 ins_encode %{
17488 __ fabs(as_FloatRegister($dst$$reg), __ T2S,
17489 as_FloatRegister($src$$reg));
17490 %}
17491 ins_pipe(vunop_fp64);
17492 %}
17493
17494 instruct vabs4F(vecX dst, vecX src)
17495 %{
17496 predicate(n->as_Vector()->length() == 4);
17497 match(Set dst (AbsVF src));
17498 ins_cost(INSN_COST * 3);
17499 format %{ "fabs $dst,$src\t# vector (4S)" %}
17500 ins_encode %{
17501 __ fabs(as_FloatRegister($dst$$reg), __ T4S,
17502 as_FloatRegister($src$$reg));
17503 %}
17504 ins_pipe(vunop_fp128);
17505 %}
17506
17507 instruct vabs2D(vecX dst, vecX src)
17508 %{
17509 predicate(n->as_Vector()->length() == 2);
17510 match(Set dst (AbsVD src));
17511 ins_cost(INSN_COST * 3);
17512 format %{ "fabs $dst,$src\t# vector (2D)" %}
17513 ins_encode %{
17514 __ fabs(as_FloatRegister($dst$$reg), __ T2D,
17515 as_FloatRegister($src$$reg));
17516 %}
17517 ins_pipe(vunop_fp128);
17518 %}
17519
17520 // --------------------------------- NEG --------------------------------------
17521
17522 instruct vneg2F(vecD dst, vecD src)
17523 %{
17524 predicate(n->as_Vector()->length() == 2);
17525 match(Set dst (NegVF src));
17526 ins_cost(INSN_COST * 3);
17527 format %{ "fneg $dst,$src\t# vector (2S)" %}
17528 ins_encode %{
17529 __ fneg(as_FloatRegister($dst$$reg), __ T2S,
17530 as_FloatRegister($src$$reg));
17531 %}
17532 ins_pipe(vunop_fp64);
17533 %}
17534
17535 instruct vneg4F(vecX dst, vecX src)
17536 %{
17537 predicate(n->as_Vector()->length() == 4);
17538 match(Set dst (NegVF src));
17539 ins_cost(INSN_COST * 3);
17540 format %{ "fneg $dst,$src\t# vector (4S)" %}
17541 ins_encode %{
17542 __ fneg(as_FloatRegister($dst$$reg), __ T4S,
17543 as_FloatRegister($src$$reg));
17544 %}
17545 ins_pipe(vunop_fp128);
17546 %}
17547
17548 instruct vneg2D(vecX dst, vecX src)
17549 %{
17550 predicate(n->as_Vector()->length() == 2);
17551 match(Set dst (NegVD src));
17552 ins_cost(INSN_COST * 3);
17553 format %{ "fneg $dst,$src\t# vector (2D)" %}
17554 ins_encode %{
17555 __ fneg(as_FloatRegister($dst$$reg), __ T2D,
17556 as_FloatRegister($src$$reg));
17557 %}
17558 ins_pipe(vunop_fp128);
17559 %}
17560
17561 // --------------------------------- AND --------------------------------------
17562
17563 instruct vand8B(vecD dst, vecD src1, vecD src2)
17564 %{
17565 predicate(n->as_Vector()->length_in_bytes() == 4 ||
17566 n->as_Vector()->length_in_bytes() == 8);
17567 match(Set dst (AndV src1 src2));
17568 ins_cost(INSN_COST);
17569 format %{ "and $dst,$src1,$src2\t# vector (8B)" %}
17570 ins_encode %{
17571 __ andr(as_FloatRegister($dst$$reg), __ T8B,
17572 as_FloatRegister($src1$$reg),
17573 as_FloatRegister($src2$$reg));
17574 %}
17575 ins_pipe(vlogical64);
17576 %}
17577
17578 instruct vand16B(vecX dst, vecX src1, vecX src2)
17579 %{
17580 predicate(n->as_Vector()->length_in_bytes() == 16);
17581 match(Set dst (AndV src1 src2));
17582 ins_cost(INSN_COST);
17583 format %{ "and $dst,$src1,$src2\t# vector (16B)" %}
17584 ins_encode %{
17585 __ andr(as_FloatRegister($dst$$reg), __ T16B,
17586 as_FloatRegister($src1$$reg),
17587 as_FloatRegister($src2$$reg));
17588 %}
17589 ins_pipe(vlogical128);
17590 %}
17591
17592 // --------------------------------- OR ---------------------------------------
17593
17594 instruct vor8B(vecD dst, vecD src1, vecD src2)
17595 %{
17596 predicate(n->as_Vector()->length_in_bytes() == 4 ||
17597 n->as_Vector()->length_in_bytes() == 8);
17598 match(Set dst (OrV src1 src2));
17599 ins_cost(INSN_COST);
17600 format %{ "and $dst,$src1,$src2\t# vector (8B)" %}
17601 ins_encode %{
17602 __ orr(as_FloatRegister($dst$$reg), __ T8B,
17603 as_FloatRegister($src1$$reg),
17604 as_FloatRegister($src2$$reg));
17605 %}
17606 ins_pipe(vlogical64);
17607 %}
17608
17609 instruct vor16B(vecX dst, vecX src1, vecX src2)
17610 %{
17611 predicate(n->as_Vector()->length_in_bytes() == 16);
17612 match(Set dst (OrV src1 src2));
17613 ins_cost(INSN_COST);
17614 format %{ "orr $dst,$src1,$src2\t# vector (16B)" %}
17615 ins_encode %{
17616 __ orr(as_FloatRegister($dst$$reg), __ T16B,
17617 as_FloatRegister($src1$$reg),
17618 as_FloatRegister($src2$$reg));
17619 %}
17620 ins_pipe(vlogical128);
17621 %}
17622
17623 // --------------------------------- XOR --------------------------------------
17624
17625 instruct vxor8B(vecD dst, vecD src1, vecD src2)
17626 %{
17627 predicate(n->as_Vector()->length_in_bytes() == 4 ||
17628 n->as_Vector()->length_in_bytes() == 8);
17629 match(Set dst (XorV src1 src2));
17630 ins_cost(INSN_COST);
17631 format %{ "xor $dst,$src1,$src2\t# vector (8B)" %}
17632 ins_encode %{
17633 __ eor(as_FloatRegister($dst$$reg), __ T8B,
17634 as_FloatRegister($src1$$reg),
17635 as_FloatRegister($src2$$reg));
17636 %}
17637 ins_pipe(vlogical64);
17638 %}
17639
17640 instruct vxor16B(vecX dst, vecX src1, vecX src2)
17641 %{
17642 predicate(n->as_Vector()->length_in_bytes() == 16);
17643 match(Set dst (XorV src1 src2));
17644 ins_cost(INSN_COST);
17645 format %{ "xor $dst,$src1,$src2\t# vector (16B)" %}
17646 ins_encode %{
17647 __ eor(as_FloatRegister($dst$$reg), __ T16B,
17648 as_FloatRegister($src1$$reg),
17649 as_FloatRegister($src2$$reg));
17650 %}
17651 ins_pipe(vlogical128);
17652 %}
17653
17654 // ------------------------------ Shift ---------------------------------------
17655 instruct vshiftcnt8B(vecD dst, iRegIorL2I cnt) %{
17656 predicate(n->as_Vector()->length_in_bytes() == 8);
17657 match(Set dst (LShiftCntV cnt));
17658 match(Set dst (RShiftCntV cnt));
17659 format %{ "dup $dst, $cnt\t# shift count vector (8B)" %}
17660 ins_encode %{
17661 __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($cnt$$reg));
17662 %}
17663 ins_pipe(vdup_reg_reg64);
17664 %}
17665
17666 instruct vshiftcnt16B(vecX dst, iRegIorL2I cnt) %{
17667 predicate(n->as_Vector()->length_in_bytes() == 16);
17668 match(Set dst (LShiftCntV cnt));
17669 match(Set dst (RShiftCntV cnt));
17670 format %{ "dup $dst, $cnt\t# shift count vector (16B)" %}
17671 ins_encode %{
17672 __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($cnt$$reg));
17673 %}
17674 ins_pipe(vdup_reg_reg128);
17675 %}
17676
17677 instruct vsll8B(vecD dst, vecD src, vecD shift) %{
17678 predicate(n->as_Vector()->length() == 4 ||
17679 n->as_Vector()->length() == 8);
17680 match(Set dst (LShiftVB src shift));
17681 ins_cost(INSN_COST);
17682 format %{ "sshl $dst,$src,$shift\t# vector (8B)" %}
17683 ins_encode %{
17684 __ sshl(as_FloatRegister($dst$$reg), __ T8B,
17685 as_FloatRegister($src$$reg),
17686 as_FloatRegister($shift$$reg));
17687 %}
17688 ins_pipe(vshift64);
17689 %}
17690
17691 instruct vsll16B(vecX dst, vecX src, vecX shift) %{
17692 predicate(n->as_Vector()->length() == 16);
17693 match(Set dst (LShiftVB src shift));
17694 ins_cost(INSN_COST);
17695 format %{ "sshl $dst,$src,$shift\t# vector (16B)" %}
17696 ins_encode %{
17697 __ sshl(as_FloatRegister($dst$$reg), __ T16B,
17698 as_FloatRegister($src$$reg),
17699 as_FloatRegister($shift$$reg));
17700 %}
17701 ins_pipe(vshift128);
17702 %}
17703
17704 // Right shifts with vector shift count on aarch64 SIMD are implemented
17705 // as left shift by negative shift count.
17706 // There are two cases for vector shift count.
17707 //
17708 // Case 1: The vector shift count is from replication.
17709 // | |
17710 // LoadVector RShiftCntV
17711 // | /
17712 // RShiftVI
17713 // Note: In inner loop, multiple neg instructions are used, which can be
17714 // moved to outer loop and merge into one neg instruction.
17715 //
17716 // Case 2: The vector shift count is from loading.
17717 // This case isn't supported by middle-end now. But it's supported by
17718 // panama/vectorIntrinsics(JEP 338: Vector API).
17719 // | |
17720 // LoadVector LoadVector
17721 // | /
17722 // RShiftVI
17723 //
17724
17725 instruct vsra8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
17726 predicate(n->as_Vector()->length() == 4 ||
17727 n->as_Vector()->length() == 8);
17728 match(Set dst (RShiftVB src shift));
17729 ins_cost(INSN_COST);
17730 effect(TEMP tmp);
17731 format %{ "negr $tmp,$shift\t"
17732 "sshl $dst,$src,$tmp\t# vector (8B)" %}
17733 ins_encode %{
17734 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17735 as_FloatRegister($shift$$reg));
17736 __ sshl(as_FloatRegister($dst$$reg), __ T8B,
17737 as_FloatRegister($src$$reg),
17738 as_FloatRegister($tmp$$reg));
17739 %}
17740 ins_pipe(vshift64);
17741 %}
17742
17743 instruct vsra16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
17744 predicate(n->as_Vector()->length() == 16);
17745 match(Set dst (RShiftVB src shift));
17746 ins_cost(INSN_COST);
17747 effect(TEMP tmp);
17748 format %{ "negr $tmp,$shift\t"
17749 "sshl $dst,$src,$tmp\t# vector (16B)" %}
17750 ins_encode %{
17751 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17752 as_FloatRegister($shift$$reg));
17753 __ sshl(as_FloatRegister($dst$$reg), __ T16B,
17754 as_FloatRegister($src$$reg),
17755 as_FloatRegister($tmp$$reg));
17756 %}
17757 ins_pipe(vshift128);
17758 %}
17759
17760 instruct vsrl8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
17761 predicate(n->as_Vector()->length() == 4 ||
17762 n->as_Vector()->length() == 8);
17763 match(Set dst (URShiftVB src shift));
17764 ins_cost(INSN_COST);
17765 effect(TEMP tmp);
17766 format %{ "negr $tmp,$shift\t"
17767 "ushl $dst,$src,$tmp\t# vector (8B)" %}
17768 ins_encode %{
17769 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17770 as_FloatRegister($shift$$reg));
17771 __ ushl(as_FloatRegister($dst$$reg), __ T8B,
17772 as_FloatRegister($src$$reg),
17773 as_FloatRegister($tmp$$reg));
17774 %}
17775 ins_pipe(vshift64);
17776 %}
17777
17778 instruct vsrl16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
17779 predicate(n->as_Vector()->length() == 16);
17780 match(Set dst (URShiftVB src shift));
17781 ins_cost(INSN_COST);
17782 effect(TEMP tmp);
17783 format %{ "negr $tmp,$shift\t"
17784 "ushl $dst,$src,$tmp\t# vector (16B)" %}
17785 ins_encode %{
17786 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17787 as_FloatRegister($shift$$reg));
17788 __ ushl(as_FloatRegister($dst$$reg), __ T16B,
17789 as_FloatRegister($src$$reg),
17790 as_FloatRegister($tmp$$reg));
17791 %}
17792 ins_pipe(vshift128);
17793 %}
17794
17795 instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
17796 predicate(n->as_Vector()->length() == 4 ||
17797 n->as_Vector()->length() == 8);
17798 match(Set dst (LShiftVB src (LShiftCntV shift)));
17799 ins_cost(INSN_COST);
17800 format %{ "shl $dst, $src, $shift\t# vector (8B)" %}
17801 ins_encode %{
17802 int sh = (int)$shift$$constant;
17803 if (sh >= 8) {
17804 __ eor(as_FloatRegister($dst$$reg), __ T8B,
17805 as_FloatRegister($src$$reg),
17806 as_FloatRegister($src$$reg));
17807 } else {
17808 __ shl(as_FloatRegister($dst$$reg), __ T8B,
17809 as_FloatRegister($src$$reg), sh);
17810 }
17811 %}
17812 ins_pipe(vshift64_imm);
17813 %}
17814
17815 instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
17816 predicate(n->as_Vector()->length() == 16);
17817 match(Set dst (LShiftVB src (LShiftCntV shift)));
17818 ins_cost(INSN_COST);
17819 format %{ "shl $dst, $src, $shift\t# vector (16B)" %}
17820 ins_encode %{
17821 int sh = (int)$shift$$constant;
17822 if (sh >= 8) {
17823 __ eor(as_FloatRegister($dst$$reg), __ T16B,
17824 as_FloatRegister($src$$reg),
17825 as_FloatRegister($src$$reg));
17826 } else {
17827 __ shl(as_FloatRegister($dst$$reg), __ T16B,
17828 as_FloatRegister($src$$reg), sh);
17829 }
17830 %}
17831 ins_pipe(vshift128_imm);
17832 %}
17833
17834 instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{
17835 predicate(n->as_Vector()->length() == 4 ||
17836 n->as_Vector()->length() == 8);
17837 match(Set dst (RShiftVB src (RShiftCntV shift)));
17838 ins_cost(INSN_COST);
17839 format %{ "sshr $dst, $src, $shift\t# vector (8B)" %}
17840 ins_encode %{
17841 int sh = (int)$shift$$constant;
17842 if (sh >= 8) sh = 7;
17843 __ sshr(as_FloatRegister($dst$$reg), __ T8B,
17844 as_FloatRegister($src$$reg), sh);
17845 %}
17846 ins_pipe(vshift64_imm);
17847 %}
17848
17849 instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
17850 predicate(n->as_Vector()->length() == 16);
17851 match(Set dst (RShiftVB src (RShiftCntV shift)));
17852 ins_cost(INSN_COST);
17853 format %{ "sshr $dst, $src, $shift\t# vector (16B)" %}
17854 ins_encode %{
17855 int sh = (int)$shift$$constant;
17856 if (sh >= 8) sh = 7;
17857 __ sshr(as_FloatRegister($dst$$reg), __ T16B,
17858 as_FloatRegister($src$$reg), sh);
17859 %}
17860 ins_pipe(vshift128_imm);
17861 %}
17862
17863 instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{
17864 predicate(n->as_Vector()->length() == 4 ||
17865 n->as_Vector()->length() == 8);
17866 match(Set dst (URShiftVB src (RShiftCntV shift)));
17867 ins_cost(INSN_COST);
17868 format %{ "ushr $dst, $src, $shift\t# vector (8B)" %}
17869 ins_encode %{
17870 int sh = (int)$shift$$constant;
17871 if (sh >= 8) {
17872 __ eor(as_FloatRegister($dst$$reg), __ T8B,
17873 as_FloatRegister($src$$reg),
17874 as_FloatRegister($src$$reg));
17875 } else {
17876 __ ushr(as_FloatRegister($dst$$reg), __ T8B,
17877 as_FloatRegister($src$$reg), sh);
17878 }
17879 %}
17880 ins_pipe(vshift64_imm);
17881 %}
17882
17883 instruct vsrl16B_imm(vecX dst, vecX src, immI shift) %{
17884 predicate(n->as_Vector()->length() == 16);
17885 match(Set dst (URShiftVB src (RShiftCntV shift)));
17886 ins_cost(INSN_COST);
17887 format %{ "ushr $dst, $src, $shift\t# vector (16B)" %}
17888 ins_encode %{
17889 int sh = (int)$shift$$constant;
17890 if (sh >= 8) {
17891 __ eor(as_FloatRegister($dst$$reg), __ T16B,
17892 as_FloatRegister($src$$reg),
17893 as_FloatRegister($src$$reg));
17894 } else {
17895 __ ushr(as_FloatRegister($dst$$reg), __ T16B,
17896 as_FloatRegister($src$$reg), sh);
17897 }
17898 %}
17899 ins_pipe(vshift128_imm);
17900 %}
17901
17902 instruct vsll4S(vecD dst, vecD src, vecD shift) %{
17903 predicate(n->as_Vector()->length() == 2 ||
17904 n->as_Vector()->length() == 4);
17905 match(Set dst (LShiftVS src shift));
17906 ins_cost(INSN_COST);
17907 format %{ "sshl $dst,$src,$shift\t# vector (4H)" %}
17908 ins_encode %{
17909 __ sshl(as_FloatRegister($dst$$reg), __ T4H,
17910 as_FloatRegister($src$$reg),
17911 as_FloatRegister($shift$$reg));
17912 %}
17913 ins_pipe(vshift64);
17914 %}
17915
17916 instruct vsll8S(vecX dst, vecX src, vecX shift) %{
17917 predicate(n->as_Vector()->length() == 8);
17918 match(Set dst (LShiftVS src shift));
17919 ins_cost(INSN_COST);
17920 format %{ "sshl $dst,$src,$shift\t# vector (8H)" %}
17921 ins_encode %{
17922 __ sshl(as_FloatRegister($dst$$reg), __ T8H,
17923 as_FloatRegister($src$$reg),
17924 as_FloatRegister($shift$$reg));
17925 %}
17926 ins_pipe(vshift128);
17927 %}
17928
17929 instruct vsra4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
17930 predicate(n->as_Vector()->length() == 2 ||
17931 n->as_Vector()->length() == 4);
17932 match(Set dst (RShiftVS src shift));
17933 ins_cost(INSN_COST);
17934 effect(TEMP tmp);
17935 format %{ "negr $tmp,$shift\t"
17936 "sshl $dst,$src,$tmp\t# vector (4H)" %}
17937 ins_encode %{
17938 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17939 as_FloatRegister($shift$$reg));
17940 __ sshl(as_FloatRegister($dst$$reg), __ T4H,
17941 as_FloatRegister($src$$reg),
17942 as_FloatRegister($tmp$$reg));
17943 %}
17944 ins_pipe(vshift64);
17945 %}
17946
17947 instruct vsra8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
17948 predicate(n->as_Vector()->length() == 8);
17949 match(Set dst (RShiftVS src shift));
17950 ins_cost(INSN_COST);
17951 effect(TEMP tmp);
17952 format %{ "negr $tmp,$shift\t"
17953 "sshl $dst,$src,$tmp\t# vector (8H)" %}
17954 ins_encode %{
17955 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17956 as_FloatRegister($shift$$reg));
17957 __ sshl(as_FloatRegister($dst$$reg), __ T8H,
17958 as_FloatRegister($src$$reg),
17959 as_FloatRegister($tmp$$reg));
17960 %}
17961 ins_pipe(vshift128);
17962 %}
17963
17964 instruct vsrl4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
17965 predicate(n->as_Vector()->length() == 2 ||
17966 n->as_Vector()->length() == 4);
17967 match(Set dst (URShiftVS src shift));
17968 ins_cost(INSN_COST);
17969 effect(TEMP tmp);
17970 format %{ "negr $tmp,$shift\t"
17971 "ushl $dst,$src,$tmp\t# vector (4H)" %}
17972 ins_encode %{
17973 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17974 as_FloatRegister($shift$$reg));
17975 __ ushl(as_FloatRegister($dst$$reg), __ T4H,
17976 as_FloatRegister($src$$reg),
17977 as_FloatRegister($tmp$$reg));
17978 %}
17979 ins_pipe(vshift64);
17980 %}
17981
17982 instruct vsrl8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
17983 predicate(n->as_Vector()->length() == 8);
17984 match(Set dst (URShiftVS src shift));
17985 ins_cost(INSN_COST);
17986 effect(TEMP tmp);
17987 format %{ "negr $tmp,$shift\t"
17988 "ushl $dst,$src,$tmp\t# vector (8H)" %}
17989 ins_encode %{
17990 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17991 as_FloatRegister($shift$$reg));
17992 __ ushl(as_FloatRegister($dst$$reg), __ T8H,
17993 as_FloatRegister($src$$reg),
17994 as_FloatRegister($tmp$$reg));
17995 %}
17996 ins_pipe(vshift128);
17997 %}
17998
17999 instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
18000 predicate(n->as_Vector()->length() == 2 ||
18001 n->as_Vector()->length() == 4);
18002 match(Set dst (LShiftVS src (LShiftCntV shift)));
18003 ins_cost(INSN_COST);
18004 format %{ "shl $dst, $src, $shift\t# vector (4H)" %}
18005 ins_encode %{
18006 int sh = (int)$shift$$constant;
18007 if (sh >= 16) {
18008 __ eor(as_FloatRegister($dst$$reg), __ T8B,
18009 as_FloatRegister($src$$reg),
18010 as_FloatRegister($src$$reg));
18011 } else {
18012 __ shl(as_FloatRegister($dst$$reg), __ T4H,
18013 as_FloatRegister($src$$reg), sh);
18014 }
18015 %}
18016 ins_pipe(vshift64_imm);
18017 %}
18018
18019 instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
18020 predicate(n->as_Vector()->length() == 8);
18021 match(Set dst (LShiftVS src (LShiftCntV shift)));
18022 ins_cost(INSN_COST);
18023 format %{ "shl $dst, $src, $shift\t# vector (8H)" %}
18024 ins_encode %{
18025 int sh = (int)$shift$$constant;
18026 if (sh >= 16) {
18027 __ eor(as_FloatRegister($dst$$reg), __ T16B,
18028 as_FloatRegister($src$$reg),
18029 as_FloatRegister($src$$reg));
18030 } else {
18031 __ shl(as_FloatRegister($dst$$reg), __ T8H,
18032 as_FloatRegister($src$$reg), sh);
18033 }
18034 %}
18035 ins_pipe(vshift128_imm);
18036 %}
18037
18038 instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{
18039 predicate(n->as_Vector()->length() == 2 ||
18040 n->as_Vector()->length() == 4);
18041 match(Set dst (RShiftVS src (RShiftCntV shift)));
18042 ins_cost(INSN_COST);
18043 format %{ "sshr $dst, $src, $shift\t# vector (4H)" %}
18044 ins_encode %{
18045 int sh = (int)$shift$$constant;
18046 if (sh >= 16) sh = 15;
18047 __ sshr(as_FloatRegister($dst$$reg), __ T4H,
18048 as_FloatRegister($src$$reg), sh);
18049 %}
18050 ins_pipe(vshift64_imm);
18051 %}
18052
18053 instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
18054 predicate(n->as_Vector()->length() == 8);
18055 match(Set dst (RShiftVS src (RShiftCntV shift)));
18056 ins_cost(INSN_COST);
18057 format %{ "sshr $dst, $src, $shift\t# vector (8H)" %}
18058 ins_encode %{
18059 int sh = (int)$shift$$constant;
18060 if (sh >= 16) sh = 15;
18061 __ sshr(as_FloatRegister($dst$$reg), __ T8H,
18062 as_FloatRegister($src$$reg), sh);
18063 %}
18064 ins_pipe(vshift128_imm);
18065 %}
18066
18067 instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{
18068 predicate(n->as_Vector()->length() == 2 ||
18069 n->as_Vector()->length() == 4);
18070 match(Set dst (URShiftVS src (RShiftCntV shift)));
18071 ins_cost(INSN_COST);
18072 format %{ "ushr $dst, $src, $shift\t# vector (4H)" %}
18073 ins_encode %{
18074 int sh = (int)$shift$$constant;
18075 if (sh >= 16) {
18076 __ eor(as_FloatRegister($dst$$reg), __ T8B,
18077 as_FloatRegister($src$$reg),
18078 as_FloatRegister($src$$reg));
18079 } else {
18080 __ ushr(as_FloatRegister($dst$$reg), __ T4H,
18081 as_FloatRegister($src$$reg), sh);
18082 }
18083 %}
18084 ins_pipe(vshift64_imm);
18085 %}
18086
18087 instruct vsrl8S_imm(vecX dst, vecX src, immI shift) %{
18088 predicate(n->as_Vector()->length() == 8);
18089 match(Set dst (URShiftVS src (RShiftCntV shift)));
18090 ins_cost(INSN_COST);
18091 format %{ "ushr $dst, $src, $shift\t# vector (8H)" %}
18092 ins_encode %{
18093 int sh = (int)$shift$$constant;
18094 if (sh >= 16) {
18095 __ eor(as_FloatRegister($dst$$reg), __ T16B,
18096 as_FloatRegister($src$$reg),
18097 as_FloatRegister($src$$reg));
18098 } else {
18099 __ ushr(as_FloatRegister($dst$$reg), __ T8H,
18100 as_FloatRegister($src$$reg), sh);
18101 }
18102 %}
18103 ins_pipe(vshift128_imm);
18104 %}
18105
18106 instruct vsll2I(vecD dst, vecD src, vecD shift) %{
18107 predicate(n->as_Vector()->length() == 2);
18108 match(Set dst (LShiftVI src shift));
18109 ins_cost(INSN_COST);
18110 format %{ "sshl $dst,$src,$shift\t# vector (2S)" %}
18111 ins_encode %{
18112 __ sshl(as_FloatRegister($dst$$reg), __ T2S,
18113 as_FloatRegister($src$$reg),
18114 as_FloatRegister($shift$$reg));
18115 %}
18116 ins_pipe(vshift64);
18117 %}
18118
18119 instruct vsll4I(vecX dst, vecX src, vecX shift) %{
18120 predicate(n->as_Vector()->length() == 4);
18121 match(Set dst (LShiftVI src shift));
18122 ins_cost(INSN_COST);
18123 format %{ "sshl $dst,$src,$shift\t# vector (4S)" %}
18124 ins_encode %{
18125 __ sshl(as_FloatRegister($dst$$reg), __ T4S,
18126 as_FloatRegister($src$$reg),
18127 as_FloatRegister($shift$$reg));
18128 %}
18129 ins_pipe(vshift128);
18130 %}
18131
18132 instruct vsra2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
18133 predicate(n->as_Vector()->length() == 2);
18134 match(Set dst (RShiftVI src shift));
18135 ins_cost(INSN_COST);
18136 effect(TEMP tmp);
18137 format %{ "negr $tmp,$shift\t"
18138 "sshl $dst,$src,$tmp\t# vector (2S)" %}
18139 ins_encode %{
18140 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
18141 as_FloatRegister($shift$$reg));
18142 __ sshl(as_FloatRegister($dst$$reg), __ T2S,
18143 as_FloatRegister($src$$reg),
18144 as_FloatRegister($tmp$$reg));
18145 %}
18146 ins_pipe(vshift64);
18147 %}
18148
18149 instruct vsra4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
18150 predicate(n->as_Vector()->length() == 4);
18151 match(Set dst (RShiftVI src shift));
18152 ins_cost(INSN_COST);
18153 effect(TEMP tmp);
18154 format %{ "negr $tmp,$shift\t"
18155 "sshl $dst,$src,$tmp\t# vector (4S)" %}
18156 ins_encode %{
18157 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
18158 as_FloatRegister($shift$$reg));
18159 __ sshl(as_FloatRegister($dst$$reg), __ T4S,
18160 as_FloatRegister($src$$reg),
18161 as_FloatRegister($tmp$$reg));
18162 %}
18163 ins_pipe(vshift128);
18164 %}
18165
18166 instruct vsrl2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
18167 predicate(n->as_Vector()->length() == 2);
18168 match(Set dst (URShiftVI src shift));
18169 ins_cost(INSN_COST);
18170 effect(TEMP tmp);
18171 format %{ "negr $tmp,$shift\t"
18172 "ushl $dst,$src,$tmp\t# vector (2S)" %}
18173 ins_encode %{
18174 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
18175 as_FloatRegister($shift$$reg));
18176 __ ushl(as_FloatRegister($dst$$reg), __ T2S,
18177 as_FloatRegister($src$$reg),
18178 as_FloatRegister($tmp$$reg));
18179 %}
18180 ins_pipe(vshift64);
18181 %}
18182
18183 instruct vsrl4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
18184 predicate(n->as_Vector()->length() == 4);
18185 match(Set dst (URShiftVI src shift));
18186 ins_cost(INSN_COST);
18187 effect(TEMP tmp);
18188 format %{ "negr $tmp,$shift\t"
18189 "ushl $dst,$src,$tmp\t# vector (4S)" %}
18190 ins_encode %{
18191 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
18192 as_FloatRegister($shift$$reg));
18193 __ ushl(as_FloatRegister($dst$$reg), __ T4S,
18194 as_FloatRegister($src$$reg),
18195 as_FloatRegister($tmp$$reg));
18196 %}
18197 ins_pipe(vshift128);
18198 %}
18199
18200 instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
18201 predicate(n->as_Vector()->length() == 2);
18202 match(Set dst (LShiftVI src (LShiftCntV shift)));
18203 ins_cost(INSN_COST);
18204 format %{ "shl $dst, $src, $shift\t# vector (2S)" %}
18205 ins_encode %{
18206 __ shl(as_FloatRegister($dst$$reg), __ T2S,
18207 as_FloatRegister($src$$reg),
18208 (int)$shift$$constant);
18209 %}
18210 ins_pipe(vshift64_imm);
18211 %}
18212
18213 instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{
18214 predicate(n->as_Vector()->length() == 4);
18215 match(Set dst (LShiftVI src (LShiftCntV shift)));
18216 ins_cost(INSN_COST);
18217 format %{ "shl $dst, $src, $shift\t# vector (4S)" %}
18218 ins_encode %{
18219 __ shl(as_FloatRegister($dst$$reg), __ T4S,
18220 as_FloatRegister($src$$reg),
18221 (int)$shift$$constant);
18222 %}
18223 ins_pipe(vshift128_imm);
18224 %}
18225
18226 instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{
18227 predicate(n->as_Vector()->length() == 2);
18228 match(Set dst (RShiftVI src (RShiftCntV shift)));
18229 ins_cost(INSN_COST);
18230 format %{ "sshr $dst, $src, $shift\t# vector (2S)" %}
18231 ins_encode %{
18232 __ sshr(as_FloatRegister($dst$$reg), __ T2S,
18233 as_FloatRegister($src$$reg),
18234 (int)$shift$$constant);
18235 %}
18236 ins_pipe(vshift64_imm);
18237 %}
18238
18239 instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{
18240 predicate(n->as_Vector()->length() == 4);
18241 match(Set dst (RShiftVI src (RShiftCntV shift)));
18242 ins_cost(INSN_COST);
18243 format %{ "sshr $dst, $src, $shift\t# vector (4S)" %}
18244 ins_encode %{
18245 __ sshr(as_FloatRegister($dst$$reg), __ T4S,
18246 as_FloatRegister($src$$reg),
18247 (int)$shift$$constant);
18248 %}
18249 ins_pipe(vshift128_imm);
18250 %}
18251
18252 instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{
18253 predicate(n->as_Vector()->length() == 2);
18254 match(Set dst (URShiftVI src (RShiftCntV shift)));
18255 ins_cost(INSN_COST);
18256 format %{ "ushr $dst, $src, $shift\t# vector (2S)" %}
18257 ins_encode %{
18258 __ ushr(as_FloatRegister($dst$$reg), __ T2S,
18259 as_FloatRegister($src$$reg),
18260 (int)$shift$$constant);
18261 %}
18262 ins_pipe(vshift64_imm);
18263 %}
18264
18265 instruct vsrl4I_imm(vecX dst, vecX src, immI shift) %{
18266 predicate(n->as_Vector()->length() == 4);
18267 match(Set dst (URShiftVI src (RShiftCntV shift)));
18268 ins_cost(INSN_COST);
18269 format %{ "ushr $dst, $src, $shift\t# vector (4S)" %}
18270 ins_encode %{
18271 __ ushr(as_FloatRegister($dst$$reg), __ T4S,
18272 as_FloatRegister($src$$reg),
18273 (int)$shift$$constant);
18274 %}
18275 ins_pipe(vshift128_imm);
18276 %}
18277
18278 instruct vsll2L(vecX dst, vecX src, vecX shift) %{
18279 predicate(n->as_Vector()->length() == 2);
18280 match(Set dst (LShiftVL src shift));
18281 ins_cost(INSN_COST);
18282 format %{ "sshl $dst,$src,$shift\t# vector (2D)" %}
18283 ins_encode %{
18284 __ sshl(as_FloatRegister($dst$$reg), __ T2D,
18285 as_FloatRegister($src$$reg),
18286 as_FloatRegister($shift$$reg));
18287 %}
18288 ins_pipe(vshift128);
18289 %}
18290
18291 instruct vsra2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
18292 predicate(n->as_Vector()->length() == 2);
18293 match(Set dst (RShiftVL src shift));
18294 ins_cost(INSN_COST);
18295 effect(TEMP tmp);
18296 format %{ "negr $tmp,$shift\t"
18297 "sshl $dst,$src,$tmp\t# vector (2D)" %}
18298 ins_encode %{
18299 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
18300 as_FloatRegister($shift$$reg));
18301 __ sshl(as_FloatRegister($dst$$reg), __ T2D,
18302 as_FloatRegister($src$$reg),
18303 as_FloatRegister($tmp$$reg));
18304 %}
18305 ins_pipe(vshift128);
18306 %}
18307
18308 instruct vsrl2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
18309 predicate(n->as_Vector()->length() == 2);
18310 match(Set dst (URShiftVL src shift));
18311 ins_cost(INSN_COST);
18312 effect(TEMP tmp);
18313 format %{ "negr $tmp,$shift\t"
18314 "ushl $dst,$src,$tmp\t# vector (2D)" %}
18315 ins_encode %{
18316 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
18317 as_FloatRegister($shift$$reg));
18318 __ ushl(as_FloatRegister($dst$$reg), __ T2D,
18319 as_FloatRegister($src$$reg),
18320 as_FloatRegister($tmp$$reg));
18321 %}
18322 ins_pipe(vshift128);
18323 %}
18324
18325 instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
18326 predicate(n->as_Vector()->length() == 2);
18327 match(Set dst (LShiftVL src (LShiftCntV shift)));
18328 ins_cost(INSN_COST);
18329 format %{ "shl $dst, $src, $shift\t# vector (2D)" %}
18330 ins_encode %{
18331 __ shl(as_FloatRegister($dst$$reg), __ T2D,
18332 as_FloatRegister($src$$reg),
18333 (int)$shift$$constant);
18334 %}
18335 ins_pipe(vshift128_imm);
18336 %}
18337
18338 instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{
18339 predicate(n->as_Vector()->length() == 2);
18340 match(Set dst (RShiftVL src (RShiftCntV shift)));
18341 ins_cost(INSN_COST);
18342 format %{ "sshr $dst, $src, $shift\t# vector (2D)" %}
18343 ins_encode %{
18344 __ sshr(as_FloatRegister($dst$$reg), __ T2D,
18345 as_FloatRegister($src$$reg),
18346 (int)$shift$$constant);
18347 %}
18348 ins_pipe(vshift128_imm);
18349 %}
18350
18351 instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{
18352 predicate(n->as_Vector()->length() == 2);
18353 match(Set dst (URShiftVL src (RShiftCntV shift)));
18354 ins_cost(INSN_COST);
18355 format %{ "ushr $dst, $src, $shift\t# vector (2D)" %}
18356 ins_encode %{
18357 __ ushr(as_FloatRegister($dst$$reg), __ T2D,
18358 as_FloatRegister($src$$reg),
18359 (int)$shift$$constant);
18360 %}
18361 ins_pipe(vshift128_imm);
18362 %}
18363
18364 instruct vmax2F(vecD dst, vecD src1, vecD src2)
18365 %{
18366 predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18367 match(Set dst (MaxV src1 src2));
18368 ins_cost(INSN_COST);
18369 format %{ "fmax $dst,$src1,$src2\t# vector (2F)" %}
18370 ins_encode %{
18371 __ fmax(as_FloatRegister($dst$$reg), __ T2S,
18372 as_FloatRegister($src1$$reg),
18373 as_FloatRegister($src2$$reg));
18374 %}
18375 ins_pipe(vdop_fp64);
18376 %}
18377
18378 instruct vmax4F(vecX dst, vecX src1, vecX src2)
18379 %{
18380 predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18381 match(Set dst (MaxV src1 src2));
18382 ins_cost(INSN_COST);
18383 format %{ "fmax $dst,$src1,$src2\t# vector (4S)" %}
18384 ins_encode %{
18385 __ fmax(as_FloatRegister($dst$$reg), __ T4S,
18386 as_FloatRegister($src1$$reg),
18387 as_FloatRegister($src2$$reg));
18388 %}
18389 ins_pipe(vdop_fp128);
18390 %}
18391
18392 instruct vmax2D(vecX dst, vecX src1, vecX src2)
18393 %{
18394 predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
18395 match(Set dst (MaxV src1 src2));
18396 ins_cost(INSN_COST);
18397 format %{ "fmax $dst,$src1,$src2\t# vector (2D)" %}
18398 ins_encode %{
18399 __ fmax(as_FloatRegister($dst$$reg), __ T2D,
18400 as_FloatRegister($src1$$reg),
18401 as_FloatRegister($src2$$reg));
18402 %}
18403 ins_pipe(vdop_fp128);
18404 %}
18405
18406 instruct vmin2F(vecD dst, vecD src1, vecD src2)
18407 %{
18408 predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18409 match(Set dst (MinV src1 src2));
18410 ins_cost(INSN_COST);
18411 format %{ "fmin $dst,$src1,$src2\t# vector (2F)" %}
18412 ins_encode %{
18413 __ fmin(as_FloatRegister($dst$$reg), __ T2S,
18414 as_FloatRegister($src1$$reg),
18415 as_FloatRegister($src2$$reg));
18416 %}
18417 ins_pipe(vdop_fp64);
18418 %}
18419
18420 instruct vmin4F(vecX dst, vecX src1, vecX src2)
18421 %{
18422 predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18423 match(Set dst (MinV src1 src2));
18424 ins_cost(INSN_COST);
18425 format %{ "fmin $dst,$src1,$src2\t# vector (4S)" %}
18426 ins_encode %{
18427 __ fmin(as_FloatRegister($dst$$reg), __ T4S,
18428 as_FloatRegister($src1$$reg),
18429 as_FloatRegister($src2$$reg));
18430 %}
18431 ins_pipe(vdop_fp128);
18432 %}
18433
18434 instruct vmin2D(vecX dst, vecX src1, vecX src2)
18435 %{
18436 predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
18437 match(Set dst (MinV src1 src2));
18438 ins_cost(INSN_COST);
18439 format %{ "fmin $dst,$src1,$src2\t# vector (2D)" %}
18440 ins_encode %{
18441 __ fmin(as_FloatRegister($dst$$reg), __ T2D,
18442 as_FloatRegister($src1$$reg),
18443 as_FloatRegister($src2$$reg));
18444 %}
18445 ins_pipe(vdop_fp128);
18446 %}
18447
18448 instruct vround2D_reg(vecX dst, vecX src, immI rmode) %{
18449 predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
18450 match(Set dst (RoundDoubleModeV src rmode));
18451 format %{ "frint $dst, $src, $rmode" %}
18452 ins_encode %{
18453 switch ($rmode$$constant) {
18454 case RoundDoubleModeNode::rmode_rint:
18455 __ frintn(as_FloatRegister($dst$$reg), __ T2D,
18456 as_FloatRegister($src$$reg));
18457 break;
18458 case RoundDoubleModeNode::rmode_floor:
18459 __ frintm(as_FloatRegister($dst$$reg), __ T2D,
18460 as_FloatRegister($src$$reg));
18461 break;
18462 case RoundDoubleModeNode::rmode_ceil:
18463 __ frintp(as_FloatRegister($dst$$reg), __ T2D,
18464 as_FloatRegister($src$$reg));
18465 break;
18466 }
18467 %}
18468 ins_pipe(vdop_fp128);
18469 %}
18470
18471 instruct vpopcount4I(vecX dst, vecX src) %{
18472 predicate(UsePopCountInstruction && n->as_Vector()->length() == 4);
18473 match(Set dst (PopCountVI src));
18474 format %{
18475 "cnt $dst, $src\t# vector (16B)\n\t"
18476 "uaddlp $dst, $dst\t# vector (16B)\n\t"
18477 "uaddlp $dst, $dst\t# vector (8H)"
18478 %}
18479 ins_encode %{
18480 __ cnt(as_FloatRegister($dst$$reg), __ T16B,
18481 as_FloatRegister($src$$reg));
18482 __ uaddlp(as_FloatRegister($dst$$reg), __ T16B,
18483 as_FloatRegister($dst$$reg));
18484 __ uaddlp(as_FloatRegister($dst$$reg), __ T8H,
18485 as_FloatRegister($dst$$reg));
18486 %}
18487 ins_pipe(pipe_class_default);
18488 %}
18489
18490 instruct vpopcount2I(vecD dst, vecD src) %{
18491 predicate(UsePopCountInstruction && n->as_Vector()->length() == 2);
18492 match(Set dst (PopCountVI src));
18493 format %{
18494 "cnt $dst, $src\t# vector (8B)\n\t"
18495 "uaddlp $dst, $dst\t# vector (8B)\n\t"
18496 "uaddlp $dst, $dst\t# vector (4H)"
18497 %}
18498 ins_encode %{
18499 __ cnt(as_FloatRegister($dst$$reg), __ T8B,
18500 as_FloatRegister($src$$reg));
18501 __ uaddlp(as_FloatRegister($dst$$reg), __ T8B,
18502 as_FloatRegister($dst$$reg));
18503 __ uaddlp(as_FloatRegister($dst$$reg), __ T4H,
18504 as_FloatRegister($dst$$reg));
18505 %}
18506 ins_pipe(pipe_class_default);
18507 %}
18508
18509 //----------PEEPHOLE RULES-----------------------------------------------------
18510 // These must follow all instruction definitions as they use the names
18511 // defined in the instructions definitions.
18512 //
18513 // peepmatch ( root_instr_name [preceding_instruction]* );
18514 //
18515 // peepconstraint %{
18516 // (instruction_number.operand_name relational_op instruction_number.operand_name
18517 // [, ...] );
18518 // // instruction numbers are zero-based using left to right order in peepmatch
18519 //
18520 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
18521 // // provide an instruction_number.operand_name for each operand that appears
18522 // // in the replacement instruction's match rule
18523 //
18524 // ---------VM FLAGS---------------------------------------------------------
18525 //
18526 // All peephole optimizations can be turned off using -XX:-OptoPeephole
18527 //
18528 // Each peephole rule is given an identifying number starting with zero and
18529 // increasing by one in the order seen by the parser. An individual peephole
18530 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
18531 // on the command-line.
18532 //
18533 // ---------CURRENT LIMITATIONS----------------------------------------------
18534 //
18535 // Only match adjacent instructions in same basic block
18536 // Only equality constraints
18537 // Only constraints between operands, not (0.dest_reg == RAX_enc)
18538 // Only one replacement instruction
18539 //
18540 // ---------EXAMPLE----------------------------------------------------------
18541 //
18542 // // pertinent parts of existing instructions in architecture description
18543 // instruct movI(iRegINoSp dst, iRegI src)
18544 // %{
18545 // match(Set dst (CopyI src));
18546 // %}
18547 //
18548 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
18549 // %{
18550 // match(Set dst (AddI dst src));
18551 // effect(KILL cr);
18552 // %}
18553 //
18554 // // Change (inc mov) to lea
18555 // peephole %{
18556 // // increment preceeded by register-register move
18557 // peepmatch ( incI_iReg movI );
18558 // // require that the destination register of the increment
18559 // // match the destination register of the move
18560 // peepconstraint ( 0.dst == 1.dst );
18561 // // construct a replacement instruction that sets
18562 // // the destination to ( move's source register + one )
18563 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
18564 // %}
18565 //
18566
18567 // Implementation no longer uses movX instructions since
18568 // machine-independent system no longer uses CopyX nodes.
18569 //
18570 // peephole
18571 // %{
18572 // peepmatch (incI_iReg movI);
18573 // peepconstraint (0.dst == 1.dst);
18574 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
18575 // %}
18576
18577 // peephole
18578 // %{
18579 // peepmatch (decI_iReg movI);
18580 // peepconstraint (0.dst == 1.dst);
18581 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
18582 // %}
18583
18584 // peephole
18585 // %{
18586 // peepmatch (addI_iReg_imm movI);
18587 // peepconstraint (0.dst == 1.dst);
18588 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
18589 // %}
18590
18591 // peephole
18592 // %{
18593 // peepmatch (incL_iReg movL);
18594 // peepconstraint (0.dst == 1.dst);
18595 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
18596 // %}
18597
18598 // peephole
18599 // %{
18600 // peepmatch (decL_iReg movL);
18601 // peepconstraint (0.dst == 1.dst);
18602 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
18603 // %}
18604
18605 // peephole
18606 // %{
18607 // peepmatch (addL_iReg_imm movL);
18608 // peepconstraint (0.dst == 1.dst);
18609 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
18610 // %}
18611
18612 // peephole
18613 // %{
18614 // peepmatch (addP_iReg_imm movP);
18615 // peepconstraint (0.dst == 1.dst);
18616 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
18617 // %}
18618
18619 // // Change load of spilled value to only a spill
18620 // instruct storeI(memory mem, iRegI src)
18621 // %{
18622 // match(Set mem (StoreI mem src));
18623 // %}
18624 //
18625 // instruct loadI(iRegINoSp dst, memory mem)
18626 // %{
18627 // match(Set dst (LoadI mem));
18628 // %}
18629 //
18630
18631 //----------SMARTSPILL RULES---------------------------------------------------
18632 // These must follow all instruction definitions as they use the names
18633 // defined in the instructions definitions.
18634
18635 // Local Variables:
18636 // mode: c++
18637 // End: