1 //
2 // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2019, Red Hat Inc.
4 // All rights reserved.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // archtecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 invisible to the allocator (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
99 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
100 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
101 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
102 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
103 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
104 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
105 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
106 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
107 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
108 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
109 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
110 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
111 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
112 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
113 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
114 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18->as_VMReg() );
115 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
116 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
117 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
118 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
119 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
120 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
121 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
122 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
123 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
124 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
125 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
126 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
127 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
128 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
129 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
130 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
131 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
132 reg_def R27 ( NS, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
133 reg_def R27_H ( NS, SOE, Op_RegI, 27, r27->as_VMReg()->next());
134 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
135 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
136 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
137 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
138 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
139 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
140 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
141 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
142
143 // ----------------------------
144 // Float/Double Registers
145 // ----------------------------
146
147 // Double Registers
148
149 // The rules of ADL require that double registers be defined in pairs.
150 // Each pair must be two 32-bit values, but not necessarily a pair of
151 // single float registers. In each pair, ADLC-assigned register numbers
152 // must be adjacent, with the lower number even. Finally, when the
153 // CPU stores such a register pair to memory, the word associated with
154 // the lower ADLC-assigned number must be stored to the lower address.
155
156 // AArch64 has 32 floating-point registers. Each can store a vector of
157 // single or double precision floating-point values up to 8 * 32
158 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
159 // use the first float or double element of the vector.
160
161 // for Java use float registers v0-v15 are always save on call whereas
162 // the platform ABI treats v8-v15 as callee save). float registers
163 // v16-v31 are SOC as per the platform spec
164
165 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
166 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
167 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
168 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
169
170 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
171 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
172 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
173 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
174
175 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
176 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
177 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
178 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
179
180 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
181 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
182 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
183 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
184
185 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
186 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
187 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
188 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
189
190 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
191 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
192 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
193 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
194
195 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
196 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
197 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
198 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
199
200 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
201 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
202 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
203 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
204
205 reg_def V8 ( SOC, SOC, Op_RegF, 8, v8->as_VMReg() );
206 reg_def V8_H ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next() );
207 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
208 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
209
210 reg_def V9 ( SOC, SOC, Op_RegF, 9, v9->as_VMReg() );
211 reg_def V9_H ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next() );
212 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
213 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
214
215 reg_def V10 ( SOC, SOC, Op_RegF, 10, v10->as_VMReg() );
216 reg_def V10_H( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next() );
217 reg_def V10_J( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2));
218 reg_def V10_K( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3));
219
220 reg_def V11 ( SOC, SOC, Op_RegF, 11, v11->as_VMReg() );
221 reg_def V11_H( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next() );
222 reg_def V11_J( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2));
223 reg_def V11_K( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3));
224
225 reg_def V12 ( SOC, SOC, Op_RegF, 12, v12->as_VMReg() );
226 reg_def V12_H( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next() );
227 reg_def V12_J( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2));
228 reg_def V12_K( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3));
229
230 reg_def V13 ( SOC, SOC, Op_RegF, 13, v13->as_VMReg() );
231 reg_def V13_H( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next() );
232 reg_def V13_J( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2));
233 reg_def V13_K( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3));
234
235 reg_def V14 ( SOC, SOC, Op_RegF, 14, v14->as_VMReg() );
236 reg_def V14_H( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next() );
237 reg_def V14_J( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2));
238 reg_def V14_K( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3));
239
240 reg_def V15 ( SOC, SOC, Op_RegF, 15, v15->as_VMReg() );
241 reg_def V15_H( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next() );
242 reg_def V15_J( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2));
243 reg_def V15_K( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3));
244
245 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
246 reg_def V16_H( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
247 reg_def V16_J( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2));
248 reg_def V16_K( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3));
249
250 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
251 reg_def V17_H( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
252 reg_def V17_J( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2));
253 reg_def V17_K( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3));
254
255 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
256 reg_def V18_H( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
257 reg_def V18_J( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2));
258 reg_def V18_K( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3));
259
260 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
261 reg_def V19_H( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
262 reg_def V19_J( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2));
263 reg_def V19_K( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3));
264
265 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
266 reg_def V20_H( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
267 reg_def V20_J( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2));
268 reg_def V20_K( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3));
269
270 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
271 reg_def V21_H( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
272 reg_def V21_J( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2));
273 reg_def V21_K( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3));
274
275 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
276 reg_def V22_H( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
277 reg_def V22_J( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2));
278 reg_def V22_K( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3));
279
280 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
281 reg_def V23_H( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
282 reg_def V23_J( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2));
283 reg_def V23_K( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3));
284
285 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
286 reg_def V24_H( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
287 reg_def V24_J( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2));
288 reg_def V24_K( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3));
289
290 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
291 reg_def V25_H( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
292 reg_def V25_J( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2));
293 reg_def V25_K( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3));
294
295 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
296 reg_def V26_H( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
297 reg_def V26_J( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2));
298 reg_def V26_K( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3));
299
300 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
301 reg_def V27_H( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
302 reg_def V27_J( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2));
303 reg_def V27_K( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3));
304
305 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
306 reg_def V28_H( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
307 reg_def V28_J( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2));
308 reg_def V28_K( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3));
309
310 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
311 reg_def V29_H( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
312 reg_def V29_J( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2));
313 reg_def V29_K( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3));
314
315 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
316 reg_def V30_H( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
317 reg_def V30_J( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2));
318 reg_def V30_K( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3));
319
320 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
321 reg_def V31_H( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
322 reg_def V31_J( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2));
323 reg_def V31_K( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3));
324
325 // ----------------------------
326 // Special Registers
327 // ----------------------------
328
329 // the AArch64 CSPR status flag register is not directly acessible as
330 // instruction operand. the FPSR status flag register is a system
331 // register which can be written/read using MSR/MRS but again does not
332 // appear as an operand (a code identifying the FSPR occurs as an
333 // immediate value in the instruction).
334
335 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
336
337
338 // Specify priority of register selection within phases of register
339 // allocation. Highest priority is first. A useful heuristic is to
340 // give registers a low priority when they are required by machine
341 // instructions, like EAX and EDX on I486, and choose no-save registers
342 // before save-on-call, & save-on-call before save-on-entry. Registers
343 // which participate in fixed calling sequences should come last.
344 // Registers which are used as pairs must fall on an even boundary.
345
346 alloc_class chunk0(
347 // volatiles
348 R10, R10_H,
349 R11, R11_H,
350 R12, R12_H,
351 R13, R13_H,
352 R14, R14_H,
353 R15, R15_H,
354 R16, R16_H,
355 R17, R17_H,
356 R18, R18_H,
357
358 // arg registers
359 R0, R0_H,
360 R1, R1_H,
361 R2, R2_H,
362 R3, R3_H,
363 R4, R4_H,
364 R5, R5_H,
365 R6, R6_H,
366 R7, R7_H,
367
368 // non-volatiles
369 R19, R19_H,
370 R20, R20_H,
371 R21, R21_H,
372 R22, R22_H,
373 R23, R23_H,
374 R24, R24_H,
375 R25, R25_H,
376 R26, R26_H,
377
378 // non-allocatable registers
379
380 R27, R27_H, // heapbase
381 R28, R28_H, // thread
382 R29, R29_H, // fp
383 R30, R30_H, // lr
384 R31, R31_H, // sp
385 );
386
387 alloc_class chunk1(
388
389 // no save
390 V16, V16_H, V16_J, V16_K,
391 V17, V17_H, V17_J, V17_K,
392 V18, V18_H, V18_J, V18_K,
393 V19, V19_H, V19_J, V19_K,
394 V20, V20_H, V20_J, V20_K,
395 V21, V21_H, V21_J, V21_K,
396 V22, V22_H, V22_J, V22_K,
397 V23, V23_H, V23_J, V23_K,
398 V24, V24_H, V24_J, V24_K,
399 V25, V25_H, V25_J, V25_K,
400 V26, V26_H, V26_J, V26_K,
401 V27, V27_H, V27_J, V27_K,
402 V28, V28_H, V28_J, V28_K,
403 V29, V29_H, V29_J, V29_K,
404 V30, V30_H, V30_J, V30_K,
405 V31, V31_H, V31_J, V31_K,
406
407 // arg registers
408 V0, V0_H, V0_J, V0_K,
409 V1, V1_H, V1_J, V1_K,
410 V2, V2_H, V2_J, V2_K,
411 V3, V3_H, V3_J, V3_K,
412 V4, V4_H, V4_J, V4_K,
413 V5, V5_H, V5_J, V5_K,
414 V6, V6_H, V6_J, V6_K,
415 V7, V7_H, V7_J, V7_K,
416
417 // non-volatiles
418 V8, V8_H, V8_J, V8_K,
419 V9, V9_H, V9_J, V9_K,
420 V10, V10_H, V10_J, V10_K,
421 V11, V11_H, V11_J, V11_K,
422 V12, V12_H, V12_J, V12_K,
423 V13, V13_H, V13_J, V13_K,
424 V14, V14_H, V14_J, V14_K,
425 V15, V15_H, V15_J, V15_K,
426 );
427
428 alloc_class chunk2(RFLAGS);
429
430 //----------Architecture Description Register Classes--------------------------
431 // Several register classes are automatically defined based upon information in
432 // this architecture description.
433 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
434 // 2) reg_class compiler_method_oop_reg ( /* as def'd in frame section */ )
435 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
436 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
437 //
438
439 // Class for all 32 bit integer registers -- excludes SP which will
440 // never be used as an integer register
441 reg_class any_reg32(
442 R0,
443 R1,
444 R2,
445 R3,
446 R4,
447 R5,
448 R6,
449 R7,
450 R10,
451 R11,
452 R12,
453 R13,
454 R14,
455 R15,
456 R16,
457 R17,
458 R18,
459 R19,
460 R20,
461 R21,
462 R22,
463 R23,
464 R24,
465 R25,
466 R26,
467 R27,
468 R28,
469 R29,
470 R30
471 );
472
473 // Singleton class for R0 int register
474 reg_class int_r0_reg(R0);
475
476 // Singleton class for R2 int register
477 reg_class int_r2_reg(R2);
478
479 // Singleton class for R3 int register
480 reg_class int_r3_reg(R3);
481
482 // Singleton class for R4 int register
483 reg_class int_r4_reg(R4);
484
485 // Class for all long integer registers (including RSP)
486 reg_class any_reg(
487 R0, R0_H,
488 R1, R1_H,
489 R2, R2_H,
490 R3, R3_H,
491 R4, R4_H,
492 R5, R5_H,
493 R6, R6_H,
494 R7, R7_H,
495 R10, R10_H,
496 R11, R11_H,
497 R12, R12_H,
498 R13, R13_H,
499 R14, R14_H,
500 R15, R15_H,
501 R16, R16_H,
502 R17, R17_H,
503 R18, R18_H,
504 R19, R19_H,
505 R20, R20_H,
506 R21, R21_H,
507 R22, R22_H,
508 R23, R23_H,
509 R24, R24_H,
510 R25, R25_H,
511 R26, R26_H,
512 R27, R27_H,
513 R28, R28_H,
514 R29, R29_H,
515 R30, R30_H,
516 R31, R31_H
517 );
518
519 // Class for all non-special integer registers
520 reg_class no_special_reg32(
521 R0,
522 R1,
523 R2,
524 R3,
525 R4,
526 R5,
527 R6,
528 R7,
529 R10,
530 R11,
531 R12, // rmethod
532 R13,
533 R14,
534 R15,
535 R16,
536 R17,
537 R18,
538 R19,
539 R20,
540 R21,
541 R22,
542 R23,
543 R24,
544 R25,
545 R26
546 /* R27, */ // heapbase
547 /* R28, */ // thread
548 /* R29, */ // fp
549 /* R30, */ // lr
550 /* R31 */ // sp
551 );
552
553 // Class for all non-special long integer registers
554 reg_class no_special_reg(
555 R0, R0_H,
556 R1, R1_H,
557 R2, R2_H,
558 R3, R3_H,
559 R4, R4_H,
560 R5, R5_H,
561 R6, R6_H,
562 R7, R7_H,
563 R10, R10_H,
564 R11, R11_H,
565 R12, R12_H, // rmethod
566 R13, R13_H,
567 R14, R14_H,
568 R15, R15_H,
569 R16, R16_H,
570 R17, R17_H,
571 R18, R18_H,
572 R19, R19_H,
573 R20, R20_H,
574 R21, R21_H,
575 R22, R22_H,
576 R23, R23_H,
577 R24, R24_H,
578 R25, R25_H,
579 R26, R26_H,
580 /* R27, R27_H, */ // heapbase
581 /* R28, R28_H, */ // thread
582 /* R29, R29_H, */ // fp
583 /* R30, R30_H, */ // lr
584 /* R31, R31_H */ // sp
585 );
586
587 // Class for 64 bit register r0
588 reg_class r0_reg(
589 R0, R0_H
590 );
591
592 // Class for 64 bit register r1
593 reg_class r1_reg(
594 R1, R1_H
595 );
596
597 // Class for 64 bit register r2
598 reg_class r2_reg(
599 R2, R2_H
600 );
601
602 // Class for 64 bit register r3
603 reg_class r3_reg(
604 R3, R3_H
605 );
606
607 // Class for 64 bit register r4
608 reg_class r4_reg(
609 R4, R4_H
610 );
611
612 // Class for 64 bit register r5
613 reg_class r5_reg(
614 R5, R5_H
615 );
616
617 // Class for 64 bit register r10
618 reg_class r10_reg(
619 R10, R10_H
620 );
621
622 // Class for 64 bit register r11
623 reg_class r11_reg(
624 R11, R11_H
625 );
626
627 // Class for method register
628 reg_class method_reg(
629 R12, R12_H
630 );
631
632 // Class for heapbase register
633 reg_class heapbase_reg(
634 R27, R27_H
635 );
636
637 // Class for thread register
638 reg_class thread_reg(
639 R28, R28_H
640 );
641
642 // Class for frame pointer register
643 reg_class fp_reg(
644 R29, R29_H
645 );
646
647 // Class for link register
648 reg_class lr_reg(
649 R30, R30_H
650 );
651
652 // Class for long sp register
653 reg_class sp_reg(
654 R31, R31_H
655 );
656
657 // Class for all pointer registers
658 reg_class ptr_reg(
659 R0, R0_H,
660 R1, R1_H,
661 R2, R2_H,
662 R3, R3_H,
663 R4, R4_H,
664 R5, R5_H,
665 R6, R6_H,
666 R7, R7_H,
667 R10, R10_H,
668 R11, R11_H,
669 R12, R12_H,
670 R13, R13_H,
671 R14, R14_H,
672 R15, R15_H,
673 R16, R16_H,
674 R17, R17_H,
675 R18, R18_H,
676 R19, R19_H,
677 R20, R20_H,
678 R21, R21_H,
679 R22, R22_H,
680 R23, R23_H,
681 R24, R24_H,
682 R25, R25_H,
683 R26, R26_H,
684 R27, R27_H,
685 R28, R28_H,
686 R29, R29_H,
687 R30, R30_H,
688 R31, R31_H
689 );
690
691 // Class for all non_special pointer registers
692 reg_class no_special_ptr_reg(
693 R0, R0_H,
694 R1, R1_H,
695 R2, R2_H,
696 R3, R3_H,
697 R4, R4_H,
698 R5, R5_H,
699 R6, R6_H,
700 R7, R7_H,
701 R10, R10_H,
702 R11, R11_H,
703 R12, R12_H,
704 R13, R13_H,
705 R14, R14_H,
706 R15, R15_H,
707 R16, R16_H,
708 R17, R17_H,
709 R18, R18_H,
710 R19, R19_H,
711 R20, R20_H,
712 R21, R21_H,
713 R22, R22_H,
714 R23, R23_H,
715 R24, R24_H,
716 R25, R25_H,
717 R26, R26_H,
718 /* R27, R27_H, */ // heapbase
719 /* R28, R28_H, */ // thread
720 /* R29, R29_H, */ // fp
721 /* R30, R30_H, */ // lr
722 /* R31, R31_H */ // sp
723 );
724
725 // Class for all float registers
726 reg_class float_reg(
727 V0,
728 V1,
729 V2,
730 V3,
731 V4,
732 V5,
733 V6,
734 V7,
735 V8,
736 V9,
737 V10,
738 V11,
739 V12,
740 V13,
741 V14,
742 V15,
743 V16,
744 V17,
745 V18,
746 V19,
747 V20,
748 V21,
749 V22,
750 V23,
751 V24,
752 V25,
753 V26,
754 V27,
755 V28,
756 V29,
757 V30,
758 V31
759 );
760
761 // Double precision float registers have virtual `high halves' that
762 // are needed by the allocator.
763 // Class for all double registers
764 reg_class double_reg(
765 V0, V0_H,
766 V1, V1_H,
767 V2, V2_H,
768 V3, V3_H,
769 V4, V4_H,
770 V5, V5_H,
771 V6, V6_H,
772 V7, V7_H,
773 V8, V8_H,
774 V9, V9_H,
775 V10, V10_H,
776 V11, V11_H,
777 V12, V12_H,
778 V13, V13_H,
779 V14, V14_H,
780 V15, V15_H,
781 V16, V16_H,
782 V17, V17_H,
783 V18, V18_H,
784 V19, V19_H,
785 V20, V20_H,
786 V21, V21_H,
787 V22, V22_H,
788 V23, V23_H,
789 V24, V24_H,
790 V25, V25_H,
791 V26, V26_H,
792 V27, V27_H,
793 V28, V28_H,
794 V29, V29_H,
795 V30, V30_H,
796 V31, V31_H
797 );
798
799 // Class for all 64bit vector registers
800 reg_class vectord_reg(
801 V0, V0_H,
802 V1, V1_H,
803 V2, V2_H,
804 V3, V3_H,
805 V4, V4_H,
806 V5, V5_H,
807 V6, V6_H,
808 V7, V7_H,
809 V8, V8_H,
810 V9, V9_H,
811 V10, V10_H,
812 V11, V11_H,
813 V12, V12_H,
814 V13, V13_H,
815 V14, V14_H,
816 V15, V15_H,
817 V16, V16_H,
818 V17, V17_H,
819 V18, V18_H,
820 V19, V19_H,
821 V20, V20_H,
822 V21, V21_H,
823 V22, V22_H,
824 V23, V23_H,
825 V24, V24_H,
826 V25, V25_H,
827 V26, V26_H,
828 V27, V27_H,
829 V28, V28_H,
830 V29, V29_H,
831 V30, V30_H,
832 V31, V31_H
833 );
834
835 // Class for all 128bit vector registers
836 reg_class vectorx_reg(
837 V0, V0_H, V0_J, V0_K,
838 V1, V1_H, V1_J, V1_K,
839 V2, V2_H, V2_J, V2_K,
840 V3, V3_H, V3_J, V3_K,
841 V4, V4_H, V4_J, V4_K,
842 V5, V5_H, V5_J, V5_K,
843 V6, V6_H, V6_J, V6_K,
844 V7, V7_H, V7_J, V7_K,
845 V8, V8_H, V8_J, V8_K,
846 V9, V9_H, V9_J, V9_K,
847 V10, V10_H, V10_J, V10_K,
848 V11, V11_H, V11_J, V11_K,
849 V12, V12_H, V12_J, V12_K,
850 V13, V13_H, V13_J, V13_K,
851 V14, V14_H, V14_J, V14_K,
852 V15, V15_H, V15_J, V15_K,
853 V16, V16_H, V16_J, V16_K,
854 V17, V17_H, V17_J, V17_K,
855 V18, V18_H, V18_J, V18_K,
856 V19, V19_H, V19_J, V19_K,
857 V20, V20_H, V20_J, V20_K,
858 V21, V21_H, V21_J, V21_K,
859 V22, V22_H, V22_J, V22_K,
860 V23, V23_H, V23_J, V23_K,
861 V24, V24_H, V24_J, V24_K,
862 V25, V25_H, V25_J, V25_K,
863 V26, V26_H, V26_J, V26_K,
864 V27, V27_H, V27_J, V27_K,
865 V28, V28_H, V28_J, V28_K,
866 V29, V29_H, V29_J, V29_K,
867 V30, V30_H, V30_J, V30_K,
868 V31, V31_H, V31_J, V31_K
869 );
870
871 // Class for 128 bit register v0
872 reg_class v0_reg(
873 V0, V0_H
874 );
875
876 // Class for 128 bit register v1
877 reg_class v1_reg(
878 V1, V1_H
879 );
880
881 // Class for 128 bit register v2
882 reg_class v2_reg(
883 V2, V2_H
884 );
885
886 // Class for 128 bit register v3
887 reg_class v3_reg(
888 V3, V3_H
889 );
890
891 // Singleton class for condition codes
892 reg_class int_flags(RFLAGS);
893
894 %}
895
896 //----------DEFINITION BLOCK---------------------------------------------------
897 // Define name --> value mappings to inform the ADLC of an integer valued name
898 // Current support includes integer values in the range [0, 0x7FFFFFFF]
899 // Format:
900 // int_def <name> ( <int_value>, <expression>);
901 // Generated Code in ad_<arch>.hpp
902 // #define <name> (<expression>)
903 // // value == <int_value>
904 // Generated code in ad_<arch>.cpp adlc_verification()
905 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
906 //
907
908 // we follow the ppc-aix port in using a simple cost model which ranks
909 // register operations as cheap, memory ops as more expensive and
910 // branches as most expensive. the first two have a low as well as a
911 // normal cost. huge cost appears to be a way of saying don't do
912 // something
913
914 definitions %{
915 // The default cost (of a register move instruction).
916 int_def INSN_COST ( 100, 100);
917 int_def BRANCH_COST ( 200, 2 * INSN_COST);
918 int_def CALL_COST ( 200, 2 * INSN_COST);
919 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
920 %}
921
922
923 //----------SOURCE BLOCK-------------------------------------------------------
924 // This is a block of C++ code which provides values, functions, and
925 // definitions necessary in the rest of the architecture description
926
927 source_hpp %{
928
929 #if INCLUDE_ALL_GCS
930 #include "shenandoahBarrierSetAssembler_aarch64.hpp"
931 #endif
932
933 class CallStubImpl {
934
935 //--------------------------------------------------------------
936 //---< Used for optimization in Compile::shorten_branches >---
937 //--------------------------------------------------------------
938
939 public:
940 // Size of call trampoline stub.
941 static uint size_call_trampoline() {
942 return 0; // no call trampolines on this platform
943 }
944
945 // number of relocations needed by a call trampoline stub
946 static uint reloc_call_trampoline() {
947 return 0; // no call trampolines on this platform
948 }
949 };
950
951 class HandlerImpl {
952
953 public:
954
955 static int emit_exception_handler(CodeBuffer &cbuf);
956 static int emit_deopt_handler(CodeBuffer& cbuf);
957
958 static uint size_exception_handler() {
959 return MacroAssembler::far_branch_size();
960 }
961
962 static uint size_deopt_handler() {
963 // count one adr and one far branch instruction
964 // return 4 * NativeInstruction::instruction_size;
965 return NativeInstruction::instruction_size + MacroAssembler::far_branch_size();
966 }
967 };
968
969 bool is_CAS(int opcode);
970
971 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
972
973 bool unnecessary_acquire(const Node *barrier);
974 bool needs_acquiring_load(const Node *load);
975
976 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
977
978 bool unnecessary_release(const Node *barrier);
979 bool unnecessary_volatile(const Node *barrier);
980 bool needs_releasing_store(const Node *store);
981
982 // predicate controlling translation of CompareAndSwapX
983 bool needs_acquiring_load_exclusive(const Node *load);
984
985 // predicate controlling translation of StoreCM
986 bool unnecessary_storestore(const Node *storecm);
987 %}
988
989 source %{
990
991 // Optimizaton of volatile gets and puts
992 // -------------------------------------
993 //
994 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
995 // use to implement volatile reads and writes. For a volatile read
996 // we simply need
997 //
998 // ldar<x>
999 //
1000 // and for a volatile write we need
1001 //
1002 // stlr<x>
1003 //
1004 // Alternatively, we can implement them by pairing a normal
1005 // load/store with a memory barrier. For a volatile read we need
1006 //
1007 // ldr<x>
1008 // dmb ishld
1009 //
1010 // for a volatile write
1011 //
1012 // dmb ish
1013 // str<x>
1014 // dmb ish
1015 //
1016 // We can also use ldaxr and stlxr to implement compare and swap CAS
1017 // sequences. These are normally translated to an instruction
1018 // sequence like the following
1019 //
1020 // dmb ish
1021 // retry:
1022 // ldxr<x> rval raddr
1023 // cmp rval rold
1024 // b.ne done
1025 // stlxr<x> rval, rnew, rold
1026 // cbnz rval retry
1027 // done:
1028 // cset r0, eq
1029 // dmb ishld
1030 //
1031 // Note that the exclusive store is already using an stlxr
1032 // instruction. That is required to ensure visibility to other
1033 // threads of the exclusive write (assuming it succeeds) before that
1034 // of any subsequent writes.
1035 //
1036 // The following instruction sequence is an improvement on the above
1037 //
1038 // retry:
1039 // ldaxr<x> rval raddr
1040 // cmp rval rold
1041 // b.ne done
1042 // stlxr<x> rval, rnew, rold
1043 // cbnz rval retry
1044 // done:
1045 // cset r0, eq
1046 //
1047 // We don't need the leading dmb ish since the stlxr guarantees
1048 // visibility of prior writes in the case that the swap is
1049 // successful. Crucially we don't have to worry about the case where
1050 // the swap is not successful since no valid program should be
1051 // relying on visibility of prior changes by the attempting thread
1052 // in the case where the CAS fails.
1053 //
1054 // Similarly, we don't need the trailing dmb ishld if we substitute
1055 // an ldaxr instruction since that will provide all the guarantees we
1056 // require regarding observation of changes made by other threads
1057 // before any change to the CAS address observed by the load.
1058 //
1059 // In order to generate the desired instruction sequence we need to
1060 // be able to identify specific 'signature' ideal graph node
1061 // sequences which i) occur as a translation of a volatile reads or
1062 // writes or CAS operations and ii) do not occur through any other
1063 // translation or graph transformation. We can then provide
1064 // alternative aldc matching rules which translate these node
1065 // sequences to the desired machine code sequences. Selection of the
1066 // alternative rules can be implemented by predicates which identify
1067 // the relevant node sequences.
1068 //
1069 // The ideal graph generator translates a volatile read to the node
1070 // sequence
1071 //
1072 // LoadX[mo_acquire]
1073 // MemBarAcquire
1074 //
1075 // As a special case when using the compressed oops optimization we
1076 // may also see this variant
1077 //
1078 // LoadN[mo_acquire]
1079 // DecodeN
1080 // MemBarAcquire
1081 //
1082 // A volatile write is translated to the node sequence
1083 //
1084 // MemBarRelease
1085 // StoreX[mo_release] {CardMark}-optional
1086 // MemBarVolatile
1087 //
1088 // n.b. the above node patterns are generated with a strict
1089 // 'signature' configuration of input and output dependencies (see
1090 // the predicates below for exact details). The card mark may be as
1091 // simple as a few extra nodes or, in a few GC configurations, may
1092 // include more complex control flow between the leading and
1093 // trailing memory barriers. However, whatever the card mark
1094 // configuration these signatures are unique to translated volatile
1095 // reads/stores -- they will not appear as a result of any other
1096 // bytecode translation or inlining nor as a consequence of
1097 // optimizing transforms.
1098 //
1099 // We also want to catch inlined unsafe volatile gets and puts and
1100 // be able to implement them using either ldar<x>/stlr<x> or some
1101 // combination of ldr<x>/stlr<x> and dmb instructions.
1102 //
1103 // Inlined unsafe volatiles puts manifest as a minor variant of the
1104 // normal volatile put node sequence containing an extra cpuorder
1105 // membar
1106 //
1107 // MemBarRelease
1108 // MemBarCPUOrder
1109 // StoreX[mo_release] {CardMark}-optional
1110 // MemBarVolatile
1111 //
1112 // n.b. as an aside, the cpuorder membar is not itself subject to
1113 // matching and translation by adlc rules. However, the rule
1114 // predicates need to detect its presence in order to correctly
1115 // select the desired adlc rules.
1116 //
1117 // Inlined unsafe volatile gets manifest as a somewhat different
1118 // node sequence to a normal volatile get
1119 //
1120 // MemBarCPUOrder
1121 // || \\
1122 // MemBarAcquire LoadX[mo_acquire]
1123 // ||
1124 // MemBarCPUOrder
1125 //
1126 // In this case the acquire membar does not directly depend on the
1127 // load. However, we can be sure that the load is generated from an
1128 // inlined unsafe volatile get if we see it dependent on this unique
1129 // sequence of membar nodes. Similarly, given an acquire membar we
1130 // can know that it was added because of an inlined unsafe volatile
1131 // get if it is fed and feeds a cpuorder membar and if its feed
1132 // membar also feeds an acquiring load.
1133 //
1134 // Finally an inlined (Unsafe) CAS operation is translated to the
1135 // following ideal graph
1136 //
1137 // MemBarRelease
1138 // MemBarCPUOrder
1139 // CompareAndSwapX {CardMark}-optional
1140 // MemBarCPUOrder
1141 // MemBarAcquire
1142 //
1143 // So, where we can identify these volatile read and write
1144 // signatures we can choose to plant either of the above two code
1145 // sequences. For a volatile read we can simply plant a normal
1146 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1147 // also choose to inhibit translation of the MemBarAcquire and
1148 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1149 //
1150 // When we recognise a volatile store signature we can choose to
1151 // plant at a dmb ish as a translation for the MemBarRelease, a
1152 // normal str<x> and then a dmb ish for the MemBarVolatile.
1153 // Alternatively, we can inhibit translation of the MemBarRelease
1154 // and MemBarVolatile and instead plant a simple stlr<x>
1155 // instruction.
1156 //
1157 // when we recognise a CAS signature we can choose to plant a dmb
1158 // ish as a translation for the MemBarRelease, the conventional
1159 // macro-instruction sequence for the CompareAndSwap node (which
1160 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1161 // Alternatively, we can elide generation of the dmb instructions
1162 // and plant the alternative CompareAndSwap macro-instruction
1163 // sequence (which uses ldaxr<x>).
1164 //
1165 // Of course, the above only applies when we see these signature
1166 // configurations. We still want to plant dmb instructions in any
1167 // other cases where we may see a MemBarAcquire, MemBarRelease or
1168 // MemBarVolatile. For example, at the end of a constructor which
1169 // writes final/volatile fields we will see a MemBarRelease
1170 // instruction and this needs a 'dmb ish' lest we risk the
1171 // constructed object being visible without making the
1172 // final/volatile field writes visible.
1173 //
1174 // n.b. the translation rules below which rely on detection of the
1175 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1176 // If we see anything other than the signature configurations we
1177 // always just translate the loads and stores to ldr<x> and str<x>
1178 // and translate acquire, release and volatile membars to the
1179 // relevant dmb instructions.
1180 //
1181
1182 // is_CAS(int opcode)
1183 //
1184 // return true if opcode is one of the possible CompareAndSwapX
1185 // values otherwise false.
1186
1187 bool is_CAS(int opcode)
1188 {
1189 switch(opcode) {
1190 // We handle these
1191 case Op_CompareAndSwapI:
1192 case Op_CompareAndSwapL:
1193 case Op_CompareAndSwapP:
1194 case Op_CompareAndSwapN:
1195 case Op_GetAndSetI:
1196 case Op_GetAndSetL:
1197 case Op_GetAndSetP:
1198 case Op_GetAndSetN:
1199 case Op_GetAndAddI:
1200 case Op_GetAndAddL:
1201 return true;
1202 default:
1203 return false;
1204 }
1205 }
1206
1207 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1208
1209 bool unnecessary_acquire(const Node *barrier)
1210 {
1211 assert(barrier->is_MemBar(), "expecting a membar");
1212
1213 if (UseBarriersForVolatile) {
1214 // we need to plant a dmb
1215 return false;
1216 }
1217
1218 MemBarNode* mb = barrier->as_MemBar();
1219
1220 if (mb->trailing_load()) {
1221 return true;
1222 }
1223
1224 if (mb->trailing_load_store()) {
1225 Node* load_store = mb->in(MemBarNode::Precedent);
1226 assert(load_store->is_LoadStore(), "unexpected graph shape");
1227 return is_CAS(load_store->Opcode());
1228 }
1229
1230 return false;
1231 }
1232
1233 bool needs_acquiring_load(const Node *n)
1234 {
1235 assert(n->is_Load(), "expecting a load");
1236 if (UseBarriersForVolatile) {
1237 // we use a normal load and a dmb
1238 return false;
1239 }
1240
1241 LoadNode *ld = n->as_Load();
1242
1243 return ld->is_acquire();
1244 }
1245
1246 bool unnecessary_release(const Node *n)
1247 {
1248 assert((n->is_MemBar() &&
1249 n->Opcode() == Op_MemBarRelease),
1250 "expecting a release membar");
1251
1252 if (UseBarriersForVolatile) {
1253 // we need to plant a dmb
1254 return false;
1255 }
1256
1257 MemBarNode *barrier = n->as_MemBar();
1258
1259 if (!barrier->leading()) {
1260 return false;
1261 } else {
1262 Node* trailing = barrier->trailing_membar();
1263 MemBarNode* trailing_mb = trailing->as_MemBar();
1264 assert(trailing_mb->trailing(), "Not a trailing membar?");
1265 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1266
1267 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1268 if (mem->is_Store()) {
1269 assert(mem->as_Store()->is_release(), "");
1270 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1271 return true;
1272 } else {
1273 assert(mem->is_LoadStore(), "");
1274 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1275 return is_CAS(mem->Opcode());
1276 }
1277 }
1278
1279 return false;
1280 }
1281
1282 bool unnecessary_volatile(const Node *n)
1283 {
1284 // assert n->is_MemBar();
1285 if (UseBarriersForVolatile) {
1286 // we need to plant a dmb
1287 return false;
1288 }
1289
1290 MemBarNode *mbvol = n->as_MemBar();
1291
1292 bool release = mbvol->trailing_store();
1293 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1294 #ifdef ASSERT
1295 if (release) {
1296 Node* leading = mbvol->leading_membar();
1297 assert(leading->Opcode() == Op_MemBarRelease, "");
1298 assert(leading->as_MemBar()->leading_store(), "");
1299 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1300 }
1301 #endif
1302
1303 return release;
1304 }
1305
1306 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1307
1308 bool needs_releasing_store(const Node *n)
1309 {
1310 // assert n->is_Store();
1311 if (UseBarriersForVolatile) {
1312 // we use a normal store and dmb combination
1313 return false;
1314 }
1315
1316 StoreNode *st = n->as_Store();
1317
1318 return st->trailing_membar() != NULL;
1319 }
1320
1321 // predicate controlling translation of CAS
1322 //
1323 // returns true if CAS needs to use an acquiring load otherwise false
1324
1325 bool needs_acquiring_load_exclusive(const Node *n)
1326 {
1327 assert(is_CAS(n->Opcode()), "expecting a compare and swap");
1328 if (UseBarriersForVolatile) {
1329 return false;
1330 }
1331
1332 LoadStoreNode* ldst = n->as_LoadStore();
1333 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1334
1335 // so we can just return true here
1336 return true;
1337 }
1338
1339 // predicate controlling translation of StoreCM
1340 //
1341 // returns true if a StoreStore must precede the card write otherwise
1342 // false
1343
1344 bool unnecessary_storestore(const Node *storecm)
1345 {
1346 assert(storecm->Opcode() == Op_StoreCM, "expecting a StoreCM");
1347
1348 // we need to generate a dmb ishst between an object put and the
1349 // associated card mark when we are using CMS without conditional
1350 // card marking
1351
1352 if (UseConcMarkSweepGC && !UseCondCardMark) {
1353 return false;
1354 }
1355
1356 // a storestore is unnecesary in all other cases
1357
1358 return true;
1359 }
1360
1361
1362 #define __ _masm.
1363
1364 // advance declaratuons for helper functions to convert register
1365 // indices to register objects
1366
1367 // the ad file has to provide implementations of certain methods
1368 // expected by the generic code
1369 //
1370 // REQUIRED FUNCTIONALITY
1371
1372 //=============================================================================
1373
1374 // !!!!! Special hack to get all types of calls to specify the byte offset
1375 // from the start of the call to the point where the return address
1376 // will point.
1377
1378 int MachCallStaticJavaNode::ret_addr_offset()
1379 {
1380 // call should be a simple bl
1381 // unless this is a method handle invoke in which case it is
1382 // mov(rfp, sp), bl, mov(sp, rfp)
1383 int off = 4;
1384 if (_method_handle_invoke) {
1385 off += 4;
1386 }
1387 return off;
1388 }
1389
1390 int MachCallDynamicJavaNode::ret_addr_offset()
1391 {
1392 return 16; // movz, movk, movk, bl
1393 }
1394
1395 int MachCallRuntimeNode::ret_addr_offset() {
1396 // for generated stubs the call will be
1397 // bl(addr)
1398 // for real runtime callouts it will be six instructions
1399 // see aarch64_enc_java_to_runtime
1400 // adr(rscratch2, retaddr)
1401 // lea(rscratch1, RuntimeAddress(addr)
1402 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1403 // blr(rscratch1)
1404 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1405 if (cb) {
1406 return MacroAssembler::far_branch_size();
1407 } else {
1408 return 6 * NativeInstruction::instruction_size;
1409 }
1410 }
1411
1412 // Indicate if the safepoint node needs the polling page as an input
1413
1414 // the shared code plants the oop data at the start of the generated
1415 // code for the safepoint node and that needs ot be at the load
1416 // instruction itself. so we cannot plant a mov of the safepoint poll
1417 // address followed by a load. setting this to true means the mov is
1418 // scheduled as a prior instruction. that's better for scheduling
1419 // anyway.
1420
1421 bool SafePointNode::needs_polling_address_input()
1422 {
1423 return true;
1424 }
1425
1426 //=============================================================================
1427
1428 #ifndef PRODUCT
1429 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1430 st->print("BREAKPOINT");
1431 }
1432 #endif
1433
1434 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1435 MacroAssembler _masm(&cbuf);
1436 __ brk(0);
1437 }
1438
1439 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1440 return MachNode::size(ra_);
1441 }
1442
1443 //=============================================================================
1444
1445 #ifndef PRODUCT
1446 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1447 st->print("nop \t# %d bytes pad for loops and calls", _count);
1448 }
1449 #endif
1450
1451 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1452 MacroAssembler _masm(&cbuf);
1453 for (int i = 0; i < _count; i++) {
1454 __ nop();
1455 }
1456 }
1457
1458 uint MachNopNode::size(PhaseRegAlloc*) const {
1459 return _count * NativeInstruction::instruction_size;
1460 }
1461
1462 //=============================================================================
1463 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1464
1465 int Compile::ConstantTable::calculate_table_base_offset() const {
1466 return 0; // absolute addressing, no offset
1467 }
1468
1469 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1470 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1471 ShouldNotReachHere();
1472 }
1473
1474 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1475 // Empty encoding
1476 }
1477
1478 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1479 return 0;
1480 }
1481
1482 #ifndef PRODUCT
1483 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1484 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1485 }
1486 #endif
1487
1488 #ifndef PRODUCT
1489 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1490 Compile* C = ra_->C;
1491
1492 int framesize = C->frame_slots() << LogBytesPerInt;
1493
1494 if (C->need_stack_bang(framesize))
1495 st->print("# stack bang size=%d\n\t", framesize);
1496
1497 if (framesize == 0) {
1498 // Is this even possible?
1499 st->print("stp lr, rfp, [sp, #%d]!", -(2 * wordSize));
1500 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1501 st->print("sub sp, sp, #%d\n\t", framesize);
1502 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1503 } else {
1504 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1505 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1506 st->print("sub sp, sp, rscratch1");
1507 }
1508 }
1509 #endif
1510
1511 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1512 Compile* C = ra_->C;
1513 MacroAssembler _masm(&cbuf);
1514
1515 // n.b. frame size includes space for return pc and rfp
1516 long framesize = ((long)C->frame_slots()) << LogBytesPerInt;
1517 assert(framesize%(2*wordSize) == 0, "must preserve 2*wordSize alignment");
1518
1519 // insert a nop at the start of the prolog so we can patch in a
1520 // branch if we need to invalidate the method later
1521 __ nop();
1522
1523 if (C->need_stack_bang(framesize))
1524 __ generate_stack_overflow_check(framesize);
1525
1526 __ build_frame(framesize);
1527
1528 if (VerifyStackAtCalls) {
1529 Unimplemented();
1530 }
1531
1532 C->set_frame_complete(cbuf.insts_size());
1533
1534 if (C->has_mach_constant_base_node()) {
1535 // NOTE: We set the table base offset here because users might be
1536 // emitted before MachConstantBaseNode.
1537 Compile::ConstantTable& constant_table = C->constant_table();
1538 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1539 }
1540 }
1541
1542 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1543 {
1544 return MachNode::size(ra_); // too many variables; just compute it
1545 // the hard way
1546 }
1547
1548 int MachPrologNode::reloc() const
1549 {
1550 return 0;
1551 }
1552
1553 //=============================================================================
1554
1555 #ifndef PRODUCT
1556 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1557 Compile* C = ra_->C;
1558 int framesize = C->frame_slots() << LogBytesPerInt;
1559
1560 st->print("# pop frame %d\n\t",framesize);
1561
1562 if (framesize == 0) {
1563 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1564 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1565 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1566 st->print("add sp, sp, #%d\n\t", framesize);
1567 } else {
1568 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1569 st->print("add sp, sp, rscratch1\n\t");
1570 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1571 }
1572
1573 if (do_polling() && C->is_method_compilation()) {
1574 st->print("# touch polling page\n\t");
1575 st->print("mov rscratch1, #" INTPTR_FORMAT "\n\t", p2i(os::get_polling_page()));
1576 st->print("ldr zr, [rscratch1]");
1577 }
1578 }
1579 #endif
1580
1581 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1582 Compile* C = ra_->C;
1583 MacroAssembler _masm(&cbuf);
1584 int framesize = C->frame_slots() << LogBytesPerInt;
1585
1586 __ remove_frame(framesize);
1587
1588 if (do_polling() && C->is_method_compilation()) {
1589 __ read_polling_page(rscratch1, os::get_polling_page(), relocInfo::poll_return_type);
1590 }
1591 }
1592
1593 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1594 // Variable size. Determine dynamically.
1595 return MachNode::size(ra_);
1596 }
1597
1598 int MachEpilogNode::reloc() const {
1599 // Return number of relocatable values contained in this instruction.
1600 return 1; // 1 for polling page.
1601 }
1602
1603 const Pipeline * MachEpilogNode::pipeline() const {
1604 return MachNode::pipeline_class();
1605 }
1606
1607 // This method seems to be obsolete. It is declared in machnode.hpp
1608 // and defined in all *.ad files, but it is never called. Should we
1609 // get rid of it?
1610 int MachEpilogNode::safepoint_offset() const {
1611 assert(do_polling(), "no return for this epilog node");
1612 return 4;
1613 }
1614
1615 //=============================================================================
1616
1617 // Figure out which register class each belongs in: rc_int, rc_float or
1618 // rc_stack.
1619 enum RC { rc_bad, rc_int, rc_float, rc_stack };
1620
1621 static enum RC rc_class(OptoReg::Name reg) {
1622
1623 if (reg == OptoReg::Bad) {
1624 return rc_bad;
1625 }
1626
1627 // we have 30 int registers * 2 halves
1628 // (rscratch1 and rscratch2 are omitted)
1629
1630 if (reg < 60) {
1631 return rc_int;
1632 }
1633
1634 // we have 32 float register * 2 halves
1635 if (reg < 60 + 128) {
1636 return rc_float;
1637 }
1638
1639 // Between float regs & stack is the flags regs.
1640 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1641
1642 return rc_stack;
1643 }
1644
1645 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1646 Compile* C = ra_->C;
1647
1648 // Get registers to move.
1649 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1650 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1651 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1652 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1653
1654 enum RC src_hi_rc = rc_class(src_hi);
1655 enum RC src_lo_rc = rc_class(src_lo);
1656 enum RC dst_hi_rc = rc_class(dst_hi);
1657 enum RC dst_lo_rc = rc_class(dst_lo);
1658
1659 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1660
1661 if (src_hi != OptoReg::Bad) {
1662 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1663 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1664 "expected aligned-adjacent pairs");
1665 }
1666
1667 if (src_lo == dst_lo && src_hi == dst_hi) {
1668 return 0; // Self copy, no move.
1669 }
1670
1671 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1672 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1673 int src_offset = ra_->reg2offset(src_lo);
1674 int dst_offset = ra_->reg2offset(dst_lo);
1675
1676 if (bottom_type()->isa_vect() != NULL) {
1677 uint ireg = ideal_reg();
1678 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1679 if (cbuf) {
1680 MacroAssembler _masm(cbuf);
1681 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1682 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1683 // stack->stack
1684 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1685 if (ireg == Op_VecD) {
1686 __ unspill(rscratch1, true, src_offset);
1687 __ spill(rscratch1, true, dst_offset);
1688 } else {
1689 __ spill_copy128(src_offset, dst_offset);
1690 }
1691 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1692 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1693 ireg == Op_VecD ? __ T8B : __ T16B,
1694 as_FloatRegister(Matcher::_regEncode[src_lo]));
1695 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1696 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1697 ireg == Op_VecD ? __ D : __ Q,
1698 ra_->reg2offset(dst_lo));
1699 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1700 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1701 ireg == Op_VecD ? __ D : __ Q,
1702 ra_->reg2offset(src_lo));
1703 } else {
1704 ShouldNotReachHere();
1705 }
1706 }
1707 } else if (cbuf) {
1708 MacroAssembler _masm(cbuf);
1709 switch (src_lo_rc) {
1710 case rc_int:
1711 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1712 if (is64) {
1713 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1714 as_Register(Matcher::_regEncode[src_lo]));
1715 } else {
1716 MacroAssembler _masm(cbuf);
1717 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1718 as_Register(Matcher::_regEncode[src_lo]));
1719 }
1720 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1721 if (is64) {
1722 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1723 as_Register(Matcher::_regEncode[src_lo]));
1724 } else {
1725 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1726 as_Register(Matcher::_regEncode[src_lo]));
1727 }
1728 } else { // gpr --> stack spill
1729 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1730 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
1731 }
1732 break;
1733 case rc_float:
1734 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
1735 if (is64) {
1736 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
1737 as_FloatRegister(Matcher::_regEncode[src_lo]));
1738 } else {
1739 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
1740 as_FloatRegister(Matcher::_regEncode[src_lo]));
1741 }
1742 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
1743 if (cbuf) {
1744 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1745 as_FloatRegister(Matcher::_regEncode[src_lo]));
1746 } else {
1747 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1748 as_FloatRegister(Matcher::_regEncode[src_lo]));
1749 }
1750 } else { // fpr --> stack spill
1751 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1752 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1753 is64 ? __ D : __ S, dst_offset);
1754 }
1755 break;
1756 case rc_stack:
1757 if (dst_lo_rc == rc_int) { // stack --> gpr load
1758 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
1759 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
1760 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1761 is64 ? __ D : __ S, src_offset);
1762 } else { // stack --> stack copy
1763 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1764 __ unspill(rscratch1, is64, src_offset);
1765 __ spill(rscratch1, is64, dst_offset);
1766 }
1767 break;
1768 default:
1769 assert(false, "bad rc_class for spill");
1770 ShouldNotReachHere();
1771 }
1772 }
1773
1774 if (st) {
1775 st->print("spill ");
1776 if (src_lo_rc == rc_stack) {
1777 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
1778 } else {
1779 st->print("%s -> ", Matcher::regName[src_lo]);
1780 }
1781 if (dst_lo_rc == rc_stack) {
1782 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
1783 } else {
1784 st->print("%s", Matcher::regName[dst_lo]);
1785 }
1786 if (bottom_type()->isa_vect() != NULL) {
1787 st->print("\t# vector spill size = %d", ideal_reg()==Op_VecD ? 64:128);
1788 } else {
1789 st->print("\t# spill size = %d", is64 ? 64:32);
1790 }
1791 }
1792
1793 return 0;
1794
1795 }
1796
1797 #ifndef PRODUCT
1798 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1799 if (!ra_)
1800 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
1801 else
1802 implementation(NULL, ra_, false, st);
1803 }
1804 #endif
1805
1806 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1807 implementation(&cbuf, ra_, false, NULL);
1808 }
1809
1810 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1811 return MachNode::size(ra_);
1812 }
1813
1814 //=============================================================================
1815
1816 #ifndef PRODUCT
1817 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1818 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1819 int reg = ra_->get_reg_first(this);
1820 st->print("add %s, rsp, #%d]\t# box lock",
1821 Matcher::regName[reg], offset);
1822 }
1823 #endif
1824
1825 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1826 MacroAssembler _masm(&cbuf);
1827
1828 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1829 int reg = ra_->get_encode(this);
1830
1831 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
1832 __ add(as_Register(reg), sp, offset);
1833 } else {
1834 ShouldNotReachHere();
1835 }
1836 }
1837
1838 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
1839 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
1840 return 4;
1841 }
1842
1843 //=============================================================================
1844
1845 #ifndef PRODUCT
1846 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1847 {
1848 st->print_cr("# MachUEPNode");
1849 if (UseCompressedClassPointers) {
1850 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1851 if (Universe::narrow_klass_shift() != 0) {
1852 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1853 }
1854 } else {
1855 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1856 }
1857 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
1858 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
1859 }
1860 #endif
1861
1862 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1863 {
1864 // This is the unverified entry point.
1865 MacroAssembler _masm(&cbuf);
1866
1867 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
1868 Label skip;
1869 // TODO
1870 // can we avoid this skip and still use a reloc?
1871 __ br(Assembler::EQ, skip);
1872 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1873 __ bind(skip);
1874 }
1875
1876 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1877 {
1878 return MachNode::size(ra_);
1879 }
1880
1881 // REQUIRED EMIT CODE
1882
1883 //=============================================================================
1884
1885 // Emit exception handler code.
1886 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
1887 {
1888 // mov rscratch1 #exception_blob_entry_point
1889 // br rscratch1
1890 // Note that the code buffer's insts_mark is always relative to insts.
1891 // That's why we must use the macroassembler to generate a handler.
1892 MacroAssembler _masm(&cbuf);
1893 address base = __ start_a_stub(size_exception_handler());
1894 if (base == NULL) {
1895 ciEnv::current()->record_failure("CodeCache is full");
1896 return 0; // CodeBuffer::expand failed
1897 }
1898 int offset = __ offset();
1899 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
1900 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1901 __ end_a_stub();
1902 return offset;
1903 }
1904
1905 // Emit deopt handler code.
1906 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
1907 {
1908 // Note that the code buffer's insts_mark is always relative to insts.
1909 // That's why we must use the macroassembler to generate a handler.
1910 MacroAssembler _masm(&cbuf);
1911 address base = __ start_a_stub(size_deopt_handler());
1912 if (base == NULL) {
1913 ciEnv::current()->record_failure("CodeCache is full");
1914 return 0; // CodeBuffer::expand failed
1915 }
1916 int offset = __ offset();
1917
1918 __ adr(lr, __ pc());
1919 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
1920
1921 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1922 __ end_a_stub();
1923 return offset;
1924 }
1925
1926 // REQUIRED MATCHER CODE
1927
1928 //=============================================================================
1929
1930 const bool Matcher::match_rule_supported(int opcode) {
1931
1932 // TODO
1933 // identify extra cases that we might want to provide match rules for
1934 // e.g. Op_StrEquals and other intrinsics
1935 if (!has_match_rule(opcode)) {
1936 return false;
1937 }
1938
1939 return true; // Per default match rules are supported.
1940 }
1941
1942 int Matcher::regnum_to_fpu_offset(int regnum)
1943 {
1944 Unimplemented();
1945 return 0;
1946 }
1947
1948 // Is this branch offset short enough that a short branch can be used?
1949 //
1950 // NOTE: If the platform does not provide any short branch variants, then
1951 // this method should return false for offset 0.
1952 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1953 // The passed offset is relative to address of the branch.
1954
1955 return (-32768 <= offset && offset < 32768);
1956 }
1957
1958 const bool Matcher::isSimpleConstant64(jlong value) {
1959 // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1960 // Probably always true, even if a temp register is required.
1961 return true;
1962 }
1963
1964 // true just means we have fast l2f conversion
1965 const bool Matcher::convL2FSupported(void) {
1966 return true;
1967 }
1968
1969 // Vector width in bytes.
1970 const int Matcher::vector_width_in_bytes(BasicType bt) {
1971 int size = MIN2(16,(int)MaxVectorSize);
1972 // Minimum 2 values in vector
1973 if (size < 2*type2aelembytes(bt)) size = 0;
1974 // But never < 4
1975 if (size < 4) size = 0;
1976 return size;
1977 }
1978
1979 // Limits on vector size (number of elements) loaded into vector.
1980 const int Matcher::max_vector_size(const BasicType bt) {
1981 return vector_width_in_bytes(bt)/type2aelembytes(bt);
1982 }
1983 const int Matcher::min_vector_size(const BasicType bt) {
1984 // For the moment limit the vector size to 8 bytes
1985 int size = 8 / type2aelembytes(bt);
1986 if (size < 2) size = 2;
1987 return size;
1988 }
1989
1990 // Vector ideal reg.
1991 const uint Matcher::vector_ideal_reg(int len) {
1992 switch(len) {
1993 case 8: return Op_VecD;
1994 case 16: return Op_VecX;
1995 }
1996 ShouldNotReachHere();
1997 return 0;
1998 }
1999
2000 const uint Matcher::vector_shift_count_ideal_reg(int size) {
2001 switch(size) {
2002 case 8: return Op_VecD;
2003 case 16: return Op_VecX;
2004 }
2005 ShouldNotReachHere();
2006 return 0;
2007 }
2008
2009 // AES support not yet implemented
2010 const bool Matcher::pass_original_key_for_aes() {
2011 return false;
2012 }
2013
2014 // x86 supports misaligned vectors store/load.
2015 const bool Matcher::misaligned_vectors_ok() {
2016 return !AlignVector; // can be changed by flag
2017 }
2018
2019 // false => size gets scaled to BytesPerLong, ok.
2020 const bool Matcher::init_array_count_is_in_bytes = false;
2021
2022 // Threshold size for cleararray.
2023 const int Matcher::init_array_short_size = 4 * BytesPerLong;
2024
2025 // Use conditional move (CMOVL)
2026 const int Matcher::long_cmove_cost() {
2027 // long cmoves are no more expensive than int cmoves
2028 return 0;
2029 }
2030
2031 const int Matcher::float_cmove_cost() {
2032 // float cmoves are no more expensive than int cmoves
2033 return 0;
2034 }
2035
2036 // Does the CPU require late expand (see block.cpp for description of late expand)?
2037 const bool Matcher::require_postalloc_expand = false;
2038
2039 // Should the Matcher clone shifts on addressing modes, expecting them
2040 // to be subsumed into complex addressing expressions or compute them
2041 // into registers? True for Intel but false for most RISCs
2042 const bool Matcher::clone_shift_expressions = false;
2043
2044 // Do we need to mask the count passed to shift instructions or does
2045 // the cpu only look at the lower 5/6 bits anyway?
2046 const bool Matcher::need_masked_shift_count = false;
2047
2048 // This affects two different things:
2049 // - how Decode nodes are matched
2050 // - how ImplicitNullCheck opportunities are recognized
2051 // If true, the matcher will try to remove all Decodes and match them
2052 // (as operands) into nodes. NullChecks are not prepared to deal with
2053 // Decodes by final_graph_reshaping().
2054 // If false, final_graph_reshaping() forces the decode behind the Cmp
2055 // for a NullCheck. The matcher matches the Decode node into a register.
2056 // Implicit_null_check optimization moves the Decode along with the
2057 // memory operation back up before the NullCheck.
2058 bool Matcher::narrow_oop_use_complex_address() {
2059 return Universe::narrow_oop_shift() == 0;
2060 }
2061
2062 bool Matcher::narrow_klass_use_complex_address() {
2063 // TODO
2064 // decide whether we need to set this to true
2065 return false;
2066 }
2067
2068 // Is it better to copy float constants, or load them directly from
2069 // memory? Intel can load a float constant from a direct address,
2070 // requiring no extra registers. Most RISCs will have to materialize
2071 // an address into a register first, so they would do better to copy
2072 // the constant from stack.
2073 const bool Matcher::rematerialize_float_constants = false;
2074
2075 // If CPU can load and store mis-aligned doubles directly then no
2076 // fixup is needed. Else we split the double into 2 integer pieces
2077 // and move it piece-by-piece. Only happens when passing doubles into
2078 // C code as the Java calling convention forces doubles to be aligned.
2079 const bool Matcher::misaligned_doubles_ok = true;
2080
2081 // No-op on amd64
2082 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
2083 Unimplemented();
2084 }
2085
2086 // Advertise here if the CPU requires explicit rounding operations to
2087 // implement the UseStrictFP mode.
2088 const bool Matcher::strict_fp_requires_explicit_rounding = false;
2089
2090 // Are floats converted to double when stored to stack during
2091 // deoptimization?
2092 bool Matcher::float_in_double() { return true; }
2093
2094 // Do ints take an entire long register or just half?
2095 // The relevant question is how the int is callee-saved:
2096 // the whole long is written but de-opt'ing will have to extract
2097 // the relevant 32 bits.
2098 const bool Matcher::int_in_long = true;
2099
2100 // Return whether or not this register is ever used as an argument.
2101 // This function is used on startup to build the trampoline stubs in
2102 // generateOptoStub. Registers not mentioned will be killed by the VM
2103 // call in the trampoline, and arguments in those registers not be
2104 // available to the callee.
2105 bool Matcher::can_be_java_arg(int reg)
2106 {
2107 return
2108 reg == R0_num || reg == R0_H_num ||
2109 reg == R1_num || reg == R1_H_num ||
2110 reg == R2_num || reg == R2_H_num ||
2111 reg == R3_num || reg == R3_H_num ||
2112 reg == R4_num || reg == R4_H_num ||
2113 reg == R5_num || reg == R5_H_num ||
2114 reg == R6_num || reg == R6_H_num ||
2115 reg == R7_num || reg == R7_H_num ||
2116 reg == V0_num || reg == V0_H_num ||
2117 reg == V1_num || reg == V1_H_num ||
2118 reg == V2_num || reg == V2_H_num ||
2119 reg == V3_num || reg == V3_H_num ||
2120 reg == V4_num || reg == V4_H_num ||
2121 reg == V5_num || reg == V5_H_num ||
2122 reg == V6_num || reg == V6_H_num ||
2123 reg == V7_num || reg == V7_H_num;
2124 }
2125
2126 bool Matcher::is_spillable_arg(int reg)
2127 {
2128 return can_be_java_arg(reg);
2129 }
2130
2131 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2132 return false;
2133 }
2134
2135 RegMask Matcher::divI_proj_mask() {
2136 ShouldNotReachHere();
2137 return RegMask();
2138 }
2139
2140 // Register for MODI projection of divmodI.
2141 RegMask Matcher::modI_proj_mask() {
2142 ShouldNotReachHere();
2143 return RegMask();
2144 }
2145
2146 // Register for DIVL projection of divmodL.
2147 RegMask Matcher::divL_proj_mask() {
2148 ShouldNotReachHere();
2149 return RegMask();
2150 }
2151
2152 // Register for MODL projection of divmodL.
2153 RegMask Matcher::modL_proj_mask() {
2154 ShouldNotReachHere();
2155 return RegMask();
2156 }
2157
2158 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2159 return FP_REG_mask();
2160 }
2161
2162
2163 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2164 MacroAssembler _masm(&cbuf); \
2165 { \
2166 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2167 guarantee(DISP == 0, "mode not permitted for volatile"); \
2168 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2169 __ INSN(REG, as_Register(BASE)); \
2170 }
2171
2172 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2173 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2174 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2175 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2176
2177 // Used for all non-volatile memory accesses. The use of
2178 // $mem->opcode() to discover whether this pattern uses sign-extended
2179 // offsets is something of a kludge.
2180 static void loadStore(MacroAssembler masm, mem_insn insn,
2181 Register reg, int opcode,
2182 Register base, int index, int size, int disp)
2183 {
2184 Address::extend scale;
2185
2186 // Hooboy, this is fugly. We need a way to communicate to the
2187 // encoder that the index needs to be sign extended, so we have to
2188 // enumerate all the cases.
2189 switch (opcode) {
2190 case INDINDEXSCALEDOFFSETI2L:
2191 case INDINDEXSCALEDI2L:
2192 case INDINDEXSCALEDOFFSETI2LN:
2193 case INDINDEXSCALEDI2LN:
2194 case INDINDEXOFFSETI2L:
2195 case INDINDEXOFFSETI2LN:
2196 scale = Address::sxtw(size);
2197 break;
2198 default:
2199 scale = Address::lsl(size);
2200 }
2201
2202 if (index == -1) {
2203 (masm.*insn)(reg, Address(base, disp));
2204 } else {
2205 if (disp == 0) {
2206 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2207 } else {
2208 masm.lea(rscratch1, Address(base, disp));
2209 (masm.*insn)(reg, Address(rscratch1, as_Register(index), scale));
2210 }
2211 }
2212 }
2213
2214 static void loadStore(MacroAssembler masm, mem_float_insn insn,
2215 FloatRegister reg, int opcode,
2216 Register base, int index, int size, int disp)
2217 {
2218 Address::extend scale;
2219
2220 switch (opcode) {
2221 case INDINDEXSCALEDOFFSETI2L:
2222 case INDINDEXSCALEDI2L:
2223 case INDINDEXSCALEDOFFSETI2LN:
2224 case INDINDEXSCALEDI2LN:
2225 scale = Address::sxtw(size);
2226 break;
2227 default:
2228 scale = Address::lsl(size);
2229 }
2230
2231 if (index == -1) {
2232 (masm.*insn)(reg, Address(base, disp));
2233 } else {
2234 if (disp == 0) {
2235 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2236 } else {
2237 masm.lea(rscratch1, Address(base, disp));
2238 (masm.*insn)(reg, Address(rscratch1, as_Register(index), scale));
2239 }
2240 }
2241 }
2242
2243 static void loadStore(MacroAssembler masm, mem_vector_insn insn,
2244 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2245 int opcode, Register base, int index, int size, int disp)
2246 {
2247 if (index == -1) {
2248 (masm.*insn)(reg, T, Address(base, disp));
2249 } else {
2250 assert(disp == 0, "unsupported address mode");
2251 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2252 }
2253 }
2254
2255 %}
2256
2257
2258
2259 //----------ENCODING BLOCK-----------------------------------------------------
2260 // This block specifies the encoding classes used by the compiler to
2261 // output byte streams. Encoding classes are parameterized macros
2262 // used by Machine Instruction Nodes in order to generate the bit
2263 // encoding of the instruction. Operands specify their base encoding
2264 // interface with the interface keyword. There are currently
2265 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2266 // COND_INTER. REG_INTER causes an operand to generate a function
2267 // which returns its register number when queried. CONST_INTER causes
2268 // an operand to generate a function which returns the value of the
2269 // constant when queried. MEMORY_INTER causes an operand to generate
2270 // four functions which return the Base Register, the Index Register,
2271 // the Scale Value, and the Offset Value of the operand when queried.
2272 // COND_INTER causes an operand to generate six functions which return
2273 // the encoding code (ie - encoding bits for the instruction)
2274 // associated with each basic boolean condition for a conditional
2275 // instruction.
2276 //
2277 // Instructions specify two basic values for encoding. Again, a
2278 // function is available to check if the constant displacement is an
2279 // oop. They use the ins_encode keyword to specify their encoding
2280 // classes (which must be a sequence of enc_class names, and their
2281 // parameters, specified in the encoding block), and they use the
2282 // opcode keyword to specify, in order, their primary, secondary, and
2283 // tertiary opcode. Only the opcode sections which a particular
2284 // instruction needs for encoding need to be specified.
2285 encode %{
2286 // Build emit functions for each basic byte or larger field in the
2287 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2288 // from C++ code in the enc_class source block. Emit functions will
2289 // live in the main source block for now. In future, we can
2290 // generalize this by adding a syntax that specifies the sizes of
2291 // fields in an order, so that the adlc can build the emit functions
2292 // automagically
2293
2294 // catch all for unimplemented encodings
2295 enc_class enc_unimplemented %{
2296 MacroAssembler _masm(&cbuf);
2297 __ unimplemented("C2 catch all");
2298 %}
2299
2300 // BEGIN Non-volatile memory access
2301
2302 enc_class aarch64_enc_ldrsbw(iRegI dst, memory mem) %{
2303 Register dst_reg = as_Register($dst$$reg);
2304 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2305 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2306 %}
2307
2308 enc_class aarch64_enc_ldrsb(iRegI dst, memory mem) %{
2309 Register dst_reg = as_Register($dst$$reg);
2310 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2311 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2312 %}
2313
2314 enc_class aarch64_enc_ldrb(iRegI dst, memory mem) %{
2315 Register dst_reg = as_Register($dst$$reg);
2316 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2317 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2318 %}
2319
2320 enc_class aarch64_enc_ldrb(iRegL dst, memory mem) %{
2321 Register dst_reg = as_Register($dst$$reg);
2322 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2323 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2324 %}
2325
2326 enc_class aarch64_enc_ldrshw(iRegI dst, memory mem) %{
2327 Register dst_reg = as_Register($dst$$reg);
2328 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2329 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2330 %}
2331
2332 enc_class aarch64_enc_ldrsh(iRegI dst, memory mem) %{
2333 Register dst_reg = as_Register($dst$$reg);
2334 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2335 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2336 %}
2337
2338 enc_class aarch64_enc_ldrh(iRegI dst, memory mem) %{
2339 Register dst_reg = as_Register($dst$$reg);
2340 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2341 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2342 %}
2343
2344 enc_class aarch64_enc_ldrh(iRegL dst, memory mem) %{
2345 Register dst_reg = as_Register($dst$$reg);
2346 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2347 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2348 %}
2349
2350 enc_class aarch64_enc_ldrw(iRegI dst, memory mem) %{
2351 Register dst_reg = as_Register($dst$$reg);
2352 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2353 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2354 %}
2355
2356 enc_class aarch64_enc_ldrw(iRegL dst, memory mem) %{
2357 Register dst_reg = as_Register($dst$$reg);
2358 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2359 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2360 %}
2361
2362 enc_class aarch64_enc_ldrsw(iRegL dst, memory mem) %{
2363 Register dst_reg = as_Register($dst$$reg);
2364 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2365 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2366 %}
2367
2368 enc_class aarch64_enc_ldr(iRegL dst, memory mem) %{
2369 Register dst_reg = as_Register($dst$$reg);
2370 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2371 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2372 %}
2373
2374 enc_class aarch64_enc_ldrs(vRegF dst, memory mem) %{
2375 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2376 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2377 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2378 %}
2379
2380 enc_class aarch64_enc_ldrd(vRegD dst, memory mem) %{
2381 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2382 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2383 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2384 %}
2385
2386 enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
2387 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2388 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
2389 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2390 %}
2391
2392 enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
2393 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2394 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
2395 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2396 %}
2397
2398 enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
2399 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2400 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
2401 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2402 %}
2403
2404 enc_class aarch64_enc_strb(iRegI src, memory mem) %{
2405 Register src_reg = as_Register($src$$reg);
2406 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2407 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2408 %}
2409
2410 enc_class aarch64_enc_strb0(memory mem) %{
2411 MacroAssembler _masm(&cbuf);
2412 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2413 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2414 %}
2415
2416 enc_class aarch64_enc_strb0_ordered(memory mem) %{
2417 MacroAssembler _masm(&cbuf);
2418 __ membar(Assembler::StoreStore);
2419 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2420 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2421 %}
2422
2423 enc_class aarch64_enc_strh(iRegI src, memory mem) %{
2424 Register src_reg = as_Register($src$$reg);
2425 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2426 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2427 %}
2428
2429 enc_class aarch64_enc_strh0(memory mem) %{
2430 MacroAssembler _masm(&cbuf);
2431 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2432 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2433 %}
2434
2435 enc_class aarch64_enc_strw(iRegI src, memory mem) %{
2436 Register src_reg = as_Register($src$$reg);
2437 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2438 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2439 %}
2440
2441 enc_class aarch64_enc_strw0(memory mem) %{
2442 MacroAssembler _masm(&cbuf);
2443 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2444 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2445 %}
2446
2447 enc_class aarch64_enc_str(iRegL src, memory mem) %{
2448 Register src_reg = as_Register($src$$reg);
2449 // we sometimes get asked to store the stack pointer into the
2450 // current thread -- we cannot do that directly on AArch64
2451 if (src_reg == r31_sp) {
2452 MacroAssembler _masm(&cbuf);
2453 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2454 __ mov(rscratch2, sp);
2455 src_reg = rscratch2;
2456 }
2457 loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2458 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2459 %}
2460
2461 enc_class aarch64_enc_str0(memory mem) %{
2462 MacroAssembler _masm(&cbuf);
2463 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2464 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2465 %}
2466
2467 enc_class aarch64_enc_strs(vRegF src, memory mem) %{
2468 FloatRegister src_reg = as_FloatRegister($src$$reg);
2469 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2470 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2471 %}
2472
2473 enc_class aarch64_enc_strd(vRegD src, memory mem) %{
2474 FloatRegister src_reg = as_FloatRegister($src$$reg);
2475 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2476 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2477 %}
2478
2479 enc_class aarch64_enc_strvS(vecD src, memory mem) %{
2480 FloatRegister src_reg = as_FloatRegister($src$$reg);
2481 loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
2482 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2483 %}
2484
2485 enc_class aarch64_enc_strvD(vecD src, memory mem) %{
2486 FloatRegister src_reg = as_FloatRegister($src$$reg);
2487 loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
2488 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2489 %}
2490
2491 enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
2492 FloatRegister src_reg = as_FloatRegister($src$$reg);
2493 loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
2494 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2495 %}
2496
2497 // END Non-volatile memory access
2498
2499 // this encoding writes the address of the first instruction in the
2500 // call sequence for the runtime call into the anchor pc slot. this
2501 // address allows the runtime to i) locate the code buffer for the
2502 // caller (any address in the buffer would do) and ii) find the oop
2503 // map associated with the call (has to address the instruction
2504 // following the call). note that we have to store the address which
2505 // follows the actual call.
2506 //
2507 // the offset from the current pc can be computed by considering
2508 // what gets generated between this point up to and including the
2509 // call. it looks like this
2510 //
2511 // movz xscratch1 0xnnnn <-- current pc is here
2512 // movk xscratch1 0xnnnn
2513 // movk xscratch1 0xnnnn
2514 // str xscratch1, [xthread,#anchor_pc_off]
2515 // mov xscratch2, sp
2516 // str xscratch2, [xthread,#anchor_sp_off
2517 // mov x0, x1
2518 // . . .
2519 // mov xn-1, xn
2520 // mov xn, thread <-- always passed
2521 // mov xn+1, rfp <-- optional iff primary == 1
2522 // movz xscratch1 0xnnnn
2523 // movk xscratch1 0xnnnn
2524 // movk xscratch1 0xnnnn
2525 // blr xscratch1
2526 // . . .
2527 //
2528 // where the called routine has n args (including the thread and,
2529 // possibly the stub's caller return address currently in rfp). we
2530 // can compute n by looking at the number of args passed into the
2531 // stub. we assert that nargs is < 7.
2532 //
2533 // so the offset we need to add to the pc (in 32-bit words) is
2534 // 3 + <-- load 48-bit constant return pc
2535 // 1 + <-- write anchor pc
2536 // 1 + <-- copy sp
2537 // 1 + <-- write anchor sp
2538 // nargs + <-- java stub arg count
2539 // 1 + <-- extra thread arg
2540 // [ 1 + ] <-- optional ret address of stub caller
2541 // 3 + <-- load 64 bit call target address
2542 // 1 <-- blr instruction
2543 //
2544 // i.e we need to add (nargs + 11) * 4 bytes or (nargs + 12) * 4 bytes
2545 //
2546
2547 enc_class aarch64_enc_save_pc() %{
2548 Compile* C = ra_->C;
2549 int nargs = C->tf()->domain()->cnt() - TypeFunc::Parms;
2550 if ($primary) { nargs++; }
2551 assert(nargs <= 8, "opto runtime stub has more than 8 args!");
2552 MacroAssembler _masm(&cbuf);
2553 address pc = __ pc();
2554 int call_offset = (nargs + 11) * 4;
2555 int field_offset = in_bytes(JavaThread::frame_anchor_offset()) +
2556 in_bytes(JavaFrameAnchor::last_Java_pc_offset());
2557 __ lea(rscratch1, InternalAddress(pc + call_offset));
2558 __ str(rscratch1, Address(rthread, field_offset));
2559 %}
2560
2561 // volatile loads and stores
2562
2563 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
2564 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2565 rscratch1, stlrb);
2566 if (VM_Version::cpu_cpuFeatures() & VM_Version::CPU_DMB_ATOMICS)
2567 __ dmb(__ ISH);
2568 %}
2569
2570 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
2571 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2572 rscratch1, stlrh);
2573 if (VM_Version::cpu_cpuFeatures() & VM_Version::CPU_DMB_ATOMICS)
2574 __ dmb(__ ISH);
2575 %}
2576
2577 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
2578 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2579 rscratch1, stlrw);
2580 if (VM_Version::cpu_cpuFeatures() & VM_Version::CPU_DMB_ATOMICS)
2581 __ dmb(__ ISH);
2582 %}
2583
2584
2585 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
2586 Register dst_reg = as_Register($dst$$reg);
2587 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2588 rscratch1, ldarb);
2589 __ sxtbw(dst_reg, dst_reg);
2590 %}
2591
2592 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
2593 Register dst_reg = as_Register($dst$$reg);
2594 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2595 rscratch1, ldarb);
2596 __ sxtb(dst_reg, dst_reg);
2597 %}
2598
2599 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
2600 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2601 rscratch1, ldarb);
2602 %}
2603
2604 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
2605 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2606 rscratch1, ldarb);
2607 %}
2608
2609 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
2610 Register dst_reg = as_Register($dst$$reg);
2611 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2612 rscratch1, ldarh);
2613 __ sxthw(dst_reg, dst_reg);
2614 %}
2615
2616 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
2617 Register dst_reg = as_Register($dst$$reg);
2618 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2619 rscratch1, ldarh);
2620 __ sxth(dst_reg, dst_reg);
2621 %}
2622
2623 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
2624 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2625 rscratch1, ldarh);
2626 %}
2627
2628 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
2629 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2630 rscratch1, ldarh);
2631 %}
2632
2633 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
2634 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2635 rscratch1, ldarw);
2636 %}
2637
2638 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
2639 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2640 rscratch1, ldarw);
2641 %}
2642
2643 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
2644 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2645 rscratch1, ldar);
2646 %}
2647
2648 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
2649 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2650 rscratch1, ldarw);
2651 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
2652 %}
2653
2654 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
2655 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2656 rscratch1, ldar);
2657 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
2658 %}
2659
2660 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
2661 Register src_reg = as_Register($src$$reg);
2662 // we sometimes get asked to store the stack pointer into the
2663 // current thread -- we cannot do that directly on AArch64
2664 if (src_reg == r31_sp) {
2665 MacroAssembler _masm(&cbuf);
2666 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2667 __ mov(rscratch2, sp);
2668 src_reg = rscratch2;
2669 }
2670 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2671 rscratch1, stlr);
2672 if (VM_Version::cpu_cpuFeatures() & VM_Version::CPU_DMB_ATOMICS)
2673 __ dmb(__ ISH);
2674 %}
2675
2676 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
2677 {
2678 MacroAssembler _masm(&cbuf);
2679 FloatRegister src_reg = as_FloatRegister($src$$reg);
2680 __ fmovs(rscratch2, src_reg);
2681 }
2682 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2683 rscratch1, stlrw);
2684 if (VM_Version::cpu_cpuFeatures() & VM_Version::CPU_DMB_ATOMICS)
2685 __ dmb(__ ISH);
2686 %}
2687
2688 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
2689 {
2690 MacroAssembler _masm(&cbuf);
2691 FloatRegister src_reg = as_FloatRegister($src$$reg);
2692 __ fmovd(rscratch2, src_reg);
2693 }
2694 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2695 rscratch1, stlr);
2696 if (VM_Version::cpu_cpuFeatures() & VM_Version::CPU_DMB_ATOMICS)
2697 __ dmb(__ ISH);
2698 %}
2699
2700 // synchronized read/update encodings
2701
2702 enc_class aarch64_enc_ldaxr(iRegL dst, memory mem) %{
2703 MacroAssembler _masm(&cbuf);
2704 Register dst_reg = as_Register($dst$$reg);
2705 Register base = as_Register($mem$$base);
2706 int index = $mem$$index;
2707 int scale = $mem$$scale;
2708 int disp = $mem$$disp;
2709 if (index == -1) {
2710 if (disp != 0) {
2711 __ lea(rscratch1, Address(base, disp));
2712 __ ldaxr(dst_reg, rscratch1);
2713 } else {
2714 // TODO
2715 // should we ever get anything other than this case?
2716 __ ldaxr(dst_reg, base);
2717 }
2718 } else {
2719 Register index_reg = as_Register(index);
2720 if (disp == 0) {
2721 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
2722 __ ldaxr(dst_reg, rscratch1);
2723 } else {
2724 __ lea(rscratch1, Address(base, disp));
2725 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
2726 __ ldaxr(dst_reg, rscratch1);
2727 }
2728 }
2729 %}
2730
2731 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory mem) %{
2732 MacroAssembler _masm(&cbuf);
2733 Register src_reg = as_Register($src$$reg);
2734 Register base = as_Register($mem$$base);
2735 int index = $mem$$index;
2736 int scale = $mem$$scale;
2737 int disp = $mem$$disp;
2738 if (index == -1) {
2739 if (disp != 0) {
2740 __ lea(rscratch2, Address(base, disp));
2741 __ stlxr(rscratch1, src_reg, rscratch2);
2742 } else {
2743 // TODO
2744 // should we ever get anything other than this case?
2745 __ stlxr(rscratch1, src_reg, base);
2746 }
2747 } else {
2748 Register index_reg = as_Register(index);
2749 if (disp == 0) {
2750 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
2751 __ stlxr(rscratch1, src_reg, rscratch2);
2752 } else {
2753 __ lea(rscratch2, Address(base, disp));
2754 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
2755 __ stlxr(rscratch1, src_reg, rscratch2);
2756 }
2757 }
2758 __ cmpw(rscratch1, zr);
2759 %}
2760
2761 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2762 MacroAssembler _masm(&cbuf);
2763 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2764 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2765 Assembler::xword, /*acquire*/ false, /*release*/ true);
2766 %}
2767
2768 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2769 MacroAssembler _masm(&cbuf);
2770 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2771 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2772 Assembler::word, /*acquire*/ false, /*release*/ true);
2773 %}
2774
2775
2776 enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
2777 MacroAssembler _masm(&cbuf);
2778 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2779 Register tmp = $tmp$$Register;
2780 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
2781 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
2782 /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
2783 %}
2784
2785 // The only difference between aarch64_enc_cmpxchg and
2786 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
2787 // CompareAndSwap sequence to serve as a barrier on acquiring a
2788 // lock.
2789 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2790 MacroAssembler _masm(&cbuf);
2791 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2792 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2793 Assembler::xword, /*acquire*/ true, /*release*/ true);
2794 %}
2795
2796 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2797 MacroAssembler _masm(&cbuf);
2798 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2799 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2800 Assembler::word, /*acquire*/ true, /*release*/ true);
2801 %}
2802
2803 enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
2804 MacroAssembler _masm(&cbuf);
2805 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2806 Register tmp = $tmp$$Register;
2807 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
2808 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
2809 /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false,
2810 $res$$Register);
2811 %}
2812
2813 // auxiliary used for CompareAndSwapX to set result register
2814 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
2815 MacroAssembler _masm(&cbuf);
2816 Register res_reg = as_Register($res$$reg);
2817 __ cset(res_reg, Assembler::EQ);
2818 %}
2819
2820 // prefetch encodings
2821
2822 enc_class aarch64_enc_prefetchr(memory mem) %{
2823 MacroAssembler _masm(&cbuf);
2824 Register base = as_Register($mem$$base);
2825 int index = $mem$$index;
2826 int scale = $mem$$scale;
2827 int disp = $mem$$disp;
2828 if (index == -1) {
2829 __ prfm(Address(base, disp), PLDL1KEEP);
2830 } else {
2831 Register index_reg = as_Register(index);
2832 if (disp == 0) {
2833 __ prfm(Address(base, index_reg, Address::lsl(scale)), PLDL1KEEP);
2834 } else {
2835 __ lea(rscratch1, Address(base, disp));
2836 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PLDL1KEEP);
2837 }
2838 }
2839 %}
2840
2841 enc_class aarch64_enc_prefetchw(memory mem) %{
2842 MacroAssembler _masm(&cbuf);
2843 Register base = as_Register($mem$$base);
2844 int index = $mem$$index;
2845 int scale = $mem$$scale;
2846 int disp = $mem$$disp;
2847 if (index == -1) {
2848 __ prfm(Address(base, disp), PSTL1KEEP);
2849 } else {
2850 Register index_reg = as_Register(index);
2851 if (disp == 0) {
2852 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
2853 } else {
2854 __ lea(rscratch1, Address(base, disp));
2855 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
2856 }
2857 }
2858 %}
2859
2860 enc_class aarch64_enc_prefetchnta(memory mem) %{
2861 MacroAssembler _masm(&cbuf);
2862 Register base = as_Register($mem$$base);
2863 int index = $mem$$index;
2864 int scale = $mem$$scale;
2865 int disp = $mem$$disp;
2866 if (index == -1) {
2867 __ prfm(Address(base, disp), PSTL1STRM);
2868 } else {
2869 Register index_reg = as_Register(index);
2870 if (disp == 0) {
2871 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1STRM);
2872 __ nop();
2873 } else {
2874 __ lea(rscratch1, Address(base, disp));
2875 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1STRM);
2876 }
2877 }
2878 %}
2879
2880 /// mov envcodings
2881
2882 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
2883 MacroAssembler _masm(&cbuf);
2884 u_int32_t con = (u_int32_t)$src$$constant;
2885 Register dst_reg = as_Register($dst$$reg);
2886 if (con == 0) {
2887 __ movw(dst_reg, zr);
2888 } else {
2889 __ movw(dst_reg, con);
2890 }
2891 %}
2892
2893 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
2894 MacroAssembler _masm(&cbuf);
2895 Register dst_reg = as_Register($dst$$reg);
2896 u_int64_t con = (u_int64_t)$src$$constant;
2897 if (con == 0) {
2898 __ mov(dst_reg, zr);
2899 } else {
2900 __ mov(dst_reg, con);
2901 }
2902 %}
2903
2904 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
2905 MacroAssembler _masm(&cbuf);
2906 Register dst_reg = as_Register($dst$$reg);
2907 address con = (address)$src$$constant;
2908 if (con == NULL || con == (address)1) {
2909 ShouldNotReachHere();
2910 } else {
2911 relocInfo::relocType rtype = $src->constant_reloc();
2912 if (rtype == relocInfo::oop_type) {
2913 __ movoop(dst_reg, (jobject)con, /*immediate*/true);
2914 } else if (rtype == relocInfo::metadata_type) {
2915 __ mov_metadata(dst_reg, (Metadata*)con);
2916 } else {
2917 assert(rtype == relocInfo::none, "unexpected reloc type");
2918 if (con < (address)(uintptr_t)os::vm_page_size()) {
2919 __ mov(dst_reg, con);
2920 } else {
2921 unsigned long offset;
2922 __ adrp(dst_reg, con, offset);
2923 __ add(dst_reg, dst_reg, offset);
2924 }
2925 }
2926 }
2927 %}
2928
2929 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
2930 MacroAssembler _masm(&cbuf);
2931 Register dst_reg = as_Register($dst$$reg);
2932 __ mov(dst_reg, zr);
2933 %}
2934
2935 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
2936 MacroAssembler _masm(&cbuf);
2937 Register dst_reg = as_Register($dst$$reg);
2938 __ mov(dst_reg, (u_int64_t)1);
2939 %}
2940
2941 enc_class aarch64_enc_mov_poll_page(iRegP dst, immPollPage src) %{
2942 MacroAssembler _masm(&cbuf);
2943 address page = (address)$src$$constant;
2944 Register dst_reg = as_Register($dst$$reg);
2945 unsigned long off;
2946 __ adrp(dst_reg, Address(page, relocInfo::poll_type), off);
2947 assert(off == 0, "assumed offset == 0");
2948 %}
2949
2950 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
2951 MacroAssembler _masm(&cbuf);
2952 __ load_byte_map_base($dst$$Register);
2953 %}
2954
2955 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
2956 MacroAssembler _masm(&cbuf);
2957 Register dst_reg = as_Register($dst$$reg);
2958 address con = (address)$src$$constant;
2959 if (con == NULL) {
2960 ShouldNotReachHere();
2961 } else {
2962 relocInfo::relocType rtype = $src->constant_reloc();
2963 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
2964 __ set_narrow_oop(dst_reg, (jobject)con);
2965 }
2966 %}
2967
2968 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
2969 MacroAssembler _masm(&cbuf);
2970 Register dst_reg = as_Register($dst$$reg);
2971 __ mov(dst_reg, zr);
2972 %}
2973
2974 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
2975 MacroAssembler _masm(&cbuf);
2976 Register dst_reg = as_Register($dst$$reg);
2977 address con = (address)$src$$constant;
2978 if (con == NULL) {
2979 ShouldNotReachHere();
2980 } else {
2981 relocInfo::relocType rtype = $src->constant_reloc();
2982 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
2983 __ set_narrow_klass(dst_reg, (Klass *)con);
2984 }
2985 %}
2986
2987 // arithmetic encodings
2988
2989 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
2990 MacroAssembler _masm(&cbuf);
2991 Register dst_reg = as_Register($dst$$reg);
2992 Register src_reg = as_Register($src1$$reg);
2993 int32_t con = (int32_t)$src2$$constant;
2994 // add has primary == 0, subtract has primary == 1
2995 if ($primary) { con = -con; }
2996 if (con < 0) {
2997 __ subw(dst_reg, src_reg, -con);
2998 } else {
2999 __ addw(dst_reg, src_reg, con);
3000 }
3001 %}
3002
3003 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3004 MacroAssembler _masm(&cbuf);
3005 Register dst_reg = as_Register($dst$$reg);
3006 Register src_reg = as_Register($src1$$reg);
3007 int32_t con = (int32_t)$src2$$constant;
3008 // add has primary == 0, subtract has primary == 1
3009 if ($primary) { con = -con; }
3010 if (con < 0) {
3011 __ sub(dst_reg, src_reg, -con);
3012 } else {
3013 __ add(dst_reg, src_reg, con);
3014 }
3015 %}
3016
3017 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3018 MacroAssembler _masm(&cbuf);
3019 Register dst_reg = as_Register($dst$$reg);
3020 Register src1_reg = as_Register($src1$$reg);
3021 Register src2_reg = as_Register($src2$$reg);
3022 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3023 %}
3024
3025 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3026 MacroAssembler _masm(&cbuf);
3027 Register dst_reg = as_Register($dst$$reg);
3028 Register src1_reg = as_Register($src1$$reg);
3029 Register src2_reg = as_Register($src2$$reg);
3030 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3031 %}
3032
3033 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3034 MacroAssembler _masm(&cbuf);
3035 Register dst_reg = as_Register($dst$$reg);
3036 Register src1_reg = as_Register($src1$$reg);
3037 Register src2_reg = as_Register($src2$$reg);
3038 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3039 %}
3040
3041 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3042 MacroAssembler _masm(&cbuf);
3043 Register dst_reg = as_Register($dst$$reg);
3044 Register src1_reg = as_Register($src1$$reg);
3045 Register src2_reg = as_Register($src2$$reg);
3046 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3047 %}
3048
3049 // compare instruction encodings
3050
3051 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3052 MacroAssembler _masm(&cbuf);
3053 Register reg1 = as_Register($src1$$reg);
3054 Register reg2 = as_Register($src2$$reg);
3055 __ cmpw(reg1, reg2);
3056 %}
3057
3058 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3059 MacroAssembler _masm(&cbuf);
3060 Register reg = as_Register($src1$$reg);
3061 int32_t val = $src2$$constant;
3062 if (val >= 0) {
3063 __ subsw(zr, reg, val);
3064 } else {
3065 __ addsw(zr, reg, -val);
3066 }
3067 %}
3068
3069 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3070 MacroAssembler _masm(&cbuf);
3071 Register reg1 = as_Register($src1$$reg);
3072 u_int32_t val = (u_int32_t)$src2$$constant;
3073 __ movw(rscratch1, val);
3074 __ cmpw(reg1, rscratch1);
3075 %}
3076
3077 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3078 MacroAssembler _masm(&cbuf);
3079 Register reg1 = as_Register($src1$$reg);
3080 Register reg2 = as_Register($src2$$reg);
3081 __ cmp(reg1, reg2);
3082 %}
3083
3084 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3085 MacroAssembler _masm(&cbuf);
3086 Register reg = as_Register($src1$$reg);
3087 int64_t val = $src2$$constant;
3088 if (val >= 0) {
3089 __ subs(zr, reg, val);
3090 } else if (val != -val) {
3091 __ adds(zr, reg, -val);
3092 } else {
3093 // aargh, Long.MIN_VALUE is a special case
3094 __ orr(rscratch1, zr, (u_int64_t)val);
3095 __ subs(zr, reg, rscratch1);
3096 }
3097 %}
3098
3099 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3100 MacroAssembler _masm(&cbuf);
3101 Register reg1 = as_Register($src1$$reg);
3102 u_int64_t val = (u_int64_t)$src2$$constant;
3103 __ mov(rscratch1, val);
3104 __ cmp(reg1, rscratch1);
3105 %}
3106
3107 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3108 MacroAssembler _masm(&cbuf);
3109 Register reg1 = as_Register($src1$$reg);
3110 Register reg2 = as_Register($src2$$reg);
3111 __ cmp(reg1, reg2);
3112 %}
3113
3114 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3115 MacroAssembler _masm(&cbuf);
3116 Register reg1 = as_Register($src1$$reg);
3117 Register reg2 = as_Register($src2$$reg);
3118 __ cmpw(reg1, reg2);
3119 %}
3120
3121 enc_class aarch64_enc_testp(iRegP src) %{
3122 MacroAssembler _masm(&cbuf);
3123 Register reg = as_Register($src$$reg);
3124 __ cmp(reg, zr);
3125 %}
3126
3127 enc_class aarch64_enc_testn(iRegN src) %{
3128 MacroAssembler _masm(&cbuf);
3129 Register reg = as_Register($src$$reg);
3130 __ cmpw(reg, zr);
3131 %}
3132
3133 enc_class aarch64_enc_b(label lbl) %{
3134 MacroAssembler _masm(&cbuf);
3135 Label *L = $lbl$$label;
3136 __ b(*L);
3137 %}
3138
3139 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3140 MacroAssembler _masm(&cbuf);
3141 Label *L = $lbl$$label;
3142 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3143 %}
3144
3145 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3146 MacroAssembler _masm(&cbuf);
3147 Label *L = $lbl$$label;
3148 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3149 %}
3150
3151 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3152 %{
3153 Register sub_reg = as_Register($sub$$reg);
3154 Register super_reg = as_Register($super$$reg);
3155 Register temp_reg = as_Register($temp$$reg);
3156 Register result_reg = as_Register($result$$reg);
3157
3158 Label miss;
3159 MacroAssembler _masm(&cbuf);
3160 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3161 NULL, &miss,
3162 /*set_cond_codes:*/ true);
3163 if ($primary) {
3164 __ mov(result_reg, zr);
3165 }
3166 __ bind(miss);
3167 %}
3168
3169 enc_class aarch64_enc_java_static_call(method meth) %{
3170 MacroAssembler _masm(&cbuf);
3171
3172 address mark = __ pc();
3173 address addr = (address)$meth$$method;
3174 address call;
3175 if (!_method) {
3176 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3177 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3178 } else if (_optimized_virtual) {
3179 call = __ trampoline_call(Address(addr, relocInfo::opt_virtual_call_type), &cbuf);
3180 } else {
3181 call = __ trampoline_call(Address(addr, relocInfo::static_call_type), &cbuf);
3182 }
3183 if (call == NULL) {
3184 ciEnv::current()->record_failure("CodeCache is full");
3185 return;
3186 }
3187
3188 if (_method) {
3189 // Emit stub for static call
3190 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
3191 if (stub == NULL) {
3192 ciEnv::current()->record_failure("CodeCache is full");
3193 return;
3194 }
3195 }
3196 %}
3197
3198 enc_class aarch64_enc_java_handle_call(method meth) %{
3199 MacroAssembler _masm(&cbuf);
3200 relocInfo::relocType reloc;
3201
3202 // RFP is preserved across all calls, even compiled calls.
3203 // Use it to preserve SP.
3204 __ mov(rfp, sp);
3205
3206 address mark = __ pc();
3207 address addr = (address)$meth$$method;
3208 address call;
3209 if (!_method) {
3210 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3211 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3212 } else if (_optimized_virtual) {
3213 call = __ trampoline_call(Address(addr, relocInfo::opt_virtual_call_type), &cbuf);
3214 } else {
3215 call = __ trampoline_call(Address(addr, relocInfo::static_call_type), &cbuf);
3216 }
3217 if (call == NULL) {
3218 ciEnv::current()->record_failure("CodeCache is full");
3219 return;
3220 }
3221
3222 if (_method) {
3223 // Emit stub for static call
3224 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
3225 if (stub == NULL) {
3226 ciEnv::current()->record_failure("CodeCache is full");
3227 return;
3228 }
3229 }
3230
3231 // now restore sp
3232 __ mov(sp, rfp);
3233 %}
3234
3235 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3236 MacroAssembler _masm(&cbuf);
3237 address call = __ ic_call((address)$meth$$method);
3238 if (call == NULL) {
3239 ciEnv::current()->record_failure("CodeCache is full");
3240 return;
3241 }
3242 %}
3243
3244 enc_class aarch64_enc_call_epilog() %{
3245 MacroAssembler _masm(&cbuf);
3246 if (VerifyStackAtCalls) {
3247 // Check that stack depth is unchanged: find majik cookie on stack
3248 __ call_Unimplemented();
3249 }
3250 %}
3251
3252 enc_class aarch64_enc_java_to_runtime(method meth) %{
3253 MacroAssembler _masm(&cbuf);
3254
3255 // some calls to generated routines (arraycopy code) are scheduled
3256 // by C2 as runtime calls. if so we can call them using a br (they
3257 // will be in a reachable segment) otherwise we have to use a blr
3258 // which loads the absolute address into a register.
3259 address entry = (address)$meth$$method;
3260 CodeBlob *cb = CodeCache::find_blob(entry);
3261 if (cb) {
3262 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3263 if (call == NULL) {
3264 ciEnv::current()->record_failure("CodeCache is full");
3265 return;
3266 }
3267 } else {
3268 Label retaddr;
3269 __ adr(rscratch2, retaddr);
3270 __ lea(rscratch1, RuntimeAddress(entry));
3271 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3272 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3273 __ blr(rscratch1);
3274 __ bind(retaddr);
3275 __ add(sp, sp, 2 * wordSize);
3276 }
3277 %}
3278
3279 enc_class aarch64_enc_rethrow() %{
3280 MacroAssembler _masm(&cbuf);
3281 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3282 %}
3283
3284 enc_class aarch64_enc_ret() %{
3285 MacroAssembler _masm(&cbuf);
3286 __ ret(lr);
3287 %}
3288
3289 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3290 MacroAssembler _masm(&cbuf);
3291 Register target_reg = as_Register($jump_target$$reg);
3292 __ br(target_reg);
3293 %}
3294
3295 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3296 MacroAssembler _masm(&cbuf);
3297 Register target_reg = as_Register($jump_target$$reg);
3298 // exception oop should be in r0
3299 // ret addr has been popped into lr
3300 // callee expects it in r3
3301 __ mov(r3, lr);
3302 __ br(target_reg);
3303 %}
3304
3305 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3306 MacroAssembler _masm(&cbuf);
3307 Register oop = as_Register($object$$reg);
3308 Register box = as_Register($box$$reg);
3309 Register disp_hdr = as_Register($tmp$$reg);
3310 Register tmp = as_Register($tmp2$$reg);
3311 Label cont;
3312 Label object_has_monitor;
3313 Label cas_failed;
3314
3315 assert_different_registers(oop, box, tmp, disp_hdr);
3316
3317 // Load markOop from object into displaced_header.
3318 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3319
3320 // Always do locking in runtime.
3321 if (EmitSync & 0x01) {
3322 __ cmp(oop, zr);
3323 return;
3324 }
3325
3326 if (UseBiasedLocking && !UseOptoBiasInlining) {
3327 __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
3328 }
3329
3330 // Handle existing monitor
3331 if ((EmitSync & 0x02) == 0) {
3332 __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3333 }
3334
3335 // Set tmp to be (markOop of object | UNLOCK_VALUE).
3336 __ orr(tmp, disp_hdr, markOopDesc::unlocked_value);
3337
3338 // Load Compare Value application register.
3339
3340 // Initialize the box. (Must happen before we update the object mark!)
3341 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3342
3343 // Compare object markOop with an unlocked value (tmp) and if
3344 // equal exchange the stack address of our box with object markOop.
3345 // On failure disp_hdr contains the possibly locked markOop.
3346 if (UseLSE) {
3347 __ mov(disp_hdr, tmp);
3348 __ casal(Assembler::xword, disp_hdr, box, oop); // Updates disp_hdr
3349 __ cmp(tmp, disp_hdr);
3350 __ br(Assembler::EQ, cont);
3351 } else {
3352 Label retry_load;
3353 if ((VM_Version::cpu_cpuFeatures() & VM_Version::CPU_STXR_PREFETCH))
3354 __ prfm(Address(oop), PSTL1STRM);
3355 __ bind(retry_load);
3356 __ ldaxr(disp_hdr, oop);
3357 __ cmp(tmp, disp_hdr);
3358 __ br(Assembler::NE, cas_failed);
3359 // use stlxr to ensure update is immediately visible
3360 __ stlxr(disp_hdr, box, oop);
3361 __ cbzw(disp_hdr, cont);
3362 __ b(retry_load);
3363 }
3364
3365 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3366
3367 // If the compare-and-exchange succeeded, then we found an unlocked
3368 // object, will have now locked it will continue at label cont
3369
3370 __ bind(cas_failed);
3371 // We did not see an unlocked object so try the fast recursive case.
3372
3373 // Check if the owner is self by comparing the value in the
3374 // markOop of object (disp_hdr) with the stack pointer.
3375 __ mov(rscratch1, sp);
3376 __ sub(disp_hdr, disp_hdr, rscratch1);
3377 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place));
3378 // If condition is true we are cont and hence we can store 0 as the
3379 // displaced header in the box, which indicates that it is a recursive lock.
3380 __ ands(tmp/*==0?*/, disp_hdr, tmp);
3381 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3382
3383 // Handle existing monitor.
3384 if ((EmitSync & 0x02) == 0) {
3385 __ b(cont);
3386
3387 __ bind(object_has_monitor);
3388 // The object's monitor m is unlocked iff m->owner == NULL,
3389 // otherwise m->owner may contain a thread or a stack address.
3390 //
3391 // Try to CAS m->owner from NULL to current thread.
3392 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
3393 __ mov(disp_hdr, zr);
3394
3395 if (UseLSE) {
3396 __ mov(rscratch1, disp_hdr);
3397 __ casal(Assembler::xword, rscratch1, rthread, tmp);
3398 __ cmp(rscratch1, disp_hdr);
3399 } else {
3400 Label retry_load, fail;
3401 if ((VM_Version::cpu_cpuFeatures() & VM_Version::CPU_STXR_PREFETCH))
3402 __ prfm(Address(tmp), PSTL1STRM);
3403 __ bind(retry_load);
3404 __ ldaxr(rscratch1, tmp);
3405 __ cmp(disp_hdr, rscratch1);
3406 __ br(Assembler::NE, fail);
3407 // use stlxr to ensure update is immediately visible
3408 __ stlxr(rscratch1, rthread, tmp);
3409 __ cbnzw(rscratch1, retry_load);
3410 __ bind(fail);
3411 }
3412
3413 // Store a non-null value into the box to avoid looking like a re-entrant
3414 // lock. The fast-path monitor unlock code checks for
3415 // markOopDesc::monitor_value so use markOopDesc::unused_mark which has the
3416 // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
3417 __ mov(tmp, (address)markOopDesc::unused_mark());
3418 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3419 }
3420
3421 __ bind(cont);
3422 // flag == EQ indicates success
3423 // flag == NE indicates failure
3424 %}
3425
3426 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3427 MacroAssembler _masm(&cbuf);
3428 Register oop = as_Register($object$$reg);
3429 Register box = as_Register($box$$reg);
3430 Register disp_hdr = as_Register($tmp$$reg);
3431 Register tmp = as_Register($tmp2$$reg);
3432 Label cont;
3433 Label object_has_monitor;
3434
3435 assert_different_registers(oop, box, tmp, disp_hdr);
3436
3437 // Always do locking in runtime.
3438 if (EmitSync & 0x01) {
3439 __ cmp(oop, zr); // Oop can't be 0 here => always false.
3440 return;
3441 }
3442
3443 if (UseBiasedLocking && !UseOptoBiasInlining) {
3444 __ biased_locking_exit(oop, tmp, cont);
3445 }
3446
3447 // Find the lock address and load the displaced header from the stack.
3448 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3449
3450 // If the displaced header is 0, we have a recursive unlock.
3451 __ cmp(disp_hdr, zr);
3452 __ br(Assembler::EQ, cont);
3453
3454 // Handle existing monitor.
3455 if ((EmitSync & 0x02) == 0) {
3456 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3457 __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3458 }
3459
3460 // Check if it is still a light weight lock, this is is true if we
3461 // see the stack address of the basicLock in the markOop of the
3462 // object.
3463
3464 if (UseLSE) {
3465 __ mov(tmp, box);
3466 __ casl(Assembler::xword, tmp, disp_hdr, oop);
3467 __ cmp(tmp, box);
3468 __ b(cont);
3469 } else {
3470 Label retry_load;
3471 if ((VM_Version::cpu_cpuFeatures() & VM_Version::CPU_STXR_PREFETCH))
3472 __ prfm(Address(oop), PSTL1STRM);
3473 __ bind(retry_load);
3474 __ ldxr(tmp, oop);
3475 __ cmp(box, tmp);
3476 __ br(Assembler::NE, cont);
3477 // use stlxr to ensure update is immediately visible
3478 __ stlxr(tmp, disp_hdr, oop);
3479 __ cbzw(tmp, cont);
3480 __ b(retry_load);
3481 }
3482
3483 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3484
3485 // Handle existing monitor.
3486 if ((EmitSync & 0x02) == 0) {
3487 __ bind(object_has_monitor);
3488 __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor
3489 __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3490 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3491 __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3492 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3493 __ cmp(rscratch1, zr);
3494 __ br(Assembler::NE, cont);
3495
3496 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3497 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3498 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3499 __ cmp(rscratch1, zr);
3500 __ br(Assembler::NE, cont);
3501 // need a release store here
3502 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3503 __ stlr(zr, tmp); // set unowned
3504 }
3505
3506 __ bind(cont);
3507 // flag == EQ indicates success
3508 // flag == NE indicates failure
3509 %}
3510
3511 %}
3512
3513 //----------FRAME--------------------------------------------------------------
3514 // Definition of frame structure and management information.
3515 //
3516 // S T A C K L A Y O U T Allocators stack-slot number
3517 // | (to get allocators register number
3518 // G Owned by | | v add OptoReg::stack0())
3519 // r CALLER | |
3520 // o | +--------+ pad to even-align allocators stack-slot
3521 // w V | pad0 | numbers; owned by CALLER
3522 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3523 // h ^ | in | 5
3524 // | | args | 4 Holes in incoming args owned by SELF
3525 // | | | | 3
3526 // | | +--------+
3527 // V | | old out| Empty on Intel, window on Sparc
3528 // | old |preserve| Must be even aligned.
3529 // | SP-+--------+----> Matcher::_old_SP, even aligned
3530 // | | in | 3 area for Intel ret address
3531 // Owned by |preserve| Empty on Sparc.
3532 // SELF +--------+
3533 // | | pad2 | 2 pad to align old SP
3534 // | +--------+ 1
3535 // | | locks | 0
3536 // | +--------+----> OptoReg::stack0(), even aligned
3537 // | | pad1 | 11 pad to align new SP
3538 // | +--------+
3539 // | | | 10
3540 // | | spills | 9 spills
3541 // V | | 8 (pad0 slot for callee)
3542 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3543 // ^ | out | 7
3544 // | | args | 6 Holes in outgoing args owned by CALLEE
3545 // Owned by +--------+
3546 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3547 // | new |preserve| Must be even-aligned.
3548 // | SP-+--------+----> Matcher::_new_SP, even aligned
3549 // | | |
3550 //
3551 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3552 // known from SELF's arguments and the Java calling convention.
3553 // Region 6-7 is determined per call site.
3554 // Note 2: If the calling convention leaves holes in the incoming argument
3555 // area, those holes are owned by SELF. Holes in the outgoing area
3556 // are owned by the CALLEE. Holes should not be nessecary in the
3557 // incoming area, as the Java calling convention is completely under
3558 // the control of the AD file. Doubles can be sorted and packed to
3559 // avoid holes. Holes in the outgoing arguments may be nessecary for
3560 // varargs C calling conventions.
3561 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3562 // even aligned with pad0 as needed.
3563 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3564 // (the latter is true on Intel but is it false on AArch64?)
3565 // region 6-11 is even aligned; it may be padded out more so that
3566 // the region from SP to FP meets the minimum stack alignment.
3567 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3568 // alignment. Region 11, pad1, may be dynamically extended so that
3569 // SP meets the minimum alignment.
3570
3571 frame %{
3572 // What direction does stack grow in (assumed to be same for C & Java)
3573 stack_direction(TOWARDS_LOW);
3574
3575 // These three registers define part of the calling convention
3576 // between compiled code and the interpreter.
3577
3578 // Inline Cache Register or methodOop for I2C.
3579 inline_cache_reg(R12);
3580
3581 // Method Oop Register when calling interpreter.
3582 interpreter_method_oop_reg(R12);
3583
3584 // Number of stack slots consumed by locking an object
3585 sync_stack_slots(2);
3586
3587 // Compiled code's Frame Pointer
3588 frame_pointer(R31);
3589
3590 // Interpreter stores its frame pointer in a register which is
3591 // stored to the stack by I2CAdaptors.
3592 // I2CAdaptors convert from interpreted java to compiled java.
3593 interpreter_frame_pointer(R29);
3594
3595 // Stack alignment requirement
3596 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3597
3598 // Number of stack slots between incoming argument block and the start of
3599 // a new frame. The PROLOG must add this many slots to the stack. The
3600 // EPILOG must remove this many slots. aarch64 needs two slots for
3601 // return address and fp.
3602 // TODO think this is correct but check
3603 in_preserve_stack_slots(4);
3604
3605 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3606 // for calls to C. Supports the var-args backing area for register parms.
3607 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3608
3609 // The after-PROLOG location of the return address. Location of
3610 // return address specifies a type (REG or STACK) and a number
3611 // representing the register number (i.e. - use a register name) or
3612 // stack slot.
3613 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3614 // Otherwise, it is above the locks and verification slot and alignment word
3615 // TODO this may well be correct but need to check why that - 2 is there
3616 // ppc port uses 0 but we definitely need to allow for fixed_slots
3617 // which folds in the space used for monitors
3618 return_addr(STACK - 2 +
3619 round_to((Compile::current()->in_preserve_stack_slots() +
3620 Compile::current()->fixed_slots()),
3621 stack_alignment_in_slots()));
3622
3623 // Body of function which returns an integer array locating
3624 // arguments either in registers or in stack slots. Passed an array
3625 // of ideal registers called "sig" and a "length" count. Stack-slot
3626 // offsets are based on outgoing arguments, i.e. a CALLER setting up
3627 // arguments for a CALLEE. Incoming stack arguments are
3628 // automatically biased by the preserve_stack_slots field above.
3629
3630 calling_convention
3631 %{
3632 // No difference between ingoing/outgoing just pass false
3633 SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3634 %}
3635
3636 c_calling_convention
3637 %{
3638 // This is obviously always outgoing
3639 (void) SharedRuntime::c_calling_convention(sig_bt, regs, NULL, length);
3640 %}
3641
3642 // Location of compiled Java return values. Same as C for now.
3643 return_value
3644 %{
3645 // TODO do we allow ideal_reg == Op_RegN???
3646 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3647 "only return normal values");
3648
3649 static const int lo[Op_RegL + 1] = { // enum name
3650 0, // Op_Node
3651 0, // Op_Set
3652 R0_num, // Op_RegN
3653 R0_num, // Op_RegI
3654 R0_num, // Op_RegP
3655 V0_num, // Op_RegF
3656 V0_num, // Op_RegD
3657 R0_num // Op_RegL
3658 };
3659
3660 static const int hi[Op_RegL + 1] = { // enum name
3661 0, // Op_Node
3662 0, // Op_Set
3663 OptoReg::Bad, // Op_RegN
3664 OptoReg::Bad, // Op_RegI
3665 R0_H_num, // Op_RegP
3666 OptoReg::Bad, // Op_RegF
3667 V0_H_num, // Op_RegD
3668 R0_H_num // Op_RegL
3669 };
3670
3671 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3672 %}
3673 %}
3674
3675 //----------ATTRIBUTES---------------------------------------------------------
3676 //----------Operand Attributes-------------------------------------------------
3677 op_attrib op_cost(1); // Required cost attribute
3678
3679 //----------Instruction Attributes---------------------------------------------
3680 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3681 ins_attrib ins_size(32); // Required size attribute (in bits)
3682 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3683 // a non-matching short branch variant
3684 // of some long branch?
3685 ins_attrib ins_alignment(4); // Required alignment attribute (must
3686 // be a power of 2) specifies the
3687 // alignment that some part of the
3688 // instruction (not necessarily the
3689 // start) requires. If > 1, a
3690 // compute_padding() function must be
3691 // provided for the instruction
3692
3693 //----------OPERANDS-----------------------------------------------------------
3694 // Operand definitions must precede instruction definitions for correct parsing
3695 // in the ADLC because operands constitute user defined types which are used in
3696 // instruction definitions.
3697
3698 //----------Simple Operands----------------------------------------------------
3699
3700 // Integer operands 32 bit
3701 // 32 bit immediate
3702 operand immI()
3703 %{
3704 match(ConI);
3705
3706 op_cost(0);
3707 format %{ %}
3708 interface(CONST_INTER);
3709 %}
3710
3711 // 32 bit zero
3712 operand immI0()
3713 %{
3714 predicate(n->get_int() == 0);
3715 match(ConI);
3716
3717 op_cost(0);
3718 format %{ %}
3719 interface(CONST_INTER);
3720 %}
3721
3722 // 32 bit unit increment
3723 operand immI_1()
3724 %{
3725 predicate(n->get_int() == 1);
3726 match(ConI);
3727
3728 op_cost(0);
3729 format %{ %}
3730 interface(CONST_INTER);
3731 %}
3732
3733 // 32 bit unit decrement
3734 operand immI_M1()
3735 %{
3736 predicate(n->get_int() == -1);
3737 match(ConI);
3738
3739 op_cost(0);
3740 format %{ %}
3741 interface(CONST_INTER);
3742 %}
3743
3744 operand immI_le_4()
3745 %{
3746 predicate(n->get_int() <= 4);
3747 match(ConI);
3748
3749 op_cost(0);
3750 format %{ %}
3751 interface(CONST_INTER);
3752 %}
3753
3754 operand immI_31()
3755 %{
3756 predicate(n->get_int() == 31);
3757 match(ConI);
3758
3759 op_cost(0);
3760 format %{ %}
3761 interface(CONST_INTER);
3762 %}
3763
3764 operand immI_8()
3765 %{
3766 predicate(n->get_int() == 8);
3767 match(ConI);
3768
3769 op_cost(0);
3770 format %{ %}
3771 interface(CONST_INTER);
3772 %}
3773
3774 operand immI_16()
3775 %{
3776 predicate(n->get_int() == 16);
3777 match(ConI);
3778
3779 op_cost(0);
3780 format %{ %}
3781 interface(CONST_INTER);
3782 %}
3783
3784 operand immI_24()
3785 %{
3786 predicate(n->get_int() == 24);
3787 match(ConI);
3788
3789 op_cost(0);
3790 format %{ %}
3791 interface(CONST_INTER);
3792 %}
3793
3794 operand immI_32()
3795 %{
3796 predicate(n->get_int() == 32);
3797 match(ConI);
3798
3799 op_cost(0);
3800 format %{ %}
3801 interface(CONST_INTER);
3802 %}
3803
3804 operand immI_48()
3805 %{
3806 predicate(n->get_int() == 48);
3807 match(ConI);
3808
3809 op_cost(0);
3810 format %{ %}
3811 interface(CONST_INTER);
3812 %}
3813
3814 operand immI_56()
3815 %{
3816 predicate(n->get_int() == 56);
3817 match(ConI);
3818
3819 op_cost(0);
3820 format %{ %}
3821 interface(CONST_INTER);
3822 %}
3823
3824 operand immI_64()
3825 %{
3826 predicate(n->get_int() == 64);
3827 match(ConI);
3828
3829 op_cost(0);
3830 format %{ %}
3831 interface(CONST_INTER);
3832 %}
3833
3834 operand immI_255()
3835 %{
3836 predicate(n->get_int() == 255);
3837 match(ConI);
3838
3839 op_cost(0);
3840 format %{ %}
3841 interface(CONST_INTER);
3842 %}
3843
3844 operand immI_65535()
3845 %{
3846 predicate(n->get_int() == 65535);
3847 match(ConI);
3848
3849 op_cost(0);
3850 format %{ %}
3851 interface(CONST_INTER);
3852 %}
3853
3854 operand immL_63()
3855 %{
3856 predicate(n->get_int() == 63);
3857 match(ConI);
3858
3859 op_cost(0);
3860 format %{ %}
3861 interface(CONST_INTER);
3862 %}
3863
3864 operand immL_255()
3865 %{
3866 predicate(n->get_int() == 255);
3867 match(ConI);
3868
3869 op_cost(0);
3870 format %{ %}
3871 interface(CONST_INTER);
3872 %}
3873
3874 operand immL_65535()
3875 %{
3876 predicate(n->get_long() == 65535L);
3877 match(ConL);
3878
3879 op_cost(0);
3880 format %{ %}
3881 interface(CONST_INTER);
3882 %}
3883
3884 operand immL_4294967295()
3885 %{
3886 predicate(n->get_long() == 4294967295L);
3887 match(ConL);
3888
3889 op_cost(0);
3890 format %{ %}
3891 interface(CONST_INTER);
3892 %}
3893
3894 operand immL_bitmask()
3895 %{
3896 predicate((n->get_long() != 0)
3897 && ((n->get_long() & 0xc000000000000000l) == 0)
3898 && is_power_of_2(n->get_long() + 1));
3899 match(ConL);
3900
3901 op_cost(0);
3902 format %{ %}
3903 interface(CONST_INTER);
3904 %}
3905
3906 operand immI_bitmask()
3907 %{
3908 predicate((n->get_int() != 0)
3909 && ((n->get_int() & 0xc0000000) == 0)
3910 && is_power_of_2(n->get_int() + 1));
3911 match(ConI);
3912
3913 op_cost(0);
3914 format %{ %}
3915 interface(CONST_INTER);
3916 %}
3917
3918 // Scale values for scaled offset addressing modes (up to long but not quad)
3919 operand immIScale()
3920 %{
3921 predicate(0 <= n->get_int() && (n->get_int() <= 3));
3922 match(ConI);
3923
3924 op_cost(0);
3925 format %{ %}
3926 interface(CONST_INTER);
3927 %}
3928
3929 // 26 bit signed offset -- for pc-relative branches
3930 operand immI26()
3931 %{
3932 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
3933 match(ConI);
3934
3935 op_cost(0);
3936 format %{ %}
3937 interface(CONST_INTER);
3938 %}
3939
3940 // 19 bit signed offset -- for pc-relative loads
3941 operand immI19()
3942 %{
3943 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
3944 match(ConI);
3945
3946 op_cost(0);
3947 format %{ %}
3948 interface(CONST_INTER);
3949 %}
3950
3951 // 12 bit unsigned offset -- for base plus immediate loads
3952 operand immIU12()
3953 %{
3954 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
3955 match(ConI);
3956
3957 op_cost(0);
3958 format %{ %}
3959 interface(CONST_INTER);
3960 %}
3961
3962 operand immLU12()
3963 %{
3964 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
3965 match(ConL);
3966
3967 op_cost(0);
3968 format %{ %}
3969 interface(CONST_INTER);
3970 %}
3971
3972 // Offset for scaled or unscaled immediate loads and stores
3973 operand immIOffset()
3974 %{
3975 predicate(Address::offset_ok_for_immed(n->get_int()));
3976 match(ConI);
3977
3978 op_cost(0);
3979 format %{ %}
3980 interface(CONST_INTER);
3981 %}
3982
3983 operand immIOffset4()
3984 %{
3985 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
3986 match(ConI);
3987
3988 op_cost(0);
3989 format %{ %}
3990 interface(CONST_INTER);
3991 %}
3992
3993 operand immIOffset8()
3994 %{
3995 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
3996 match(ConI);
3997
3998 op_cost(0);
3999 format %{ %}
4000 interface(CONST_INTER);
4001 %}
4002
4003 operand immIOffset16()
4004 %{
4005 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4006 match(ConI);
4007
4008 op_cost(0);
4009 format %{ %}
4010 interface(CONST_INTER);
4011 %}
4012
4013 operand immLoffset()
4014 %{
4015 predicate(Address::offset_ok_for_immed(n->get_long()));
4016 match(ConL);
4017
4018 op_cost(0);
4019 format %{ %}
4020 interface(CONST_INTER);
4021 %}
4022
4023 operand immLoffset4()
4024 %{
4025 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4026 match(ConL);
4027
4028 op_cost(0);
4029 format %{ %}
4030 interface(CONST_INTER);
4031 %}
4032
4033 operand immLoffset8()
4034 %{
4035 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4036 match(ConL);
4037
4038 op_cost(0);
4039 format %{ %}
4040 interface(CONST_INTER);
4041 %}
4042
4043 operand immLoffset16()
4044 %{
4045 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4046 match(ConL);
4047
4048 op_cost(0);
4049 format %{ %}
4050 interface(CONST_INTER);
4051 %}
4052
4053 // 32 bit integer valid for add sub immediate
4054 operand immIAddSub()
4055 %{
4056 predicate(Assembler::operand_valid_for_add_sub_immediate((long)n->get_int()));
4057 match(ConI);
4058 op_cost(0);
4059 format %{ %}
4060 interface(CONST_INTER);
4061 %}
4062
4063 // 32 bit unsigned integer valid for logical immediate
4064 // TODO -- check this is right when e.g the mask is 0x80000000
4065 operand immILog()
4066 %{
4067 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (unsigned long)n->get_int()));
4068 match(ConI);
4069
4070 op_cost(0);
4071 format %{ %}
4072 interface(CONST_INTER);
4073 %}
4074
4075 // Integer operands 64 bit
4076 // 64 bit immediate
4077 operand immL()
4078 %{
4079 match(ConL);
4080
4081 op_cost(0);
4082 format %{ %}
4083 interface(CONST_INTER);
4084 %}
4085
4086 // 64 bit zero
4087 operand immL0()
4088 %{
4089 predicate(n->get_long() == 0);
4090 match(ConL);
4091
4092 op_cost(0);
4093 format %{ %}
4094 interface(CONST_INTER);
4095 %}
4096
4097 // 64 bit unit increment
4098 operand immL_1()
4099 %{
4100 predicate(n->get_long() == 1);
4101 match(ConL);
4102
4103 op_cost(0);
4104 format %{ %}
4105 interface(CONST_INTER);
4106 %}
4107
4108 // 64 bit unit decrement
4109 operand immL_M1()
4110 %{
4111 predicate(n->get_long() == -1);
4112 match(ConL);
4113
4114 op_cost(0);
4115 format %{ %}
4116 interface(CONST_INTER);
4117 %}
4118
4119 // 32 bit offset of pc in thread anchor
4120
4121 operand immL_pc_off()
4122 %{
4123 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4124 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4125 match(ConL);
4126
4127 op_cost(0);
4128 format %{ %}
4129 interface(CONST_INTER);
4130 %}
4131
4132 // 64 bit integer valid for add sub immediate
4133 operand immLAddSub()
4134 %{
4135 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4136 match(ConL);
4137 op_cost(0);
4138 format %{ %}
4139 interface(CONST_INTER);
4140 %}
4141
4142 // 64 bit integer valid for logical immediate
4143 operand immLLog()
4144 %{
4145 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (unsigned long)n->get_long()));
4146 match(ConL);
4147 op_cost(0);
4148 format %{ %}
4149 interface(CONST_INTER);
4150 %}
4151
4152 // Long Immediate: low 32-bit mask
4153 operand immL_32bits()
4154 %{
4155 predicate(n->get_long() == 0xFFFFFFFFL);
4156 match(ConL);
4157 op_cost(0);
4158 format %{ %}
4159 interface(CONST_INTER);
4160 %}
4161
4162 // Pointer operands
4163 // Pointer Immediate
4164 operand immP()
4165 %{
4166 match(ConP);
4167
4168 op_cost(0);
4169 format %{ %}
4170 interface(CONST_INTER);
4171 %}
4172
4173 // NULL Pointer Immediate
4174 operand immP0()
4175 %{
4176 predicate(n->get_ptr() == 0);
4177 match(ConP);
4178
4179 op_cost(0);
4180 format %{ %}
4181 interface(CONST_INTER);
4182 %}
4183
4184 // Pointer Immediate One
4185 // this is used in object initialization (initial object header)
4186 operand immP_1()
4187 %{
4188 predicate(n->get_ptr() == 1);
4189 match(ConP);
4190
4191 op_cost(0);
4192 format %{ %}
4193 interface(CONST_INTER);
4194 %}
4195
4196 // Polling Page Pointer Immediate
4197 operand immPollPage()
4198 %{
4199 predicate((address)n->get_ptr() == os::get_polling_page());
4200 match(ConP);
4201
4202 op_cost(0);
4203 format %{ %}
4204 interface(CONST_INTER);
4205 %}
4206
4207 // Card Table Byte Map Base
4208 operand immByteMapBase()
4209 %{
4210 // Get base of card map
4211 predicate(!UseShenandoahGC && // TODO: Should really check for BS::is_a, see JDK-8193193
4212 (jbyte*)n->get_ptr() == ((CardTableModRefBS*)(Universe::heap()->barrier_set()))->byte_map_base);
4213 match(ConP);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 // Pointer Immediate Minus One
4221 // this is used when we want to write the current PC to the thread anchor
4222 operand immP_M1()
4223 %{
4224 predicate(n->get_ptr() == -1);
4225 match(ConP);
4226
4227 op_cost(0);
4228 format %{ %}
4229 interface(CONST_INTER);
4230 %}
4231
4232 // Pointer Immediate Minus Two
4233 // this is used when we want to write the current PC to the thread anchor
4234 operand immP_M2()
4235 %{
4236 predicate(n->get_ptr() == -2);
4237 match(ConP);
4238
4239 op_cost(0);
4240 format %{ %}
4241 interface(CONST_INTER);
4242 %}
4243
4244 // Float and Double operands
4245 // Double Immediate
4246 operand immD()
4247 %{
4248 match(ConD);
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 // constant 'double +0.0'.
4255 operand immD0()
4256 %{
4257 predicate((n->getd() == 0) &&
4258 (fpclassify(n->getd()) == FP_ZERO) && (signbit(n->getd()) == 0));
4259 match(ConD);
4260 op_cost(0);
4261 format %{ %}
4262 interface(CONST_INTER);
4263 %}
4264
4265 // constant 'double +0.0'.
4266 operand immDPacked()
4267 %{
4268 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4269 match(ConD);
4270 op_cost(0);
4271 format %{ %}
4272 interface(CONST_INTER);
4273 %}
4274
4275 // Float Immediate
4276 operand immF()
4277 %{
4278 match(ConF);
4279 op_cost(0);
4280 format %{ %}
4281 interface(CONST_INTER);
4282 %}
4283
4284 // constant 'float +0.0'.
4285 operand immF0()
4286 %{
4287 predicate((n->getf() == 0) &&
4288 (fpclassify(n->getf()) == FP_ZERO) && (signbit(n->getf()) == 0));
4289 match(ConF);
4290 op_cost(0);
4291 format %{ %}
4292 interface(CONST_INTER);
4293 %}
4294
4295 //
4296 operand immFPacked()
4297 %{
4298 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4299 match(ConF);
4300 op_cost(0);
4301 format %{ %}
4302 interface(CONST_INTER);
4303 %}
4304
4305 // Narrow pointer operands
4306 // Narrow Pointer Immediate
4307 operand immN()
4308 %{
4309 match(ConN);
4310
4311 op_cost(0);
4312 format %{ %}
4313 interface(CONST_INTER);
4314 %}
4315
4316 // Narrow NULL Pointer Immediate
4317 operand immN0()
4318 %{
4319 predicate(n->get_narrowcon() == 0);
4320 match(ConN);
4321
4322 op_cost(0);
4323 format %{ %}
4324 interface(CONST_INTER);
4325 %}
4326
4327 operand immNKlass()
4328 %{
4329 match(ConNKlass);
4330
4331 op_cost(0);
4332 format %{ %}
4333 interface(CONST_INTER);
4334 %}
4335
4336 // Integer 32 bit Register Operands
4337 // Integer 32 bitRegister (excludes SP)
4338 operand iRegI()
4339 %{
4340 constraint(ALLOC_IN_RC(any_reg32));
4341 match(RegI);
4342 match(iRegINoSp);
4343 op_cost(0);
4344 format %{ %}
4345 interface(REG_INTER);
4346 %}
4347
4348 // Integer 32 bit Register not Special
4349 operand iRegINoSp()
4350 %{
4351 constraint(ALLOC_IN_RC(no_special_reg32));
4352 match(RegI);
4353 op_cost(0);
4354 format %{ %}
4355 interface(REG_INTER);
4356 %}
4357
4358 // Integer 64 bit Register Operands
4359 // Integer 64 bit Register (includes SP)
4360 operand iRegL()
4361 %{
4362 constraint(ALLOC_IN_RC(any_reg));
4363 match(RegL);
4364 match(iRegLNoSp);
4365 op_cost(0);
4366 format %{ %}
4367 interface(REG_INTER);
4368 %}
4369
4370 // Integer 64 bit Register not Special
4371 operand iRegLNoSp()
4372 %{
4373 constraint(ALLOC_IN_RC(no_special_reg));
4374 match(RegL);
4375 format %{ %}
4376 interface(REG_INTER);
4377 %}
4378
4379 // Pointer Register Operands
4380 // Pointer Register
4381 operand iRegP()
4382 %{
4383 constraint(ALLOC_IN_RC(ptr_reg));
4384 match(RegP);
4385 match(iRegPNoSp);
4386 match(iRegP_R0);
4387 //match(iRegP_R2);
4388 //match(iRegP_R4);
4389 //match(iRegP_R5);
4390 match(thread_RegP);
4391 op_cost(0);
4392 format %{ %}
4393 interface(REG_INTER);
4394 %}
4395
4396 // Pointer 64 bit Register not Special
4397 operand iRegPNoSp()
4398 %{
4399 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4400 match(RegP);
4401 // match(iRegP);
4402 // match(iRegP_R0);
4403 // match(iRegP_R2);
4404 // match(iRegP_R4);
4405 // match(iRegP_R5);
4406 // match(thread_RegP);
4407 op_cost(0);
4408 format %{ %}
4409 interface(REG_INTER);
4410 %}
4411
4412 // Pointer 64 bit Register R0 only
4413 operand iRegP_R0()
4414 %{
4415 constraint(ALLOC_IN_RC(r0_reg));
4416 match(RegP);
4417 // match(iRegP);
4418 match(iRegPNoSp);
4419 op_cost(0);
4420 format %{ %}
4421 interface(REG_INTER);
4422 %}
4423
4424 // Pointer 64 bit Register R1 only
4425 operand iRegP_R1()
4426 %{
4427 constraint(ALLOC_IN_RC(r1_reg));
4428 match(RegP);
4429 // match(iRegP);
4430 match(iRegPNoSp);
4431 op_cost(0);
4432 format %{ %}
4433 interface(REG_INTER);
4434 %}
4435
4436 // Pointer 64 bit Register R2 only
4437 operand iRegP_R2()
4438 %{
4439 constraint(ALLOC_IN_RC(r2_reg));
4440 match(RegP);
4441 // match(iRegP);
4442 match(iRegPNoSp);
4443 op_cost(0);
4444 format %{ %}
4445 interface(REG_INTER);
4446 %}
4447
4448 // Pointer 64 bit Register R3 only
4449 operand iRegP_R3()
4450 %{
4451 constraint(ALLOC_IN_RC(r3_reg));
4452 match(RegP);
4453 // match(iRegP);
4454 match(iRegPNoSp);
4455 op_cost(0);
4456 format %{ %}
4457 interface(REG_INTER);
4458 %}
4459
4460 // Pointer 64 bit Register R4 only
4461 operand iRegP_R4()
4462 %{
4463 constraint(ALLOC_IN_RC(r4_reg));
4464 match(RegP);
4465 // match(iRegP);
4466 match(iRegPNoSp);
4467 op_cost(0);
4468 format %{ %}
4469 interface(REG_INTER);
4470 %}
4471
4472 // Pointer 64 bit Register R5 only
4473 operand iRegP_R5()
4474 %{
4475 constraint(ALLOC_IN_RC(r5_reg));
4476 match(RegP);
4477 // match(iRegP);
4478 match(iRegPNoSp);
4479 op_cost(0);
4480 format %{ %}
4481 interface(REG_INTER);
4482 %}
4483
4484 // Pointer 64 bit Register R10 only
4485 operand iRegP_R10()
4486 %{
4487 constraint(ALLOC_IN_RC(r10_reg));
4488 match(RegP);
4489 // match(iRegP);
4490 match(iRegPNoSp);
4491 op_cost(0);
4492 format %{ %}
4493 interface(REG_INTER);
4494 %}
4495
4496 // Long 64 bit Register R11 only
4497 operand iRegL_R11()
4498 %{
4499 constraint(ALLOC_IN_RC(r11_reg));
4500 match(RegL);
4501 match(iRegLNoSp);
4502 op_cost(0);
4503 format %{ %}
4504 interface(REG_INTER);
4505 %}
4506
4507 // Pointer 64 bit Register FP only
4508 operand iRegP_FP()
4509 %{
4510 constraint(ALLOC_IN_RC(fp_reg));
4511 match(RegP);
4512 // match(iRegP);
4513 op_cost(0);
4514 format %{ %}
4515 interface(REG_INTER);
4516 %}
4517
4518 // Register R0 only
4519 operand iRegI_R0()
4520 %{
4521 constraint(ALLOC_IN_RC(int_r0_reg));
4522 match(RegI);
4523 match(iRegINoSp);
4524 op_cost(0);
4525 format %{ %}
4526 interface(REG_INTER);
4527 %}
4528
4529 // Register R2 only
4530 operand iRegI_R2()
4531 %{
4532 constraint(ALLOC_IN_RC(int_r2_reg));
4533 match(RegI);
4534 match(iRegINoSp);
4535 op_cost(0);
4536 format %{ %}
4537 interface(REG_INTER);
4538 %}
4539
4540 // Register R3 only
4541 operand iRegI_R3()
4542 %{
4543 constraint(ALLOC_IN_RC(int_r3_reg));
4544 match(RegI);
4545 match(iRegINoSp);
4546 op_cost(0);
4547 format %{ %}
4548 interface(REG_INTER);
4549 %}
4550
4551
4552 // Register R2 only
4553 operand iRegI_R4()
4554 %{
4555 constraint(ALLOC_IN_RC(int_r4_reg));
4556 match(RegI);
4557 match(iRegINoSp);
4558 op_cost(0);
4559 format %{ %}
4560 interface(REG_INTER);
4561 %}
4562
4563
4564 // Pointer Register Operands
4565 // Narrow Pointer Register
4566 operand iRegN()
4567 %{
4568 constraint(ALLOC_IN_RC(any_reg32));
4569 match(RegN);
4570 match(iRegNNoSp);
4571 op_cost(0);
4572 format %{ %}
4573 interface(REG_INTER);
4574 %}
4575
4576 // Integer 64 bit Register not Special
4577 operand iRegNNoSp()
4578 %{
4579 constraint(ALLOC_IN_RC(no_special_reg32));
4580 match(RegN);
4581 op_cost(0);
4582 format %{ %}
4583 interface(REG_INTER);
4584 %}
4585
4586 // heap base register -- used for encoding immN0
4587
4588 operand iRegIHeapbase()
4589 %{
4590 constraint(ALLOC_IN_RC(heapbase_reg));
4591 match(RegI);
4592 op_cost(0);
4593 format %{ %}
4594 interface(REG_INTER);
4595 %}
4596
4597 // Float Register
4598 // Float register operands
4599 operand vRegF()
4600 %{
4601 constraint(ALLOC_IN_RC(float_reg));
4602 match(RegF);
4603
4604 op_cost(0);
4605 format %{ %}
4606 interface(REG_INTER);
4607 %}
4608
4609 // Double Register
4610 // Double register operands
4611 operand vRegD()
4612 %{
4613 constraint(ALLOC_IN_RC(double_reg));
4614 match(RegD);
4615
4616 op_cost(0);
4617 format %{ %}
4618 interface(REG_INTER);
4619 %}
4620
4621 operand vecD()
4622 %{
4623 constraint(ALLOC_IN_RC(vectord_reg));
4624 match(VecD);
4625
4626 op_cost(0);
4627 format %{ %}
4628 interface(REG_INTER);
4629 %}
4630
4631 operand vecX()
4632 %{
4633 constraint(ALLOC_IN_RC(vectorx_reg));
4634 match(VecX);
4635
4636 op_cost(0);
4637 format %{ %}
4638 interface(REG_INTER);
4639 %}
4640
4641 operand vRegD_V0()
4642 %{
4643 constraint(ALLOC_IN_RC(v0_reg));
4644 match(RegD);
4645 op_cost(0);
4646 format %{ %}
4647 interface(REG_INTER);
4648 %}
4649
4650 operand vRegD_V1()
4651 %{
4652 constraint(ALLOC_IN_RC(v1_reg));
4653 match(RegD);
4654 op_cost(0);
4655 format %{ %}
4656 interface(REG_INTER);
4657 %}
4658
4659 operand vRegD_V2()
4660 %{
4661 constraint(ALLOC_IN_RC(v2_reg));
4662 match(RegD);
4663 op_cost(0);
4664 format %{ %}
4665 interface(REG_INTER);
4666 %}
4667
4668 operand vRegD_V3()
4669 %{
4670 constraint(ALLOC_IN_RC(v3_reg));
4671 match(RegD);
4672 op_cost(0);
4673 format %{ %}
4674 interface(REG_INTER);
4675 %}
4676
4677 // Flags register, used as output of signed compare instructions
4678
4679 // note that on AArch64 we also use this register as the output for
4680 // for floating point compare instructions (CmpF CmpD). this ensures
4681 // that ordered inequality tests use GT, GE, LT or LE none of which
4682 // pass through cases where the result is unordered i.e. one or both
4683 // inputs to the compare is a NaN. this means that the ideal code can
4684 // replace e.g. a GT with an LE and not end up capturing the NaN case
4685 // (where the comparison should always fail). EQ and NE tests are
4686 // always generated in ideal code so that unordered folds into the NE
4687 // case, matching the behaviour of AArch64 NE.
4688 //
4689 // This differs from x86 where the outputs of FP compares use a
4690 // special FP flags registers and where compares based on this
4691 // register are distinguished into ordered inequalities (cmpOpUCF) and
4692 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
4693 // to explicitly handle the unordered case in branches. x86 also has
4694 // to include extra CMoveX rules to accept a cmpOpUCF input.
4695
4696 operand rFlagsReg()
4697 %{
4698 constraint(ALLOC_IN_RC(int_flags));
4699 match(RegFlags);
4700
4701 op_cost(0);
4702 format %{ "RFLAGS" %}
4703 interface(REG_INTER);
4704 %}
4705
4706 // Flags register, used as output of unsigned compare instructions
4707 operand rFlagsRegU()
4708 %{
4709 constraint(ALLOC_IN_RC(int_flags));
4710 match(RegFlags);
4711
4712 op_cost(0);
4713 format %{ "RFLAGSU" %}
4714 interface(REG_INTER);
4715 %}
4716
4717 // Special Registers
4718
4719 // Method Register
4720 operand inline_cache_RegP(iRegP reg)
4721 %{
4722 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
4723 match(reg);
4724 match(iRegPNoSp);
4725 op_cost(0);
4726 format %{ %}
4727 interface(REG_INTER);
4728 %}
4729
4730 operand interpreter_method_oop_RegP(iRegP reg)
4731 %{
4732 constraint(ALLOC_IN_RC(method_reg)); // interpreter_method_oop_reg
4733 match(reg);
4734 match(iRegPNoSp);
4735 op_cost(0);
4736 format %{ %}
4737 interface(REG_INTER);
4738 %}
4739
4740 // Thread Register
4741 operand thread_RegP(iRegP reg)
4742 %{
4743 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
4744 match(reg);
4745 op_cost(0);
4746 format %{ %}
4747 interface(REG_INTER);
4748 %}
4749
4750 operand lr_RegP(iRegP reg)
4751 %{
4752 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
4753 match(reg);
4754 op_cost(0);
4755 format %{ %}
4756 interface(REG_INTER);
4757 %}
4758
4759 //----------Memory Operands----------------------------------------------------
4760
4761 operand indirect(iRegP reg)
4762 %{
4763 constraint(ALLOC_IN_RC(ptr_reg));
4764 match(reg);
4765 op_cost(0);
4766 format %{ "[$reg]" %}
4767 interface(MEMORY_INTER) %{
4768 base($reg);
4769 index(0xffffffff);
4770 scale(0x0);
4771 disp(0x0);
4772 %}
4773 %}
4774
4775 operand indIndexScaledOffsetI(iRegP reg, iRegL lreg, immIScale scale, immIU12 off)
4776 %{
4777 constraint(ALLOC_IN_RC(ptr_reg));
4778 match(AddP (AddP reg (LShiftL lreg scale)) off);
4779 op_cost(INSN_COST);
4780 format %{ "$reg, $lreg lsl($scale), $off" %}
4781 interface(MEMORY_INTER) %{
4782 base($reg);
4783 index($lreg);
4784 scale($scale);
4785 disp($off);
4786 %}
4787 %}
4788
4789 operand indIndexScaledOffsetL(iRegP reg, iRegL lreg, immIScale scale, immLU12 off)
4790 %{
4791 constraint(ALLOC_IN_RC(ptr_reg));
4792 match(AddP (AddP reg (LShiftL lreg scale)) off);
4793 op_cost(INSN_COST);
4794 format %{ "$reg, $lreg lsl($scale), $off" %}
4795 interface(MEMORY_INTER) %{
4796 base($reg);
4797 index($lreg);
4798 scale($scale);
4799 disp($off);
4800 %}
4801 %}
4802
4803 operand indIndexOffsetI2L(iRegP reg, iRegI ireg, immLU12 off)
4804 %{
4805 constraint(ALLOC_IN_RC(ptr_reg));
4806 match(AddP (AddP reg (ConvI2L ireg)) off);
4807 op_cost(INSN_COST);
4808 format %{ "$reg, $ireg, $off I2L" %}
4809 interface(MEMORY_INTER) %{
4810 base($reg);
4811 index($ireg);
4812 scale(0x0);
4813 disp($off);
4814 %}
4815 %}
4816
4817 operand indIndexScaledOffsetI2L(iRegP reg, iRegI ireg, immIScale scale, immLU12 off)
4818 %{
4819 constraint(ALLOC_IN_RC(ptr_reg));
4820 match(AddP (AddP reg (LShiftL (ConvI2L ireg) scale)) off);
4821 op_cost(INSN_COST);
4822 format %{ "$reg, $ireg sxtw($scale), $off I2L" %}
4823 interface(MEMORY_INTER) %{
4824 base($reg);
4825 index($ireg);
4826 scale($scale);
4827 disp($off);
4828 %}
4829 %}
4830
4831 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
4832 %{
4833 constraint(ALLOC_IN_RC(ptr_reg));
4834 match(AddP reg (LShiftL (ConvI2L ireg) scale));
4835 op_cost(0);
4836 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
4837 interface(MEMORY_INTER) %{
4838 base($reg);
4839 index($ireg);
4840 scale($scale);
4841 disp(0x0);
4842 %}
4843 %}
4844
4845 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
4846 %{
4847 constraint(ALLOC_IN_RC(ptr_reg));
4848 match(AddP reg (LShiftL lreg scale));
4849 op_cost(0);
4850 format %{ "$reg, $lreg lsl($scale)" %}
4851 interface(MEMORY_INTER) %{
4852 base($reg);
4853 index($lreg);
4854 scale($scale);
4855 disp(0x0);
4856 %}
4857 %}
4858
4859 operand indIndex(iRegP reg, iRegL lreg)
4860 %{
4861 constraint(ALLOC_IN_RC(ptr_reg));
4862 match(AddP reg lreg);
4863 op_cost(0);
4864 format %{ "$reg, $lreg" %}
4865 interface(MEMORY_INTER) %{
4866 base($reg);
4867 index($lreg);
4868 scale(0x0);
4869 disp(0x0);
4870 %}
4871 %}
4872
4873 operand indOffI(iRegP reg, immIOffset off)
4874 %{
4875 constraint(ALLOC_IN_RC(ptr_reg));
4876 match(AddP reg off);
4877 op_cost(0);
4878 format %{ "[$reg, $off]" %}
4879 interface(MEMORY_INTER) %{
4880 base($reg);
4881 index(0xffffffff);
4882 scale(0x0);
4883 disp($off);
4884 %}
4885 %}
4886
4887 operand indOffI4(iRegP reg, immIOffset4 off)
4888 %{
4889 constraint(ALLOC_IN_RC(ptr_reg));
4890 match(AddP reg off);
4891 op_cost(0);
4892 format %{ "[$reg, $off]" %}
4893 interface(MEMORY_INTER) %{
4894 base($reg);
4895 index(0xffffffff);
4896 scale(0x0);
4897 disp($off);
4898 %}
4899 %}
4900
4901 operand indOffI8(iRegP reg, immIOffset8 off)
4902 %{
4903 constraint(ALLOC_IN_RC(ptr_reg));
4904 match(AddP reg off);
4905 op_cost(0);
4906 format %{ "[$reg, $off]" %}
4907 interface(MEMORY_INTER) %{
4908 base($reg);
4909 index(0xffffffff);
4910 scale(0x0);
4911 disp($off);
4912 %}
4913 %}
4914
4915 operand indOffI16(iRegP reg, immIOffset16 off)
4916 %{
4917 constraint(ALLOC_IN_RC(ptr_reg));
4918 match(AddP reg off);
4919 op_cost(0);
4920 format %{ "[$reg, $off]" %}
4921 interface(MEMORY_INTER) %{
4922 base($reg);
4923 index(0xffffffff);
4924 scale(0x0);
4925 disp($off);
4926 %}
4927 %}
4928
4929 operand indOffL(iRegP reg, immLoffset off)
4930 %{
4931 constraint(ALLOC_IN_RC(ptr_reg));
4932 match(AddP reg off);
4933 op_cost(0);
4934 format %{ "[$reg, $off]" %}
4935 interface(MEMORY_INTER) %{
4936 base($reg);
4937 index(0xffffffff);
4938 scale(0x0);
4939 disp($off);
4940 %}
4941 %}
4942
4943 operand indOffL4(iRegP reg, immLoffset4 off)
4944 %{
4945 constraint(ALLOC_IN_RC(ptr_reg));
4946 match(AddP reg off);
4947 op_cost(0);
4948 format %{ "[$reg, $off]" %}
4949 interface(MEMORY_INTER) %{
4950 base($reg);
4951 index(0xffffffff);
4952 scale(0x0);
4953 disp($off);
4954 %}
4955 %}
4956
4957 operand indOffL8(iRegP reg, immLoffset8 off)
4958 %{
4959 constraint(ALLOC_IN_RC(ptr_reg));
4960 match(AddP reg off);
4961 op_cost(0);
4962 format %{ "[$reg, $off]" %}
4963 interface(MEMORY_INTER) %{
4964 base($reg);
4965 index(0xffffffff);
4966 scale(0x0);
4967 disp($off);
4968 %}
4969 %}
4970
4971 operand indOffL16(iRegP reg, immLoffset16 off)
4972 %{
4973 constraint(ALLOC_IN_RC(ptr_reg));
4974 match(AddP reg off);
4975 op_cost(0);
4976 format %{ "[$reg, $off]" %}
4977 interface(MEMORY_INTER) %{
4978 base($reg);
4979 index(0xffffffff);
4980 scale(0x0);
4981 disp($off);
4982 %}
4983 %}
4984
4985 operand indirectN(iRegN reg)
4986 %{
4987 predicate(Universe::narrow_oop_shift() == 0);
4988 constraint(ALLOC_IN_RC(ptr_reg));
4989 match(DecodeN reg);
4990 op_cost(0);
4991 format %{ "[$reg]\t# narrow" %}
4992 interface(MEMORY_INTER) %{
4993 base($reg);
4994 index(0xffffffff);
4995 scale(0x0);
4996 disp(0x0);
4997 %}
4998 %}
4999
5000 operand indIndexScaledOffsetIN(iRegN reg, iRegL lreg, immIScale scale, immIU12 off)
5001 %{
5002 predicate(Universe::narrow_oop_shift() == 0);
5003 constraint(ALLOC_IN_RC(ptr_reg));
5004 match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5005 op_cost(0);
5006 format %{ "$reg, $lreg lsl($scale), $off\t# narrow" %}
5007 interface(MEMORY_INTER) %{
5008 base($reg);
5009 index($lreg);
5010 scale($scale);
5011 disp($off);
5012 %}
5013 %}
5014
5015 operand indIndexScaledOffsetLN(iRegN reg, iRegL lreg, immIScale scale, immLU12 off)
5016 %{
5017 predicate(Universe::narrow_oop_shift() == 0);
5018 constraint(ALLOC_IN_RC(ptr_reg));
5019 match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5020 op_cost(INSN_COST);
5021 format %{ "$reg, $lreg lsl($scale), $off\t# narrow" %}
5022 interface(MEMORY_INTER) %{
5023 base($reg);
5024 index($lreg);
5025 scale($scale);
5026 disp($off);
5027 %}
5028 %}
5029
5030 operand indIndexOffsetI2LN(iRegN reg, iRegI ireg, immLU12 off)
5031 %{
5032 predicate(Universe::narrow_oop_shift() == 0);
5033 constraint(ALLOC_IN_RC(ptr_reg));
5034 match(AddP (AddP (DecodeN reg) (ConvI2L ireg)) off);
5035 op_cost(INSN_COST);
5036 format %{ "$reg, $ireg, $off I2L\t# narrow" %}
5037 interface(MEMORY_INTER) %{
5038 base($reg);
5039 index($ireg);
5040 scale(0x0);
5041 disp($off);
5042 %}
5043 %}
5044
5045 operand indIndexScaledOffsetI2LN(iRegN reg, iRegI ireg, immIScale scale, immLU12 off)
5046 %{
5047 predicate(Universe::narrow_oop_shift() == 0);
5048 constraint(ALLOC_IN_RC(ptr_reg));
5049 match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale)) off);
5050 op_cost(INSN_COST);
5051 format %{ "$reg, $ireg sxtw($scale), $off I2L\t# narrow" %}
5052 interface(MEMORY_INTER) %{
5053 base($reg);
5054 index($ireg);
5055 scale($scale);
5056 disp($off);
5057 %}
5058 %}
5059
5060 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5061 %{
5062 predicate(Universe::narrow_oop_shift() == 0);
5063 constraint(ALLOC_IN_RC(ptr_reg));
5064 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5065 op_cost(0);
5066 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5067 interface(MEMORY_INTER) %{
5068 base($reg);
5069 index($ireg);
5070 scale($scale);
5071 disp(0x0);
5072 %}
5073 %}
5074
5075 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5076 %{
5077 predicate(Universe::narrow_oop_shift() == 0);
5078 constraint(ALLOC_IN_RC(ptr_reg));
5079 match(AddP (DecodeN reg) (LShiftL lreg scale));
5080 op_cost(0);
5081 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5082 interface(MEMORY_INTER) %{
5083 base($reg);
5084 index($lreg);
5085 scale($scale);
5086 disp(0x0);
5087 %}
5088 %}
5089
5090 operand indIndexN(iRegN reg, iRegL lreg)
5091 %{
5092 predicate(Universe::narrow_oop_shift() == 0);
5093 constraint(ALLOC_IN_RC(ptr_reg));
5094 match(AddP (DecodeN reg) lreg);
5095 op_cost(0);
5096 format %{ "$reg, $lreg\t# narrow" %}
5097 interface(MEMORY_INTER) %{
5098 base($reg);
5099 index($lreg);
5100 scale(0x0);
5101 disp(0x0);
5102 %}
5103 %}
5104
5105 operand indOffIN(iRegN reg, immIOffset off)
5106 %{
5107 predicate(Universe::narrow_oop_shift() == 0);
5108 constraint(ALLOC_IN_RC(ptr_reg));
5109 match(AddP (DecodeN reg) off);
5110 op_cost(0);
5111 format %{ "[$reg, $off]\t# narrow" %}
5112 interface(MEMORY_INTER) %{
5113 base($reg);
5114 index(0xffffffff);
5115 scale(0x0);
5116 disp($off);
5117 %}
5118 %}
5119
5120 operand indOffLN(iRegN reg, immLoffset off)
5121 %{
5122 predicate(Universe::narrow_oop_shift() == 0);
5123 constraint(ALLOC_IN_RC(ptr_reg));
5124 match(AddP (DecodeN reg) off);
5125 op_cost(0);
5126 format %{ "[$reg, $off]\t# narrow" %}
5127 interface(MEMORY_INTER) %{
5128 base($reg);
5129 index(0xffffffff);
5130 scale(0x0);
5131 disp($off);
5132 %}
5133 %}
5134
5135
5136
5137 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5138 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5139 %{
5140 constraint(ALLOC_IN_RC(ptr_reg));
5141 match(AddP reg off);
5142 op_cost(0);
5143 format %{ "[$reg, $off]" %}
5144 interface(MEMORY_INTER) %{
5145 base($reg);
5146 index(0xffffffff);
5147 scale(0x0);
5148 disp($off);
5149 %}
5150 %}
5151
5152 //----------Special Memory Operands--------------------------------------------
5153 // Stack Slot Operand - This operand is used for loading and storing temporary
5154 // values on the stack where a match requires a value to
5155 // flow through memory.
5156 operand stackSlotP(sRegP reg)
5157 %{
5158 constraint(ALLOC_IN_RC(stack_slots));
5159 op_cost(100);
5160 // No match rule because this operand is only generated in matching
5161 // match(RegP);
5162 format %{ "[$reg]" %}
5163 interface(MEMORY_INTER) %{
5164 base(0x1e); // RSP
5165 index(0x0); // No Index
5166 scale(0x0); // No Scale
5167 disp($reg); // Stack Offset
5168 %}
5169 %}
5170
5171 operand stackSlotI(sRegI reg)
5172 %{
5173 constraint(ALLOC_IN_RC(stack_slots));
5174 // No match rule because this operand is only generated in matching
5175 // match(RegI);
5176 format %{ "[$reg]" %}
5177 interface(MEMORY_INTER) %{
5178 base(0x1e); // RSP
5179 index(0x0); // No Index
5180 scale(0x0); // No Scale
5181 disp($reg); // Stack Offset
5182 %}
5183 %}
5184
5185 operand stackSlotF(sRegF reg)
5186 %{
5187 constraint(ALLOC_IN_RC(stack_slots));
5188 // No match rule because this operand is only generated in matching
5189 // match(RegF);
5190 format %{ "[$reg]" %}
5191 interface(MEMORY_INTER) %{
5192 base(0x1e); // RSP
5193 index(0x0); // No Index
5194 scale(0x0); // No Scale
5195 disp($reg); // Stack Offset
5196 %}
5197 %}
5198
5199 operand stackSlotD(sRegD reg)
5200 %{
5201 constraint(ALLOC_IN_RC(stack_slots));
5202 // No match rule because this operand is only generated in matching
5203 // match(RegD);
5204 format %{ "[$reg]" %}
5205 interface(MEMORY_INTER) %{
5206 base(0x1e); // RSP
5207 index(0x0); // No Index
5208 scale(0x0); // No Scale
5209 disp($reg); // Stack Offset
5210 %}
5211 %}
5212
5213 operand stackSlotL(sRegL reg)
5214 %{
5215 constraint(ALLOC_IN_RC(stack_slots));
5216 // No match rule because this operand is only generated in matching
5217 // match(RegL);
5218 format %{ "[$reg]" %}
5219 interface(MEMORY_INTER) %{
5220 base(0x1e); // RSP
5221 index(0x0); // No Index
5222 scale(0x0); // No Scale
5223 disp($reg); // Stack Offset
5224 %}
5225 %}
5226
5227 // Operands for expressing Control Flow
5228 // NOTE: Label is a predefined operand which should not be redefined in
5229 // the AD file. It is generically handled within the ADLC.
5230
5231 //----------Conditional Branch Operands----------------------------------------
5232 // Comparison Op - This is the operation of the comparison, and is limited to
5233 // the following set of codes:
5234 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5235 //
5236 // Other attributes of the comparison, such as unsignedness, are specified
5237 // by the comparison instruction that sets a condition code flags register.
5238 // That result is represented by a flags operand whose subtype is appropriate
5239 // to the unsignedness (etc.) of the comparison.
5240 //
5241 // Later, the instruction which matches both the Comparison Op (a Bool) and
5242 // the flags (produced by the Cmp) specifies the coding of the comparison op
5243 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5244
5245 // used for signed integral comparisons and fp comparisons
5246
5247 operand cmpOp()
5248 %{
5249 match(Bool);
5250
5251 format %{ "" %}
5252 interface(COND_INTER) %{
5253 equal(0x0, "eq");
5254 not_equal(0x1, "ne");
5255 less(0xb, "lt");
5256 greater_equal(0xa, "ge");
5257 less_equal(0xd, "le");
5258 greater(0xc, "gt");
5259 overflow(0x6, "vs");
5260 no_overflow(0x7, "vc");
5261 %}
5262 %}
5263
5264 // used for unsigned integral comparisons
5265
5266 operand cmpOpU()
5267 %{
5268 match(Bool);
5269
5270 format %{ "" %}
5271 interface(COND_INTER) %{
5272 equal(0x0, "eq");
5273 not_equal(0x1, "ne");
5274 less(0x3, "lo");
5275 greater_equal(0x2, "hs");
5276 less_equal(0x9, "ls");
5277 greater(0x8, "hi");
5278 overflow(0x6, "vs");
5279 no_overflow(0x7, "vc");
5280 %}
5281 %}
5282
5283 // Special operand allowing long args to int ops to be truncated for free
5284
5285 operand iRegL2I(iRegL reg) %{
5286
5287 op_cost(0);
5288
5289 match(ConvL2I reg);
5290
5291 format %{ "l2i($reg)" %}
5292
5293 interface(REG_INTER)
5294 %}
5295
5296 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5297 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5298 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5299
5300 //----------OPERAND CLASSES----------------------------------------------------
5301 // Operand Classes are groups of operands that are used as to simplify
5302 // instruction definitions by not requiring the AD writer to specify
5303 // separate instructions for every form of operand when the
5304 // instruction accepts multiple operand types with the same basic
5305 // encoding and format. The classic case of this is memory operands.
5306
5307 // memory is used to define read/write location for load/store
5308 // instruction defs. we can turn a memory op into an Address
5309
5310 opclass memory(indirect, indIndexScaledOffsetI, indIndexScaledOffsetL, indIndexOffsetI2L, indIndexScaledOffsetI2L, indIndexScaled, indIndexScaledI2L, indIndex, indOffI, indOffL,
5311 indirectN, indIndexScaledOffsetIN, indIndexScaledOffsetLN, indIndexOffsetI2LN, indIndexScaledOffsetI2LN, indIndexScaledN, indIndexScaledI2LN, indIndexN, indOffIN, indOffLN);
5312
5313 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5314
5315
5316 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5317 // operations. it allows the src to be either an iRegI or a (ConvL2I
5318 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5319 // can be elided because the 32-bit instruction will just employ the
5320 // lower 32 bits anyway.
5321 //
5322 // n.b. this does not elide all L2I conversions. if the truncated
5323 // value is consumed by more than one operation then the ConvL2I
5324 // cannot be bundled into the consuming nodes so an l2i gets planted
5325 // (actually a movw $dst $src) and the downstream instructions consume
5326 // the result of the l2i as an iRegI input. That's a shame since the
5327 // movw is actually redundant but its not too costly.
5328
5329 opclass iRegIorL2I(iRegI, iRegL2I);
5330
5331 //----------PIPELINE-----------------------------------------------------------
5332 // Rules which define the behavior of the target architectures pipeline.
5333
5334 // For specific pipelines, eg A53, define the stages of that pipeline
5335 //pipe_desc(ISS, EX1, EX2, WR);
5336 #define ISS S0
5337 #define EX1 S1
5338 #define EX2 S2
5339 #define WR S3
5340
5341 // Integer ALU reg operation
5342 pipeline %{
5343
5344 attributes %{
5345 // ARM instructions are of fixed length
5346 fixed_size_instructions; // Fixed size instructions TODO does
5347 max_instructions_per_bundle = 2; // A53 = 2, A57 = 4
5348 // ARM instructions come in 32-bit word units
5349 instruction_unit_size = 4; // An instruction is 4 bytes long
5350 instruction_fetch_unit_size = 64; // The processor fetches one line
5351 instruction_fetch_units = 1; // of 64 bytes
5352
5353 // List of nop instructions
5354 nops( MachNop );
5355 %}
5356
5357 // We don't use an actual pipeline model so don't care about resources
5358 // or description. we do use pipeline classes to introduce fixed
5359 // latencies
5360
5361 //----------RESOURCES----------------------------------------------------------
5362 // Resources are the functional units available to the machine
5363
5364 resources( INS0, INS1, INS01 = INS0 | INS1,
5365 ALU0, ALU1, ALU = ALU0 | ALU1,
5366 MAC,
5367 DIV,
5368 BRANCH,
5369 LDST,
5370 NEON_FP);
5371
5372 //----------PIPELINE DESCRIPTION-----------------------------------------------
5373 // Pipeline Description specifies the stages in the machine's pipeline
5374
5375 // Define the pipeline as a generic 6 stage pipeline
5376 pipe_desc(S0, S1, S2, S3, S4, S5);
5377
5378 //----------PIPELINE CLASSES---------------------------------------------------
5379 // Pipeline Classes describe the stages in which input and output are
5380 // referenced by the hardware pipeline.
5381
5382 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5383 %{
5384 single_instruction;
5385 src1 : S1(read);
5386 src2 : S2(read);
5387 dst : S5(write);
5388 INS01 : ISS;
5389 NEON_FP : S5;
5390 %}
5391
5392 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5393 %{
5394 single_instruction;
5395 src1 : S1(read);
5396 src2 : S2(read);
5397 dst : S5(write);
5398 INS01 : ISS;
5399 NEON_FP : S5;
5400 %}
5401
5402 pipe_class fp_uop_s(vRegF dst, vRegF src)
5403 %{
5404 single_instruction;
5405 src : S1(read);
5406 dst : S5(write);
5407 INS01 : ISS;
5408 NEON_FP : S5;
5409 %}
5410
5411 pipe_class fp_uop_d(vRegD dst, vRegD src)
5412 %{
5413 single_instruction;
5414 src : S1(read);
5415 dst : S5(write);
5416 INS01 : ISS;
5417 NEON_FP : S5;
5418 %}
5419
5420 pipe_class fp_d2f(vRegF dst, vRegD src)
5421 %{
5422 single_instruction;
5423 src : S1(read);
5424 dst : S5(write);
5425 INS01 : ISS;
5426 NEON_FP : S5;
5427 %}
5428
5429 pipe_class fp_f2d(vRegD dst, vRegF src)
5430 %{
5431 single_instruction;
5432 src : S1(read);
5433 dst : S5(write);
5434 INS01 : ISS;
5435 NEON_FP : S5;
5436 %}
5437
5438 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5439 %{
5440 single_instruction;
5441 src : S1(read);
5442 dst : S5(write);
5443 INS01 : ISS;
5444 NEON_FP : S5;
5445 %}
5446
5447 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5448 %{
5449 single_instruction;
5450 src : S1(read);
5451 dst : S5(write);
5452 INS01 : ISS;
5453 NEON_FP : S5;
5454 %}
5455
5456 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5457 %{
5458 single_instruction;
5459 src : S1(read);
5460 dst : S5(write);
5461 INS01 : ISS;
5462 NEON_FP : S5;
5463 %}
5464
5465 pipe_class fp_l2f(vRegF dst, iRegL src)
5466 %{
5467 single_instruction;
5468 src : S1(read);
5469 dst : S5(write);
5470 INS01 : ISS;
5471 NEON_FP : S5;
5472 %}
5473
5474 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5475 %{
5476 single_instruction;
5477 src : S1(read);
5478 dst : S5(write);
5479 INS01 : ISS;
5480 NEON_FP : S5;
5481 %}
5482
5483 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5484 %{
5485 single_instruction;
5486 src : S1(read);
5487 dst : S5(write);
5488 INS01 : ISS;
5489 NEON_FP : S5;
5490 %}
5491
5492 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5493 %{
5494 single_instruction;
5495 src : S1(read);
5496 dst : S5(write);
5497 INS01 : ISS;
5498 NEON_FP : S5;
5499 %}
5500
5501 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5502 %{
5503 single_instruction;
5504 src : S1(read);
5505 dst : S5(write);
5506 INS01 : ISS;
5507 NEON_FP : S5;
5508 %}
5509
5510 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5511 %{
5512 single_instruction;
5513 src1 : S1(read);
5514 src2 : S2(read);
5515 dst : S5(write);
5516 INS0 : ISS;
5517 NEON_FP : S5;
5518 %}
5519
5520 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5521 %{
5522 single_instruction;
5523 src1 : S1(read);
5524 src2 : S2(read);
5525 dst : S5(write);
5526 INS0 : ISS;
5527 NEON_FP : S5;
5528 %}
5529
5530 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5531 %{
5532 single_instruction;
5533 cr : S1(read);
5534 src1 : S1(read);
5535 src2 : S1(read);
5536 dst : S3(write);
5537 INS01 : ISS;
5538 NEON_FP : S3;
5539 %}
5540
5541 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
5542 %{
5543 single_instruction;
5544 cr : S1(read);
5545 src1 : S1(read);
5546 src2 : S1(read);
5547 dst : S3(write);
5548 INS01 : ISS;
5549 NEON_FP : S3;
5550 %}
5551
5552 pipe_class fp_imm_s(vRegF dst)
5553 %{
5554 single_instruction;
5555 dst : S3(write);
5556 INS01 : ISS;
5557 NEON_FP : S3;
5558 %}
5559
5560 pipe_class fp_imm_d(vRegD dst)
5561 %{
5562 single_instruction;
5563 dst : S3(write);
5564 INS01 : ISS;
5565 NEON_FP : S3;
5566 %}
5567
5568 pipe_class fp_load_constant_s(vRegF dst)
5569 %{
5570 single_instruction;
5571 dst : S4(write);
5572 INS01 : ISS;
5573 NEON_FP : S4;
5574 %}
5575
5576 pipe_class fp_load_constant_d(vRegD dst)
5577 %{
5578 single_instruction;
5579 dst : S4(write);
5580 INS01 : ISS;
5581 NEON_FP : S4;
5582 %}
5583
5584 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
5585 %{
5586 single_instruction;
5587 dst : S5(write);
5588 src1 : S1(read);
5589 src2 : S1(read);
5590 INS01 : ISS;
5591 NEON_FP : S5;
5592 %}
5593
5594 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
5595 %{
5596 single_instruction;
5597 dst : S5(write);
5598 src1 : S1(read);
5599 src2 : S1(read);
5600 INS0 : ISS;
5601 NEON_FP : S5;
5602 %}
5603
5604 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
5605 %{
5606 single_instruction;
5607 dst : S5(write);
5608 src1 : S1(read);
5609 src2 : S1(read);
5610 dst : S1(read);
5611 INS01 : ISS;
5612 NEON_FP : S5;
5613 %}
5614
5615 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
5616 %{
5617 single_instruction;
5618 dst : S5(write);
5619 src1 : S1(read);
5620 src2 : S1(read);
5621 dst : S1(read);
5622 INS0 : ISS;
5623 NEON_FP : S5;
5624 %}
5625
5626 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
5627 %{
5628 single_instruction;
5629 dst : S4(write);
5630 src1 : S2(read);
5631 src2 : S2(read);
5632 INS01 : ISS;
5633 NEON_FP : S4;
5634 %}
5635
5636 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
5637 %{
5638 single_instruction;
5639 dst : S4(write);
5640 src1 : S2(read);
5641 src2 : S2(read);
5642 INS0 : ISS;
5643 NEON_FP : S4;
5644 %}
5645
5646 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
5647 %{
5648 single_instruction;
5649 dst : S3(write);
5650 src1 : S2(read);
5651 src2 : S2(read);
5652 INS01 : ISS;
5653 NEON_FP : S3;
5654 %}
5655
5656 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
5657 %{
5658 single_instruction;
5659 dst : S3(write);
5660 src1 : S2(read);
5661 src2 : S2(read);
5662 INS0 : ISS;
5663 NEON_FP : S3;
5664 %}
5665
5666 pipe_class vshift64(vecD dst, vecD src, vecX shift)
5667 %{
5668 single_instruction;
5669 dst : S3(write);
5670 src : S1(read);
5671 shift : S1(read);
5672 INS01 : ISS;
5673 NEON_FP : S3;
5674 %}
5675
5676 pipe_class vshift128(vecX dst, vecX src, vecX shift)
5677 %{
5678 single_instruction;
5679 dst : S3(write);
5680 src : S1(read);
5681 shift : S1(read);
5682 INS0 : ISS;
5683 NEON_FP : S3;
5684 %}
5685
5686 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
5687 %{
5688 single_instruction;
5689 dst : S3(write);
5690 src : S1(read);
5691 INS01 : ISS;
5692 NEON_FP : S3;
5693 %}
5694
5695 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
5696 %{
5697 single_instruction;
5698 dst : S3(write);
5699 src : S1(read);
5700 INS0 : ISS;
5701 NEON_FP : S3;
5702 %}
5703
5704 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
5705 %{
5706 single_instruction;
5707 dst : S5(write);
5708 src1 : S1(read);
5709 src2 : S1(read);
5710 INS01 : ISS;
5711 NEON_FP : S5;
5712 %}
5713
5714 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
5715 %{
5716 single_instruction;
5717 dst : S5(write);
5718 src1 : S1(read);
5719 src2 : S1(read);
5720 INS0 : ISS;
5721 NEON_FP : S5;
5722 %}
5723
5724 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
5725 %{
5726 single_instruction;
5727 dst : S5(write);
5728 src1 : S1(read);
5729 src2 : S1(read);
5730 INS0 : ISS;
5731 NEON_FP : S5;
5732 %}
5733
5734 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
5735 %{
5736 single_instruction;
5737 dst : S5(write);
5738 src1 : S1(read);
5739 src2 : S1(read);
5740 INS0 : ISS;
5741 NEON_FP : S5;
5742 %}
5743
5744 pipe_class vsqrt_fp128(vecX dst, vecX src)
5745 %{
5746 single_instruction;
5747 dst : S5(write);
5748 src : S1(read);
5749 INS0 : ISS;
5750 NEON_FP : S5;
5751 %}
5752
5753 pipe_class vunop_fp64(vecD dst, vecD src)
5754 %{
5755 single_instruction;
5756 dst : S5(write);
5757 src : S1(read);
5758 INS01 : ISS;
5759 NEON_FP : S5;
5760 %}
5761
5762 pipe_class vunop_fp128(vecX dst, vecX src)
5763 %{
5764 single_instruction;
5765 dst : S5(write);
5766 src : S1(read);
5767 INS0 : ISS;
5768 NEON_FP : S5;
5769 %}
5770
5771 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
5772 %{
5773 single_instruction;
5774 dst : S3(write);
5775 src : S1(read);
5776 INS01 : ISS;
5777 NEON_FP : S3;
5778 %}
5779
5780 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
5781 %{
5782 single_instruction;
5783 dst : S3(write);
5784 src : S1(read);
5785 INS01 : ISS;
5786 NEON_FP : S3;
5787 %}
5788
5789 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
5790 %{
5791 single_instruction;
5792 dst : S3(write);
5793 src : S1(read);
5794 INS01 : ISS;
5795 NEON_FP : S3;
5796 %}
5797
5798 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
5799 %{
5800 single_instruction;
5801 dst : S3(write);
5802 src : S1(read);
5803 INS01 : ISS;
5804 NEON_FP : S3;
5805 %}
5806
5807 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
5808 %{
5809 single_instruction;
5810 dst : S3(write);
5811 src : S1(read);
5812 INS01 : ISS;
5813 NEON_FP : S3;
5814 %}
5815
5816 pipe_class vmovi_reg_imm64(vecD dst)
5817 %{
5818 single_instruction;
5819 dst : S3(write);
5820 INS01 : ISS;
5821 NEON_FP : S3;
5822 %}
5823
5824 pipe_class vmovi_reg_imm128(vecX dst)
5825 %{
5826 single_instruction;
5827 dst : S3(write);
5828 INS0 : ISS;
5829 NEON_FP : S3;
5830 %}
5831
5832 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
5833 %{
5834 single_instruction;
5835 dst : S5(write);
5836 mem : ISS(read);
5837 INS01 : ISS;
5838 NEON_FP : S3;
5839 %}
5840
5841 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
5842 %{
5843 single_instruction;
5844 dst : S5(write);
5845 mem : ISS(read);
5846 INS01 : ISS;
5847 NEON_FP : S3;
5848 %}
5849
5850 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
5851 %{
5852 single_instruction;
5853 mem : ISS(read);
5854 src : S2(read);
5855 INS01 : ISS;
5856 NEON_FP : S3;
5857 %}
5858
5859 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
5860 %{
5861 single_instruction;
5862 mem : ISS(read);
5863 src : S2(read);
5864 INS01 : ISS;
5865 NEON_FP : S3;
5866 %}
5867
5868 //------- Integer ALU operations --------------------------
5869
5870 // Integer ALU reg-reg operation
5871 // Operands needed in EX1, result generated in EX2
5872 // Eg. ADD x0, x1, x2
5873 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5874 %{
5875 single_instruction;
5876 dst : EX2(write);
5877 src1 : EX1(read);
5878 src2 : EX1(read);
5879 INS01 : ISS; // Dual issue as instruction 0 or 1
5880 ALU : EX2;
5881 %}
5882
5883 // Integer ALU reg-reg operation with constant shift
5884 // Shifted register must be available in LATE_ISS instead of EX1
5885 // Eg. ADD x0, x1, x2, LSL #2
5886 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
5887 %{
5888 single_instruction;
5889 dst : EX2(write);
5890 src1 : EX1(read);
5891 src2 : ISS(read);
5892 INS01 : ISS;
5893 ALU : EX2;
5894 %}
5895
5896 // Integer ALU reg operation with constant shift
5897 // Eg. LSL x0, x1, #shift
5898 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
5899 %{
5900 single_instruction;
5901 dst : EX2(write);
5902 src1 : ISS(read);
5903 INS01 : ISS;
5904 ALU : EX2;
5905 %}
5906
5907 // Integer ALU reg-reg operation with variable shift
5908 // Both operands must be available in LATE_ISS instead of EX1
5909 // Result is available in EX1 instead of EX2
5910 // Eg. LSLV x0, x1, x2
5911 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
5912 %{
5913 single_instruction;
5914 dst : EX1(write);
5915 src1 : ISS(read);
5916 src2 : ISS(read);
5917 INS01 : ISS;
5918 ALU : EX1;
5919 %}
5920
5921 // Integer ALU reg-reg operation with extract
5922 // As for _vshift above, but result generated in EX2
5923 // Eg. EXTR x0, x1, x2, #N
5924 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
5925 %{
5926 single_instruction;
5927 dst : EX2(write);
5928 src1 : ISS(read);
5929 src2 : ISS(read);
5930 INS1 : ISS; // Can only dual issue as Instruction 1
5931 ALU : EX1;
5932 %}
5933
5934 // Integer ALU reg operation
5935 // Eg. NEG x0, x1
5936 pipe_class ialu_reg(iRegI dst, iRegI src)
5937 %{
5938 single_instruction;
5939 dst : EX2(write);
5940 src : EX1(read);
5941 INS01 : ISS;
5942 ALU : EX2;
5943 %}
5944
5945 // Integer ALU reg mmediate operation
5946 // Eg. ADD x0, x1, #N
5947 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
5948 %{
5949 single_instruction;
5950 dst : EX2(write);
5951 src1 : EX1(read);
5952 INS01 : ISS;
5953 ALU : EX2;
5954 %}
5955
5956 // Integer ALU immediate operation (no source operands)
5957 // Eg. MOV x0, #N
5958 pipe_class ialu_imm(iRegI dst)
5959 %{
5960 single_instruction;
5961 dst : EX1(write);
5962 INS01 : ISS;
5963 ALU : EX1;
5964 %}
5965
5966 //------- Compare operation -------------------------------
5967
5968 // Compare reg-reg
5969 // Eg. CMP x0, x1
5970 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
5971 %{
5972 single_instruction;
5973 // fixed_latency(16);
5974 cr : EX2(write);
5975 op1 : EX1(read);
5976 op2 : EX1(read);
5977 INS01 : ISS;
5978 ALU : EX2;
5979 %}
5980
5981 // Compare reg-reg
5982 // Eg. CMP x0, #N
5983 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
5984 %{
5985 single_instruction;
5986 // fixed_latency(16);
5987 cr : EX2(write);
5988 op1 : EX1(read);
5989 INS01 : ISS;
5990 ALU : EX2;
5991 %}
5992
5993 //------- Conditional instructions ------------------------
5994
5995 // Conditional no operands
5996 // Eg. CSINC x0, zr, zr, <cond>
5997 pipe_class icond_none(iRegI dst, rFlagsReg cr)
5998 %{
5999 single_instruction;
6000 cr : EX1(read);
6001 dst : EX2(write);
6002 INS01 : ISS;
6003 ALU : EX2;
6004 %}
6005
6006 // Conditional 2 operand
6007 // EG. CSEL X0, X1, X2, <cond>
6008 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6009 %{
6010 single_instruction;
6011 cr : EX1(read);
6012 src1 : EX1(read);
6013 src2 : EX1(read);
6014 dst : EX2(write);
6015 INS01 : ISS;
6016 ALU : EX2;
6017 %}
6018
6019 // Conditional 2 operand
6020 // EG. CSEL X0, X1, X2, <cond>
6021 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6022 %{
6023 single_instruction;
6024 cr : EX1(read);
6025 src : EX1(read);
6026 dst : EX2(write);
6027 INS01 : ISS;
6028 ALU : EX2;
6029 %}
6030
6031 //------- Multiply pipeline operations --------------------
6032
6033 // Multiply reg-reg
6034 // Eg. MUL w0, w1, w2
6035 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6036 %{
6037 single_instruction;
6038 dst : WR(write);
6039 src1 : ISS(read);
6040 src2 : ISS(read);
6041 INS01 : ISS;
6042 MAC : WR;
6043 %}
6044
6045 // Multiply accumulate
6046 // Eg. MADD w0, w1, w2, w3
6047 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6048 %{
6049 single_instruction;
6050 dst : WR(write);
6051 src1 : ISS(read);
6052 src2 : ISS(read);
6053 src3 : ISS(read);
6054 INS01 : ISS;
6055 MAC : WR;
6056 %}
6057
6058 // Eg. MUL w0, w1, w2
6059 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6060 %{
6061 single_instruction;
6062 fixed_latency(3); // Maximum latency for 64 bit mul
6063 dst : WR(write);
6064 src1 : ISS(read);
6065 src2 : ISS(read);
6066 INS01 : ISS;
6067 MAC : WR;
6068 %}
6069
6070 // Multiply accumulate
6071 // Eg. MADD w0, w1, w2, w3
6072 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6073 %{
6074 single_instruction;
6075 fixed_latency(3); // Maximum latency for 64 bit mul
6076 dst : WR(write);
6077 src1 : ISS(read);
6078 src2 : ISS(read);
6079 src3 : ISS(read);
6080 INS01 : ISS;
6081 MAC : WR;
6082 %}
6083
6084 //------- Divide pipeline operations --------------------
6085
6086 // Eg. SDIV w0, w1, w2
6087 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6088 %{
6089 single_instruction;
6090 fixed_latency(8); // Maximum latency for 32 bit divide
6091 dst : WR(write);
6092 src1 : ISS(read);
6093 src2 : ISS(read);
6094 INS0 : ISS; // Can only dual issue as instruction 0
6095 DIV : WR;
6096 %}
6097
6098 // Eg. SDIV x0, x1, x2
6099 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6100 %{
6101 single_instruction;
6102 fixed_latency(16); // Maximum latency for 64 bit divide
6103 dst : WR(write);
6104 src1 : ISS(read);
6105 src2 : ISS(read);
6106 INS0 : ISS; // Can only dual issue as instruction 0
6107 DIV : WR;
6108 %}
6109
6110 //------- Load pipeline operations ------------------------
6111
6112 // Load - prefetch
6113 // Eg. PFRM <mem>
6114 pipe_class iload_prefetch(memory mem)
6115 %{
6116 single_instruction;
6117 mem : ISS(read);
6118 INS01 : ISS;
6119 LDST : WR;
6120 %}
6121
6122 // Load - reg, mem
6123 // Eg. LDR x0, <mem>
6124 pipe_class iload_reg_mem(iRegI dst, memory mem)
6125 %{
6126 single_instruction;
6127 dst : WR(write);
6128 mem : ISS(read);
6129 INS01 : ISS;
6130 LDST : WR;
6131 %}
6132
6133 // Load - reg, reg
6134 // Eg. LDR x0, [sp, x1]
6135 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6136 %{
6137 single_instruction;
6138 dst : WR(write);
6139 src : ISS(read);
6140 INS01 : ISS;
6141 LDST : WR;
6142 %}
6143
6144 //------- Store pipeline operations -----------------------
6145
6146 // Store - zr, mem
6147 // Eg. STR zr, <mem>
6148 pipe_class istore_mem(memory mem)
6149 %{
6150 single_instruction;
6151 mem : ISS(read);
6152 INS01 : ISS;
6153 LDST : WR;
6154 %}
6155
6156 // Store - reg, mem
6157 // Eg. STR x0, <mem>
6158 pipe_class istore_reg_mem(iRegI src, memory mem)
6159 %{
6160 single_instruction;
6161 mem : ISS(read);
6162 src : EX2(read);
6163 INS01 : ISS;
6164 LDST : WR;
6165 %}
6166
6167 // Store - reg, reg
6168 // Eg. STR x0, [sp, x1]
6169 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6170 %{
6171 single_instruction;
6172 dst : ISS(read);
6173 src : EX2(read);
6174 INS01 : ISS;
6175 LDST : WR;
6176 %}
6177
6178 //------- Store pipeline operations -----------------------
6179
6180 // Branch
6181 pipe_class pipe_branch()
6182 %{
6183 single_instruction;
6184 INS01 : ISS;
6185 BRANCH : EX1;
6186 %}
6187
6188 // Conditional branch
6189 pipe_class pipe_branch_cond(rFlagsReg cr)
6190 %{
6191 single_instruction;
6192 cr : EX1(read);
6193 INS01 : ISS;
6194 BRANCH : EX1;
6195 %}
6196
6197 // Compare & Branch
6198 // EG. CBZ/CBNZ
6199 pipe_class pipe_cmp_branch(iRegI op1)
6200 %{
6201 single_instruction;
6202 op1 : EX1(read);
6203 INS01 : ISS;
6204 BRANCH : EX1;
6205 %}
6206
6207 //------- Synchronisation operations ----------------------
6208
6209 // Any operation requiring serialization.
6210 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6211 pipe_class pipe_serial()
6212 %{
6213 single_instruction;
6214 force_serialization;
6215 fixed_latency(16);
6216 INS01 : ISS(2); // Cannot dual issue with any other instruction
6217 LDST : WR;
6218 %}
6219
6220 // Generic big/slow expanded idiom - also serialized
6221 pipe_class pipe_slow()
6222 %{
6223 instruction_count(10);
6224 multiple_bundles;
6225 force_serialization;
6226 fixed_latency(16);
6227 INS01 : ISS(2); // Cannot dual issue with any other instruction
6228 LDST : WR;
6229 %}
6230
6231 // Empty pipeline class
6232 pipe_class pipe_class_empty()
6233 %{
6234 single_instruction;
6235 fixed_latency(0);
6236 %}
6237
6238 // Default pipeline class.
6239 pipe_class pipe_class_default()
6240 %{
6241 single_instruction;
6242 fixed_latency(2);
6243 %}
6244
6245 // Pipeline class for compares.
6246 pipe_class pipe_class_compare()
6247 %{
6248 single_instruction;
6249 fixed_latency(16);
6250 %}
6251
6252 // Pipeline class for memory operations.
6253 pipe_class pipe_class_memory()
6254 %{
6255 single_instruction;
6256 fixed_latency(16);
6257 %}
6258
6259 // Pipeline class for call.
6260 pipe_class pipe_class_call()
6261 %{
6262 single_instruction;
6263 fixed_latency(100);
6264 %}
6265
6266 // Define the class for the Nop node.
6267 define %{
6268 MachNop = pipe_class_empty;
6269 %}
6270
6271 %}
6272 //----------INSTRUCTIONS-------------------------------------------------------
6273 //
6274 // match -- States which machine-independent subtree may be replaced
6275 // by this instruction.
6276 // ins_cost -- The estimated cost of this instruction is used by instruction
6277 // selection to identify a minimum cost tree of machine
6278 // instructions that matches a tree of machine-independent
6279 // instructions.
6280 // format -- A string providing the disassembly for this instruction.
6281 // The value of an instruction's operand may be inserted
6282 // by referring to it with a '$' prefix.
6283 // opcode -- Three instruction opcodes may be provided. These are referred
6284 // to within an encode class as $primary, $secondary, and $tertiary
6285 // rrspectively. The primary opcode is commonly used to
6286 // indicate the type of machine instruction, while secondary
6287 // and tertiary are often used for prefix options or addressing
6288 // modes.
6289 // ins_encode -- A list of encode classes with parameters. The encode class
6290 // name must have been defined in an 'enc_class' specification
6291 // in the encode section of the architecture description.
6292
6293 // ============================================================================
6294 // Memory (Load/Store) Instructions
6295
6296 // Load Instructions
6297
6298 // Load Byte (8 bit signed)
6299 instruct loadB(iRegINoSp dst, memory mem)
6300 %{
6301 match(Set dst (LoadB mem));
6302 predicate(!needs_acquiring_load(n));
6303
6304 ins_cost(4 * INSN_COST);
6305 format %{ "ldrsbw $dst, $mem\t# byte" %}
6306
6307 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6308
6309 ins_pipe(iload_reg_mem);
6310 %}
6311
6312 // Load Byte (8 bit signed) into long
6313 instruct loadB2L(iRegLNoSp dst, memory mem)
6314 %{
6315 match(Set dst (ConvI2L (LoadB mem)));
6316 predicate(!needs_acquiring_load(n->in(1)));
6317
6318 ins_cost(4 * INSN_COST);
6319 format %{ "ldrsb $dst, $mem\t# byte" %}
6320
6321 ins_encode(aarch64_enc_ldrsb(dst, mem));
6322
6323 ins_pipe(iload_reg_mem);
6324 %}
6325
6326 // Load Byte (8 bit unsigned)
6327 instruct loadUB(iRegINoSp dst, memory mem)
6328 %{
6329 match(Set dst (LoadUB mem));
6330 predicate(!needs_acquiring_load(n));
6331
6332 ins_cost(4 * INSN_COST);
6333 format %{ "ldrbw $dst, $mem\t# byte" %}
6334
6335 ins_encode(aarch64_enc_ldrb(dst, mem));
6336
6337 ins_pipe(iload_reg_mem);
6338 %}
6339
6340 // Load Byte (8 bit unsigned) into long
6341 instruct loadUB2L(iRegLNoSp dst, memory mem)
6342 %{
6343 match(Set dst (ConvI2L (LoadUB mem)));
6344 predicate(!needs_acquiring_load(n->in(1)));
6345
6346 ins_cost(4 * INSN_COST);
6347 format %{ "ldrb $dst, $mem\t# byte" %}
6348
6349 ins_encode(aarch64_enc_ldrb(dst, mem));
6350
6351 ins_pipe(iload_reg_mem);
6352 %}
6353
6354 // Load Short (16 bit signed)
6355 instruct loadS(iRegINoSp dst, memory mem)
6356 %{
6357 match(Set dst (LoadS mem));
6358 predicate(!needs_acquiring_load(n));
6359
6360 ins_cost(4 * INSN_COST);
6361 format %{ "ldrshw $dst, $mem\t# short" %}
6362
6363 ins_encode(aarch64_enc_ldrshw(dst, mem));
6364
6365 ins_pipe(iload_reg_mem);
6366 %}
6367
6368 // Load Short (16 bit signed) into long
6369 instruct loadS2L(iRegLNoSp dst, memory mem)
6370 %{
6371 match(Set dst (ConvI2L (LoadS mem)));
6372 predicate(!needs_acquiring_load(n->in(1)));
6373
6374 ins_cost(4 * INSN_COST);
6375 format %{ "ldrsh $dst, $mem\t# short" %}
6376
6377 ins_encode(aarch64_enc_ldrsh(dst, mem));
6378
6379 ins_pipe(iload_reg_mem);
6380 %}
6381
6382 // Load Char (16 bit unsigned)
6383 instruct loadUS(iRegINoSp dst, memory mem)
6384 %{
6385 match(Set dst (LoadUS mem));
6386 predicate(!needs_acquiring_load(n));
6387
6388 ins_cost(4 * INSN_COST);
6389 format %{ "ldrh $dst, $mem\t# short" %}
6390
6391 ins_encode(aarch64_enc_ldrh(dst, mem));
6392
6393 ins_pipe(iload_reg_mem);
6394 %}
6395
6396 // Load Short/Char (16 bit unsigned) into long
6397 instruct loadUS2L(iRegLNoSp dst, memory mem)
6398 %{
6399 match(Set dst (ConvI2L (LoadUS mem)));
6400 predicate(!needs_acquiring_load(n->in(1)));
6401
6402 ins_cost(4 * INSN_COST);
6403 format %{ "ldrh $dst, $mem\t# short" %}
6404
6405 ins_encode(aarch64_enc_ldrh(dst, mem));
6406
6407 ins_pipe(iload_reg_mem);
6408 %}
6409
6410 // Load Integer (32 bit signed)
6411 instruct loadI(iRegINoSp dst, memory mem)
6412 %{
6413 match(Set dst (LoadI mem));
6414 predicate(!needs_acquiring_load(n));
6415
6416 ins_cost(4 * INSN_COST);
6417 format %{ "ldrw $dst, $mem\t# int" %}
6418
6419 ins_encode(aarch64_enc_ldrw(dst, mem));
6420
6421 ins_pipe(iload_reg_mem);
6422 %}
6423
6424 // Load Integer (32 bit signed) into long
6425 instruct loadI2L(iRegLNoSp dst, memory mem)
6426 %{
6427 match(Set dst (ConvI2L (LoadI mem)));
6428 predicate(!needs_acquiring_load(n->in(1)));
6429
6430 ins_cost(4 * INSN_COST);
6431 format %{ "ldrsw $dst, $mem\t# int" %}
6432
6433 ins_encode(aarch64_enc_ldrsw(dst, mem));
6434
6435 ins_pipe(iload_reg_mem);
6436 %}
6437
6438 // Load Integer (32 bit unsigned) into long
6439 instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
6440 %{
6441 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6442 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6443
6444 ins_cost(4 * INSN_COST);
6445 format %{ "ldrw $dst, $mem\t# int" %}
6446
6447 ins_encode(aarch64_enc_ldrw(dst, mem));
6448
6449 ins_pipe(iload_reg_mem);
6450 %}
6451
6452 // Load Long (64 bit signed)
6453 instruct loadL(iRegLNoSp dst, memory mem)
6454 %{
6455 match(Set dst (LoadL mem));
6456 predicate(!needs_acquiring_load(n));
6457
6458 ins_cost(4 * INSN_COST);
6459 format %{ "ldr $dst, $mem\t# int" %}
6460
6461 ins_encode(aarch64_enc_ldr(dst, mem));
6462
6463 ins_pipe(iload_reg_mem);
6464 %}
6465
6466 // Load Range
6467 instruct loadRange(iRegINoSp dst, memory mem)
6468 %{
6469 match(Set dst (LoadRange mem));
6470
6471 ins_cost(4 * INSN_COST);
6472 format %{ "ldrw $dst, $mem\t# range" %}
6473
6474 ins_encode(aarch64_enc_ldrw(dst, mem));
6475
6476 ins_pipe(iload_reg_mem);
6477 %}
6478
6479 // Load Pointer
6480 instruct loadP(iRegPNoSp dst, memory mem)
6481 %{
6482 match(Set dst (LoadP mem));
6483 predicate(!needs_acquiring_load(n));
6484
6485 ins_cost(4 * INSN_COST);
6486 format %{ "ldr $dst, $mem\t# ptr" %}
6487
6488 ins_encode(aarch64_enc_ldr(dst, mem));
6489
6490 ins_pipe(iload_reg_mem);
6491 %}
6492
6493 // Load Compressed Pointer
6494 instruct loadN(iRegNNoSp dst, memory mem)
6495 %{
6496 match(Set dst (LoadN mem));
6497 predicate(!needs_acquiring_load(n));
6498
6499 ins_cost(4 * INSN_COST);
6500 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6501
6502 ins_encode(aarch64_enc_ldrw(dst, mem));
6503
6504 ins_pipe(iload_reg_mem);
6505 %}
6506
6507 // Load Klass Pointer
6508 instruct loadKlass(iRegPNoSp dst, memory mem)
6509 %{
6510 match(Set dst (LoadKlass mem));
6511 predicate(!needs_acquiring_load(n));
6512
6513 ins_cost(4 * INSN_COST);
6514 format %{ "ldr $dst, $mem\t# class" %}
6515
6516 ins_encode(aarch64_enc_ldr(dst, mem));
6517
6518 ins_pipe(iload_reg_mem);
6519 %}
6520
6521 // Load Narrow Klass Pointer
6522 instruct loadNKlass(iRegNNoSp dst, memory mem)
6523 %{
6524 match(Set dst (LoadNKlass mem));
6525 predicate(!needs_acquiring_load(n));
6526
6527 ins_cost(4 * INSN_COST);
6528 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6529
6530 ins_encode(aarch64_enc_ldrw(dst, mem));
6531
6532 ins_pipe(iload_reg_mem);
6533 %}
6534
6535 // Load Float
6536 instruct loadF(vRegF dst, memory mem)
6537 %{
6538 match(Set dst (LoadF mem));
6539 predicate(!needs_acquiring_load(n));
6540
6541 ins_cost(4 * INSN_COST);
6542 format %{ "ldrs $dst, $mem\t# float" %}
6543
6544 ins_encode( aarch64_enc_ldrs(dst, mem) );
6545
6546 ins_pipe(pipe_class_memory);
6547 %}
6548
6549 // Load Double
6550 instruct loadD(vRegD dst, memory mem)
6551 %{
6552 match(Set dst (LoadD mem));
6553 predicate(!needs_acquiring_load(n));
6554
6555 ins_cost(4 * INSN_COST);
6556 format %{ "ldrd $dst, $mem\t# double" %}
6557
6558 ins_encode( aarch64_enc_ldrd(dst, mem) );
6559
6560 ins_pipe(pipe_class_memory);
6561 %}
6562
6563
6564 // Load Int Constant
6565 instruct loadConI(iRegINoSp dst, immI src)
6566 %{
6567 match(Set dst src);
6568
6569 ins_cost(INSN_COST);
6570 format %{ "mov $dst, $src\t# int" %}
6571
6572 ins_encode( aarch64_enc_movw_imm(dst, src) );
6573
6574 ins_pipe(ialu_imm);
6575 %}
6576
6577 // Load Long Constant
6578 instruct loadConL(iRegLNoSp dst, immL src)
6579 %{
6580 match(Set dst src);
6581
6582 ins_cost(INSN_COST);
6583 format %{ "mov $dst, $src\t# long" %}
6584
6585 ins_encode( aarch64_enc_mov_imm(dst, src) );
6586
6587 ins_pipe(ialu_imm);
6588 %}
6589
6590 // Load Pointer Constant
6591
6592 instruct loadConP(iRegPNoSp dst, immP con)
6593 %{
6594 match(Set dst con);
6595
6596 ins_cost(INSN_COST * 4);
6597 format %{
6598 "mov $dst, $con\t# ptr\n\t"
6599 %}
6600
6601 ins_encode(aarch64_enc_mov_p(dst, con));
6602
6603 ins_pipe(ialu_imm);
6604 %}
6605
6606 // Load Null Pointer Constant
6607
6608 instruct loadConP0(iRegPNoSp dst, immP0 con)
6609 %{
6610 match(Set dst con);
6611
6612 ins_cost(INSN_COST);
6613 format %{ "mov $dst, $con\t# NULL ptr" %}
6614
6615 ins_encode(aarch64_enc_mov_p0(dst, con));
6616
6617 ins_pipe(ialu_imm);
6618 %}
6619
6620 // Load Pointer Constant One
6621
6622 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6623 %{
6624 match(Set dst con);
6625
6626 ins_cost(INSN_COST);
6627 format %{ "mov $dst, $con\t# NULL ptr" %}
6628
6629 ins_encode(aarch64_enc_mov_p1(dst, con));
6630
6631 ins_pipe(ialu_imm);
6632 %}
6633
6634 // Load Poll Page Constant
6635
6636 instruct loadConPollPage(iRegPNoSp dst, immPollPage con)
6637 %{
6638 match(Set dst con);
6639
6640 ins_cost(INSN_COST);
6641 format %{ "adr $dst, $con\t# Poll Page Ptr" %}
6642
6643 ins_encode(aarch64_enc_mov_poll_page(dst, con));
6644
6645 ins_pipe(ialu_imm);
6646 %}
6647
6648 // Load Byte Map Base Constant
6649
6650 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6651 %{
6652 match(Set dst con);
6653
6654 ins_cost(INSN_COST);
6655 format %{ "adr $dst, $con\t# Byte Map Base" %}
6656
6657 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6658
6659 ins_pipe(ialu_imm);
6660 %}
6661
6662 // Load Narrow Pointer Constant
6663
6664 instruct loadConN(iRegNNoSp dst, immN con)
6665 %{
6666 match(Set dst con);
6667
6668 ins_cost(INSN_COST * 4);
6669 format %{ "mov $dst, $con\t# compressed ptr" %}
6670
6671 ins_encode(aarch64_enc_mov_n(dst, con));
6672
6673 ins_pipe(ialu_imm);
6674 %}
6675
6676 // Load Narrow Null Pointer Constant
6677
6678 instruct loadConN0(iRegNNoSp dst, immN0 con)
6679 %{
6680 match(Set dst con);
6681
6682 ins_cost(INSN_COST);
6683 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
6684
6685 ins_encode(aarch64_enc_mov_n0(dst, con));
6686
6687 ins_pipe(ialu_imm);
6688 %}
6689
6690 // Load Narrow Klass Constant
6691
6692 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6693 %{
6694 match(Set dst con);
6695
6696 ins_cost(INSN_COST);
6697 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6698
6699 ins_encode(aarch64_enc_mov_nk(dst, con));
6700
6701 ins_pipe(ialu_imm);
6702 %}
6703
6704 // Load Packed Float Constant
6705
6706 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6707 match(Set dst con);
6708 ins_cost(INSN_COST * 4);
6709 format %{ "fmovs $dst, $con"%}
6710 ins_encode %{
6711 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6712 %}
6713
6714 ins_pipe(fp_imm_s);
6715 %}
6716
6717 // Load Float Constant
6718
6719 instruct loadConF(vRegF dst, immF con) %{
6720 match(Set dst con);
6721
6722 ins_cost(INSN_COST * 4);
6723
6724 format %{
6725 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6726 %}
6727
6728 ins_encode %{
6729 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6730 %}
6731
6732 ins_pipe(fp_load_constant_s);
6733 %}
6734
6735 // Load Packed Double Constant
6736
6737 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6738 match(Set dst con);
6739 ins_cost(INSN_COST);
6740 format %{ "fmovd $dst, $con"%}
6741 ins_encode %{
6742 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6743 %}
6744
6745 ins_pipe(fp_imm_d);
6746 %}
6747
6748 // Load Double Constant
6749
6750 instruct loadConD(vRegD dst, immD con) %{
6751 match(Set dst con);
6752
6753 ins_cost(INSN_COST * 5);
6754 format %{
6755 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6756 %}
6757
6758 ins_encode %{
6759 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6760 %}
6761
6762 ins_pipe(fp_load_constant_d);
6763 %}
6764
6765 // Store Instructions
6766
6767 // Store CMS card-mark Immediate
6768 instruct storeimmCM0(immI0 zero, memory mem)
6769 %{
6770 match(Set mem (StoreCM mem zero));
6771 predicate(unnecessary_storestore(n));
6772
6773 ins_cost(INSN_COST);
6774 format %{ "strb zr, $mem\t# byte" %}
6775
6776 ins_encode(aarch64_enc_strb0(mem));
6777
6778 ins_pipe(istore_mem);
6779 %}
6780
6781 // Store CMS card-mark Immediate with intervening StoreStore
6782 // needed when using CMS with no conditional card marking
6783 instruct storeimmCM0_ordered(immI0 zero, memory mem)
6784 %{
6785 match(Set mem (StoreCM mem zero));
6786
6787 ins_cost(INSN_COST * 2);
6788 format %{ "dmb ishst"
6789 "\n\tstrb zr, $mem\t# byte" %}
6790
6791 ins_encode(aarch64_enc_strb0_ordered(mem));
6792
6793 ins_pipe(istore_mem);
6794 %}
6795
6796 // Store Byte
6797 instruct storeB(iRegIorL2I src, memory mem)
6798 %{
6799 match(Set mem (StoreB mem src));
6800 predicate(!needs_releasing_store(n));
6801
6802 ins_cost(INSN_COST);
6803 format %{ "strb $src, $mem\t# byte" %}
6804
6805 ins_encode(aarch64_enc_strb(src, mem));
6806
6807 ins_pipe(istore_reg_mem);
6808 %}
6809
6810
6811 instruct storeimmB0(immI0 zero, memory mem)
6812 %{
6813 match(Set mem (StoreB mem zero));
6814 predicate(!needs_releasing_store(n));
6815
6816 ins_cost(INSN_COST);
6817 format %{ "strb zr, $mem\t# byte" %}
6818
6819 ins_encode(aarch64_enc_strb0(mem));
6820
6821 ins_pipe(istore_mem);
6822 %}
6823
6824 // Store Char/Short
6825 instruct storeC(iRegIorL2I src, memory mem)
6826 %{
6827 match(Set mem (StoreC mem src));
6828 predicate(!needs_releasing_store(n));
6829
6830 ins_cost(INSN_COST);
6831 format %{ "strh $src, $mem\t# short" %}
6832
6833 ins_encode(aarch64_enc_strh(src, mem));
6834
6835 ins_pipe(istore_reg_mem);
6836 %}
6837
6838 instruct storeimmC0(immI0 zero, memory mem)
6839 %{
6840 match(Set mem (StoreC mem zero));
6841 predicate(!needs_releasing_store(n));
6842
6843 ins_cost(INSN_COST);
6844 format %{ "strh zr, $mem\t# short" %}
6845
6846 ins_encode(aarch64_enc_strh0(mem));
6847
6848 ins_pipe(istore_mem);
6849 %}
6850
6851 // Store Integer
6852
6853 instruct storeI(iRegIorL2I src, memory mem)
6854 %{
6855 match(Set mem(StoreI mem src));
6856 predicate(!needs_releasing_store(n));
6857
6858 ins_cost(INSN_COST);
6859 format %{ "strw $src, $mem\t# int" %}
6860
6861 ins_encode(aarch64_enc_strw(src, mem));
6862
6863 ins_pipe(istore_reg_mem);
6864 %}
6865
6866 instruct storeimmI0(immI0 zero, memory mem)
6867 %{
6868 match(Set mem(StoreI mem zero));
6869 predicate(!needs_releasing_store(n));
6870
6871 ins_cost(INSN_COST);
6872 format %{ "strw zr, $mem\t# int" %}
6873
6874 ins_encode(aarch64_enc_strw0(mem));
6875
6876 ins_pipe(istore_mem);
6877 %}
6878
6879 // Store Long (64 bit signed)
6880 instruct storeL(iRegL src, memory mem)
6881 %{
6882 match(Set mem (StoreL mem src));
6883 predicate(!needs_releasing_store(n));
6884
6885 ins_cost(INSN_COST);
6886 format %{ "str $src, $mem\t# int" %}
6887
6888 ins_encode(aarch64_enc_str(src, mem));
6889
6890 ins_pipe(istore_reg_mem);
6891 %}
6892
6893 // Store Long (64 bit signed)
6894 instruct storeimmL0(immL0 zero, memory mem)
6895 %{
6896 match(Set mem (StoreL mem zero));
6897 predicate(!needs_releasing_store(n));
6898
6899 ins_cost(INSN_COST);
6900 format %{ "str zr, $mem\t# int" %}
6901
6902 ins_encode(aarch64_enc_str0(mem));
6903
6904 ins_pipe(istore_mem);
6905 %}
6906
6907 // Store Pointer
6908 instruct storeP(iRegP src, memory mem)
6909 %{
6910 match(Set mem (StoreP mem src));
6911 predicate(!needs_releasing_store(n));
6912
6913 ins_cost(INSN_COST);
6914 format %{ "str $src, $mem\t# ptr" %}
6915
6916 ins_encode %{
6917 int opcode = $mem->opcode();
6918 Register base = as_Register($mem$$base);
6919 int index = $mem$$index;
6920 int size = $mem$$scale;
6921 int disp = $mem$$disp;
6922 Register reg = as_Register($src$$reg);
6923
6924 // we sometimes get asked to store the stack pointer into the
6925 // current thread -- we cannot do that directly on AArch64
6926 if (reg == r31_sp) {
6927 MacroAssembler _masm(&cbuf);
6928 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
6929 __ mov(rscratch2, sp);
6930 reg = rscratch2;
6931 }
6932 Address::extend scale;
6933
6934 // Hooboy, this is fugly. We need a way to communicate to the
6935 // encoder that the index needs to be sign extended, so we have to
6936 // enumerate all the cases.
6937 switch (opcode) {
6938 case INDINDEXSCALEDOFFSETI2L:
6939 case INDINDEXSCALEDI2L:
6940 case INDINDEXSCALEDOFFSETI2LN:
6941 case INDINDEXSCALEDI2LN:
6942 case INDINDEXOFFSETI2L:
6943 case INDINDEXOFFSETI2LN:
6944 scale = Address::sxtw(size);
6945 break;
6946 default:
6947 scale = Address::lsl(size);
6948 }
6949
6950 Address adr;
6951 if (index == -1) {
6952 adr = Address(base, disp);
6953 } else {
6954 if (disp == 0) {
6955 adr = Address(base, as_Register(index), scale);
6956 } else {
6957 __ lea(rscratch1, Address(base, disp));
6958 adr = Address(rscratch1, as_Register(index), scale);
6959 }
6960 }
6961
6962 __ str(reg, adr);
6963 %}
6964
6965 ins_pipe(istore_reg_mem);
6966 %}
6967
6968 // Store Pointer
6969 instruct storeimmP0(immP0 zero, memory mem)
6970 %{
6971 match(Set mem (StoreP mem zero));
6972 predicate(!needs_releasing_store(n));
6973
6974 ins_cost(INSN_COST);
6975 format %{ "str zr, $mem\t# ptr" %}
6976
6977 ins_encode(aarch64_enc_str0(mem));
6978
6979 ins_pipe(istore_mem);
6980 %}
6981
6982 // Store Compressed Pointer
6983 instruct storeN(iRegN src, memory mem)
6984 %{
6985 match(Set mem (StoreN mem src));
6986 predicate(!needs_releasing_store(n));
6987
6988 ins_cost(INSN_COST);
6989 format %{ "strw $src, $mem\t# compressed ptr" %}
6990
6991 ins_encode(aarch64_enc_strw(src, mem));
6992
6993 ins_pipe(istore_reg_mem);
6994 %}
6995
6996 instruct storeImmN0(iRegIHeapbase heapbase, immN0 zero, memory mem)
6997 %{
6998 match(Set mem (StoreN mem zero));
6999 predicate(Universe::narrow_oop_base() == NULL &&
7000 Universe::narrow_klass_base() == NULL &&
7001 (!needs_releasing_store(n)));
7002
7003 ins_cost(INSN_COST);
7004 format %{ "strw rheapbase, $mem\t# compressed ptr (rheapbase==0)" %}
7005
7006 ins_encode(aarch64_enc_strw(heapbase, mem));
7007
7008 ins_pipe(istore_reg_mem);
7009 %}
7010
7011 // Store Float
7012 instruct storeF(vRegF src, memory mem)
7013 %{
7014 match(Set mem (StoreF mem src));
7015 predicate(!needs_releasing_store(n));
7016
7017 ins_cost(INSN_COST);
7018 format %{ "strs $src, $mem\t# float" %}
7019
7020 ins_encode( aarch64_enc_strs(src, mem) );
7021
7022 ins_pipe(pipe_class_memory);
7023 %}
7024
7025 // TODO
7026 // implement storeImmF0 and storeFImmPacked
7027
7028 // Store Double
7029 instruct storeD(vRegD src, memory mem)
7030 %{
7031 match(Set mem (StoreD mem src));
7032 predicate(!needs_releasing_store(n));
7033
7034 ins_cost(INSN_COST);
7035 format %{ "strd $src, $mem\t# double" %}
7036
7037 ins_encode( aarch64_enc_strd(src, mem) );
7038
7039 ins_pipe(pipe_class_memory);
7040 %}
7041
7042 // Store Compressed Klass Pointer
7043 instruct storeNKlass(iRegN src, memory mem)
7044 %{
7045 predicate(!needs_releasing_store(n));
7046 match(Set mem (StoreNKlass mem src));
7047
7048 ins_cost(INSN_COST);
7049 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7050
7051 ins_encode(aarch64_enc_strw(src, mem));
7052
7053 ins_pipe(istore_reg_mem);
7054 %}
7055
7056 // TODO
7057 // implement storeImmD0 and storeDImmPacked
7058
7059 // prefetch instructions
7060 // Must be safe to execute with invalid address (cannot fault).
7061
7062 instruct prefetchr( memory mem ) %{
7063 match(PrefetchRead mem);
7064
7065 ins_cost(INSN_COST);
7066 format %{ "prfm $mem, PLDL1KEEP\t# Prefetch into level 1 cache read keep" %}
7067
7068 ins_encode( aarch64_enc_prefetchr(mem) );
7069
7070 ins_pipe(iload_prefetch);
7071 %}
7072
7073 instruct prefetchw( memory mem ) %{
7074 match(PrefetchAllocation mem);
7075
7076 ins_cost(INSN_COST);
7077 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7078
7079 ins_encode( aarch64_enc_prefetchw(mem) );
7080
7081 ins_pipe(iload_prefetch);
7082 %}
7083
7084 instruct prefetchnta( memory mem ) %{
7085 match(PrefetchWrite mem);
7086
7087 ins_cost(INSN_COST);
7088 format %{ "prfm $mem, PSTL1STRM\t# Prefetch into level 1 cache write streaming" %}
7089
7090 ins_encode( aarch64_enc_prefetchnta(mem) );
7091
7092 ins_pipe(iload_prefetch);
7093 %}
7094
7095 // ---------------- volatile loads and stores ----------------
7096
7097 // Load Byte (8 bit signed)
7098 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7099 %{
7100 match(Set dst (LoadB mem));
7101
7102 ins_cost(VOLATILE_REF_COST);
7103 format %{ "ldarsb $dst, $mem\t# byte" %}
7104
7105 ins_encode(aarch64_enc_ldarsb(dst, mem));
7106
7107 ins_pipe(pipe_serial);
7108 %}
7109
7110 // Load Byte (8 bit signed) into long
7111 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7112 %{
7113 match(Set dst (ConvI2L (LoadB mem)));
7114
7115 ins_cost(VOLATILE_REF_COST);
7116 format %{ "ldarsb $dst, $mem\t# byte" %}
7117
7118 ins_encode(aarch64_enc_ldarsb(dst, mem));
7119
7120 ins_pipe(pipe_serial);
7121 %}
7122
7123 // Load Byte (8 bit unsigned)
7124 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7125 %{
7126 match(Set dst (LoadUB mem));
7127
7128 ins_cost(VOLATILE_REF_COST);
7129 format %{ "ldarb $dst, $mem\t# byte" %}
7130
7131 ins_encode(aarch64_enc_ldarb(dst, mem));
7132
7133 ins_pipe(pipe_serial);
7134 %}
7135
7136 // Load Byte (8 bit unsigned) into long
7137 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7138 %{
7139 match(Set dst (ConvI2L (LoadUB mem)));
7140
7141 ins_cost(VOLATILE_REF_COST);
7142 format %{ "ldarb $dst, $mem\t# byte" %}
7143
7144 ins_encode(aarch64_enc_ldarb(dst, mem));
7145
7146 ins_pipe(pipe_serial);
7147 %}
7148
7149 // Load Short (16 bit signed)
7150 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7151 %{
7152 match(Set dst (LoadS mem));
7153
7154 ins_cost(VOLATILE_REF_COST);
7155 format %{ "ldarshw $dst, $mem\t# short" %}
7156
7157 ins_encode(aarch64_enc_ldarshw(dst, mem));
7158
7159 ins_pipe(pipe_serial);
7160 %}
7161
7162 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7163 %{
7164 match(Set dst (LoadUS mem));
7165
7166 ins_cost(VOLATILE_REF_COST);
7167 format %{ "ldarhw $dst, $mem\t# short" %}
7168
7169 ins_encode(aarch64_enc_ldarhw(dst, mem));
7170
7171 ins_pipe(pipe_serial);
7172 %}
7173
7174 // Load Short/Char (16 bit unsigned) into long
7175 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7176 %{
7177 match(Set dst (ConvI2L (LoadUS mem)));
7178
7179 ins_cost(VOLATILE_REF_COST);
7180 format %{ "ldarh $dst, $mem\t# short" %}
7181
7182 ins_encode(aarch64_enc_ldarh(dst, mem));
7183
7184 ins_pipe(pipe_serial);
7185 %}
7186
7187 // Load Short/Char (16 bit signed) into long
7188 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7189 %{
7190 match(Set dst (ConvI2L (LoadS mem)));
7191
7192 ins_cost(VOLATILE_REF_COST);
7193 format %{ "ldarh $dst, $mem\t# short" %}
7194
7195 ins_encode(aarch64_enc_ldarsh(dst, mem));
7196
7197 ins_pipe(pipe_serial);
7198 %}
7199
7200 // Load Integer (32 bit signed)
7201 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7202 %{
7203 match(Set dst (LoadI mem));
7204
7205 ins_cost(VOLATILE_REF_COST);
7206 format %{ "ldarw $dst, $mem\t# int" %}
7207
7208 ins_encode(aarch64_enc_ldarw(dst, mem));
7209
7210 ins_pipe(pipe_serial);
7211 %}
7212
7213 // Load Integer (32 bit unsigned) into long
7214 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7215 %{
7216 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7217
7218 ins_cost(VOLATILE_REF_COST);
7219 format %{ "ldarw $dst, $mem\t# int" %}
7220
7221 ins_encode(aarch64_enc_ldarw(dst, mem));
7222
7223 ins_pipe(pipe_serial);
7224 %}
7225
7226 // Load Long (64 bit signed)
7227 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7228 %{
7229 match(Set dst (LoadL mem));
7230
7231 ins_cost(VOLATILE_REF_COST);
7232 format %{ "ldar $dst, $mem\t# int" %}
7233
7234 ins_encode(aarch64_enc_ldar(dst, mem));
7235
7236 ins_pipe(pipe_serial);
7237 %}
7238
7239 // Load Pointer
7240 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7241 %{
7242 match(Set dst (LoadP mem));
7243
7244 ins_cost(VOLATILE_REF_COST);
7245 format %{ "ldar $dst, $mem\t# ptr" %}
7246
7247 ins_encode(aarch64_enc_ldar(dst, mem));
7248
7249 ins_pipe(pipe_serial);
7250 %}
7251
7252 // Load Compressed Pointer
7253 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7254 %{
7255 match(Set dst (LoadN mem));
7256
7257 ins_cost(VOLATILE_REF_COST);
7258 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7259
7260 ins_encode(aarch64_enc_ldarw(dst, mem));
7261
7262 ins_pipe(pipe_serial);
7263 %}
7264
7265 // Load Float
7266 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7267 %{
7268 match(Set dst (LoadF mem));
7269
7270 ins_cost(VOLATILE_REF_COST);
7271 format %{ "ldars $dst, $mem\t# float" %}
7272
7273 ins_encode( aarch64_enc_fldars(dst, mem) );
7274
7275 ins_pipe(pipe_serial);
7276 %}
7277
7278 // Load Double
7279 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7280 %{
7281 match(Set dst (LoadD mem));
7282
7283 ins_cost(VOLATILE_REF_COST);
7284 format %{ "ldard $dst, $mem\t# double" %}
7285
7286 ins_encode( aarch64_enc_fldard(dst, mem) );
7287
7288 ins_pipe(pipe_serial);
7289 %}
7290
7291 // Store Byte
7292 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7293 %{
7294 match(Set mem (StoreB mem src));
7295
7296 ins_cost(VOLATILE_REF_COST);
7297 format %{ "stlrb $src, $mem\t# byte" %}
7298
7299 ins_encode(aarch64_enc_stlrb(src, mem));
7300
7301 ins_pipe(pipe_class_memory);
7302 %}
7303
7304 // Store Char/Short
7305 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7306 %{
7307 match(Set mem (StoreC mem src));
7308
7309 ins_cost(VOLATILE_REF_COST);
7310 format %{ "stlrh $src, $mem\t# short" %}
7311
7312 ins_encode(aarch64_enc_stlrh(src, mem));
7313
7314 ins_pipe(pipe_class_memory);
7315 %}
7316
7317 // Store Integer
7318
7319 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7320 %{
7321 match(Set mem(StoreI mem src));
7322
7323 ins_cost(VOLATILE_REF_COST);
7324 format %{ "stlrw $src, $mem\t# int" %}
7325
7326 ins_encode(aarch64_enc_stlrw(src, mem));
7327
7328 ins_pipe(pipe_class_memory);
7329 %}
7330
7331 // Store Long (64 bit signed)
7332 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7333 %{
7334 match(Set mem (StoreL mem src));
7335
7336 ins_cost(VOLATILE_REF_COST);
7337 format %{ "stlr $src, $mem\t# int" %}
7338
7339 ins_encode(aarch64_enc_stlr(src, mem));
7340
7341 ins_pipe(pipe_class_memory);
7342 %}
7343
7344 // Store Pointer
7345 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7346 %{
7347 match(Set mem (StoreP mem src));
7348
7349 ins_cost(VOLATILE_REF_COST);
7350 format %{ "stlr $src, $mem\t# ptr" %}
7351
7352 ins_encode(aarch64_enc_stlr(src, mem));
7353
7354 ins_pipe(pipe_class_memory);
7355 %}
7356
7357 // Store Compressed Pointer
7358 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7359 %{
7360 match(Set mem (StoreN mem src));
7361
7362 ins_cost(VOLATILE_REF_COST);
7363 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7364
7365 ins_encode(aarch64_enc_stlrw(src, mem));
7366
7367 ins_pipe(pipe_class_memory);
7368 %}
7369
7370 // Store Float
7371 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7372 %{
7373 match(Set mem (StoreF mem src));
7374
7375 ins_cost(VOLATILE_REF_COST);
7376 format %{ "stlrs $src, $mem\t# float" %}
7377
7378 ins_encode( aarch64_enc_fstlrs(src, mem) );
7379
7380 ins_pipe(pipe_class_memory);
7381 %}
7382
7383 // TODO
7384 // implement storeImmF0 and storeFImmPacked
7385
7386 // Store Double
7387 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7388 %{
7389 match(Set mem (StoreD mem src));
7390
7391 ins_cost(VOLATILE_REF_COST);
7392 format %{ "stlrd $src, $mem\t# double" %}
7393
7394 ins_encode( aarch64_enc_fstlrd(src, mem) );
7395
7396 ins_pipe(pipe_class_memory);
7397 %}
7398
7399 // ---------------- end of volatile loads and stores ----------------
7400
7401 // ============================================================================
7402 // BSWAP Instructions
7403
7404 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7405 match(Set dst (ReverseBytesI src));
7406
7407 ins_cost(INSN_COST);
7408 format %{ "revw $dst, $src" %}
7409
7410 ins_encode %{
7411 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7412 %}
7413
7414 ins_pipe(ialu_reg);
7415 %}
7416
7417 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7418 match(Set dst (ReverseBytesL src));
7419
7420 ins_cost(INSN_COST);
7421 format %{ "rev $dst, $src" %}
7422
7423 ins_encode %{
7424 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7425 %}
7426
7427 ins_pipe(ialu_reg);
7428 %}
7429
7430 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7431 match(Set dst (ReverseBytesUS src));
7432
7433 ins_cost(INSN_COST);
7434 format %{ "rev16w $dst, $src" %}
7435
7436 ins_encode %{
7437 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7438 %}
7439
7440 ins_pipe(ialu_reg);
7441 %}
7442
7443 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7444 match(Set dst (ReverseBytesS src));
7445
7446 ins_cost(INSN_COST);
7447 format %{ "rev16w $dst, $src\n\t"
7448 "sbfmw $dst, $dst, #0, #15" %}
7449
7450 ins_encode %{
7451 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7452 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7453 %}
7454
7455 ins_pipe(ialu_reg);
7456 %}
7457
7458 // ============================================================================
7459 // Zero Count Instructions
7460
7461 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7462 match(Set dst (CountLeadingZerosI src));
7463
7464 ins_cost(INSN_COST);
7465 format %{ "clzw $dst, $src" %}
7466 ins_encode %{
7467 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7468 %}
7469
7470 ins_pipe(ialu_reg);
7471 %}
7472
7473 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7474 match(Set dst (CountLeadingZerosL src));
7475
7476 ins_cost(INSN_COST);
7477 format %{ "clz $dst, $src" %}
7478 ins_encode %{
7479 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7480 %}
7481
7482 ins_pipe(ialu_reg);
7483 %}
7484
7485 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7486 match(Set dst (CountTrailingZerosI src));
7487
7488 ins_cost(INSN_COST * 2);
7489 format %{ "rbitw $dst, $src\n\t"
7490 "clzw $dst, $dst" %}
7491 ins_encode %{
7492 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7493 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7494 %}
7495
7496 ins_pipe(ialu_reg);
7497 %}
7498
7499 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7500 match(Set dst (CountTrailingZerosL src));
7501
7502 ins_cost(INSN_COST * 2);
7503 format %{ "rbit $dst, $src\n\t"
7504 "clz $dst, $dst" %}
7505 ins_encode %{
7506 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7507 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7508 %}
7509
7510 ins_pipe(ialu_reg);
7511 %}
7512
7513 //---------- Population Count Instructions -------------------------------------
7514 //
7515
7516 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7517 predicate(UsePopCountInstruction);
7518 match(Set dst (PopCountI src));
7519 effect(TEMP tmp);
7520 ins_cost(INSN_COST * 13);
7521
7522 format %{ "movw $src, $src\n\t"
7523 "mov $tmp, $src\t# vector (1D)\n\t"
7524 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7525 "addv $tmp, $tmp\t# vector (8B)\n\t"
7526 "mov $dst, $tmp\t# vector (1D)" %}
7527 ins_encode %{
7528 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7529 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
7530 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7531 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7532 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7533 %}
7534
7535 ins_pipe(pipe_class_default);
7536 %}
7537
7538 instruct popCountI_mem(iRegINoSp dst, memory mem, vRegF tmp) %{
7539 predicate(UsePopCountInstruction);
7540 match(Set dst (PopCountI (LoadI mem)));
7541 effect(TEMP tmp);
7542 ins_cost(INSN_COST * 13);
7543
7544 format %{ "ldrs $tmp, $mem\n\t"
7545 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7546 "addv $tmp, $tmp\t# vector (8B)\n\t"
7547 "mov $dst, $tmp\t# vector (1D)" %}
7548 ins_encode %{
7549 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7550 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7551 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
7552 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7553 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7554 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7555 %}
7556
7557 ins_pipe(pipe_class_default);
7558 %}
7559
7560 // Note: Long.bitCount(long) returns an int.
7561 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7562 predicate(UsePopCountInstruction);
7563 match(Set dst (PopCountL src));
7564 effect(TEMP tmp);
7565 ins_cost(INSN_COST * 13);
7566
7567 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7568 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7569 "addv $tmp, $tmp\t# vector (8B)\n\t"
7570 "mov $dst, $tmp\t# vector (1D)" %}
7571 ins_encode %{
7572 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
7573 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7574 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7575 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7576 %}
7577
7578 ins_pipe(pipe_class_default);
7579 %}
7580
7581 instruct popCountL_mem(iRegINoSp dst, memory mem, vRegD tmp) %{
7582 predicate(UsePopCountInstruction);
7583 match(Set dst (PopCountL (LoadL mem)));
7584 effect(TEMP tmp);
7585 ins_cost(INSN_COST * 13);
7586
7587 format %{ "ldrd $tmp, $mem\n\t"
7588 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7589 "addv $tmp, $tmp\t# vector (8B)\n\t"
7590 "mov $dst, $tmp\t# vector (1D)" %}
7591 ins_encode %{
7592 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7593 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7594 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
7595 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7596 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7597 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7598 %}
7599
7600 ins_pipe(pipe_class_default);
7601 %}
7602
7603 // ============================================================================
7604 // MemBar Instruction
7605
7606 instruct load_fence() %{
7607 match(LoadFence);
7608 ins_cost(VOLATILE_REF_COST);
7609
7610 format %{ "load_fence" %}
7611
7612 ins_encode %{
7613 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7614 %}
7615 ins_pipe(pipe_serial);
7616 %}
7617
7618 instruct unnecessary_membar_acquire() %{
7619 predicate(unnecessary_acquire(n));
7620 match(MemBarAcquire);
7621 ins_cost(0);
7622
7623 format %{ "membar_acquire (elided)" %}
7624
7625 ins_encode %{
7626 __ block_comment("membar_acquire (elided)");
7627 %}
7628
7629 ins_pipe(pipe_class_empty);
7630 %}
7631
7632 instruct membar_acquire() %{
7633 match(MemBarAcquire);
7634 ins_cost(VOLATILE_REF_COST);
7635
7636 format %{ "membar_acquire" %}
7637
7638 ins_encode %{
7639 __ block_comment("membar_acquire");
7640 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7641 %}
7642
7643 ins_pipe(pipe_serial);
7644 %}
7645
7646
7647 instruct membar_acquire_lock() %{
7648 match(MemBarAcquireLock);
7649 ins_cost(VOLATILE_REF_COST);
7650
7651 format %{ "membar_acquire_lock (elided)" %}
7652
7653 ins_encode %{
7654 __ block_comment("membar_acquire_lock (elided)");
7655 %}
7656
7657 ins_pipe(pipe_serial);
7658 %}
7659
7660 instruct store_fence() %{
7661 match(StoreFence);
7662 ins_cost(VOLATILE_REF_COST);
7663
7664 format %{ "store_fence" %}
7665
7666 ins_encode %{
7667 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7668 %}
7669 ins_pipe(pipe_serial);
7670 %}
7671
7672 instruct unnecessary_membar_release() %{
7673 predicate(unnecessary_release(n));
7674 match(MemBarRelease);
7675 ins_cost(0);
7676
7677 format %{ "membar_release (elided)" %}
7678
7679 ins_encode %{
7680 __ block_comment("membar_release (elided)");
7681 %}
7682 ins_pipe(pipe_serial);
7683 %}
7684
7685 instruct membar_release() %{
7686 match(MemBarRelease);
7687 ins_cost(VOLATILE_REF_COST);
7688
7689 format %{ "membar_release" %}
7690
7691 ins_encode %{
7692 __ block_comment("membar_release");
7693 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7694 %}
7695 ins_pipe(pipe_serial);
7696 %}
7697
7698 instruct membar_storestore() %{
7699 match(MemBarStoreStore);
7700 ins_cost(VOLATILE_REF_COST);
7701
7702 format %{ "MEMBAR-store-store" %}
7703
7704 ins_encode %{
7705 __ membar(Assembler::StoreStore);
7706 %}
7707 ins_pipe(pipe_serial);
7708 %}
7709
7710 instruct membar_release_lock() %{
7711 match(MemBarReleaseLock);
7712 ins_cost(VOLATILE_REF_COST);
7713
7714 format %{ "membar_release_lock (elided)" %}
7715
7716 ins_encode %{
7717 __ block_comment("membar_release_lock (elided)");
7718 %}
7719
7720 ins_pipe(pipe_serial);
7721 %}
7722
7723 instruct unnecessary_membar_volatile() %{
7724 predicate(unnecessary_volatile(n));
7725 match(MemBarVolatile);
7726 ins_cost(0);
7727
7728 format %{ "membar_volatile (elided)" %}
7729
7730 ins_encode %{
7731 __ block_comment("membar_volatile (elided)");
7732 %}
7733
7734 ins_pipe(pipe_serial);
7735 %}
7736
7737 instruct membar_volatile() %{
7738 match(MemBarVolatile);
7739 ins_cost(VOLATILE_REF_COST*100);
7740
7741 format %{ "membar_volatile" %}
7742
7743 ins_encode %{
7744 __ block_comment("membar_volatile");
7745 __ membar(Assembler::StoreLoad);
7746 %}
7747
7748 ins_pipe(pipe_serial);
7749 %}
7750
7751 // ============================================================================
7752 // Cast/Convert Instructions
7753
7754 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7755 match(Set dst (CastX2P src));
7756
7757 ins_cost(INSN_COST);
7758 format %{ "mov $dst, $src\t# long -> ptr" %}
7759
7760 ins_encode %{
7761 if ($dst$$reg != $src$$reg) {
7762 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7763 }
7764 %}
7765
7766 ins_pipe(ialu_reg);
7767 %}
7768
7769 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7770 match(Set dst (CastP2X src));
7771
7772 ins_cost(INSN_COST);
7773 format %{ "mov $dst, $src\t# ptr -> long" %}
7774
7775 ins_encode %{
7776 if ($dst$$reg != $src$$reg) {
7777 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7778 }
7779 %}
7780
7781 ins_pipe(ialu_reg);
7782 %}
7783
7784 // Convert oop into int for vectors alignment masking
7785 instruct convP2I(iRegINoSp dst, iRegP src) %{
7786 match(Set dst (ConvL2I (CastP2X src)));
7787
7788 ins_cost(INSN_COST);
7789 format %{ "movw $dst, $src\t# ptr -> int" %}
7790 ins_encode %{
7791 __ movw($dst$$Register, $src$$Register);
7792 %}
7793
7794 ins_pipe(ialu_reg);
7795 %}
7796
7797 // Convert compressed oop into int for vectors alignment masking
7798 // in case of 32bit oops (heap < 4Gb).
7799 instruct convN2I(iRegINoSp dst, iRegN src)
7800 %{
7801 predicate(Universe::narrow_oop_shift() == 0);
7802 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
7803
7804 ins_cost(INSN_COST);
7805 format %{ "mov dst, $src\t# compressed ptr -> int" %}
7806 ins_encode %{
7807 __ movw($dst$$Register, $src$$Register);
7808 %}
7809
7810 ins_pipe(ialu_reg);
7811 %}
7812
7813 // Convert oop pointer into compressed form
7814 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7815 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
7816 match(Set dst (EncodeP src));
7817 effect(KILL cr);
7818 ins_cost(INSN_COST * 3);
7819 format %{ "encode_heap_oop $dst, $src" %}
7820 ins_encode %{
7821 Register s = $src$$Register;
7822 Register d = $dst$$Register;
7823 __ encode_heap_oop(d, s);
7824 %}
7825 ins_pipe(ialu_reg);
7826 %}
7827
7828 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7829 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
7830 match(Set dst (EncodeP src));
7831 ins_cost(INSN_COST * 3);
7832 format %{ "encode_heap_oop_not_null $dst, $src" %}
7833 ins_encode %{
7834 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
7835 %}
7836 ins_pipe(ialu_reg);
7837 %}
7838
7839 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7840 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
7841 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
7842 match(Set dst (DecodeN src));
7843 ins_cost(INSN_COST * 3);
7844 format %{ "decode_heap_oop $dst, $src" %}
7845 ins_encode %{
7846 Register s = $src$$Register;
7847 Register d = $dst$$Register;
7848 __ decode_heap_oop(d, s);
7849 %}
7850 ins_pipe(ialu_reg);
7851 %}
7852
7853 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7854 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
7855 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
7856 match(Set dst (DecodeN src));
7857 ins_cost(INSN_COST * 3);
7858 format %{ "decode_heap_oop_not_null $dst, $src" %}
7859 ins_encode %{
7860 Register s = $src$$Register;
7861 Register d = $dst$$Register;
7862 __ decode_heap_oop_not_null(d, s);
7863 %}
7864 ins_pipe(ialu_reg);
7865 %}
7866
7867 // n.b. AArch64 implementations of encode_klass_not_null and
7868 // decode_klass_not_null do not modify the flags register so, unlike
7869 // Intel, we don't kill CR as a side effect here
7870
7871 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
7872 match(Set dst (EncodePKlass src));
7873
7874 ins_cost(INSN_COST * 3);
7875 format %{ "encode_klass_not_null $dst,$src" %}
7876
7877 ins_encode %{
7878 Register src_reg = as_Register($src$$reg);
7879 Register dst_reg = as_Register($dst$$reg);
7880 __ encode_klass_not_null(dst_reg, src_reg);
7881 %}
7882
7883 ins_pipe(ialu_reg);
7884 %}
7885
7886 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
7887 match(Set dst (DecodeNKlass src));
7888
7889 ins_cost(INSN_COST * 3);
7890 format %{ "decode_klass_not_null $dst,$src" %}
7891
7892 ins_encode %{
7893 Register src_reg = as_Register($src$$reg);
7894 Register dst_reg = as_Register($dst$$reg);
7895 if (dst_reg != src_reg) {
7896 __ decode_klass_not_null(dst_reg, src_reg);
7897 } else {
7898 __ decode_klass_not_null(dst_reg);
7899 }
7900 %}
7901
7902 ins_pipe(ialu_reg);
7903 %}
7904
7905 instruct checkCastPP(iRegPNoSp dst)
7906 %{
7907 match(Set dst (CheckCastPP dst));
7908
7909 size(0);
7910 format %{ "# checkcastPP of $dst" %}
7911 ins_encode(/* empty encoding */);
7912 ins_pipe(pipe_class_empty);
7913 %}
7914
7915 instruct castPP(iRegPNoSp dst)
7916 %{
7917 match(Set dst (CastPP dst));
7918
7919 size(0);
7920 format %{ "# castPP of $dst" %}
7921 ins_encode(/* empty encoding */);
7922 ins_pipe(pipe_class_empty);
7923 %}
7924
7925 instruct castII(iRegI dst)
7926 %{
7927 match(Set dst (CastII dst));
7928
7929 size(0);
7930 format %{ "# castII of $dst" %}
7931 ins_encode(/* empty encoding */);
7932 ins_cost(0);
7933 ins_pipe(pipe_class_empty);
7934 %}
7935
7936 // ============================================================================
7937 // Atomic operation instructions
7938 //
7939 // Intel and SPARC both implement Ideal Node LoadPLocked and
7940 // Store{PIL}Conditional instructions using a normal load for the
7941 // LoadPLocked and a CAS for the Store{PIL}Conditional.
7942 //
7943 // The ideal code appears only to use LoadPLocked/StorePLocked as a
7944 // pair to lock object allocations from Eden space when not using
7945 // TLABs.
7946 //
7947 // There does not appear to be a Load{IL}Locked Ideal Node and the
7948 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
7949 // and to use StoreIConditional only for 32-bit and StoreLConditional
7950 // only for 64-bit.
7951 //
7952 // We implement LoadPLocked and StorePLocked instructions using,
7953 // respectively the AArch64 hw load-exclusive and store-conditional
7954 // instructions. Whereas we must implement each of
7955 // Store{IL}Conditional using a CAS which employs a pair of
7956 // instructions comprising a load-exclusive followed by a
7957 // store-conditional.
7958
7959
7960 // Locked-load (linked load) of the current heap-top
7961 // used when updating the eden heap top
7962 // implemented using ldaxr on AArch64
7963
7964 instruct loadPLocked(iRegPNoSp dst, indirect mem)
7965 %{
7966 match(Set dst (LoadPLocked mem));
7967
7968 ins_cost(VOLATILE_REF_COST);
7969
7970 format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
7971
7972 ins_encode(aarch64_enc_ldaxr(dst, mem));
7973
7974 ins_pipe(pipe_serial);
7975 %}
7976
7977 // Conditional-store of the updated heap-top.
7978 // Used during allocation of the shared heap.
7979 // Sets flag (EQ) on success.
7980 // implemented using stlxr on AArch64.
7981
7982 instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
7983 %{
7984 match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7985
7986 ins_cost(VOLATILE_REF_COST);
7987
7988 // TODO
7989 // do we need to do a store-conditional release or can we just use a
7990 // plain store-conditional?
7991
7992 format %{
7993 "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
7994 "cmpw rscratch1, zr\t# EQ on successful write"
7995 %}
7996
7997 ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
7998
7999 ins_pipe(pipe_serial);
8000 %}
8001
8002
8003 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
8004 // when attempting to rebias a lock towards the current thread. We
8005 // must use the acquire form of cmpxchg in order to guarantee acquire
8006 // semantics in this case.
8007 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
8008 %{
8009 match(Set cr (StoreLConditional mem (Binary oldval newval)));
8010
8011 ins_cost(VOLATILE_REF_COST);
8012
8013 format %{
8014 "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8015 "cmpw rscratch1, zr\t# EQ on successful write"
8016 %}
8017
8018 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
8019
8020 ins_pipe(pipe_slow);
8021 %}
8022
8023 // storeIConditional also has acquire semantics, for no better reason
8024 // than matching storeLConditional. At the time of writing this
8025 // comment storeIConditional was not used anywhere by AArch64.
8026 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
8027 %{
8028 match(Set cr (StoreIConditional mem (Binary oldval newval)));
8029
8030 ins_cost(VOLATILE_REF_COST);
8031
8032 format %{
8033 "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8034 "cmpw rscratch1, zr\t# EQ on successful write"
8035 %}
8036
8037 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
8038
8039 ins_pipe(pipe_slow);
8040 %}
8041
8042 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8043 // can't match them
8044
8045 // standard CompareAndSwapX when we are using barriers
8046 // these have higher priority than the rules selected by a predicate
8047
8048 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8049
8050 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8051 ins_cost(2 * VOLATILE_REF_COST);
8052
8053 effect(KILL cr);
8054
8055 format %{
8056 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8057 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8058 %}
8059
8060 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8061 aarch64_enc_cset_eq(res));
8062
8063 ins_pipe(pipe_slow);
8064 %}
8065
8066 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8067
8068 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8069 ins_cost(2 * VOLATILE_REF_COST);
8070
8071 effect(KILL cr);
8072
8073 format %{
8074 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8075 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8076 %}
8077
8078 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8079 aarch64_enc_cset_eq(res));
8080
8081 ins_pipe(pipe_slow);
8082 %}
8083
8084 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8085
8086 predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
8087 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8088 ins_cost(2 * VOLATILE_REF_COST);
8089
8090 effect(KILL cr);
8091
8092 format %{
8093 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8094 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8095 %}
8096
8097 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8098 aarch64_enc_cset_eq(res));
8099
8100 ins_pipe(pipe_slow);
8101 %}
8102
8103 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
8104
8105 predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR);
8106 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8107 ins_cost(2 * VOLATILE_REF_COST);
8108
8109 effect(TEMP tmp, KILL cr);
8110
8111 format %{
8112 "cmpxchg_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
8113 %}
8114
8115 ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp, res));
8116
8117 ins_pipe(pipe_slow);
8118 %}
8119
8120 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8121
8122 predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR);
8123 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8124 ins_cost(2 * VOLATILE_REF_COST);
8125
8126 effect(KILL cr);
8127
8128 format %{
8129 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8130 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8131 %}
8132
8133 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8134 aarch64_enc_cset_eq(res));
8135
8136 ins_pipe(pipe_slow);
8137 %}
8138
8139 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
8140
8141 predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR);
8142 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8143 ins_cost(2 * VOLATILE_REF_COST);
8144
8145 effect(TEMP tmp, KILL cr);
8146
8147 format %{
8148 "cmpxchgw_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
8149 %}
8150
8151 ins_encode %{
8152 Register tmp = $tmp$$Register;
8153 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
8154 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
8155 %}
8156
8157 ins_pipe(pipe_slow);
8158 %}
8159
8160 // alternative CompareAndSwapX when we are eliding barriers
8161
8162 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8163
8164 predicate(needs_acquiring_load_exclusive(n));
8165 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8166 ins_cost(VOLATILE_REF_COST);
8167
8168 effect(KILL cr);
8169
8170 format %{
8171 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8172 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8173 %}
8174
8175 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8176 aarch64_enc_cset_eq(res));
8177
8178 ins_pipe(pipe_slow);
8179 %}
8180
8181 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8182
8183 predicate(needs_acquiring_load_exclusive(n));
8184 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8185 ins_cost(VOLATILE_REF_COST);
8186
8187 effect(KILL cr);
8188
8189 format %{
8190 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8191 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8192 %}
8193
8194 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8195 aarch64_enc_cset_eq(res));
8196
8197 ins_pipe(pipe_slow);
8198 %}
8199
8200 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8201
8202 predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR));
8203 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8204 ins_cost(VOLATILE_REF_COST);
8205
8206 effect(KILL cr);
8207
8208 format %{
8209 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8210 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8211 %}
8212
8213 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8214 aarch64_enc_cset_eq(res));
8215
8216 ins_pipe(pipe_slow);
8217 %}
8218
8219 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
8220
8221 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR);
8222 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8223 ins_cost(VOLATILE_REF_COST);
8224
8225 effect(TEMP tmp, KILL cr);
8226
8227 format %{
8228 "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
8229 %}
8230
8231 ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp, res));
8232
8233 ins_pipe(pipe_slow);
8234 %}
8235
8236 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8237
8238 predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier|| n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR));
8239 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8240 ins_cost(VOLATILE_REF_COST);
8241
8242 effect(KILL cr);
8243
8244 format %{
8245 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8246 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8247 %}
8248
8249 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8250 aarch64_enc_cset_eq(res));
8251
8252 ins_pipe(pipe_slow);
8253 %}
8254
8255 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
8256
8257 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR);
8258 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8259 ins_cost(VOLATILE_REF_COST);
8260
8261 effect(TEMP tmp, KILL cr);
8262
8263 format %{
8264 "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
8265 %}
8266
8267 ins_encode %{
8268 Register tmp = $tmp$$Register;
8269 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
8270 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
8271 %}
8272
8273 ins_pipe(pipe_slow);
8274 %}
8275
8276 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8277 match(Set prev (GetAndSetI mem newv));
8278 ins_cost(2 * VOLATILE_REF_COST);
8279 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8280 ins_encode %{
8281 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8282 %}
8283 ins_pipe(pipe_serial);
8284 %}
8285
8286 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8287 match(Set prev (GetAndSetL mem newv));
8288 ins_cost(2 * VOLATILE_REF_COST);
8289 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8290 ins_encode %{
8291 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8292 %}
8293 ins_pipe(pipe_serial);
8294 %}
8295
8296 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8297 match(Set prev (GetAndSetN mem newv));
8298 ins_cost(2 * VOLATILE_REF_COST);
8299 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8300 ins_encode %{
8301 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8302 %}
8303 ins_pipe(pipe_serial);
8304 %}
8305
8306 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8307 match(Set prev (GetAndSetP mem newv));
8308 ins_cost(2 * VOLATILE_REF_COST);
8309 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8310 ins_encode %{
8311 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8312 %}
8313 ins_pipe(pipe_serial);
8314 %}
8315
8316 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8317 predicate(needs_acquiring_load_exclusive(n));
8318 match(Set prev (GetAndSetI mem newv));
8319 ins_cost(VOLATILE_REF_COST);
8320 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8321 ins_encode %{
8322 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8323 %}
8324 ins_pipe(pipe_serial);
8325 %}
8326
8327 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8328 predicate(needs_acquiring_load_exclusive(n));
8329 match(Set prev (GetAndSetL mem newv));
8330 ins_cost(VOLATILE_REF_COST);
8331 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8332 ins_encode %{
8333 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8334 %}
8335 ins_pipe(pipe_serial);
8336 %}
8337
8338 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8339 predicate(needs_acquiring_load_exclusive(n));
8340 match(Set prev (GetAndSetN mem newv));
8341 ins_cost(VOLATILE_REF_COST);
8342 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8343 ins_encode %{
8344 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8345 %}
8346 ins_pipe(pipe_serial);
8347 %}
8348
8349 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8350 predicate(needs_acquiring_load_exclusive(n));
8351 match(Set prev (GetAndSetP mem newv));
8352 ins_cost(VOLATILE_REF_COST);
8353 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8354 ins_encode %{
8355 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8356 %}
8357 ins_pipe(pipe_serial);
8358 %}
8359
8360
8361 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
8362 match(Set newval (GetAndAddL mem incr));
8363 ins_cost(2 * VOLATILE_REF_COST + 1);
8364 format %{ "get_and_addL $newval, [$mem], $incr" %}
8365 ins_encode %{
8366 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
8367 %}
8368 ins_pipe(pipe_serial);
8369 %}
8370
8371 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
8372 predicate(n->as_LoadStore()->result_not_used());
8373 match(Set dummy (GetAndAddL mem incr));
8374 ins_cost(2 * VOLATILE_REF_COST);
8375 format %{ "get_and_addL [$mem], $incr" %}
8376 ins_encode %{
8377 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
8378 %}
8379 ins_pipe(pipe_serial);
8380 %}
8381
8382 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
8383 match(Set newval (GetAndAddL mem incr));
8384 ins_cost(2 * VOLATILE_REF_COST + 1);
8385 format %{ "get_and_addL $newval, [$mem], $incr" %}
8386 ins_encode %{
8387 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
8388 %}
8389 ins_pipe(pipe_serial);
8390 %}
8391
8392 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
8393 predicate(n->as_LoadStore()->result_not_used());
8394 match(Set dummy (GetAndAddL mem incr));
8395 ins_cost(2 * VOLATILE_REF_COST);
8396 format %{ "get_and_addL [$mem], $incr" %}
8397 ins_encode %{
8398 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
8399 %}
8400 ins_pipe(pipe_serial);
8401 %}
8402
8403 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
8404 match(Set newval (GetAndAddI mem incr));
8405 ins_cost(2 * VOLATILE_REF_COST + 1);
8406 format %{ "get_and_addI $newval, [$mem], $incr" %}
8407 ins_encode %{
8408 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
8409 %}
8410 ins_pipe(pipe_serial);
8411 %}
8412
8413 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
8414 predicate(n->as_LoadStore()->result_not_used());
8415 match(Set dummy (GetAndAddI mem incr));
8416 ins_cost(2 * VOLATILE_REF_COST);
8417 format %{ "get_and_addI [$mem], $incr" %}
8418 ins_encode %{
8419 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
8420 %}
8421 ins_pipe(pipe_serial);
8422 %}
8423
8424 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
8425 match(Set newval (GetAndAddI mem incr));
8426 ins_cost(2 * VOLATILE_REF_COST + 1);
8427 format %{ "get_and_addI $newval, [$mem], $incr" %}
8428 ins_encode %{
8429 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
8430 %}
8431 ins_pipe(pipe_serial);
8432 %}
8433
8434 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
8435 predicate(n->as_LoadStore()->result_not_used());
8436 match(Set dummy (GetAndAddI mem incr));
8437 ins_cost(2 * VOLATILE_REF_COST);
8438 format %{ "get_and_addI [$mem], $incr" %}
8439 ins_encode %{
8440 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
8441 %}
8442 ins_pipe(pipe_serial);
8443 %}
8444
8445 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
8446 predicate(needs_acquiring_load_exclusive(n));
8447 match(Set newval (GetAndAddL mem incr));
8448 ins_cost(VOLATILE_REF_COST + 1);
8449 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
8450 ins_encode %{
8451 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
8452 %}
8453 ins_pipe(pipe_serial);
8454 %}
8455
8456 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
8457 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8458 match(Set dummy (GetAndAddL mem incr));
8459 ins_cost(VOLATILE_REF_COST);
8460 format %{ "get_and_addL_acq [$mem], $incr" %}
8461 ins_encode %{
8462 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
8463 %}
8464 ins_pipe(pipe_serial);
8465 %}
8466
8467 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
8468 predicate(needs_acquiring_load_exclusive(n));
8469 match(Set newval (GetAndAddL mem incr));
8470 ins_cost(VOLATILE_REF_COST + 1);
8471 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
8472 ins_encode %{
8473 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
8474 %}
8475 ins_pipe(pipe_serial);
8476 %}
8477
8478 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
8479 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8480 match(Set dummy (GetAndAddL mem incr));
8481 ins_cost(VOLATILE_REF_COST);
8482 format %{ "get_and_addL_acq [$mem], $incr" %}
8483 ins_encode %{
8484 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
8485 %}
8486 ins_pipe(pipe_serial);
8487 %}
8488
8489 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
8490 predicate(needs_acquiring_load_exclusive(n));
8491 match(Set newval (GetAndAddI mem incr));
8492 ins_cost(VOLATILE_REF_COST + 1);
8493 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
8494 ins_encode %{
8495 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
8496 %}
8497 ins_pipe(pipe_serial);
8498 %}
8499
8500 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
8501 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8502 match(Set dummy (GetAndAddI mem incr));
8503 ins_cost(VOLATILE_REF_COST);
8504 format %{ "get_and_addI_acq [$mem], $incr" %}
8505 ins_encode %{
8506 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
8507 %}
8508 ins_pipe(pipe_serial);
8509 %}
8510
8511 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
8512 predicate(needs_acquiring_load_exclusive(n));
8513 match(Set newval (GetAndAddI mem incr));
8514 ins_cost(VOLATILE_REF_COST + 1);
8515 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
8516 ins_encode %{
8517 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
8518 %}
8519 ins_pipe(pipe_serial);
8520 %}
8521
8522 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
8523 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8524 match(Set dummy (GetAndAddI mem incr));
8525 ins_cost(VOLATILE_REF_COST);
8526 format %{ "get_and_addI_acq [$mem], $incr" %}
8527 ins_encode %{
8528 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
8529 %}
8530 ins_pipe(pipe_serial);
8531 %}
8532
8533 // ============================================================================
8534 // Conditional Move Instructions
8535
8536 // n.b. we have identical rules for both a signed compare op (cmpOp)
8537 // and an unsigned compare op (cmpOpU). it would be nice if we could
8538 // define an op class which merged both inputs and use it to type the
8539 // argument to a single rule. unfortunatelyt his fails because the
8540 // opclass does not live up to the COND_INTER interface of its
8541 // component operands. When the generic code tries to negate the
8542 // operand it ends up running the generci Machoper::negate method
8543 // which throws a ShouldNotHappen. So, we have to provide two flavours
8544 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8545
8546 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8547 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8548
8549 ins_cost(INSN_COST * 2);
8550 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8551
8552 ins_encode %{
8553 __ cselw(as_Register($dst$$reg),
8554 as_Register($src2$$reg),
8555 as_Register($src1$$reg),
8556 (Assembler::Condition)$cmp$$cmpcode);
8557 %}
8558
8559 ins_pipe(icond_reg_reg);
8560 %}
8561
8562 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8563 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8564
8565 ins_cost(INSN_COST * 2);
8566 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8567
8568 ins_encode %{
8569 __ cselw(as_Register($dst$$reg),
8570 as_Register($src2$$reg),
8571 as_Register($src1$$reg),
8572 (Assembler::Condition)$cmp$$cmpcode);
8573 %}
8574
8575 ins_pipe(icond_reg_reg);
8576 %}
8577
8578 // special cases where one arg is zero
8579
8580 // n.b. this is selected in preference to the rule above because it
8581 // avoids loading constant 0 into a source register
8582
8583 // TODO
8584 // we ought only to be able to cull one of these variants as the ideal
8585 // transforms ought always to order the zero consistently (to left/right?)
8586
8587 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8588 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8589
8590 ins_cost(INSN_COST * 2);
8591 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8592
8593 ins_encode %{
8594 __ cselw(as_Register($dst$$reg),
8595 as_Register($src$$reg),
8596 zr,
8597 (Assembler::Condition)$cmp$$cmpcode);
8598 %}
8599
8600 ins_pipe(icond_reg);
8601 %}
8602
8603 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8604 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8605
8606 ins_cost(INSN_COST * 2);
8607 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8608
8609 ins_encode %{
8610 __ cselw(as_Register($dst$$reg),
8611 as_Register($src$$reg),
8612 zr,
8613 (Assembler::Condition)$cmp$$cmpcode);
8614 %}
8615
8616 ins_pipe(icond_reg);
8617 %}
8618
8619 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8620 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8621
8622 ins_cost(INSN_COST * 2);
8623 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8624
8625 ins_encode %{
8626 __ cselw(as_Register($dst$$reg),
8627 zr,
8628 as_Register($src$$reg),
8629 (Assembler::Condition)$cmp$$cmpcode);
8630 %}
8631
8632 ins_pipe(icond_reg);
8633 %}
8634
8635 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8636 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8637
8638 ins_cost(INSN_COST * 2);
8639 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8640
8641 ins_encode %{
8642 __ cselw(as_Register($dst$$reg),
8643 zr,
8644 as_Register($src$$reg),
8645 (Assembler::Condition)$cmp$$cmpcode);
8646 %}
8647
8648 ins_pipe(icond_reg);
8649 %}
8650
8651 // special case for creating a boolean 0 or 1
8652
8653 // n.b. this is selected in preference to the rule above because it
8654 // avoids loading constants 0 and 1 into a source register
8655
8656 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8657 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8658
8659 ins_cost(INSN_COST * 2);
8660 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8661
8662 ins_encode %{
8663 // equivalently
8664 // cset(as_Register($dst$$reg),
8665 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8666 __ csincw(as_Register($dst$$reg),
8667 zr,
8668 zr,
8669 (Assembler::Condition)$cmp$$cmpcode);
8670 %}
8671
8672 ins_pipe(icond_none);
8673 %}
8674
8675 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8676 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8677
8678 ins_cost(INSN_COST * 2);
8679 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8680
8681 ins_encode %{
8682 // equivalently
8683 // cset(as_Register($dst$$reg),
8684 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8685 __ csincw(as_Register($dst$$reg),
8686 zr,
8687 zr,
8688 (Assembler::Condition)$cmp$$cmpcode);
8689 %}
8690
8691 ins_pipe(icond_none);
8692 %}
8693
8694 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8695 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8696
8697 ins_cost(INSN_COST * 2);
8698 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8699
8700 ins_encode %{
8701 __ csel(as_Register($dst$$reg),
8702 as_Register($src2$$reg),
8703 as_Register($src1$$reg),
8704 (Assembler::Condition)$cmp$$cmpcode);
8705 %}
8706
8707 ins_pipe(icond_reg_reg);
8708 %}
8709
8710 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8711 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8712
8713 ins_cost(INSN_COST * 2);
8714 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8715
8716 ins_encode %{
8717 __ csel(as_Register($dst$$reg),
8718 as_Register($src2$$reg),
8719 as_Register($src1$$reg),
8720 (Assembler::Condition)$cmp$$cmpcode);
8721 %}
8722
8723 ins_pipe(icond_reg_reg);
8724 %}
8725
8726 // special cases where one arg is zero
8727
8728 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8729 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8730
8731 ins_cost(INSN_COST * 2);
8732 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8733
8734 ins_encode %{
8735 __ csel(as_Register($dst$$reg),
8736 zr,
8737 as_Register($src$$reg),
8738 (Assembler::Condition)$cmp$$cmpcode);
8739 %}
8740
8741 ins_pipe(icond_reg);
8742 %}
8743
8744 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8745 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8746
8747 ins_cost(INSN_COST * 2);
8748 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8749
8750 ins_encode %{
8751 __ csel(as_Register($dst$$reg),
8752 zr,
8753 as_Register($src$$reg),
8754 (Assembler::Condition)$cmp$$cmpcode);
8755 %}
8756
8757 ins_pipe(icond_reg);
8758 %}
8759
8760 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8761 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8762
8763 ins_cost(INSN_COST * 2);
8764 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8765
8766 ins_encode %{
8767 __ csel(as_Register($dst$$reg),
8768 as_Register($src$$reg),
8769 zr,
8770 (Assembler::Condition)$cmp$$cmpcode);
8771 %}
8772
8773 ins_pipe(icond_reg);
8774 %}
8775
8776 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8777 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8778
8779 ins_cost(INSN_COST * 2);
8780 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8781
8782 ins_encode %{
8783 __ csel(as_Register($dst$$reg),
8784 as_Register($src$$reg),
8785 zr,
8786 (Assembler::Condition)$cmp$$cmpcode);
8787 %}
8788
8789 ins_pipe(icond_reg);
8790 %}
8791
8792 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8793 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8794
8795 ins_cost(INSN_COST * 2);
8796 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8797
8798 ins_encode %{
8799 __ csel(as_Register($dst$$reg),
8800 as_Register($src2$$reg),
8801 as_Register($src1$$reg),
8802 (Assembler::Condition)$cmp$$cmpcode);
8803 %}
8804
8805 ins_pipe(icond_reg_reg);
8806 %}
8807
8808 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8809 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8810
8811 ins_cost(INSN_COST * 2);
8812 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8813
8814 ins_encode %{
8815 __ csel(as_Register($dst$$reg),
8816 as_Register($src2$$reg),
8817 as_Register($src1$$reg),
8818 (Assembler::Condition)$cmp$$cmpcode);
8819 %}
8820
8821 ins_pipe(icond_reg_reg);
8822 %}
8823
8824 // special cases where one arg is zero
8825
8826 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8827 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8828
8829 ins_cost(INSN_COST * 2);
8830 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8831
8832 ins_encode %{
8833 __ csel(as_Register($dst$$reg),
8834 zr,
8835 as_Register($src$$reg),
8836 (Assembler::Condition)$cmp$$cmpcode);
8837 %}
8838
8839 ins_pipe(icond_reg);
8840 %}
8841
8842 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8843 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8844
8845 ins_cost(INSN_COST * 2);
8846 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8847
8848 ins_encode %{
8849 __ csel(as_Register($dst$$reg),
8850 zr,
8851 as_Register($src$$reg),
8852 (Assembler::Condition)$cmp$$cmpcode);
8853 %}
8854
8855 ins_pipe(icond_reg);
8856 %}
8857
8858 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8859 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8860
8861 ins_cost(INSN_COST * 2);
8862 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8863
8864 ins_encode %{
8865 __ csel(as_Register($dst$$reg),
8866 as_Register($src$$reg),
8867 zr,
8868 (Assembler::Condition)$cmp$$cmpcode);
8869 %}
8870
8871 ins_pipe(icond_reg);
8872 %}
8873
8874 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8875 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8876
8877 ins_cost(INSN_COST * 2);
8878 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8879
8880 ins_encode %{
8881 __ csel(as_Register($dst$$reg),
8882 as_Register($src$$reg),
8883 zr,
8884 (Assembler::Condition)$cmp$$cmpcode);
8885 %}
8886
8887 ins_pipe(icond_reg);
8888 %}
8889
8890 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8891 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8892
8893 ins_cost(INSN_COST * 2);
8894 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8895
8896 ins_encode %{
8897 __ cselw(as_Register($dst$$reg),
8898 as_Register($src2$$reg),
8899 as_Register($src1$$reg),
8900 (Assembler::Condition)$cmp$$cmpcode);
8901 %}
8902
8903 ins_pipe(icond_reg_reg);
8904 %}
8905
8906 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8907 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8908
8909 ins_cost(INSN_COST * 2);
8910 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8911
8912 ins_encode %{
8913 __ cselw(as_Register($dst$$reg),
8914 as_Register($src2$$reg),
8915 as_Register($src1$$reg),
8916 (Assembler::Condition)$cmp$$cmpcode);
8917 %}
8918
8919 ins_pipe(icond_reg_reg);
8920 %}
8921
8922 // special cases where one arg is zero
8923
8924 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8925 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8926
8927 ins_cost(INSN_COST * 2);
8928 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8929
8930 ins_encode %{
8931 __ cselw(as_Register($dst$$reg),
8932 zr,
8933 as_Register($src$$reg),
8934 (Assembler::Condition)$cmp$$cmpcode);
8935 %}
8936
8937 ins_pipe(icond_reg);
8938 %}
8939
8940 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8941 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8942
8943 ins_cost(INSN_COST * 2);
8944 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8945
8946 ins_encode %{
8947 __ cselw(as_Register($dst$$reg),
8948 zr,
8949 as_Register($src$$reg),
8950 (Assembler::Condition)$cmp$$cmpcode);
8951 %}
8952
8953 ins_pipe(icond_reg);
8954 %}
8955
8956 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8957 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8958
8959 ins_cost(INSN_COST * 2);
8960 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8961
8962 ins_encode %{
8963 __ cselw(as_Register($dst$$reg),
8964 as_Register($src$$reg),
8965 zr,
8966 (Assembler::Condition)$cmp$$cmpcode);
8967 %}
8968
8969 ins_pipe(icond_reg);
8970 %}
8971
8972 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8973 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8974
8975 ins_cost(INSN_COST * 2);
8976 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8977
8978 ins_encode %{
8979 __ cselw(as_Register($dst$$reg),
8980 as_Register($src$$reg),
8981 zr,
8982 (Assembler::Condition)$cmp$$cmpcode);
8983 %}
8984
8985 ins_pipe(icond_reg);
8986 %}
8987
8988 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8989 %{
8990 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8991
8992 ins_cost(INSN_COST * 3);
8993
8994 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8995 ins_encode %{
8996 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8997 __ fcsels(as_FloatRegister($dst$$reg),
8998 as_FloatRegister($src2$$reg),
8999 as_FloatRegister($src1$$reg),
9000 cond);
9001 %}
9002
9003 ins_pipe(fp_cond_reg_reg_s);
9004 %}
9005
9006 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9007 %{
9008 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9009
9010 ins_cost(INSN_COST * 3);
9011
9012 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9013 ins_encode %{
9014 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9015 __ fcsels(as_FloatRegister($dst$$reg),
9016 as_FloatRegister($src2$$reg),
9017 as_FloatRegister($src1$$reg),
9018 cond);
9019 %}
9020
9021 ins_pipe(fp_cond_reg_reg_s);
9022 %}
9023
9024 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9025 %{
9026 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9027
9028 ins_cost(INSN_COST * 3);
9029
9030 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9031 ins_encode %{
9032 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9033 __ fcseld(as_FloatRegister($dst$$reg),
9034 as_FloatRegister($src2$$reg),
9035 as_FloatRegister($src1$$reg),
9036 cond);
9037 %}
9038
9039 ins_pipe(fp_cond_reg_reg_d);
9040 %}
9041
9042 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9043 %{
9044 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9045
9046 ins_cost(INSN_COST * 3);
9047
9048 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9049 ins_encode %{
9050 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9051 __ fcseld(as_FloatRegister($dst$$reg),
9052 as_FloatRegister($src2$$reg),
9053 as_FloatRegister($src1$$reg),
9054 cond);
9055 %}
9056
9057 ins_pipe(fp_cond_reg_reg_d);
9058 %}
9059
9060 // ============================================================================
9061 // Arithmetic Instructions
9062 //
9063
9064 // Integer Addition
9065
9066 // TODO
9067 // these currently employ operations which do not set CR and hence are
9068 // not flagged as killing CR but we would like to isolate the cases
9069 // where we want to set flags from those where we don't. need to work
9070 // out how to do that.
9071
9072 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9073 match(Set dst (AddI src1 src2));
9074
9075 ins_cost(INSN_COST);
9076 format %{ "addw $dst, $src1, $src2" %}
9077
9078 ins_encode %{
9079 __ addw(as_Register($dst$$reg),
9080 as_Register($src1$$reg),
9081 as_Register($src2$$reg));
9082 %}
9083
9084 ins_pipe(ialu_reg_reg);
9085 %}
9086
9087 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9088 match(Set dst (AddI src1 src2));
9089
9090 ins_cost(INSN_COST);
9091 format %{ "addw $dst, $src1, $src2" %}
9092
9093 // use opcode to indicate that this is an add not a sub
9094 opcode(0x0);
9095
9096 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9097
9098 ins_pipe(ialu_reg_imm);
9099 %}
9100
9101 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9102 match(Set dst (AddI (ConvL2I src1) src2));
9103
9104 ins_cost(INSN_COST);
9105 format %{ "addw $dst, $src1, $src2" %}
9106
9107 // use opcode to indicate that this is an add not a sub
9108 opcode(0x0);
9109
9110 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9111
9112 ins_pipe(ialu_reg_imm);
9113 %}
9114
9115 // Pointer Addition
9116 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
9117 match(Set dst (AddP src1 src2));
9118
9119 ins_cost(INSN_COST);
9120 format %{ "add $dst, $src1, $src2\t# ptr" %}
9121
9122 ins_encode %{
9123 __ add(as_Register($dst$$reg),
9124 as_Register($src1$$reg),
9125 as_Register($src2$$reg));
9126 %}
9127
9128 ins_pipe(ialu_reg_reg);
9129 %}
9130
9131 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
9132 match(Set dst (AddP src1 (ConvI2L src2)));
9133
9134 ins_cost(1.9 * INSN_COST);
9135 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9136
9137 ins_encode %{
9138 __ add(as_Register($dst$$reg),
9139 as_Register($src1$$reg),
9140 as_Register($src2$$reg), ext::sxtw);
9141 %}
9142
9143 ins_pipe(ialu_reg_reg);
9144 %}
9145
9146 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
9147 match(Set dst (AddP src1 (LShiftL src2 scale)));
9148
9149 ins_cost(1.9 * INSN_COST);
9150 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9151
9152 ins_encode %{
9153 __ lea(as_Register($dst$$reg),
9154 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9155 Address::lsl($scale$$constant)));
9156 %}
9157
9158 ins_pipe(ialu_reg_reg_shift);
9159 %}
9160
9161 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
9162 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9163
9164 ins_cost(1.9 * INSN_COST);
9165 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9166
9167 ins_encode %{
9168 __ lea(as_Register($dst$$reg),
9169 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9170 Address::sxtw($scale$$constant)));
9171 %}
9172
9173 ins_pipe(ialu_reg_reg_shift);
9174 %}
9175
9176 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9177 match(Set dst (LShiftL (ConvI2L src) scale));
9178
9179 ins_cost(INSN_COST);
9180 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9181
9182 ins_encode %{
9183 __ sbfiz(as_Register($dst$$reg),
9184 as_Register($src$$reg),
9185 $scale$$constant & 63, MIN(32, (-$scale$$constant) & 63));
9186 %}
9187
9188 ins_pipe(ialu_reg_shift);
9189 %}
9190
9191 // Pointer Immediate Addition
9192 // n.b. this needs to be more expensive than using an indirect memory
9193 // operand
9194 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
9195 match(Set dst (AddP src1 src2));
9196
9197 ins_cost(INSN_COST);
9198 format %{ "add $dst, $src1, $src2\t# ptr" %}
9199
9200 // use opcode to indicate that this is an add not a sub
9201 opcode(0x0);
9202
9203 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9204
9205 ins_pipe(ialu_reg_imm);
9206 %}
9207
9208 // Long Addition
9209 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9210
9211 match(Set dst (AddL src1 src2));
9212
9213 ins_cost(INSN_COST);
9214 format %{ "add $dst, $src1, $src2" %}
9215
9216 ins_encode %{
9217 __ add(as_Register($dst$$reg),
9218 as_Register($src1$$reg),
9219 as_Register($src2$$reg));
9220 %}
9221
9222 ins_pipe(ialu_reg_reg);
9223 %}
9224
9225 // No constant pool entries requiredLong Immediate Addition.
9226 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9227 match(Set dst (AddL src1 src2));
9228
9229 ins_cost(INSN_COST);
9230 format %{ "add $dst, $src1, $src2" %}
9231
9232 // use opcode to indicate that this is an add not a sub
9233 opcode(0x0);
9234
9235 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9236
9237 ins_pipe(ialu_reg_imm);
9238 %}
9239
9240 // Integer Subtraction
9241 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9242 match(Set dst (SubI src1 src2));
9243
9244 ins_cost(INSN_COST);
9245 format %{ "subw $dst, $src1, $src2" %}
9246
9247 ins_encode %{
9248 __ subw(as_Register($dst$$reg),
9249 as_Register($src1$$reg),
9250 as_Register($src2$$reg));
9251 %}
9252
9253 ins_pipe(ialu_reg_reg);
9254 %}
9255
9256 // Immediate Subtraction
9257 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9258 match(Set dst (SubI src1 src2));
9259
9260 ins_cost(INSN_COST);
9261 format %{ "subw $dst, $src1, $src2" %}
9262
9263 // use opcode to indicate that this is a sub not an add
9264 opcode(0x1);
9265
9266 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9267
9268 ins_pipe(ialu_reg_imm);
9269 %}
9270
9271 // Long Subtraction
9272 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9273
9274 match(Set dst (SubL src1 src2));
9275
9276 ins_cost(INSN_COST);
9277 format %{ "sub $dst, $src1, $src2" %}
9278
9279 ins_encode %{
9280 __ sub(as_Register($dst$$reg),
9281 as_Register($src1$$reg),
9282 as_Register($src2$$reg));
9283 %}
9284
9285 ins_pipe(ialu_reg_reg);
9286 %}
9287
9288 // No constant pool entries requiredLong Immediate Subtraction.
9289 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9290 match(Set dst (SubL src1 src2));
9291
9292 ins_cost(INSN_COST);
9293 format %{ "sub$dst, $src1, $src2" %}
9294
9295 // use opcode to indicate that this is a sub not an add
9296 opcode(0x1);
9297
9298 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9299
9300 ins_pipe(ialu_reg_imm);
9301 %}
9302
9303 // Integer Negation (special case for sub)
9304
9305 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9306 match(Set dst (SubI zero src));
9307
9308 ins_cost(INSN_COST);
9309 format %{ "negw $dst, $src\t# int" %}
9310
9311 ins_encode %{
9312 __ negw(as_Register($dst$$reg),
9313 as_Register($src$$reg));
9314 %}
9315
9316 ins_pipe(ialu_reg);
9317 %}
9318
9319 // Long Negation
9320
9321 instruct negL_reg(iRegLNoSp dst, iRegIorL2I src, immL0 zero, rFlagsReg cr) %{
9322 match(Set dst (SubL zero src));
9323
9324 ins_cost(INSN_COST);
9325 format %{ "neg $dst, $src\t# long" %}
9326
9327 ins_encode %{
9328 __ neg(as_Register($dst$$reg),
9329 as_Register($src$$reg));
9330 %}
9331
9332 ins_pipe(ialu_reg);
9333 %}
9334
9335 // Integer Multiply
9336
9337 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9338 match(Set dst (MulI src1 src2));
9339
9340 ins_cost(INSN_COST * 3);
9341 format %{ "mulw $dst, $src1, $src2" %}
9342
9343 ins_encode %{
9344 __ mulw(as_Register($dst$$reg),
9345 as_Register($src1$$reg),
9346 as_Register($src2$$reg));
9347 %}
9348
9349 ins_pipe(imul_reg_reg);
9350 %}
9351
9352 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9353 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9354
9355 ins_cost(INSN_COST * 3);
9356 format %{ "smull $dst, $src1, $src2" %}
9357
9358 ins_encode %{
9359 __ smull(as_Register($dst$$reg),
9360 as_Register($src1$$reg),
9361 as_Register($src2$$reg));
9362 %}
9363
9364 ins_pipe(imul_reg_reg);
9365 %}
9366
9367 // Long Multiply
9368
9369 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9370 match(Set dst (MulL src1 src2));
9371
9372 ins_cost(INSN_COST * 5);
9373 format %{ "mul $dst, $src1, $src2" %}
9374
9375 ins_encode %{
9376 __ mul(as_Register($dst$$reg),
9377 as_Register($src1$$reg),
9378 as_Register($src2$$reg));
9379 %}
9380
9381 ins_pipe(lmul_reg_reg);
9382 %}
9383
9384 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9385 %{
9386 match(Set dst (MulHiL src1 src2));
9387
9388 ins_cost(INSN_COST * 7);
9389 format %{ "smulh $dst, $src1, $src2, \t# mulhi" %}
9390
9391 ins_encode %{
9392 __ smulh(as_Register($dst$$reg),
9393 as_Register($src1$$reg),
9394 as_Register($src2$$reg));
9395 %}
9396
9397 ins_pipe(lmul_reg_reg);
9398 %}
9399
9400 // Combined Integer Multiply & Add/Sub
9401
9402 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9403 match(Set dst (AddI src3 (MulI src1 src2)));
9404
9405 ins_cost(INSN_COST * 3);
9406 format %{ "madd $dst, $src1, $src2, $src3" %}
9407
9408 ins_encode %{
9409 __ maddw(as_Register($dst$$reg),
9410 as_Register($src1$$reg),
9411 as_Register($src2$$reg),
9412 as_Register($src3$$reg));
9413 %}
9414
9415 ins_pipe(imac_reg_reg);
9416 %}
9417
9418 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9419 match(Set dst (SubI src3 (MulI src1 src2)));
9420
9421 ins_cost(INSN_COST * 3);
9422 format %{ "msub $dst, $src1, $src2, $src3" %}
9423
9424 ins_encode %{
9425 __ msubw(as_Register($dst$$reg),
9426 as_Register($src1$$reg),
9427 as_Register($src2$$reg),
9428 as_Register($src3$$reg));
9429 %}
9430
9431 ins_pipe(imac_reg_reg);
9432 %}
9433
9434 // Combined Long Multiply & Add/Sub
9435
9436 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9437 match(Set dst (AddL src3 (MulL src1 src2)));
9438
9439 ins_cost(INSN_COST * 5);
9440 format %{ "madd $dst, $src1, $src2, $src3" %}
9441
9442 ins_encode %{
9443 __ madd(as_Register($dst$$reg),
9444 as_Register($src1$$reg),
9445 as_Register($src2$$reg),
9446 as_Register($src3$$reg));
9447 %}
9448
9449 ins_pipe(lmac_reg_reg);
9450 %}
9451
9452 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9453 match(Set dst (SubL src3 (MulL src1 src2)));
9454
9455 ins_cost(INSN_COST * 5);
9456 format %{ "msub $dst, $src1, $src2, $src3" %}
9457
9458 ins_encode %{
9459 __ msub(as_Register($dst$$reg),
9460 as_Register($src1$$reg),
9461 as_Register($src2$$reg),
9462 as_Register($src3$$reg));
9463 %}
9464
9465 ins_pipe(lmac_reg_reg);
9466 %}
9467
9468 // Integer Divide
9469
9470 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9471 match(Set dst (DivI src1 src2));
9472
9473 ins_cost(INSN_COST * 19);
9474 format %{ "sdivw $dst, $src1, $src2" %}
9475
9476 ins_encode(aarch64_enc_divw(dst, src1, src2));
9477 ins_pipe(idiv_reg_reg);
9478 %}
9479
9480 instruct signExtract(iRegINoSp dst, iRegIorL2I src1, immI_31 div1, immI_31 div2) %{
9481 match(Set dst (URShiftI (RShiftI src1 div1) div2));
9482 ins_cost(INSN_COST);
9483 format %{ "lsrw $dst, $src1, $div1" %}
9484 ins_encode %{
9485 __ lsrw(as_Register($dst$$reg), as_Register($src1$$reg), 31);
9486 %}
9487 ins_pipe(ialu_reg_shift);
9488 %}
9489
9490 instruct div2Round(iRegINoSp dst, iRegIorL2I src, immI_31 div1, immI_31 div2) %{
9491 match(Set dst (AddI src (URShiftI (RShiftI src div1) div2)));
9492 ins_cost(INSN_COST);
9493 format %{ "addw $dst, $src, LSR $div1" %}
9494
9495 ins_encode %{
9496 __ addw(as_Register($dst$$reg),
9497 as_Register($src$$reg),
9498 as_Register($src$$reg),
9499 Assembler::LSR, 31);
9500 %}
9501 ins_pipe(ialu_reg);
9502 %}
9503
9504 // Long Divide
9505
9506 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9507 match(Set dst (DivL src1 src2));
9508
9509 ins_cost(INSN_COST * 35);
9510 format %{ "sdiv $dst, $src1, $src2" %}
9511
9512 ins_encode(aarch64_enc_div(dst, src1, src2));
9513 ins_pipe(ldiv_reg_reg);
9514 %}
9515
9516 instruct signExtractL(iRegLNoSp dst, iRegL src1, immL_63 div1, immL_63 div2) %{
9517 match(Set dst (URShiftL (RShiftL src1 div1) div2));
9518 ins_cost(INSN_COST);
9519 format %{ "lsr $dst, $src1, $div1" %}
9520 ins_encode %{
9521 __ lsr(as_Register($dst$$reg), as_Register($src1$$reg), 63);
9522 %}
9523 ins_pipe(ialu_reg_shift);
9524 %}
9525
9526 instruct div2RoundL(iRegLNoSp dst, iRegL src, immL_63 div1, immL_63 div2) %{
9527 match(Set dst (AddL src (URShiftL (RShiftL src div1) div2)));
9528 ins_cost(INSN_COST);
9529 format %{ "add $dst, $src, $div1" %}
9530
9531 ins_encode %{
9532 __ add(as_Register($dst$$reg),
9533 as_Register($src$$reg),
9534 as_Register($src$$reg),
9535 Assembler::LSR, 63);
9536 %}
9537 ins_pipe(ialu_reg);
9538 %}
9539
9540 // Integer Remainder
9541
9542 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9543 match(Set dst (ModI src1 src2));
9544
9545 ins_cost(INSN_COST * 22);
9546 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9547 "msubw($dst, rscratch1, $src2, $src1" %}
9548
9549 ins_encode(aarch64_enc_modw(dst, src1, src2));
9550 ins_pipe(idiv_reg_reg);
9551 %}
9552
9553 // Long Remainder
9554
9555 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9556 match(Set dst (ModL src1 src2));
9557
9558 ins_cost(INSN_COST * 38);
9559 format %{ "sdiv rscratch1, $src1, $src2\n"
9560 "msub($dst, rscratch1, $src2, $src1" %}
9561
9562 ins_encode(aarch64_enc_mod(dst, src1, src2));
9563 ins_pipe(ldiv_reg_reg);
9564 %}
9565
9566 // Integer Shifts
9567
9568 // Shift Left Register
9569 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9570 match(Set dst (LShiftI src1 src2));
9571
9572 ins_cost(INSN_COST * 2);
9573 format %{ "lslvw $dst, $src1, $src2" %}
9574
9575 ins_encode %{
9576 __ lslvw(as_Register($dst$$reg),
9577 as_Register($src1$$reg),
9578 as_Register($src2$$reg));
9579 %}
9580
9581 ins_pipe(ialu_reg_reg_vshift);
9582 %}
9583
9584 // Shift Left Immediate
9585 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9586 match(Set dst (LShiftI src1 src2));
9587
9588 ins_cost(INSN_COST);
9589 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9590
9591 ins_encode %{
9592 __ lslw(as_Register($dst$$reg),
9593 as_Register($src1$$reg),
9594 $src2$$constant & 0x1f);
9595 %}
9596
9597 ins_pipe(ialu_reg_shift);
9598 %}
9599
9600 // Shift Right Logical Register
9601 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9602 match(Set dst (URShiftI src1 src2));
9603
9604 ins_cost(INSN_COST * 2);
9605 format %{ "lsrvw $dst, $src1, $src2" %}
9606
9607 ins_encode %{
9608 __ lsrvw(as_Register($dst$$reg),
9609 as_Register($src1$$reg),
9610 as_Register($src2$$reg));
9611 %}
9612
9613 ins_pipe(ialu_reg_reg_vshift);
9614 %}
9615
9616 // Shift Right Logical Immediate
9617 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9618 match(Set dst (URShiftI src1 src2));
9619
9620 ins_cost(INSN_COST);
9621 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9622
9623 ins_encode %{
9624 __ lsrw(as_Register($dst$$reg),
9625 as_Register($src1$$reg),
9626 $src2$$constant & 0x1f);
9627 %}
9628
9629 ins_pipe(ialu_reg_shift);
9630 %}
9631
9632 // Shift Right Arithmetic Register
9633 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9634 match(Set dst (RShiftI src1 src2));
9635
9636 ins_cost(INSN_COST * 2);
9637 format %{ "asrvw $dst, $src1, $src2" %}
9638
9639 ins_encode %{
9640 __ asrvw(as_Register($dst$$reg),
9641 as_Register($src1$$reg),
9642 as_Register($src2$$reg));
9643 %}
9644
9645 ins_pipe(ialu_reg_reg_vshift);
9646 %}
9647
9648 // Shift Right Arithmetic Immediate
9649 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9650 match(Set dst (RShiftI src1 src2));
9651
9652 ins_cost(INSN_COST);
9653 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9654
9655 ins_encode %{
9656 __ asrw(as_Register($dst$$reg),
9657 as_Register($src1$$reg),
9658 $src2$$constant & 0x1f);
9659 %}
9660
9661 ins_pipe(ialu_reg_shift);
9662 %}
9663
9664 // Combined Int Mask and Right Shift (using UBFM)
9665 // TODO
9666
9667 // Long Shifts
9668
9669 // Shift Left Register
9670 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9671 match(Set dst (LShiftL src1 src2));
9672
9673 ins_cost(INSN_COST * 2);
9674 format %{ "lslv $dst, $src1, $src2" %}
9675
9676 ins_encode %{
9677 __ lslv(as_Register($dst$$reg),
9678 as_Register($src1$$reg),
9679 as_Register($src2$$reg));
9680 %}
9681
9682 ins_pipe(ialu_reg_reg_vshift);
9683 %}
9684
9685 // Shift Left Immediate
9686 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9687 match(Set dst (LShiftL src1 src2));
9688
9689 ins_cost(INSN_COST);
9690 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9691
9692 ins_encode %{
9693 __ lsl(as_Register($dst$$reg),
9694 as_Register($src1$$reg),
9695 $src2$$constant & 0x3f);
9696 %}
9697
9698 ins_pipe(ialu_reg_shift);
9699 %}
9700
9701 // Shift Right Logical Register
9702 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9703 match(Set dst (URShiftL src1 src2));
9704
9705 ins_cost(INSN_COST * 2);
9706 format %{ "lsrv $dst, $src1, $src2" %}
9707
9708 ins_encode %{
9709 __ lsrv(as_Register($dst$$reg),
9710 as_Register($src1$$reg),
9711 as_Register($src2$$reg));
9712 %}
9713
9714 ins_pipe(ialu_reg_reg_vshift);
9715 %}
9716
9717 // Shift Right Logical Immediate
9718 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9719 match(Set dst (URShiftL src1 src2));
9720
9721 ins_cost(INSN_COST);
9722 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9723
9724 ins_encode %{
9725 __ lsr(as_Register($dst$$reg),
9726 as_Register($src1$$reg),
9727 $src2$$constant & 0x3f);
9728 %}
9729
9730 ins_pipe(ialu_reg_shift);
9731 %}
9732
9733 // A special-case pattern for card table stores.
9734 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9735 match(Set dst (URShiftL (CastP2X src1) src2));
9736
9737 ins_cost(INSN_COST);
9738 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9739
9740 ins_encode %{
9741 __ lsr(as_Register($dst$$reg),
9742 as_Register($src1$$reg),
9743 $src2$$constant & 0x3f);
9744 %}
9745
9746 ins_pipe(ialu_reg_shift);
9747 %}
9748
9749 // Shift Right Arithmetic Register
9750 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9751 match(Set dst (RShiftL src1 src2));
9752
9753 ins_cost(INSN_COST * 2);
9754 format %{ "asrv $dst, $src1, $src2" %}
9755
9756 ins_encode %{
9757 __ asrv(as_Register($dst$$reg),
9758 as_Register($src1$$reg),
9759 as_Register($src2$$reg));
9760 %}
9761
9762 ins_pipe(ialu_reg_reg_vshift);
9763 %}
9764
9765 // Shift Right Arithmetic Immediate
9766 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9767 match(Set dst (RShiftL src1 src2));
9768
9769 ins_cost(INSN_COST);
9770 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9771
9772 ins_encode %{
9773 __ asr(as_Register($dst$$reg),
9774 as_Register($src1$$reg),
9775 $src2$$constant & 0x3f);
9776 %}
9777
9778 ins_pipe(ialu_reg_shift);
9779 %}
9780
9781 // BEGIN This section of the file is automatically generated. Do not edit --------------
9782
9783 instruct regL_not_reg(iRegLNoSp dst,
9784 iRegL src1, immL_M1 m1,
9785 rFlagsReg cr) %{
9786 match(Set dst (XorL src1 m1));
9787 ins_cost(INSN_COST);
9788 format %{ "eon $dst, $src1, zr" %}
9789
9790 ins_encode %{
9791 __ eon(as_Register($dst$$reg),
9792 as_Register($src1$$reg),
9793 zr,
9794 Assembler::LSL, 0);
9795 %}
9796
9797 ins_pipe(ialu_reg);
9798 %}
9799 instruct regI_not_reg(iRegINoSp dst,
9800 iRegIorL2I src1, immI_M1 m1,
9801 rFlagsReg cr) %{
9802 match(Set dst (XorI src1 m1));
9803 ins_cost(INSN_COST);
9804 format %{ "eonw $dst, $src1, zr" %}
9805
9806 ins_encode %{
9807 __ eonw(as_Register($dst$$reg),
9808 as_Register($src1$$reg),
9809 zr,
9810 Assembler::LSL, 0);
9811 %}
9812
9813 ins_pipe(ialu_reg);
9814 %}
9815
9816 instruct AndI_reg_not_reg(iRegINoSp dst,
9817 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
9818 rFlagsReg cr) %{
9819 match(Set dst (AndI src1 (XorI src2 m1)));
9820 ins_cost(INSN_COST);
9821 format %{ "bicw $dst, $src1, $src2" %}
9822
9823 ins_encode %{
9824 __ bicw(as_Register($dst$$reg),
9825 as_Register($src1$$reg),
9826 as_Register($src2$$reg),
9827 Assembler::LSL, 0);
9828 %}
9829
9830 ins_pipe(ialu_reg_reg);
9831 %}
9832
9833 instruct AndL_reg_not_reg(iRegLNoSp dst,
9834 iRegL src1, iRegL src2, immL_M1 m1,
9835 rFlagsReg cr) %{
9836 match(Set dst (AndL src1 (XorL src2 m1)));
9837 ins_cost(INSN_COST);
9838 format %{ "bic $dst, $src1, $src2" %}
9839
9840 ins_encode %{
9841 __ bic(as_Register($dst$$reg),
9842 as_Register($src1$$reg),
9843 as_Register($src2$$reg),
9844 Assembler::LSL, 0);
9845 %}
9846
9847 ins_pipe(ialu_reg_reg);
9848 %}
9849
9850 instruct OrI_reg_not_reg(iRegINoSp dst,
9851 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
9852 rFlagsReg cr) %{
9853 match(Set dst (OrI src1 (XorI src2 m1)));
9854 ins_cost(INSN_COST);
9855 format %{ "ornw $dst, $src1, $src2" %}
9856
9857 ins_encode %{
9858 __ ornw(as_Register($dst$$reg),
9859 as_Register($src1$$reg),
9860 as_Register($src2$$reg),
9861 Assembler::LSL, 0);
9862 %}
9863
9864 ins_pipe(ialu_reg_reg);
9865 %}
9866
9867 instruct OrL_reg_not_reg(iRegLNoSp dst,
9868 iRegL src1, iRegL src2, immL_M1 m1,
9869 rFlagsReg cr) %{
9870 match(Set dst (OrL src1 (XorL src2 m1)));
9871 ins_cost(INSN_COST);
9872 format %{ "orn $dst, $src1, $src2" %}
9873
9874 ins_encode %{
9875 __ orn(as_Register($dst$$reg),
9876 as_Register($src1$$reg),
9877 as_Register($src2$$reg),
9878 Assembler::LSL, 0);
9879 %}
9880
9881 ins_pipe(ialu_reg_reg);
9882 %}
9883
9884 instruct XorI_reg_not_reg(iRegINoSp dst,
9885 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
9886 rFlagsReg cr) %{
9887 match(Set dst (XorI m1 (XorI src2 src1)));
9888 ins_cost(INSN_COST);
9889 format %{ "eonw $dst, $src1, $src2" %}
9890
9891 ins_encode %{
9892 __ eonw(as_Register($dst$$reg),
9893 as_Register($src1$$reg),
9894 as_Register($src2$$reg),
9895 Assembler::LSL, 0);
9896 %}
9897
9898 ins_pipe(ialu_reg_reg);
9899 %}
9900
9901 instruct XorL_reg_not_reg(iRegLNoSp dst,
9902 iRegL src1, iRegL src2, immL_M1 m1,
9903 rFlagsReg cr) %{
9904 match(Set dst (XorL m1 (XorL src2 src1)));
9905 ins_cost(INSN_COST);
9906 format %{ "eon $dst, $src1, $src2" %}
9907
9908 ins_encode %{
9909 __ eon(as_Register($dst$$reg),
9910 as_Register($src1$$reg),
9911 as_Register($src2$$reg),
9912 Assembler::LSL, 0);
9913 %}
9914
9915 ins_pipe(ialu_reg_reg);
9916 %}
9917
9918 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
9919 iRegIorL2I src1, iRegIorL2I src2,
9920 immI src3, immI_M1 src4, rFlagsReg cr) %{
9921 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
9922 ins_cost(1.9 * INSN_COST);
9923 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
9924
9925 ins_encode %{
9926 __ bicw(as_Register($dst$$reg),
9927 as_Register($src1$$reg),
9928 as_Register($src2$$reg),
9929 Assembler::LSR,
9930 $src3$$constant & 0x1f);
9931 %}
9932
9933 ins_pipe(ialu_reg_reg_shift);
9934 %}
9935
9936 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
9937 iRegL src1, iRegL src2,
9938 immI src3, immL_M1 src4, rFlagsReg cr) %{
9939 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
9940 ins_cost(1.9 * INSN_COST);
9941 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
9942
9943 ins_encode %{
9944 __ bic(as_Register($dst$$reg),
9945 as_Register($src1$$reg),
9946 as_Register($src2$$reg),
9947 Assembler::LSR,
9948 $src3$$constant & 0x3f);
9949 %}
9950
9951 ins_pipe(ialu_reg_reg_shift);
9952 %}
9953
9954 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
9955 iRegIorL2I src1, iRegIorL2I src2,
9956 immI src3, immI_M1 src4, rFlagsReg cr) %{
9957 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
9958 ins_cost(1.9 * INSN_COST);
9959 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
9960
9961 ins_encode %{
9962 __ bicw(as_Register($dst$$reg),
9963 as_Register($src1$$reg),
9964 as_Register($src2$$reg),
9965 Assembler::ASR,
9966 $src3$$constant & 0x1f);
9967 %}
9968
9969 ins_pipe(ialu_reg_reg_shift);
9970 %}
9971
9972 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
9973 iRegL src1, iRegL src2,
9974 immI src3, immL_M1 src4, rFlagsReg cr) %{
9975 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
9976 ins_cost(1.9 * INSN_COST);
9977 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
9978
9979 ins_encode %{
9980 __ bic(as_Register($dst$$reg),
9981 as_Register($src1$$reg),
9982 as_Register($src2$$reg),
9983 Assembler::ASR,
9984 $src3$$constant & 0x3f);
9985 %}
9986
9987 ins_pipe(ialu_reg_reg_shift);
9988 %}
9989
9990 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
9991 iRegIorL2I src1, iRegIorL2I src2,
9992 immI src3, immI_M1 src4, rFlagsReg cr) %{
9993 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
9994 ins_cost(1.9 * INSN_COST);
9995 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
9996
9997 ins_encode %{
9998 __ bicw(as_Register($dst$$reg),
9999 as_Register($src1$$reg),
10000 as_Register($src2$$reg),
10001 Assembler::LSL,
10002 $src3$$constant & 0x1f);
10003 %}
10004
10005 ins_pipe(ialu_reg_reg_shift);
10006 %}
10007
10008 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10009 iRegL src1, iRegL src2,
10010 immI src3, immL_M1 src4, rFlagsReg cr) %{
10011 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10012 ins_cost(1.9 * INSN_COST);
10013 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10014
10015 ins_encode %{
10016 __ bic(as_Register($dst$$reg),
10017 as_Register($src1$$reg),
10018 as_Register($src2$$reg),
10019 Assembler::LSL,
10020 $src3$$constant & 0x3f);
10021 %}
10022
10023 ins_pipe(ialu_reg_reg_shift);
10024 %}
10025
10026 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10027 iRegIorL2I src1, iRegIorL2I src2,
10028 immI src3, immI_M1 src4, rFlagsReg cr) %{
10029 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10030 ins_cost(1.9 * INSN_COST);
10031 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10032
10033 ins_encode %{
10034 __ eonw(as_Register($dst$$reg),
10035 as_Register($src1$$reg),
10036 as_Register($src2$$reg),
10037 Assembler::LSR,
10038 $src3$$constant & 0x1f);
10039 %}
10040
10041 ins_pipe(ialu_reg_reg_shift);
10042 %}
10043
10044 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10045 iRegL src1, iRegL src2,
10046 immI src3, immL_M1 src4, rFlagsReg cr) %{
10047 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10048 ins_cost(1.9 * INSN_COST);
10049 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10050
10051 ins_encode %{
10052 __ eon(as_Register($dst$$reg),
10053 as_Register($src1$$reg),
10054 as_Register($src2$$reg),
10055 Assembler::LSR,
10056 $src3$$constant & 0x3f);
10057 %}
10058
10059 ins_pipe(ialu_reg_reg_shift);
10060 %}
10061
10062 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10063 iRegIorL2I src1, iRegIorL2I src2,
10064 immI src3, immI_M1 src4, rFlagsReg cr) %{
10065 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10066 ins_cost(1.9 * INSN_COST);
10067 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10068
10069 ins_encode %{
10070 __ eonw(as_Register($dst$$reg),
10071 as_Register($src1$$reg),
10072 as_Register($src2$$reg),
10073 Assembler::ASR,
10074 $src3$$constant & 0x1f);
10075 %}
10076
10077 ins_pipe(ialu_reg_reg_shift);
10078 %}
10079
10080 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10081 iRegL src1, iRegL src2,
10082 immI src3, immL_M1 src4, rFlagsReg cr) %{
10083 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10084 ins_cost(1.9 * INSN_COST);
10085 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10086
10087 ins_encode %{
10088 __ eon(as_Register($dst$$reg),
10089 as_Register($src1$$reg),
10090 as_Register($src2$$reg),
10091 Assembler::ASR,
10092 $src3$$constant & 0x3f);
10093 %}
10094
10095 ins_pipe(ialu_reg_reg_shift);
10096 %}
10097
10098 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10099 iRegIorL2I src1, iRegIorL2I src2,
10100 immI src3, immI_M1 src4, rFlagsReg cr) %{
10101 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10102 ins_cost(1.9 * INSN_COST);
10103 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10104
10105 ins_encode %{
10106 __ eonw(as_Register($dst$$reg),
10107 as_Register($src1$$reg),
10108 as_Register($src2$$reg),
10109 Assembler::LSL,
10110 $src3$$constant & 0x1f);
10111 %}
10112
10113 ins_pipe(ialu_reg_reg_shift);
10114 %}
10115
10116 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10117 iRegL src1, iRegL src2,
10118 immI src3, immL_M1 src4, rFlagsReg cr) %{
10119 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10120 ins_cost(1.9 * INSN_COST);
10121 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10122
10123 ins_encode %{
10124 __ eon(as_Register($dst$$reg),
10125 as_Register($src1$$reg),
10126 as_Register($src2$$reg),
10127 Assembler::LSL,
10128 $src3$$constant & 0x3f);
10129 %}
10130
10131 ins_pipe(ialu_reg_reg_shift);
10132 %}
10133
10134 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10135 iRegIorL2I src1, iRegIorL2I src2,
10136 immI src3, immI_M1 src4, rFlagsReg cr) %{
10137 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10138 ins_cost(1.9 * INSN_COST);
10139 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10140
10141 ins_encode %{
10142 __ ornw(as_Register($dst$$reg),
10143 as_Register($src1$$reg),
10144 as_Register($src2$$reg),
10145 Assembler::LSR,
10146 $src3$$constant & 0x1f);
10147 %}
10148
10149 ins_pipe(ialu_reg_reg_shift);
10150 %}
10151
10152 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10153 iRegL src1, iRegL src2,
10154 immI src3, immL_M1 src4, rFlagsReg cr) %{
10155 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10156 ins_cost(1.9 * INSN_COST);
10157 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10158
10159 ins_encode %{
10160 __ orn(as_Register($dst$$reg),
10161 as_Register($src1$$reg),
10162 as_Register($src2$$reg),
10163 Assembler::LSR,
10164 $src3$$constant & 0x3f);
10165 %}
10166
10167 ins_pipe(ialu_reg_reg_shift);
10168 %}
10169
10170 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10171 iRegIorL2I src1, iRegIorL2I src2,
10172 immI src3, immI_M1 src4, rFlagsReg cr) %{
10173 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10174 ins_cost(1.9 * INSN_COST);
10175 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10176
10177 ins_encode %{
10178 __ ornw(as_Register($dst$$reg),
10179 as_Register($src1$$reg),
10180 as_Register($src2$$reg),
10181 Assembler::ASR,
10182 $src3$$constant & 0x1f);
10183 %}
10184
10185 ins_pipe(ialu_reg_reg_shift);
10186 %}
10187
10188 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10189 iRegL src1, iRegL src2,
10190 immI src3, immL_M1 src4, rFlagsReg cr) %{
10191 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10192 ins_cost(1.9 * INSN_COST);
10193 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10194
10195 ins_encode %{
10196 __ orn(as_Register($dst$$reg),
10197 as_Register($src1$$reg),
10198 as_Register($src2$$reg),
10199 Assembler::ASR,
10200 $src3$$constant & 0x3f);
10201 %}
10202
10203 ins_pipe(ialu_reg_reg_shift);
10204 %}
10205
10206 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10207 iRegIorL2I src1, iRegIorL2I src2,
10208 immI src3, immI_M1 src4, rFlagsReg cr) %{
10209 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10210 ins_cost(1.9 * INSN_COST);
10211 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10212
10213 ins_encode %{
10214 __ ornw(as_Register($dst$$reg),
10215 as_Register($src1$$reg),
10216 as_Register($src2$$reg),
10217 Assembler::LSL,
10218 $src3$$constant & 0x1f);
10219 %}
10220
10221 ins_pipe(ialu_reg_reg_shift);
10222 %}
10223
10224 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10225 iRegL src1, iRegL src2,
10226 immI src3, immL_M1 src4, rFlagsReg cr) %{
10227 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10228 ins_cost(1.9 * INSN_COST);
10229 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10230
10231 ins_encode %{
10232 __ orn(as_Register($dst$$reg),
10233 as_Register($src1$$reg),
10234 as_Register($src2$$reg),
10235 Assembler::LSL,
10236 $src3$$constant & 0x3f);
10237 %}
10238
10239 ins_pipe(ialu_reg_reg_shift);
10240 %}
10241
10242 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10243 iRegIorL2I src1, iRegIorL2I src2,
10244 immI src3, rFlagsReg cr) %{
10245 match(Set dst (AndI src1 (URShiftI src2 src3)));
10246
10247 ins_cost(1.9 * INSN_COST);
10248 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10249
10250 ins_encode %{
10251 __ andw(as_Register($dst$$reg),
10252 as_Register($src1$$reg),
10253 as_Register($src2$$reg),
10254 Assembler::LSR,
10255 $src3$$constant & 0x1f);
10256 %}
10257
10258 ins_pipe(ialu_reg_reg_shift);
10259 %}
10260
10261 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10262 iRegL src1, iRegL src2,
10263 immI src3, rFlagsReg cr) %{
10264 match(Set dst (AndL src1 (URShiftL src2 src3)));
10265
10266 ins_cost(1.9 * INSN_COST);
10267 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10268
10269 ins_encode %{
10270 __ andr(as_Register($dst$$reg),
10271 as_Register($src1$$reg),
10272 as_Register($src2$$reg),
10273 Assembler::LSR,
10274 $src3$$constant & 0x3f);
10275 %}
10276
10277 ins_pipe(ialu_reg_reg_shift);
10278 %}
10279
10280 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10281 iRegIorL2I src1, iRegIorL2I src2,
10282 immI src3, rFlagsReg cr) %{
10283 match(Set dst (AndI src1 (RShiftI src2 src3)));
10284
10285 ins_cost(1.9 * INSN_COST);
10286 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10287
10288 ins_encode %{
10289 __ andw(as_Register($dst$$reg),
10290 as_Register($src1$$reg),
10291 as_Register($src2$$reg),
10292 Assembler::ASR,
10293 $src3$$constant & 0x1f);
10294 %}
10295
10296 ins_pipe(ialu_reg_reg_shift);
10297 %}
10298
10299 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10300 iRegL src1, iRegL src2,
10301 immI src3, rFlagsReg cr) %{
10302 match(Set dst (AndL src1 (RShiftL src2 src3)));
10303
10304 ins_cost(1.9 * INSN_COST);
10305 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10306
10307 ins_encode %{
10308 __ andr(as_Register($dst$$reg),
10309 as_Register($src1$$reg),
10310 as_Register($src2$$reg),
10311 Assembler::ASR,
10312 $src3$$constant & 0x3f);
10313 %}
10314
10315 ins_pipe(ialu_reg_reg_shift);
10316 %}
10317
10318 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10319 iRegIorL2I src1, iRegIorL2I src2,
10320 immI src3, rFlagsReg cr) %{
10321 match(Set dst (AndI src1 (LShiftI src2 src3)));
10322
10323 ins_cost(1.9 * INSN_COST);
10324 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10325
10326 ins_encode %{
10327 __ andw(as_Register($dst$$reg),
10328 as_Register($src1$$reg),
10329 as_Register($src2$$reg),
10330 Assembler::LSL,
10331 $src3$$constant & 0x1f);
10332 %}
10333
10334 ins_pipe(ialu_reg_reg_shift);
10335 %}
10336
10337 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10338 iRegL src1, iRegL src2,
10339 immI src3, rFlagsReg cr) %{
10340 match(Set dst (AndL src1 (LShiftL src2 src3)));
10341
10342 ins_cost(1.9 * INSN_COST);
10343 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10344
10345 ins_encode %{
10346 __ andr(as_Register($dst$$reg),
10347 as_Register($src1$$reg),
10348 as_Register($src2$$reg),
10349 Assembler::LSL,
10350 $src3$$constant & 0x3f);
10351 %}
10352
10353 ins_pipe(ialu_reg_reg_shift);
10354 %}
10355
10356 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10357 iRegIorL2I src1, iRegIorL2I src2,
10358 immI src3, rFlagsReg cr) %{
10359 match(Set dst (XorI src1 (URShiftI src2 src3)));
10360
10361 ins_cost(1.9 * INSN_COST);
10362 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10363
10364 ins_encode %{
10365 __ eorw(as_Register($dst$$reg),
10366 as_Register($src1$$reg),
10367 as_Register($src2$$reg),
10368 Assembler::LSR,
10369 $src3$$constant & 0x1f);
10370 %}
10371
10372 ins_pipe(ialu_reg_reg_shift);
10373 %}
10374
10375 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10376 iRegL src1, iRegL src2,
10377 immI src3, rFlagsReg cr) %{
10378 match(Set dst (XorL src1 (URShiftL src2 src3)));
10379
10380 ins_cost(1.9 * INSN_COST);
10381 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10382
10383 ins_encode %{
10384 __ eor(as_Register($dst$$reg),
10385 as_Register($src1$$reg),
10386 as_Register($src2$$reg),
10387 Assembler::LSR,
10388 $src3$$constant & 0x3f);
10389 %}
10390
10391 ins_pipe(ialu_reg_reg_shift);
10392 %}
10393
10394 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10395 iRegIorL2I src1, iRegIorL2I src2,
10396 immI src3, rFlagsReg cr) %{
10397 match(Set dst (XorI src1 (RShiftI src2 src3)));
10398
10399 ins_cost(1.9 * INSN_COST);
10400 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10401
10402 ins_encode %{
10403 __ eorw(as_Register($dst$$reg),
10404 as_Register($src1$$reg),
10405 as_Register($src2$$reg),
10406 Assembler::ASR,
10407 $src3$$constant & 0x1f);
10408 %}
10409
10410 ins_pipe(ialu_reg_reg_shift);
10411 %}
10412
10413 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10414 iRegL src1, iRegL src2,
10415 immI src3, rFlagsReg cr) %{
10416 match(Set dst (XorL src1 (RShiftL src2 src3)));
10417
10418 ins_cost(1.9 * INSN_COST);
10419 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10420
10421 ins_encode %{
10422 __ eor(as_Register($dst$$reg),
10423 as_Register($src1$$reg),
10424 as_Register($src2$$reg),
10425 Assembler::ASR,
10426 $src3$$constant & 0x3f);
10427 %}
10428
10429 ins_pipe(ialu_reg_reg_shift);
10430 %}
10431
10432 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10433 iRegIorL2I src1, iRegIorL2I src2,
10434 immI src3, rFlagsReg cr) %{
10435 match(Set dst (XorI src1 (LShiftI src2 src3)));
10436
10437 ins_cost(1.9 * INSN_COST);
10438 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10439
10440 ins_encode %{
10441 __ eorw(as_Register($dst$$reg),
10442 as_Register($src1$$reg),
10443 as_Register($src2$$reg),
10444 Assembler::LSL,
10445 $src3$$constant & 0x1f);
10446 %}
10447
10448 ins_pipe(ialu_reg_reg_shift);
10449 %}
10450
10451 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10452 iRegL src1, iRegL src2,
10453 immI src3, rFlagsReg cr) %{
10454 match(Set dst (XorL src1 (LShiftL src2 src3)));
10455
10456 ins_cost(1.9 * INSN_COST);
10457 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10458
10459 ins_encode %{
10460 __ eor(as_Register($dst$$reg),
10461 as_Register($src1$$reg),
10462 as_Register($src2$$reg),
10463 Assembler::LSL,
10464 $src3$$constant & 0x3f);
10465 %}
10466
10467 ins_pipe(ialu_reg_reg_shift);
10468 %}
10469
10470 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10471 iRegIorL2I src1, iRegIorL2I src2,
10472 immI src3, rFlagsReg cr) %{
10473 match(Set dst (OrI src1 (URShiftI src2 src3)));
10474
10475 ins_cost(1.9 * INSN_COST);
10476 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10477
10478 ins_encode %{
10479 __ orrw(as_Register($dst$$reg),
10480 as_Register($src1$$reg),
10481 as_Register($src2$$reg),
10482 Assembler::LSR,
10483 $src3$$constant & 0x1f);
10484 %}
10485
10486 ins_pipe(ialu_reg_reg_shift);
10487 %}
10488
10489 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10490 iRegL src1, iRegL src2,
10491 immI src3, rFlagsReg cr) %{
10492 match(Set dst (OrL src1 (URShiftL src2 src3)));
10493
10494 ins_cost(1.9 * INSN_COST);
10495 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10496
10497 ins_encode %{
10498 __ orr(as_Register($dst$$reg),
10499 as_Register($src1$$reg),
10500 as_Register($src2$$reg),
10501 Assembler::LSR,
10502 $src3$$constant & 0x3f);
10503 %}
10504
10505 ins_pipe(ialu_reg_reg_shift);
10506 %}
10507
10508 instruct OrI_reg_RShift_reg(iRegINoSp dst,
10509 iRegIorL2I src1, iRegIorL2I src2,
10510 immI src3, rFlagsReg cr) %{
10511 match(Set dst (OrI src1 (RShiftI src2 src3)));
10512
10513 ins_cost(1.9 * INSN_COST);
10514 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
10515
10516 ins_encode %{
10517 __ orrw(as_Register($dst$$reg),
10518 as_Register($src1$$reg),
10519 as_Register($src2$$reg),
10520 Assembler::ASR,
10521 $src3$$constant & 0x1f);
10522 %}
10523
10524 ins_pipe(ialu_reg_reg_shift);
10525 %}
10526
10527 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
10528 iRegL src1, iRegL src2,
10529 immI src3, rFlagsReg cr) %{
10530 match(Set dst (OrL src1 (RShiftL src2 src3)));
10531
10532 ins_cost(1.9 * INSN_COST);
10533 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
10534
10535 ins_encode %{
10536 __ orr(as_Register($dst$$reg),
10537 as_Register($src1$$reg),
10538 as_Register($src2$$reg),
10539 Assembler::ASR,
10540 $src3$$constant & 0x3f);
10541 %}
10542
10543 ins_pipe(ialu_reg_reg_shift);
10544 %}
10545
10546 instruct OrI_reg_LShift_reg(iRegINoSp dst,
10547 iRegIorL2I src1, iRegIorL2I src2,
10548 immI src3, rFlagsReg cr) %{
10549 match(Set dst (OrI src1 (LShiftI src2 src3)));
10550
10551 ins_cost(1.9 * INSN_COST);
10552 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
10553
10554 ins_encode %{
10555 __ orrw(as_Register($dst$$reg),
10556 as_Register($src1$$reg),
10557 as_Register($src2$$reg),
10558 Assembler::LSL,
10559 $src3$$constant & 0x1f);
10560 %}
10561
10562 ins_pipe(ialu_reg_reg_shift);
10563 %}
10564
10565 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
10566 iRegL src1, iRegL src2,
10567 immI src3, rFlagsReg cr) %{
10568 match(Set dst (OrL src1 (LShiftL src2 src3)));
10569
10570 ins_cost(1.9 * INSN_COST);
10571 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
10572
10573 ins_encode %{
10574 __ orr(as_Register($dst$$reg),
10575 as_Register($src1$$reg),
10576 as_Register($src2$$reg),
10577 Assembler::LSL,
10578 $src3$$constant & 0x3f);
10579 %}
10580
10581 ins_pipe(ialu_reg_reg_shift);
10582 %}
10583
10584 instruct AddI_reg_URShift_reg(iRegINoSp dst,
10585 iRegIorL2I src1, iRegIorL2I src2,
10586 immI src3, rFlagsReg cr) %{
10587 match(Set dst (AddI src1 (URShiftI src2 src3)));
10588
10589 ins_cost(1.9 * INSN_COST);
10590 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
10591
10592 ins_encode %{
10593 __ addw(as_Register($dst$$reg),
10594 as_Register($src1$$reg),
10595 as_Register($src2$$reg),
10596 Assembler::LSR,
10597 $src3$$constant & 0x1f);
10598 %}
10599
10600 ins_pipe(ialu_reg_reg_shift);
10601 %}
10602
10603 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
10604 iRegL src1, iRegL src2,
10605 immI src3, rFlagsReg cr) %{
10606 match(Set dst (AddL src1 (URShiftL src2 src3)));
10607
10608 ins_cost(1.9 * INSN_COST);
10609 format %{ "add $dst, $src1, $src2, LSR $src3" %}
10610
10611 ins_encode %{
10612 __ add(as_Register($dst$$reg),
10613 as_Register($src1$$reg),
10614 as_Register($src2$$reg),
10615 Assembler::LSR,
10616 $src3$$constant & 0x3f);
10617 %}
10618
10619 ins_pipe(ialu_reg_reg_shift);
10620 %}
10621
10622 instruct AddI_reg_RShift_reg(iRegINoSp dst,
10623 iRegIorL2I src1, iRegIorL2I src2,
10624 immI src3, rFlagsReg cr) %{
10625 match(Set dst (AddI src1 (RShiftI src2 src3)));
10626
10627 ins_cost(1.9 * INSN_COST);
10628 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
10629
10630 ins_encode %{
10631 __ addw(as_Register($dst$$reg),
10632 as_Register($src1$$reg),
10633 as_Register($src2$$reg),
10634 Assembler::ASR,
10635 $src3$$constant & 0x1f);
10636 %}
10637
10638 ins_pipe(ialu_reg_reg_shift);
10639 %}
10640
10641 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
10642 iRegL src1, iRegL src2,
10643 immI src3, rFlagsReg cr) %{
10644 match(Set dst (AddL src1 (RShiftL src2 src3)));
10645
10646 ins_cost(1.9 * INSN_COST);
10647 format %{ "add $dst, $src1, $src2, ASR $src3" %}
10648
10649 ins_encode %{
10650 __ add(as_Register($dst$$reg),
10651 as_Register($src1$$reg),
10652 as_Register($src2$$reg),
10653 Assembler::ASR,
10654 $src3$$constant & 0x3f);
10655 %}
10656
10657 ins_pipe(ialu_reg_reg_shift);
10658 %}
10659
10660 instruct AddI_reg_LShift_reg(iRegINoSp dst,
10661 iRegIorL2I src1, iRegIorL2I src2,
10662 immI src3, rFlagsReg cr) %{
10663 match(Set dst (AddI src1 (LShiftI src2 src3)));
10664
10665 ins_cost(1.9 * INSN_COST);
10666 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
10667
10668 ins_encode %{
10669 __ addw(as_Register($dst$$reg),
10670 as_Register($src1$$reg),
10671 as_Register($src2$$reg),
10672 Assembler::LSL,
10673 $src3$$constant & 0x1f);
10674 %}
10675
10676 ins_pipe(ialu_reg_reg_shift);
10677 %}
10678
10679 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
10680 iRegL src1, iRegL src2,
10681 immI src3, rFlagsReg cr) %{
10682 match(Set dst (AddL src1 (LShiftL src2 src3)));
10683
10684 ins_cost(1.9 * INSN_COST);
10685 format %{ "add $dst, $src1, $src2, LSL $src3" %}
10686
10687 ins_encode %{
10688 __ add(as_Register($dst$$reg),
10689 as_Register($src1$$reg),
10690 as_Register($src2$$reg),
10691 Assembler::LSL,
10692 $src3$$constant & 0x3f);
10693 %}
10694
10695 ins_pipe(ialu_reg_reg_shift);
10696 %}
10697
10698 instruct SubI_reg_URShift_reg(iRegINoSp dst,
10699 iRegIorL2I src1, iRegIorL2I src2,
10700 immI src3, rFlagsReg cr) %{
10701 match(Set dst (SubI src1 (URShiftI src2 src3)));
10702
10703 ins_cost(1.9 * INSN_COST);
10704 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
10705
10706 ins_encode %{
10707 __ subw(as_Register($dst$$reg),
10708 as_Register($src1$$reg),
10709 as_Register($src2$$reg),
10710 Assembler::LSR,
10711 $src3$$constant & 0x1f);
10712 %}
10713
10714 ins_pipe(ialu_reg_reg_shift);
10715 %}
10716
10717 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
10718 iRegL src1, iRegL src2,
10719 immI src3, rFlagsReg cr) %{
10720 match(Set dst (SubL src1 (URShiftL src2 src3)));
10721
10722 ins_cost(1.9 * INSN_COST);
10723 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
10724
10725 ins_encode %{
10726 __ sub(as_Register($dst$$reg),
10727 as_Register($src1$$reg),
10728 as_Register($src2$$reg),
10729 Assembler::LSR,
10730 $src3$$constant & 0x3f);
10731 %}
10732
10733 ins_pipe(ialu_reg_reg_shift);
10734 %}
10735
10736 instruct SubI_reg_RShift_reg(iRegINoSp dst,
10737 iRegIorL2I src1, iRegIorL2I src2,
10738 immI src3, rFlagsReg cr) %{
10739 match(Set dst (SubI src1 (RShiftI src2 src3)));
10740
10741 ins_cost(1.9 * INSN_COST);
10742 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
10743
10744 ins_encode %{
10745 __ subw(as_Register($dst$$reg),
10746 as_Register($src1$$reg),
10747 as_Register($src2$$reg),
10748 Assembler::ASR,
10749 $src3$$constant & 0x1f);
10750 %}
10751
10752 ins_pipe(ialu_reg_reg_shift);
10753 %}
10754
10755 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
10756 iRegL src1, iRegL src2,
10757 immI src3, rFlagsReg cr) %{
10758 match(Set dst (SubL src1 (RShiftL src2 src3)));
10759
10760 ins_cost(1.9 * INSN_COST);
10761 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
10762
10763 ins_encode %{
10764 __ sub(as_Register($dst$$reg),
10765 as_Register($src1$$reg),
10766 as_Register($src2$$reg),
10767 Assembler::ASR,
10768 $src3$$constant & 0x3f);
10769 %}
10770
10771 ins_pipe(ialu_reg_reg_shift);
10772 %}
10773
10774 instruct SubI_reg_LShift_reg(iRegINoSp dst,
10775 iRegIorL2I src1, iRegIorL2I src2,
10776 immI src3, rFlagsReg cr) %{
10777 match(Set dst (SubI src1 (LShiftI src2 src3)));
10778
10779 ins_cost(1.9 * INSN_COST);
10780 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
10781
10782 ins_encode %{
10783 __ subw(as_Register($dst$$reg),
10784 as_Register($src1$$reg),
10785 as_Register($src2$$reg),
10786 Assembler::LSL,
10787 $src3$$constant & 0x1f);
10788 %}
10789
10790 ins_pipe(ialu_reg_reg_shift);
10791 %}
10792
10793 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
10794 iRegL src1, iRegL src2,
10795 immI src3, rFlagsReg cr) %{
10796 match(Set dst (SubL src1 (LShiftL src2 src3)));
10797
10798 ins_cost(1.9 * INSN_COST);
10799 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
10800
10801 ins_encode %{
10802 __ sub(as_Register($dst$$reg),
10803 as_Register($src1$$reg),
10804 as_Register($src2$$reg),
10805 Assembler::LSL,
10806 $src3$$constant & 0x3f);
10807 %}
10808
10809 ins_pipe(ialu_reg_reg_shift);
10810 %}
10811
10812
10813
10814 // Shift Left followed by Shift Right.
10815 // This idiom is used by the compiler for the i2b bytecode etc.
10816 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
10817 %{
10818 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
10819 // Make sure we are not going to exceed what sbfm can do.
10820 predicate((unsigned int)n->in(2)->get_int() <= 63
10821 && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
10822
10823 ins_cost(INSN_COST * 2);
10824 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
10825 ins_encode %{
10826 int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10827 int s = 63 - lshift;
10828 int r = (rshift - lshift) & 63;
10829 __ sbfm(as_Register($dst$$reg),
10830 as_Register($src$$reg),
10831 r, s);
10832 %}
10833
10834 ins_pipe(ialu_reg_shift);
10835 %}
10836
10837 // Shift Left followed by Shift Right.
10838 // This idiom is used by the compiler for the i2b bytecode etc.
10839 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
10840 %{
10841 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
10842 // Make sure we are not going to exceed what sbfmw can do.
10843 predicate((unsigned int)n->in(2)->get_int() <= 31
10844 && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
10845
10846 ins_cost(INSN_COST * 2);
10847 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
10848 ins_encode %{
10849 int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10850 int s = 31 - lshift;
10851 int r = (rshift - lshift) & 31;
10852 __ sbfmw(as_Register($dst$$reg),
10853 as_Register($src$$reg),
10854 r, s);
10855 %}
10856
10857 ins_pipe(ialu_reg_shift);
10858 %}
10859
10860 // Shift Left followed by Shift Right.
10861 // This idiom is used by the compiler for the i2b bytecode etc.
10862 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
10863 %{
10864 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
10865 // Make sure we are not going to exceed what ubfm can do.
10866 predicate((unsigned int)n->in(2)->get_int() <= 63
10867 && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
10868
10869 ins_cost(INSN_COST * 2);
10870 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
10871 ins_encode %{
10872 int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10873 int s = 63 - lshift;
10874 int r = (rshift - lshift) & 63;
10875 __ ubfm(as_Register($dst$$reg),
10876 as_Register($src$$reg),
10877 r, s);
10878 %}
10879
10880 ins_pipe(ialu_reg_shift);
10881 %}
10882
10883 // Shift Left followed by Shift Right.
10884 // This idiom is used by the compiler for the i2b bytecode etc.
10885 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
10886 %{
10887 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
10888 // Make sure we are not going to exceed what ubfmw can do.
10889 predicate((unsigned int)n->in(2)->get_int() <= 31
10890 && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
10891
10892 ins_cost(INSN_COST * 2);
10893 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
10894 ins_encode %{
10895 int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10896 int s = 31 - lshift;
10897 int r = (rshift - lshift) & 31;
10898 __ ubfmw(as_Register($dst$$reg),
10899 as_Register($src$$reg),
10900 r, s);
10901 %}
10902
10903 ins_pipe(ialu_reg_shift);
10904 %}
10905 // Bitfield extract with shift & mask
10906
10907 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
10908 %{
10909 match(Set dst (AndI (URShiftI src rshift) mask));
10910 // Make sure we are not going to exceed what ubfxw can do.
10911 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
10912
10913 ins_cost(INSN_COST);
10914 format %{ "ubfxw $dst, $src, $mask" %}
10915 ins_encode %{
10916 int rshift = $rshift$$constant & 31;
10917 long mask = $mask$$constant;
10918 int width = exact_log2(mask+1);
10919 __ ubfxw(as_Register($dst$$reg),
10920 as_Register($src$$reg), rshift, width);
10921 %}
10922 ins_pipe(ialu_reg_shift);
10923 %}
10924 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
10925 %{
10926 match(Set dst (AndL (URShiftL src rshift) mask));
10927 // Make sure we are not going to exceed what ubfx can do.
10928 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
10929
10930 ins_cost(INSN_COST);
10931 format %{ "ubfx $dst, $src, $mask" %}
10932 ins_encode %{
10933 int rshift = $rshift$$constant & 63;
10934 long mask = $mask$$constant;
10935 int width = exact_log2_long(mask+1);
10936 __ ubfx(as_Register($dst$$reg),
10937 as_Register($src$$reg), rshift, width);
10938 %}
10939 ins_pipe(ialu_reg_shift);
10940 %}
10941
10942 // We can use ubfx when extending an And with a mask when we know mask
10943 // is positive. We know that because immI_bitmask guarantees it.
10944 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
10945 %{
10946 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
10947 // Make sure we are not going to exceed what ubfxw can do.
10948 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
10949
10950 ins_cost(INSN_COST * 2);
10951 format %{ "ubfx $dst, $src, $mask" %}
10952 ins_encode %{
10953 int rshift = $rshift$$constant & 31;
10954 long mask = $mask$$constant;
10955 int width = exact_log2(mask+1);
10956 __ ubfx(as_Register($dst$$reg),
10957 as_Register($src$$reg), rshift, width);
10958 %}
10959 ins_pipe(ialu_reg_shift);
10960 %}
10961
10962 // Rotations
10963
10964 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
10965 %{
10966 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
10967 predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
10968
10969 ins_cost(INSN_COST);
10970 format %{ "extr $dst, $src1, $src2, #$rshift" %}
10971
10972 ins_encode %{
10973 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
10974 $rshift$$constant & 63);
10975 %}
10976 ins_pipe(ialu_reg_reg_extr);
10977 %}
10978
10979 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
10980 %{
10981 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
10982 predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
10983
10984 ins_cost(INSN_COST);
10985 format %{ "extr $dst, $src1, $src2, #$rshift" %}
10986
10987 ins_encode %{
10988 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
10989 $rshift$$constant & 31);
10990 %}
10991 ins_pipe(ialu_reg_reg_extr);
10992 %}
10993
10994 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
10995 %{
10996 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
10997 predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
10998
10999 ins_cost(INSN_COST);
11000 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11001
11002 ins_encode %{
11003 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11004 $rshift$$constant & 63);
11005 %}
11006 ins_pipe(ialu_reg_reg_extr);
11007 %}
11008
11009 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11010 %{
11011 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11012 predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
11013
11014 ins_cost(INSN_COST);
11015 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11016
11017 ins_encode %{
11018 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11019 $rshift$$constant & 31);
11020 %}
11021 ins_pipe(ialu_reg_reg_extr);
11022 %}
11023
11024
11025 // rol expander
11026
11027 instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
11028 %{
11029 effect(DEF dst, USE src, USE shift);
11030
11031 format %{ "rol $dst, $src, $shift" %}
11032 ins_cost(INSN_COST * 3);
11033 ins_encode %{
11034 __ subw(rscratch1, zr, as_Register($shift$$reg));
11035 __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
11036 rscratch1);
11037 %}
11038 ins_pipe(ialu_reg_reg_vshift);
11039 %}
11040
11041 // rol expander
11042
11043 instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
11044 %{
11045 effect(DEF dst, USE src, USE shift);
11046
11047 format %{ "rol $dst, $src, $shift" %}
11048 ins_cost(INSN_COST * 3);
11049 ins_encode %{
11050 __ subw(rscratch1, zr, as_Register($shift$$reg));
11051 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
11052 rscratch1);
11053 %}
11054 ins_pipe(ialu_reg_reg_vshift);
11055 %}
11056
11057 instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
11058 %{
11059 match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
11060
11061 expand %{
11062 rolL_rReg(dst, src, shift, cr);
11063 %}
11064 %}
11065
11066 instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
11067 %{
11068 match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
11069
11070 expand %{
11071 rolL_rReg(dst, src, shift, cr);
11072 %}
11073 %}
11074
11075 instruct rolI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
11076 %{
11077 match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
11078
11079 expand %{
11080 rolI_rReg(dst, src, shift, cr);
11081 %}
11082 %}
11083
11084 instruct rolI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
11085 %{
11086 match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
11087
11088 expand %{
11089 rolI_rReg(dst, src, shift, cr);
11090 %}
11091 %}
11092
11093 // ror expander
11094
11095 instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
11096 %{
11097 effect(DEF dst, USE src, USE shift);
11098
11099 format %{ "ror $dst, $src, $shift" %}
11100 ins_cost(INSN_COST);
11101 ins_encode %{
11102 __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
11103 as_Register($shift$$reg));
11104 %}
11105 ins_pipe(ialu_reg_reg_vshift);
11106 %}
11107
11108 // ror expander
11109
11110 instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
11111 %{
11112 effect(DEF dst, USE src, USE shift);
11113
11114 format %{ "ror $dst, $src, $shift" %}
11115 ins_cost(INSN_COST);
11116 ins_encode %{
11117 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
11118 as_Register($shift$$reg));
11119 %}
11120 ins_pipe(ialu_reg_reg_vshift);
11121 %}
11122
11123 instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
11124 %{
11125 match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
11126
11127 expand %{
11128 rorL_rReg(dst, src, shift, cr);
11129 %}
11130 %}
11131
11132 instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
11133 %{
11134 match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
11135
11136 expand %{
11137 rorL_rReg(dst, src, shift, cr);
11138 %}
11139 %}
11140
11141 instruct rorI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
11142 %{
11143 match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
11144
11145 expand %{
11146 rorI_rReg(dst, src, shift, cr);
11147 %}
11148 %}
11149
11150 instruct rorI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
11151 %{
11152 match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
11153
11154 expand %{
11155 rorI_rReg(dst, src, shift, cr);
11156 %}
11157 %}
11158
11159 // Add/subtract (extended)
11160
11161 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11162 %{
11163 match(Set dst (AddL src1 (ConvI2L src2)));
11164 ins_cost(INSN_COST);
11165 format %{ "add $dst, $src1, sxtw $src2" %}
11166
11167 ins_encode %{
11168 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11169 as_Register($src2$$reg), ext::sxtw);
11170 %}
11171 ins_pipe(ialu_reg_reg);
11172 %};
11173
11174 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11175 %{
11176 match(Set dst (SubL src1 (ConvI2L src2)));
11177 ins_cost(INSN_COST);
11178 format %{ "sub $dst, $src1, sxtw $src2" %}
11179
11180 ins_encode %{
11181 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11182 as_Register($src2$$reg), ext::sxtw);
11183 %}
11184 ins_pipe(ialu_reg_reg);
11185 %};
11186
11187
11188 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11189 %{
11190 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11191 ins_cost(INSN_COST);
11192 format %{ "add $dst, $src1, sxth $src2" %}
11193
11194 ins_encode %{
11195 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11196 as_Register($src2$$reg), ext::sxth);
11197 %}
11198 ins_pipe(ialu_reg_reg);
11199 %}
11200
11201 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11202 %{
11203 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11204 ins_cost(INSN_COST);
11205 format %{ "add $dst, $src1, sxtb $src2" %}
11206
11207 ins_encode %{
11208 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11209 as_Register($src2$$reg), ext::sxtb);
11210 %}
11211 ins_pipe(ialu_reg_reg);
11212 %}
11213
11214 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11215 %{
11216 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11217 ins_cost(INSN_COST);
11218 format %{ "add $dst, $src1, uxtb $src2" %}
11219
11220 ins_encode %{
11221 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11222 as_Register($src2$$reg), ext::uxtb);
11223 %}
11224 ins_pipe(ialu_reg_reg);
11225 %}
11226
11227 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11228 %{
11229 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11230 ins_cost(INSN_COST);
11231 format %{ "add $dst, $src1, sxth $src2" %}
11232
11233 ins_encode %{
11234 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11235 as_Register($src2$$reg), ext::sxth);
11236 %}
11237 ins_pipe(ialu_reg_reg);
11238 %}
11239
11240 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11241 %{
11242 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11243 ins_cost(INSN_COST);
11244 format %{ "add $dst, $src1, sxtw $src2" %}
11245
11246 ins_encode %{
11247 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11248 as_Register($src2$$reg), ext::sxtw);
11249 %}
11250 ins_pipe(ialu_reg_reg);
11251 %}
11252
11253 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11254 %{
11255 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11256 ins_cost(INSN_COST);
11257 format %{ "add $dst, $src1, sxtb $src2" %}
11258
11259 ins_encode %{
11260 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11261 as_Register($src2$$reg), ext::sxtb);
11262 %}
11263 ins_pipe(ialu_reg_reg);
11264 %}
11265
11266 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11267 %{
11268 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11269 ins_cost(INSN_COST);
11270 format %{ "add $dst, $src1, uxtb $src2" %}
11271
11272 ins_encode %{
11273 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11274 as_Register($src2$$reg), ext::uxtb);
11275 %}
11276 ins_pipe(ialu_reg_reg);
11277 %}
11278
11279
11280 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11281 %{
11282 match(Set dst (AddI src1 (AndI src2 mask)));
11283 ins_cost(INSN_COST);
11284 format %{ "addw $dst, $src1, $src2, uxtb" %}
11285
11286 ins_encode %{
11287 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11288 as_Register($src2$$reg), ext::uxtb);
11289 %}
11290 ins_pipe(ialu_reg_reg);
11291 %}
11292
11293 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11294 %{
11295 match(Set dst (AddI src1 (AndI src2 mask)));
11296 ins_cost(INSN_COST);
11297 format %{ "addw $dst, $src1, $src2, uxth" %}
11298
11299 ins_encode %{
11300 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11301 as_Register($src2$$reg), ext::uxth);
11302 %}
11303 ins_pipe(ialu_reg_reg);
11304 %}
11305
11306 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
11307 %{
11308 match(Set dst (AddL src1 (AndL src2 mask)));
11309 ins_cost(INSN_COST);
11310 format %{ "add $dst, $src1, $src2, uxtb" %}
11311
11312 ins_encode %{
11313 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11314 as_Register($src2$$reg), ext::uxtb);
11315 %}
11316 ins_pipe(ialu_reg_reg);
11317 %}
11318
11319 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
11320 %{
11321 match(Set dst (AddL src1 (AndL src2 mask)));
11322 ins_cost(INSN_COST);
11323 format %{ "add $dst, $src1, $src2, uxth" %}
11324
11325 ins_encode %{
11326 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11327 as_Register($src2$$reg), ext::uxth);
11328 %}
11329 ins_pipe(ialu_reg_reg);
11330 %}
11331
11332 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
11333 %{
11334 match(Set dst (AddL src1 (AndL src2 mask)));
11335 ins_cost(INSN_COST);
11336 format %{ "add $dst, $src1, $src2, uxtw" %}
11337
11338 ins_encode %{
11339 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11340 as_Register($src2$$reg), ext::uxtw);
11341 %}
11342 ins_pipe(ialu_reg_reg);
11343 %}
11344
11345 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11346 %{
11347 match(Set dst (SubI src1 (AndI src2 mask)));
11348 ins_cost(INSN_COST);
11349 format %{ "subw $dst, $src1, $src2, uxtb" %}
11350
11351 ins_encode %{
11352 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
11353 as_Register($src2$$reg), ext::uxtb);
11354 %}
11355 ins_pipe(ialu_reg_reg);
11356 %}
11357
11358 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11359 %{
11360 match(Set dst (SubI src1 (AndI src2 mask)));
11361 ins_cost(INSN_COST);
11362 format %{ "subw $dst, $src1, $src2, uxth" %}
11363
11364 ins_encode %{
11365 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
11366 as_Register($src2$$reg), ext::uxth);
11367 %}
11368 ins_pipe(ialu_reg_reg);
11369 %}
11370
11371 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
11372 %{
11373 match(Set dst (SubL src1 (AndL src2 mask)));
11374 ins_cost(INSN_COST);
11375 format %{ "sub $dst, $src1, $src2, uxtb" %}
11376
11377 ins_encode %{
11378 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11379 as_Register($src2$$reg), ext::uxtb);
11380 %}
11381 ins_pipe(ialu_reg_reg);
11382 %}
11383
11384 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
11385 %{
11386 match(Set dst (SubL src1 (AndL src2 mask)));
11387 ins_cost(INSN_COST);
11388 format %{ "sub $dst, $src1, $src2, uxth" %}
11389
11390 ins_encode %{
11391 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11392 as_Register($src2$$reg), ext::uxth);
11393 %}
11394 ins_pipe(ialu_reg_reg);
11395 %}
11396
11397 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
11398 %{
11399 match(Set dst (SubL src1 (AndL src2 mask)));
11400 ins_cost(INSN_COST);
11401 format %{ "sub $dst, $src1, $src2, uxtw" %}
11402
11403 ins_encode %{
11404 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11405 as_Register($src2$$reg), ext::uxtw);
11406 %}
11407 ins_pipe(ialu_reg_reg);
11408 %}
11409
11410 // END This section of the file is automatically generated. Do not edit --------------
11411
11412 // ============================================================================
11413 // Floating Point Arithmetic Instructions
11414
11415 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
11416 match(Set dst (AddF src1 src2));
11417
11418 ins_cost(INSN_COST * 5);
11419 format %{ "fadds $dst, $src1, $src2" %}
11420
11421 ins_encode %{
11422 __ fadds(as_FloatRegister($dst$$reg),
11423 as_FloatRegister($src1$$reg),
11424 as_FloatRegister($src2$$reg));
11425 %}
11426
11427 ins_pipe(fp_dop_reg_reg_s);
11428 %}
11429
11430 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
11431 match(Set dst (AddD src1 src2));
11432
11433 ins_cost(INSN_COST * 5);
11434 format %{ "faddd $dst, $src1, $src2" %}
11435
11436 ins_encode %{
11437 __ faddd(as_FloatRegister($dst$$reg),
11438 as_FloatRegister($src1$$reg),
11439 as_FloatRegister($src2$$reg));
11440 %}
11441
11442 ins_pipe(fp_dop_reg_reg_d);
11443 %}
11444
11445 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
11446 match(Set dst (SubF src1 src2));
11447
11448 ins_cost(INSN_COST * 5);
11449 format %{ "fsubs $dst, $src1, $src2" %}
11450
11451 ins_encode %{
11452 __ fsubs(as_FloatRegister($dst$$reg),
11453 as_FloatRegister($src1$$reg),
11454 as_FloatRegister($src2$$reg));
11455 %}
11456
11457 ins_pipe(fp_dop_reg_reg_s);
11458 %}
11459
11460 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
11461 match(Set dst (SubD src1 src2));
11462
11463 ins_cost(INSN_COST * 5);
11464 format %{ "fsubd $dst, $src1, $src2" %}
11465
11466 ins_encode %{
11467 __ fsubd(as_FloatRegister($dst$$reg),
11468 as_FloatRegister($src1$$reg),
11469 as_FloatRegister($src2$$reg));
11470 %}
11471
11472 ins_pipe(fp_dop_reg_reg_d);
11473 %}
11474
11475 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
11476 match(Set dst (MulF src1 src2));
11477
11478 ins_cost(INSN_COST * 6);
11479 format %{ "fmuls $dst, $src1, $src2" %}
11480
11481 ins_encode %{
11482 __ fmuls(as_FloatRegister($dst$$reg),
11483 as_FloatRegister($src1$$reg),
11484 as_FloatRegister($src2$$reg));
11485 %}
11486
11487 ins_pipe(fp_dop_reg_reg_s);
11488 %}
11489
11490 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
11491 match(Set dst (MulD src1 src2));
11492
11493 ins_cost(INSN_COST * 6);
11494 format %{ "fmuld $dst, $src1, $src2" %}
11495
11496 ins_encode %{
11497 __ fmuld(as_FloatRegister($dst$$reg),
11498 as_FloatRegister($src1$$reg),
11499 as_FloatRegister($src2$$reg));
11500 %}
11501
11502 ins_pipe(fp_dop_reg_reg_d);
11503 %}
11504
11505 // We cannot use these fused mul w add/sub ops because they don't
11506 // produce the same result as the equivalent separated ops
11507 // (essentially they don't round the intermediate result). that's a
11508 // shame. leaving them here in case we can idenitfy cases where it is
11509 // legitimate to use them
11510
11511
11512 // instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
11513 // match(Set dst (AddF (MulF src1 src2) src3));
11514
11515 // format %{ "fmadds $dst, $src1, $src2, $src3" %}
11516
11517 // ins_encode %{
11518 // __ fmadds(as_FloatRegister($dst$$reg),
11519 // as_FloatRegister($src1$$reg),
11520 // as_FloatRegister($src2$$reg),
11521 // as_FloatRegister($src3$$reg));
11522 // %}
11523
11524 // ins_pipe(pipe_class_default);
11525 // %}
11526
11527 // instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
11528 // match(Set dst (AddD (MulD src1 src2) src3));
11529
11530 // format %{ "fmaddd $dst, $src1, $src2, $src3" %}
11531
11532 // ins_encode %{
11533 // __ fmaddd(as_FloatRegister($dst$$reg),
11534 // as_FloatRegister($src1$$reg),
11535 // as_FloatRegister($src2$$reg),
11536 // as_FloatRegister($src3$$reg));
11537 // %}
11538
11539 // ins_pipe(pipe_class_default);
11540 // %}
11541
11542 // instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
11543 // match(Set dst (AddF (MulF (NegF src1) src2) src3));
11544 // match(Set dst (AddF (NegF (MulF src1 src2)) src3));
11545
11546 // format %{ "fmsubs $dst, $src1, $src2, $src3" %}
11547
11548 // ins_encode %{
11549 // __ fmsubs(as_FloatRegister($dst$$reg),
11550 // as_FloatRegister($src1$$reg),
11551 // as_FloatRegister($src2$$reg),
11552 // as_FloatRegister($src3$$reg));
11553 // %}
11554
11555 // ins_pipe(pipe_class_default);
11556 // %}
11557
11558 // instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
11559 // match(Set dst (AddD (MulD (NegD src1) src2) src3));
11560 // match(Set dst (AddD (NegD (MulD src1 src2)) src3));
11561
11562 // format %{ "fmsubd $dst, $src1, $src2, $src3" %}
11563
11564 // ins_encode %{
11565 // __ fmsubd(as_FloatRegister($dst$$reg),
11566 // as_FloatRegister($src1$$reg),
11567 // as_FloatRegister($src2$$reg),
11568 // as_FloatRegister($src3$$reg));
11569 // %}
11570
11571 // ins_pipe(pipe_class_default);
11572 // %}
11573
11574 // instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
11575 // match(Set dst (SubF (MulF (NegF src1) src2) src3));
11576 // match(Set dst (SubF (NegF (MulF src1 src2)) src3));
11577
11578 // format %{ "fnmadds $dst, $src1, $src2, $src3" %}
11579
11580 // ins_encode %{
11581 // __ fnmadds(as_FloatRegister($dst$$reg),
11582 // as_FloatRegister($src1$$reg),
11583 // as_FloatRegister($src2$$reg),
11584 // as_FloatRegister($src3$$reg));
11585 // %}
11586
11587 // ins_pipe(pipe_class_default);
11588 // %}
11589
11590 // instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
11591 // match(Set dst (SubD (MulD (NegD src1) src2) src3));
11592 // match(Set dst (SubD (NegD (MulD src1 src2)) src3));
11593
11594 // format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
11595
11596 // ins_encode %{
11597 // __ fnmaddd(as_FloatRegister($dst$$reg),
11598 // as_FloatRegister($src1$$reg),
11599 // as_FloatRegister($src2$$reg),
11600 // as_FloatRegister($src3$$reg));
11601 // %}
11602
11603 // ins_pipe(pipe_class_default);
11604 // %}
11605
11606 // instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
11607 // match(Set dst (SubF (MulF src1 src2) src3));
11608
11609 // format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
11610
11611 // ins_encode %{
11612 // __ fnmsubs(as_FloatRegister($dst$$reg),
11613 // as_FloatRegister($src1$$reg),
11614 // as_FloatRegister($src2$$reg),
11615 // as_FloatRegister($src3$$reg));
11616 // %}
11617
11618 // ins_pipe(pipe_class_default);
11619 // %}
11620
11621 // instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
11622 // match(Set dst (SubD (MulD src1 src2) src3));
11623
11624 // format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
11625
11626 // ins_encode %{
11627 // // n.b. insn name should be fnmsubd
11628 // __ fnmsub(as_FloatRegister($dst$$reg),
11629 // as_FloatRegister($src1$$reg),
11630 // as_FloatRegister($src2$$reg),
11631 // as_FloatRegister($src3$$reg));
11632 // %}
11633
11634 // ins_pipe(pipe_class_default);
11635 // %}
11636
11637
11638 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
11639 match(Set dst (DivF src1 src2));
11640
11641 ins_cost(INSN_COST * 18);
11642 format %{ "fdivs $dst, $src1, $src2" %}
11643
11644 ins_encode %{
11645 __ fdivs(as_FloatRegister($dst$$reg),
11646 as_FloatRegister($src1$$reg),
11647 as_FloatRegister($src2$$reg));
11648 %}
11649
11650 ins_pipe(fp_div_s);
11651 %}
11652
11653 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
11654 match(Set dst (DivD src1 src2));
11655
11656 ins_cost(INSN_COST * 32);
11657 format %{ "fdivd $dst, $src1, $src2" %}
11658
11659 ins_encode %{
11660 __ fdivd(as_FloatRegister($dst$$reg),
11661 as_FloatRegister($src1$$reg),
11662 as_FloatRegister($src2$$reg));
11663 %}
11664
11665 ins_pipe(fp_div_d);
11666 %}
11667
11668 instruct negF_reg_reg(vRegF dst, vRegF src) %{
11669 match(Set dst (NegF src));
11670
11671 ins_cost(INSN_COST * 3);
11672 format %{ "fneg $dst, $src" %}
11673
11674 ins_encode %{
11675 __ fnegs(as_FloatRegister($dst$$reg),
11676 as_FloatRegister($src$$reg));
11677 %}
11678
11679 ins_pipe(fp_uop_s);
11680 %}
11681
11682 instruct negD_reg_reg(vRegD dst, vRegD src) %{
11683 match(Set dst (NegD src));
11684
11685 ins_cost(INSN_COST * 3);
11686 format %{ "fnegd $dst, $src" %}
11687
11688 ins_encode %{
11689 __ fnegd(as_FloatRegister($dst$$reg),
11690 as_FloatRegister($src$$reg));
11691 %}
11692
11693 ins_pipe(fp_uop_d);
11694 %}
11695
11696 instruct absF_reg(vRegF dst, vRegF src) %{
11697 match(Set dst (AbsF src));
11698
11699 ins_cost(INSN_COST * 3);
11700 format %{ "fabss $dst, $src" %}
11701 ins_encode %{
11702 __ fabss(as_FloatRegister($dst$$reg),
11703 as_FloatRegister($src$$reg));
11704 %}
11705
11706 ins_pipe(fp_uop_s);
11707 %}
11708
11709 instruct absD_reg(vRegD dst, vRegD src) %{
11710 match(Set dst (AbsD src));
11711
11712 ins_cost(INSN_COST * 3);
11713 format %{ "fabsd $dst, $src" %}
11714 ins_encode %{
11715 __ fabsd(as_FloatRegister($dst$$reg),
11716 as_FloatRegister($src$$reg));
11717 %}
11718
11719 ins_pipe(fp_uop_d);
11720 %}
11721
11722 instruct sqrtD_reg(vRegD dst, vRegD src) %{
11723 match(Set dst (SqrtD src));
11724
11725 ins_cost(INSN_COST * 50);
11726 format %{ "fsqrtd $dst, $src" %}
11727 ins_encode %{
11728 __ fsqrtd(as_FloatRegister($dst$$reg),
11729 as_FloatRegister($src$$reg));
11730 %}
11731
11732 ins_pipe(fp_div_s);
11733 %}
11734
11735 instruct sqrtF_reg(vRegF dst, vRegF src) %{
11736 match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
11737
11738 ins_cost(INSN_COST * 50);
11739 format %{ "fsqrts $dst, $src" %}
11740 ins_encode %{
11741 __ fsqrts(as_FloatRegister($dst$$reg),
11742 as_FloatRegister($src$$reg));
11743 %}
11744
11745 ins_pipe(fp_div_d);
11746 %}
11747
11748 // ============================================================================
11749 // Logical Instructions
11750
11751 // Integer Logical Instructions
11752
11753 // And Instructions
11754
11755
11756 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
11757 match(Set dst (AndI src1 src2));
11758
11759 format %{ "andw $dst, $src1, $src2\t# int" %}
11760
11761 ins_cost(INSN_COST);
11762 ins_encode %{
11763 __ andw(as_Register($dst$$reg),
11764 as_Register($src1$$reg),
11765 as_Register($src2$$reg));
11766 %}
11767
11768 ins_pipe(ialu_reg_reg);
11769 %}
11770
11771 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
11772 match(Set dst (AndI src1 src2));
11773
11774 format %{ "andsw $dst, $src1, $src2\t# int" %}
11775
11776 ins_cost(INSN_COST);
11777 ins_encode %{
11778 __ andw(as_Register($dst$$reg),
11779 as_Register($src1$$reg),
11780 (unsigned long)($src2$$constant));
11781 %}
11782
11783 ins_pipe(ialu_reg_imm);
11784 %}
11785
11786 // Or Instructions
11787
11788 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11789 match(Set dst (OrI src1 src2));
11790
11791 format %{ "orrw $dst, $src1, $src2\t# int" %}
11792
11793 ins_cost(INSN_COST);
11794 ins_encode %{
11795 __ orrw(as_Register($dst$$reg),
11796 as_Register($src1$$reg),
11797 as_Register($src2$$reg));
11798 %}
11799
11800 ins_pipe(ialu_reg_reg);
11801 %}
11802
11803 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
11804 match(Set dst (OrI src1 src2));
11805
11806 format %{ "orrw $dst, $src1, $src2\t# int" %}
11807
11808 ins_cost(INSN_COST);
11809 ins_encode %{
11810 __ orrw(as_Register($dst$$reg),
11811 as_Register($src1$$reg),
11812 (unsigned long)($src2$$constant));
11813 %}
11814
11815 ins_pipe(ialu_reg_imm);
11816 %}
11817
11818 // Xor Instructions
11819
11820 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11821 match(Set dst (XorI src1 src2));
11822
11823 format %{ "eorw $dst, $src1, $src2\t# int" %}
11824
11825 ins_cost(INSN_COST);
11826 ins_encode %{
11827 __ eorw(as_Register($dst$$reg),
11828 as_Register($src1$$reg),
11829 as_Register($src2$$reg));
11830 %}
11831
11832 ins_pipe(ialu_reg_reg);
11833 %}
11834
11835 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
11836 match(Set dst (XorI src1 src2));
11837
11838 format %{ "eorw $dst, $src1, $src2\t# int" %}
11839
11840 ins_cost(INSN_COST);
11841 ins_encode %{
11842 __ eorw(as_Register($dst$$reg),
11843 as_Register($src1$$reg),
11844 (unsigned long)($src2$$constant));
11845 %}
11846
11847 ins_pipe(ialu_reg_imm);
11848 %}
11849
11850 // Long Logical Instructions
11851 // TODO
11852
11853 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
11854 match(Set dst (AndL src1 src2));
11855
11856 format %{ "and $dst, $src1, $src2\t# int" %}
11857
11858 ins_cost(INSN_COST);
11859 ins_encode %{
11860 __ andr(as_Register($dst$$reg),
11861 as_Register($src1$$reg),
11862 as_Register($src2$$reg));
11863 %}
11864
11865 ins_pipe(ialu_reg_reg);
11866 %}
11867
11868 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
11869 match(Set dst (AndL src1 src2));
11870
11871 format %{ "and $dst, $src1, $src2\t# int" %}
11872
11873 ins_cost(INSN_COST);
11874 ins_encode %{
11875 __ andr(as_Register($dst$$reg),
11876 as_Register($src1$$reg),
11877 (unsigned long)($src2$$constant));
11878 %}
11879
11880 ins_pipe(ialu_reg_imm);
11881 %}
11882
11883 // Or Instructions
11884
11885 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11886 match(Set dst (OrL src1 src2));
11887
11888 format %{ "orr $dst, $src1, $src2\t# int" %}
11889
11890 ins_cost(INSN_COST);
11891 ins_encode %{
11892 __ orr(as_Register($dst$$reg),
11893 as_Register($src1$$reg),
11894 as_Register($src2$$reg));
11895 %}
11896
11897 ins_pipe(ialu_reg_reg);
11898 %}
11899
11900 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
11901 match(Set dst (OrL src1 src2));
11902
11903 format %{ "orr $dst, $src1, $src2\t# int" %}
11904
11905 ins_cost(INSN_COST);
11906 ins_encode %{
11907 __ orr(as_Register($dst$$reg),
11908 as_Register($src1$$reg),
11909 (unsigned long)($src2$$constant));
11910 %}
11911
11912 ins_pipe(ialu_reg_imm);
11913 %}
11914
11915 // Xor Instructions
11916
11917 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11918 match(Set dst (XorL src1 src2));
11919
11920 format %{ "eor $dst, $src1, $src2\t# int" %}
11921
11922 ins_cost(INSN_COST);
11923 ins_encode %{
11924 __ eor(as_Register($dst$$reg),
11925 as_Register($src1$$reg),
11926 as_Register($src2$$reg));
11927 %}
11928
11929 ins_pipe(ialu_reg_reg);
11930 %}
11931
11932 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
11933 match(Set dst (XorL src1 src2));
11934
11935 ins_cost(INSN_COST);
11936 format %{ "eor $dst, $src1, $src2\t# int" %}
11937
11938 ins_encode %{
11939 __ eor(as_Register($dst$$reg),
11940 as_Register($src1$$reg),
11941 (unsigned long)($src2$$constant));
11942 %}
11943
11944 ins_pipe(ialu_reg_imm);
11945 %}
11946
11947 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
11948 %{
11949 match(Set dst (ConvI2L src));
11950
11951 ins_cost(INSN_COST);
11952 format %{ "sxtw $dst, $src\t# i2l" %}
11953 ins_encode %{
11954 __ sbfm($dst$$Register, $src$$Register, 0, 31);
11955 %}
11956 ins_pipe(ialu_reg_shift);
11957 %}
11958
11959 // this pattern occurs in bigmath arithmetic
11960 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
11961 %{
11962 match(Set dst (AndL (ConvI2L src) mask));
11963
11964 ins_cost(INSN_COST);
11965 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
11966 ins_encode %{
11967 __ ubfm($dst$$Register, $src$$Register, 0, 31);
11968 %}
11969
11970 ins_pipe(ialu_reg_shift);
11971 %}
11972
11973 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
11974 match(Set dst (ConvL2I src));
11975
11976 ins_cost(INSN_COST);
11977 format %{ "movw $dst, $src \t// l2i" %}
11978
11979 ins_encode %{
11980 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
11981 %}
11982
11983 ins_pipe(ialu_reg);
11984 %}
11985
11986 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
11987 %{
11988 match(Set dst (Conv2B src));
11989 effect(KILL cr);
11990
11991 format %{
11992 "cmpw $src, zr\n\t"
11993 "cset $dst, ne"
11994 %}
11995
11996 ins_encode %{
11997 __ cmpw(as_Register($src$$reg), zr);
11998 __ cset(as_Register($dst$$reg), Assembler::NE);
11999 %}
12000
12001 ins_pipe(ialu_reg);
12002 %}
12003
12004 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
12005 %{
12006 match(Set dst (Conv2B src));
12007 effect(KILL cr);
12008
12009 format %{
12010 "cmp $src, zr\n\t"
12011 "cset $dst, ne"
12012 %}
12013
12014 ins_encode %{
12015 __ cmp(as_Register($src$$reg), zr);
12016 __ cset(as_Register($dst$$reg), Assembler::NE);
12017 %}
12018
12019 ins_pipe(ialu_reg);
12020 %}
12021
12022 instruct convD2F_reg(vRegF dst, vRegD src) %{
12023 match(Set dst (ConvD2F src));
12024
12025 ins_cost(INSN_COST * 5);
12026 format %{ "fcvtd $dst, $src \t// d2f" %}
12027
12028 ins_encode %{
12029 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
12030 %}
12031
12032 ins_pipe(fp_d2f);
12033 %}
12034
12035 instruct convF2D_reg(vRegD dst, vRegF src) %{
12036 match(Set dst (ConvF2D src));
12037
12038 ins_cost(INSN_COST * 5);
12039 format %{ "fcvts $dst, $src \t// f2d" %}
12040
12041 ins_encode %{
12042 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
12043 %}
12044
12045 ins_pipe(fp_f2d);
12046 %}
12047
12048 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
12049 match(Set dst (ConvF2I src));
12050
12051 ins_cost(INSN_COST * 5);
12052 format %{ "fcvtzsw $dst, $src \t// f2i" %}
12053
12054 ins_encode %{
12055 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
12056 %}
12057
12058 ins_pipe(fp_f2i);
12059 %}
12060
12061 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
12062 match(Set dst (ConvF2L src));
12063
12064 ins_cost(INSN_COST * 5);
12065 format %{ "fcvtzs $dst, $src \t// f2l" %}
12066
12067 ins_encode %{
12068 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
12069 %}
12070
12071 ins_pipe(fp_f2l);
12072 %}
12073
12074 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
12075 match(Set dst (ConvI2F src));
12076
12077 ins_cost(INSN_COST * 5);
12078 format %{ "scvtfws $dst, $src \t// i2f" %}
12079
12080 ins_encode %{
12081 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
12082 %}
12083
12084 ins_pipe(fp_i2f);
12085 %}
12086
12087 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
12088 match(Set dst (ConvL2F src));
12089
12090 ins_cost(INSN_COST * 5);
12091 format %{ "scvtfs $dst, $src \t// l2f" %}
12092
12093 ins_encode %{
12094 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
12095 %}
12096
12097 ins_pipe(fp_l2f);
12098 %}
12099
12100 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
12101 match(Set dst (ConvD2I src));
12102
12103 ins_cost(INSN_COST * 5);
12104 format %{ "fcvtzdw $dst, $src \t// d2i" %}
12105
12106 ins_encode %{
12107 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
12108 %}
12109
12110 ins_pipe(fp_d2i);
12111 %}
12112
12113 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
12114 match(Set dst (ConvD2L src));
12115
12116 ins_cost(INSN_COST * 5);
12117 format %{ "fcvtzd $dst, $src \t// d2l" %}
12118
12119 ins_encode %{
12120 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
12121 %}
12122
12123 ins_pipe(fp_d2l);
12124 %}
12125
12126 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
12127 match(Set dst (ConvI2D src));
12128
12129 ins_cost(INSN_COST * 5);
12130 format %{ "scvtfwd $dst, $src \t// i2d" %}
12131
12132 ins_encode %{
12133 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
12134 %}
12135
12136 ins_pipe(fp_i2d);
12137 %}
12138
12139 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
12140 match(Set dst (ConvL2D src));
12141
12142 ins_cost(INSN_COST * 5);
12143 format %{ "scvtfd $dst, $src \t// l2d" %}
12144
12145 ins_encode %{
12146 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
12147 %}
12148
12149 ins_pipe(fp_l2d);
12150 %}
12151
12152 // stack <-> reg and reg <-> reg shuffles with no conversion
12153
12154 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
12155
12156 match(Set dst (MoveF2I src));
12157
12158 effect(DEF dst, USE src);
12159
12160 ins_cost(4 * INSN_COST);
12161
12162 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
12163
12164 ins_encode %{
12165 __ ldrw($dst$$Register, Address(sp, $src$$disp));
12166 %}
12167
12168 ins_pipe(iload_reg_reg);
12169
12170 %}
12171
12172 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
12173
12174 match(Set dst (MoveI2F src));
12175
12176 effect(DEF dst, USE src);
12177
12178 ins_cost(4 * INSN_COST);
12179
12180 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
12181
12182 ins_encode %{
12183 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
12184 %}
12185
12186 ins_pipe(pipe_class_memory);
12187
12188 %}
12189
12190 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
12191
12192 match(Set dst (MoveD2L src));
12193
12194 effect(DEF dst, USE src);
12195
12196 ins_cost(4 * INSN_COST);
12197
12198 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
12199
12200 ins_encode %{
12201 __ ldr($dst$$Register, Address(sp, $src$$disp));
12202 %}
12203
12204 ins_pipe(iload_reg_reg);
12205
12206 %}
12207
12208 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
12209
12210 match(Set dst (MoveL2D src));
12211
12212 effect(DEF dst, USE src);
12213
12214 ins_cost(4 * INSN_COST);
12215
12216 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
12217
12218 ins_encode %{
12219 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
12220 %}
12221
12222 ins_pipe(pipe_class_memory);
12223
12224 %}
12225
12226 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
12227
12228 match(Set dst (MoveF2I src));
12229
12230 effect(DEF dst, USE src);
12231
12232 ins_cost(INSN_COST);
12233
12234 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
12235
12236 ins_encode %{
12237 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
12238 %}
12239
12240 ins_pipe(pipe_class_memory);
12241
12242 %}
12243
12244 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
12245
12246 match(Set dst (MoveI2F src));
12247
12248 effect(DEF dst, USE src);
12249
12250 ins_cost(INSN_COST);
12251
12252 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
12253
12254 ins_encode %{
12255 __ strw($src$$Register, Address(sp, $dst$$disp));
12256 %}
12257
12258 ins_pipe(istore_reg_reg);
12259
12260 %}
12261
12262 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
12263
12264 match(Set dst (MoveD2L src));
12265
12266 effect(DEF dst, USE src);
12267
12268 ins_cost(INSN_COST);
12269
12270 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
12271
12272 ins_encode %{
12273 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
12274 %}
12275
12276 ins_pipe(pipe_class_memory);
12277
12278 %}
12279
12280 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
12281
12282 match(Set dst (MoveL2D src));
12283
12284 effect(DEF dst, USE src);
12285
12286 ins_cost(INSN_COST);
12287
12288 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
12289
12290 ins_encode %{
12291 __ str($src$$Register, Address(sp, $dst$$disp));
12292 %}
12293
12294 ins_pipe(istore_reg_reg);
12295
12296 %}
12297
12298 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
12299
12300 match(Set dst (MoveF2I src));
12301
12302 effect(DEF dst, USE src);
12303
12304 ins_cost(INSN_COST);
12305
12306 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
12307
12308 ins_encode %{
12309 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
12310 %}
12311
12312 ins_pipe(fp_f2i);
12313
12314 %}
12315
12316 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
12317
12318 match(Set dst (MoveI2F src));
12319
12320 effect(DEF dst, USE src);
12321
12322 ins_cost(INSN_COST);
12323
12324 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
12325
12326 ins_encode %{
12327 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
12328 %}
12329
12330 ins_pipe(fp_i2f);
12331
12332 %}
12333
12334 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
12335
12336 match(Set dst (MoveD2L src));
12337
12338 effect(DEF dst, USE src);
12339
12340 ins_cost(INSN_COST);
12341
12342 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
12343
12344 ins_encode %{
12345 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
12346 %}
12347
12348 ins_pipe(fp_d2l);
12349
12350 %}
12351
12352 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
12353
12354 match(Set dst (MoveL2D src));
12355
12356 effect(DEF dst, USE src);
12357
12358 ins_cost(INSN_COST);
12359
12360 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
12361
12362 ins_encode %{
12363 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
12364 %}
12365
12366 ins_pipe(fp_l2d);
12367
12368 %}
12369
12370 // ============================================================================
12371 // clearing of an array
12372
12373 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
12374 %{
12375 match(Set dummy (ClearArray cnt base));
12376 effect(USE_KILL cnt, USE_KILL base);
12377
12378 ins_cost(4 * INSN_COST);
12379 format %{ "ClearArray $cnt, $base" %}
12380
12381 ins_encode %{
12382 __ zero_words($base$$Register, $cnt$$Register);
12383 %}
12384
12385 ins_pipe(pipe_class_memory);
12386 %}
12387
12388 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 tmp, Universe dummy, rFlagsReg cr)
12389 %{
12390 match(Set dummy (ClearArray cnt base));
12391 effect(USE_KILL base, TEMP tmp);
12392
12393 ins_cost(4 * INSN_COST);
12394 format %{ "ClearArray $cnt, $base" %}
12395
12396 ins_encode %{
12397 __ zero_words($base$$Register, (u_int64_t)$cnt$$constant);
12398 %}
12399
12400 ins_pipe(pipe_class_memory);
12401 %}
12402
12403 // ============================================================================
12404 // Overflow Math Instructions
12405
12406 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
12407 %{
12408 match(Set cr (OverflowAddI op1 op2));
12409
12410 format %{ "cmnw $op1, $op2\t# overflow check int" %}
12411 ins_cost(INSN_COST);
12412 ins_encode %{
12413 __ cmnw($op1$$Register, $op2$$Register);
12414 %}
12415
12416 ins_pipe(icmp_reg_reg);
12417 %}
12418
12419 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
12420 %{
12421 match(Set cr (OverflowAddI op1 op2));
12422
12423 format %{ "cmnw $op1, $op2\t# overflow check int" %}
12424 ins_cost(INSN_COST);
12425 ins_encode %{
12426 __ cmnw($op1$$Register, $op2$$constant);
12427 %}
12428
12429 ins_pipe(icmp_reg_imm);
12430 %}
12431
12432 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
12433 %{
12434 match(Set cr (OverflowAddL op1 op2));
12435
12436 format %{ "cmn $op1, $op2\t# overflow check long" %}
12437 ins_cost(INSN_COST);
12438 ins_encode %{
12439 __ cmn($op1$$Register, $op2$$Register);
12440 %}
12441
12442 ins_pipe(icmp_reg_reg);
12443 %}
12444
12445 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
12446 %{
12447 match(Set cr (OverflowAddL op1 op2));
12448
12449 format %{ "cmn $op1, $op2\t# overflow check long" %}
12450 ins_cost(INSN_COST);
12451 ins_encode %{
12452 __ cmn($op1$$Register, $op2$$constant);
12453 %}
12454
12455 ins_pipe(icmp_reg_imm);
12456 %}
12457
12458 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
12459 %{
12460 match(Set cr (OverflowSubI op1 op2));
12461
12462 format %{ "cmpw $op1, $op2\t# overflow check int" %}
12463 ins_cost(INSN_COST);
12464 ins_encode %{
12465 __ cmpw($op1$$Register, $op2$$Register);
12466 %}
12467
12468 ins_pipe(icmp_reg_reg);
12469 %}
12470
12471 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
12472 %{
12473 match(Set cr (OverflowSubI op1 op2));
12474
12475 format %{ "cmpw $op1, $op2\t# overflow check int" %}
12476 ins_cost(INSN_COST);
12477 ins_encode %{
12478 __ cmpw($op1$$Register, $op2$$constant);
12479 %}
12480
12481 ins_pipe(icmp_reg_imm);
12482 %}
12483
12484 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
12485 %{
12486 match(Set cr (OverflowSubL op1 op2));
12487
12488 format %{ "cmp $op1, $op2\t# overflow check long" %}
12489 ins_cost(INSN_COST);
12490 ins_encode %{
12491 __ cmp($op1$$Register, $op2$$Register);
12492 %}
12493
12494 ins_pipe(icmp_reg_reg);
12495 %}
12496
12497 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
12498 %{
12499 match(Set cr (OverflowSubL op1 op2));
12500
12501 format %{ "cmp $op1, $op2\t# overflow check long" %}
12502 ins_cost(INSN_COST);
12503 ins_encode %{
12504 __ cmp($op1$$Register, $op2$$constant);
12505 %}
12506
12507 ins_pipe(icmp_reg_imm);
12508 %}
12509
12510 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
12511 %{
12512 match(Set cr (OverflowSubI zero op1));
12513
12514 format %{ "cmpw zr, $op1\t# overflow check int" %}
12515 ins_cost(INSN_COST);
12516 ins_encode %{
12517 __ cmpw(zr, $op1$$Register);
12518 %}
12519
12520 ins_pipe(icmp_reg_imm);
12521 %}
12522
12523 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
12524 %{
12525 match(Set cr (OverflowSubL zero op1));
12526
12527 format %{ "cmp zr, $op1\t# overflow check long" %}
12528 ins_cost(INSN_COST);
12529 ins_encode %{
12530 __ cmp(zr, $op1$$Register);
12531 %}
12532
12533 ins_pipe(icmp_reg_imm);
12534 %}
12535
12536 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
12537 %{
12538 match(Set cr (OverflowMulI op1 op2));
12539
12540 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
12541 "cmp rscratch1, rscratch1, sxtw\n\t"
12542 "movw rscratch1, #0x80000000\n\t"
12543 "cselw rscratch1, rscratch1, zr, NE\n\t"
12544 "cmpw rscratch1, #1" %}
12545 ins_cost(5 * INSN_COST);
12546 ins_encode %{
12547 __ smull(rscratch1, $op1$$Register, $op2$$Register);
12548 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
12549 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
12550 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
12551 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
12552 %}
12553
12554 ins_pipe(pipe_slow);
12555 %}
12556
12557 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
12558 %{
12559 match(If cmp (OverflowMulI op1 op2));
12560 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
12561 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
12562 effect(USE labl, KILL cr);
12563
12564 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
12565 "cmp rscratch1, rscratch1, sxtw\n\t"
12566 "b$cmp $labl" %}
12567 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
12568 ins_encode %{
12569 Label* L = $labl$$label;
12570 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
12571 __ smull(rscratch1, $op1$$Register, $op2$$Register);
12572 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
12573 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
12574 %}
12575
12576 ins_pipe(pipe_serial);
12577 %}
12578
12579 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
12580 %{
12581 match(Set cr (OverflowMulL op1 op2));
12582
12583 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
12584 "smulh rscratch2, $op1, $op2\n\t"
12585 "cmp rscratch2, rscratch1, ASR #31\n\t"
12586 "movw rscratch1, #0x80000000\n\t"
12587 "cselw rscratch1, rscratch1, zr, NE\n\t"
12588 "cmpw rscratch1, #1" %}
12589 ins_cost(6 * INSN_COST);
12590 ins_encode %{
12591 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
12592 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
12593 __ cmp(rscratch2, rscratch1, Assembler::ASR, 31); // Top is pure sign ext
12594 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
12595 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
12596 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
12597 %}
12598
12599 ins_pipe(pipe_slow);
12600 %}
12601
12602 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
12603 %{
12604 match(If cmp (OverflowMulL op1 op2));
12605 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
12606 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
12607 effect(USE labl, KILL cr);
12608
12609 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
12610 "smulh rscratch2, $op1, $op2\n\t"
12611 "cmp rscratch2, rscratch1, ASR #31\n\t"
12612 "b$cmp $labl" %}
12613 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
12614 ins_encode %{
12615 Label* L = $labl$$label;
12616 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
12617 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
12618 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
12619 __ cmp(rscratch2, rscratch1, Assembler::ASR, 31); // Top is pure sign ext
12620 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
12621 %}
12622
12623 ins_pipe(pipe_serial);
12624 %}
12625
12626 // ============================================================================
12627 // Compare Instructions
12628
12629 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
12630 %{
12631 match(Set cr (CmpI op1 op2));
12632
12633 effect(DEF cr, USE op1, USE op2);
12634
12635 ins_cost(INSN_COST);
12636 format %{ "cmpw $op1, $op2" %}
12637
12638 ins_encode(aarch64_enc_cmpw(op1, op2));
12639
12640 ins_pipe(icmp_reg_reg);
12641 %}
12642
12643 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
12644 %{
12645 match(Set cr (CmpI op1 zero));
12646
12647 effect(DEF cr, USE op1);
12648
12649 ins_cost(INSN_COST);
12650 format %{ "cmpw $op1, 0" %}
12651
12652 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
12653
12654 ins_pipe(icmp_reg_imm);
12655 %}
12656
12657 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
12658 %{
12659 match(Set cr (CmpI op1 op2));
12660
12661 effect(DEF cr, USE op1);
12662
12663 ins_cost(INSN_COST);
12664 format %{ "cmpw $op1, $op2" %}
12665
12666 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
12667
12668 ins_pipe(icmp_reg_imm);
12669 %}
12670
12671 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
12672 %{
12673 match(Set cr (CmpI op1 op2));
12674
12675 effect(DEF cr, USE op1);
12676
12677 ins_cost(INSN_COST * 2);
12678 format %{ "cmpw $op1, $op2" %}
12679
12680 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
12681
12682 ins_pipe(icmp_reg_imm);
12683 %}
12684
12685 // Unsigned compare Instructions; really, same as signed compare
12686 // except it should only be used to feed an If or a CMovI which takes a
12687 // cmpOpU.
12688
12689 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
12690 %{
12691 match(Set cr (CmpU op1 op2));
12692
12693 effect(DEF cr, USE op1, USE op2);
12694
12695 ins_cost(INSN_COST);
12696 format %{ "cmpw $op1, $op2\t# unsigned" %}
12697
12698 ins_encode(aarch64_enc_cmpw(op1, op2));
12699
12700 ins_pipe(icmp_reg_reg);
12701 %}
12702
12703 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
12704 %{
12705 match(Set cr (CmpU op1 zero));
12706
12707 effect(DEF cr, USE op1);
12708
12709 ins_cost(INSN_COST);
12710 format %{ "cmpw $op1, #0\t# unsigned" %}
12711
12712 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
12713
12714 ins_pipe(icmp_reg_imm);
12715 %}
12716
12717 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
12718 %{
12719 match(Set cr (CmpU op1 op2));
12720
12721 effect(DEF cr, USE op1);
12722
12723 ins_cost(INSN_COST);
12724 format %{ "cmpw $op1, $op2\t# unsigned" %}
12725
12726 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
12727
12728 ins_pipe(icmp_reg_imm);
12729 %}
12730
12731 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
12732 %{
12733 match(Set cr (CmpU op1 op2));
12734
12735 effect(DEF cr, USE op1);
12736
12737 ins_cost(INSN_COST * 2);
12738 format %{ "cmpw $op1, $op2\t# unsigned" %}
12739
12740 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
12741
12742 ins_pipe(icmp_reg_imm);
12743 %}
12744
12745 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
12746 %{
12747 match(Set cr (CmpL op1 op2));
12748
12749 effect(DEF cr, USE op1, USE op2);
12750
12751 ins_cost(INSN_COST);
12752 format %{ "cmp $op1, $op2" %}
12753
12754 ins_encode(aarch64_enc_cmp(op1, op2));
12755
12756 ins_pipe(icmp_reg_reg);
12757 %}
12758
12759 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
12760 %{
12761 match(Set cr (CmpL op1 zero));
12762
12763 effect(DEF cr, USE op1);
12764
12765 ins_cost(INSN_COST);
12766 format %{ "tst $op1" %}
12767
12768 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
12769
12770 ins_pipe(icmp_reg_imm);
12771 %}
12772
12773 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
12774 %{
12775 match(Set cr (CmpL op1 op2));
12776
12777 effect(DEF cr, USE op1);
12778
12779 ins_cost(INSN_COST);
12780 format %{ "cmp $op1, $op2" %}
12781
12782 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
12783
12784 ins_pipe(icmp_reg_imm);
12785 %}
12786
12787 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
12788 %{
12789 match(Set cr (CmpL op1 op2));
12790
12791 effect(DEF cr, USE op1);
12792
12793 ins_cost(INSN_COST * 2);
12794 format %{ "cmp $op1, $op2" %}
12795
12796 ins_encode(aarch64_enc_cmp_imm(op1, op2));
12797
12798 ins_pipe(icmp_reg_imm);
12799 %}
12800
12801 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
12802 %{
12803 match(Set cr (CmpUL op1 op2));
12804
12805 effect(DEF cr, USE op1, USE op2);
12806
12807 ins_cost(INSN_COST);
12808 format %{ "cmp $op1, $op2" %}
12809
12810 ins_encode(aarch64_enc_cmp(op1, op2));
12811
12812 ins_pipe(icmp_reg_reg);
12813 %}
12814
12815 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
12816 %{
12817 match(Set cr (CmpUL op1 zero));
12818
12819 effect(DEF cr, USE op1);
12820
12821 ins_cost(INSN_COST);
12822 format %{ "tst $op1" %}
12823
12824 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
12825
12826 ins_pipe(icmp_reg_imm);
12827 %}
12828
12829 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
12830 %{
12831 match(Set cr (CmpUL op1 op2));
12832
12833 effect(DEF cr, USE op1);
12834
12835 ins_cost(INSN_COST);
12836 format %{ "cmp $op1, $op2" %}
12837
12838 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
12839
12840 ins_pipe(icmp_reg_imm);
12841 %}
12842
12843 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
12844 %{
12845 match(Set cr (CmpUL op1 op2));
12846
12847 effect(DEF cr, USE op1);
12848
12849 ins_cost(INSN_COST * 2);
12850 format %{ "cmp $op1, $op2" %}
12851
12852 ins_encode(aarch64_enc_cmp_imm(op1, op2));
12853
12854 ins_pipe(icmp_reg_imm);
12855 %}
12856
12857 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
12858 %{
12859 match(Set cr (CmpP op1 op2));
12860
12861 effect(DEF cr, USE op1, USE op2);
12862
12863 ins_cost(INSN_COST);
12864 format %{ "cmp $op1, $op2\t // ptr" %}
12865
12866 ins_encode(aarch64_enc_cmpp(op1, op2));
12867
12868 ins_pipe(icmp_reg_reg);
12869 %}
12870
12871 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
12872 %{
12873 match(Set cr (CmpN op1 op2));
12874
12875 effect(DEF cr, USE op1, USE op2);
12876
12877 ins_cost(INSN_COST);
12878 format %{ "cmp $op1, $op2\t // compressed ptr" %}
12879
12880 ins_encode(aarch64_enc_cmpn(op1, op2));
12881
12882 ins_pipe(icmp_reg_reg);
12883 %}
12884
12885 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
12886 %{
12887 match(Set cr (CmpP op1 zero));
12888
12889 effect(DEF cr, USE op1, USE zero);
12890
12891 ins_cost(INSN_COST);
12892 format %{ "cmp $op1, 0\t // ptr" %}
12893
12894 ins_encode(aarch64_enc_testp(op1));
12895
12896 ins_pipe(icmp_reg_imm);
12897 %}
12898
12899 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
12900 %{
12901 match(Set cr (CmpN op1 zero));
12902
12903 effect(DEF cr, USE op1, USE zero);
12904
12905 ins_cost(INSN_COST);
12906 format %{ "cmp $op1, 0\t // compressed ptr" %}
12907
12908 ins_encode(aarch64_enc_testn(op1));
12909
12910 ins_pipe(icmp_reg_imm);
12911 %}
12912
12913 // FP comparisons
12914 //
12915 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
12916 // using normal cmpOp. See declaration of rFlagsReg for details.
12917
12918 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
12919 %{
12920 match(Set cr (CmpF src1 src2));
12921
12922 ins_cost(3 * INSN_COST);
12923 format %{ "fcmps $src1, $src2" %}
12924
12925 ins_encode %{
12926 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
12927 %}
12928
12929 ins_pipe(pipe_class_compare);
12930 %}
12931
12932 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
12933 %{
12934 match(Set cr (CmpF src1 src2));
12935
12936 ins_cost(3 * INSN_COST);
12937 format %{ "fcmps $src1, 0.0" %}
12938
12939 ins_encode %{
12940 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
12941 %}
12942
12943 ins_pipe(pipe_class_compare);
12944 %}
12945 // FROM HERE
12946
12947 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
12948 %{
12949 match(Set cr (CmpD src1 src2));
12950
12951 ins_cost(3 * INSN_COST);
12952 format %{ "fcmpd $src1, $src2" %}
12953
12954 ins_encode %{
12955 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
12956 %}
12957
12958 ins_pipe(pipe_class_compare);
12959 %}
12960
12961 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
12962 %{
12963 match(Set cr (CmpD src1 src2));
12964
12965 ins_cost(3 * INSN_COST);
12966 format %{ "fcmpd $src1, 0.0" %}
12967
12968 ins_encode %{
12969 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
12970 %}
12971
12972 ins_pipe(pipe_class_compare);
12973 %}
12974
12975 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
12976 %{
12977 match(Set dst (CmpF3 src1 src2));
12978 effect(KILL cr);
12979
12980 ins_cost(5 * INSN_COST);
12981 format %{ "fcmps $src1, $src2\n\t"
12982 "csinvw($dst, zr, zr, eq\n\t"
12983 "csnegw($dst, $dst, $dst, lt)"
12984 %}
12985
12986 ins_encode %{
12987 Label done;
12988 FloatRegister s1 = as_FloatRegister($src1$$reg);
12989 FloatRegister s2 = as_FloatRegister($src2$$reg);
12990 Register d = as_Register($dst$$reg);
12991 __ fcmps(s1, s2);
12992 // installs 0 if EQ else -1
12993 __ csinvw(d, zr, zr, Assembler::EQ);
12994 // keeps -1 if less or unordered else installs 1
12995 __ csnegw(d, d, d, Assembler::LT);
12996 __ bind(done);
12997 %}
12998
12999 ins_pipe(pipe_class_default);
13000
13001 %}
13002
13003 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
13004 %{
13005 match(Set dst (CmpD3 src1 src2));
13006 effect(KILL cr);
13007
13008 ins_cost(5 * INSN_COST);
13009 format %{ "fcmpd $src1, $src2\n\t"
13010 "csinvw($dst, zr, zr, eq\n\t"
13011 "csnegw($dst, $dst, $dst, lt)"
13012 %}
13013
13014 ins_encode %{
13015 Label done;
13016 FloatRegister s1 = as_FloatRegister($src1$$reg);
13017 FloatRegister s2 = as_FloatRegister($src2$$reg);
13018 Register d = as_Register($dst$$reg);
13019 __ fcmpd(s1, s2);
13020 // installs 0 if EQ else -1
13021 __ csinvw(d, zr, zr, Assembler::EQ);
13022 // keeps -1 if less or unordered else installs 1
13023 __ csnegw(d, d, d, Assembler::LT);
13024 __ bind(done);
13025 %}
13026 ins_pipe(pipe_class_default);
13027
13028 %}
13029
13030 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
13031 %{
13032 match(Set dst (CmpF3 src1 zero));
13033 effect(KILL cr);
13034
13035 ins_cost(5 * INSN_COST);
13036 format %{ "fcmps $src1, 0.0\n\t"
13037 "csinvw($dst, zr, zr, eq\n\t"
13038 "csnegw($dst, $dst, $dst, lt)"
13039 %}
13040
13041 ins_encode %{
13042 Label done;
13043 FloatRegister s1 = as_FloatRegister($src1$$reg);
13044 Register d = as_Register($dst$$reg);
13045 __ fcmps(s1, 0.0);
13046 // installs 0 if EQ else -1
13047 __ csinvw(d, zr, zr, Assembler::EQ);
13048 // keeps -1 if less or unordered else installs 1
13049 __ csnegw(d, d, d, Assembler::LT);
13050 __ bind(done);
13051 %}
13052
13053 ins_pipe(pipe_class_default);
13054
13055 %}
13056
13057 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
13058 %{
13059 match(Set dst (CmpD3 src1 zero));
13060 effect(KILL cr);
13061
13062 ins_cost(5 * INSN_COST);
13063 format %{ "fcmpd $src1, 0.0\n\t"
13064 "csinvw($dst, zr, zr, eq\n\t"
13065 "csnegw($dst, $dst, $dst, lt)"
13066 %}
13067
13068 ins_encode %{
13069 Label done;
13070 FloatRegister s1 = as_FloatRegister($src1$$reg);
13071 Register d = as_Register($dst$$reg);
13072 __ fcmpd(s1, 0.0);
13073 // installs 0 if EQ else -1
13074 __ csinvw(d, zr, zr, Assembler::EQ);
13075 // keeps -1 if less or unordered else installs 1
13076 __ csnegw(d, d, d, Assembler::LT);
13077 __ bind(done);
13078 %}
13079 ins_pipe(pipe_class_default);
13080
13081 %}
13082
13083 // Manifest a CmpL result in an integer register.
13084 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
13085 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
13086 %{
13087 match(Set dst (CmpL3 src1 src2));
13088 effect(KILL flags);
13089
13090 ins_cost(INSN_COST * 6);
13091 format %{
13092 "cmp $src1, $src2"
13093 "csetw $dst, ne"
13094 "cnegw $dst, lt"
13095 %}
13096 // format %{ "CmpL3 $dst, $src1, $src2" %}
13097 ins_encode %{
13098 __ cmp($src1$$Register, $src2$$Register);
13099 __ csetw($dst$$Register, Assembler::NE);
13100 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
13101 %}
13102
13103 ins_pipe(ialu_reg_reg);
13104 %}
13105
13106 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
13107 %{
13108 match(Set dst (CmpLTMask p q));
13109 effect(KILL cr);
13110
13111 ins_cost(3 * INSN_COST);
13112
13113 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
13114 "csetw $dst, lt\n\t"
13115 "subw $dst, zr, $dst"
13116 %}
13117
13118 ins_encode %{
13119 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
13120 __ csetw(as_Register($dst$$reg), Assembler::LT);
13121 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
13122 %}
13123
13124 ins_pipe(ialu_reg_reg);
13125 %}
13126
13127 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
13128 %{
13129 match(Set dst (CmpLTMask src zero));
13130 effect(KILL cr);
13131
13132 ins_cost(INSN_COST);
13133
13134 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
13135
13136 ins_encode %{
13137 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
13138 %}
13139
13140 ins_pipe(ialu_reg_shift);
13141 %}
13142
13143 // ============================================================================
13144 // Max and Min
13145
13146 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13147 %{
13148 match(Set dst (MinI src1 src2));
13149
13150 effect(DEF dst, USE src1, USE src2, KILL cr);
13151 size(8);
13152
13153 ins_cost(INSN_COST * 3);
13154 format %{
13155 "cmpw $src1 $src2\t signed int\n\t"
13156 "cselw $dst, $src1, $src2 lt\t"
13157 %}
13158
13159 ins_encode %{
13160 __ cmpw(as_Register($src1$$reg),
13161 as_Register($src2$$reg));
13162 __ cselw(as_Register($dst$$reg),
13163 as_Register($src1$$reg),
13164 as_Register($src2$$reg),
13165 Assembler::LT);
13166 %}
13167
13168 ins_pipe(ialu_reg_reg);
13169 %}
13170 // FROM HERE
13171
13172 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13173 %{
13174 match(Set dst (MaxI src1 src2));
13175
13176 effect(DEF dst, USE src1, USE src2, KILL cr);
13177 size(8);
13178
13179 ins_cost(INSN_COST * 3);
13180 format %{
13181 "cmpw $src1 $src2\t signed int\n\t"
13182 "cselw $dst, $src1, $src2 gt\t"
13183 %}
13184
13185 ins_encode %{
13186 __ cmpw(as_Register($src1$$reg),
13187 as_Register($src2$$reg));
13188 __ cselw(as_Register($dst$$reg),
13189 as_Register($src1$$reg),
13190 as_Register($src2$$reg),
13191 Assembler::GT);
13192 %}
13193
13194 ins_pipe(ialu_reg_reg);
13195 %}
13196
13197 // ============================================================================
13198 // Branch Instructions
13199
13200 // Direct Branch.
13201 instruct branch(label lbl)
13202 %{
13203 match(Goto);
13204
13205 effect(USE lbl);
13206
13207 ins_cost(BRANCH_COST);
13208 format %{ "b $lbl" %}
13209
13210 ins_encode(aarch64_enc_b(lbl));
13211
13212 ins_pipe(pipe_branch);
13213 %}
13214
13215 // Conditional Near Branch
13216 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
13217 %{
13218 // Same match rule as `branchConFar'.
13219 match(If cmp cr);
13220
13221 effect(USE lbl);
13222
13223 ins_cost(BRANCH_COST);
13224 // If set to 1 this indicates that the current instruction is a
13225 // short variant of a long branch. This avoids using this
13226 // instruction in first-pass matching. It will then only be used in
13227 // the `Shorten_branches' pass.
13228 // ins_short_branch(1);
13229 format %{ "b$cmp $lbl" %}
13230
13231 ins_encode(aarch64_enc_br_con(cmp, lbl));
13232
13233 ins_pipe(pipe_branch_cond);
13234 %}
13235
13236 // Conditional Near Branch Unsigned
13237 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
13238 %{
13239 // Same match rule as `branchConFar'.
13240 match(If cmp cr);
13241
13242 effect(USE lbl);
13243
13244 ins_cost(BRANCH_COST);
13245 // If set to 1 this indicates that the current instruction is a
13246 // short variant of a long branch. This avoids using this
13247 // instruction in first-pass matching. It will then only be used in
13248 // the `Shorten_branches' pass.
13249 // ins_short_branch(1);
13250 format %{ "b$cmp $lbl\t# unsigned" %}
13251
13252 ins_encode(aarch64_enc_br_conU(cmp, lbl));
13253
13254 ins_pipe(pipe_branch_cond);
13255 %}
13256
13257 // Make use of CBZ and CBNZ. These instructions, as well as being
13258 // shorter than (cmp; branch), have the additional benefit of not
13259 // killing the flags.
13260
13261 instruct cmpI_imm0_branch(cmpOp cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
13262 match(If cmp (CmpI op1 op2));
13263 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13264 || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
13265 effect(USE labl);
13266
13267 ins_cost(BRANCH_COST);
13268 format %{ "cbw$cmp $op1, $labl" %}
13269 ins_encode %{
13270 Label* L = $labl$$label;
13271 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13272 if (cond == Assembler::EQ)
13273 __ cbzw($op1$$Register, *L);
13274 else
13275 __ cbnzw($op1$$Register, *L);
13276 %}
13277 ins_pipe(pipe_cmp_branch);
13278 %}
13279
13280 instruct cmpL_imm0_branch(cmpOp cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
13281 match(If cmp (CmpL op1 op2));
13282 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13283 || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
13284 effect(USE labl);
13285
13286 ins_cost(BRANCH_COST);
13287 format %{ "cb$cmp $op1, $labl" %}
13288 ins_encode %{
13289 Label* L = $labl$$label;
13290 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13291 if (cond == Assembler::EQ)
13292 __ cbz($op1$$Register, *L);
13293 else
13294 __ cbnz($op1$$Register, *L);
13295 %}
13296 ins_pipe(pipe_cmp_branch);
13297 %}
13298
13299 instruct cmpP_imm0_branch(cmpOp cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
13300 match(If cmp (CmpP op1 op2));
13301 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13302 || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
13303 effect(USE labl);
13304
13305 ins_cost(BRANCH_COST);
13306 format %{ "cb$cmp $op1, $labl" %}
13307 ins_encode %{
13308 Label* L = $labl$$label;
13309 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13310 if (cond == Assembler::EQ)
13311 __ cbz($op1$$Register, *L);
13312 else
13313 __ cbnz($op1$$Register, *L);
13314 %}
13315 ins_pipe(pipe_cmp_branch);
13316 %}
13317
13318 instruct cmpN_imm0_branch(cmpOp cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
13319 match(If cmp (CmpN op1 op2));
13320 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13321 || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
13322 effect(USE labl);
13323
13324 ins_cost(BRANCH_COST);
13325 format %{ "cbw$cmp $op1, $labl" %}
13326 ins_encode %{
13327 Label* L = $labl$$label;
13328 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13329 if (cond == Assembler::EQ)
13330 __ cbzw($op1$$Register, *L);
13331 else
13332 __ cbnzw($op1$$Register, *L);
13333 %}
13334 ins_pipe(pipe_cmp_branch);
13335 %}
13336
13337 instruct cmpP_narrowOop_imm0_branch(cmpOp cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
13338 match(If cmp (CmpP (DecodeN oop) zero));
13339 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13340 || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
13341 effect(USE labl);
13342
13343 ins_cost(BRANCH_COST);
13344 format %{ "cb$cmp $oop, $labl" %}
13345 ins_encode %{
13346 Label* L = $labl$$label;
13347 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13348 if (cond == Assembler::EQ)
13349 __ cbzw($oop$$Register, *L);
13350 else
13351 __ cbnzw($oop$$Register, *L);
13352 %}
13353 ins_pipe(pipe_cmp_branch);
13354 %}
13355
13356 instruct cmpUI_imm0_branch(cmpOpU cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
13357 match(If cmp (CmpU op1 op2));
13358 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13359 || n->in(1)->as_Bool()->_test._test == BoolTest::eq
13360 || n->in(1)->as_Bool()->_test._test == BoolTest::gt
13361 || n->in(1)->as_Bool()->_test._test == BoolTest::le);
13362 effect(USE labl);
13363
13364 ins_cost(BRANCH_COST);
13365 format %{ "cbw$cmp $op1, $labl" %}
13366 ins_encode %{
13367 Label* L = $labl$$label;
13368 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13369 if (cond == Assembler::EQ || cond == Assembler::LS)
13370 __ cbzw($op1$$Register, *L);
13371 else
13372 __ cbnzw($op1$$Register, *L);
13373 %}
13374 ins_pipe(pipe_cmp_branch);
13375 %}
13376
13377 instruct cmpUL_imm0_branch(cmpOpU cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
13378 match(If cmp (CmpUL op1 op2));
13379 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13380 || n->in(1)->as_Bool()->_test._test == BoolTest::eq
13381 || n->in(1)->as_Bool()->_test._test == BoolTest::gt
13382 || n->in(1)->as_Bool()->_test._test == BoolTest::le);
13383 effect(USE labl);
13384
13385 ins_cost(BRANCH_COST);
13386 format %{ "cb$cmp $op1, $labl" %}
13387 ins_encode %{
13388 Label* L = $labl$$label;
13389 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13390 if (cond == Assembler::EQ || cond == Assembler::LS)
13391 __ cbz($op1$$Register, *L);
13392 else
13393 __ cbnz($op1$$Register, *L);
13394 %}
13395 ins_pipe(pipe_cmp_branch);
13396 %}
13397
13398 // Test bit and Branch
13399
13400 // Patterns for short (< 32KiB) variants
13401 instruct cmpL_branch_sign(cmpOp cmp, iRegL op1, immL0 op2, label labl) %{
13402 match(If cmp (CmpL op1 op2));
13403 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::lt
13404 || n->in(1)->as_Bool()->_test._test == BoolTest::ge);
13405 effect(USE labl);
13406
13407 ins_cost(BRANCH_COST);
13408 format %{ "cb$cmp $op1, $labl # long" %}
13409 ins_encode %{
13410 Label* L = $labl$$label;
13411 Assembler::Condition cond =
13412 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
13413 __ tbr(cond, $op1$$Register, 63, *L);
13414 %}
13415 ins_pipe(pipe_cmp_branch);
13416 ins_short_branch(1);
13417 %}
13418
13419 instruct cmpI_branch_sign(cmpOp cmp, iRegIorL2I op1, immI0 op2, label labl) %{
13420 match(If cmp (CmpI op1 op2));
13421 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::lt
13422 || n->in(1)->as_Bool()->_test._test == BoolTest::ge);
13423 effect(USE labl);
13424
13425 ins_cost(BRANCH_COST);
13426 format %{ "cb$cmp $op1, $labl # int" %}
13427 ins_encode %{
13428 Label* L = $labl$$label;
13429 Assembler::Condition cond =
13430 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
13431 __ tbr(cond, $op1$$Register, 31, *L);
13432 %}
13433 ins_pipe(pipe_cmp_branch);
13434 ins_short_branch(1);
13435 %}
13436
13437 instruct cmpL_branch_bit(cmpOp cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
13438 match(If cmp (CmpL (AndL op1 op2) op3));
13439 predicate((n->in(1)->as_Bool()->_test._test == BoolTest::ne
13440 || n->in(1)->as_Bool()->_test._test == BoolTest::eq)
13441 && is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
13442 effect(USE labl);
13443
13444 ins_cost(BRANCH_COST);
13445 format %{ "tb$cmp $op1, $op2, $labl" %}
13446 ins_encode %{
13447 Label* L = $labl$$label;
13448 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13449 int bit = exact_log2($op2$$constant);
13450 __ tbr(cond, $op1$$Register, bit, *L);
13451 %}
13452 ins_pipe(pipe_cmp_branch);
13453 ins_short_branch(1);
13454 %}
13455
13456 instruct cmpI_branch_bit(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
13457 match(If cmp (CmpI (AndI op1 op2) op3));
13458 predicate((n->in(1)->as_Bool()->_test._test == BoolTest::ne
13459 || n->in(1)->as_Bool()->_test._test == BoolTest::eq)
13460 && is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
13461 effect(USE labl);
13462
13463 ins_cost(BRANCH_COST);
13464 format %{ "tb$cmp $op1, $op2, $labl" %}
13465 ins_encode %{
13466 Label* L = $labl$$label;
13467 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13468 int bit = exact_log2($op2$$constant);
13469 __ tbr(cond, $op1$$Register, bit, *L);
13470 %}
13471 ins_pipe(pipe_cmp_branch);
13472 ins_short_branch(1);
13473 %}
13474
13475 // And far variants
13476 instruct far_cmpL_branch_sign(cmpOp cmp, iRegL op1, immL0 op2, label labl) %{
13477 match(If cmp (CmpL op1 op2));
13478 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::lt
13479 || n->in(1)->as_Bool()->_test._test == BoolTest::ge);
13480 effect(USE labl);
13481
13482 ins_cost(BRANCH_COST);
13483 format %{ "cb$cmp $op1, $labl # long" %}
13484 ins_encode %{
13485 Label* L = $labl$$label;
13486 Assembler::Condition cond =
13487 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
13488 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
13489 %}
13490 ins_pipe(pipe_cmp_branch);
13491 %}
13492
13493 instruct far_cmpI_branch_sign(cmpOp cmp, iRegIorL2I op1, immI0 op2, label labl) %{
13494 match(If cmp (CmpI op1 op2));
13495 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::lt
13496 || n->in(1)->as_Bool()->_test._test == BoolTest::ge);
13497 effect(USE labl);
13498
13499 ins_cost(BRANCH_COST);
13500 format %{ "cb$cmp $op1, $labl # int" %}
13501 ins_encode %{
13502 Label* L = $labl$$label;
13503 Assembler::Condition cond =
13504 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
13505 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
13506 %}
13507 ins_pipe(pipe_cmp_branch);
13508 %}
13509
13510 instruct far_cmpL_branch_bit(cmpOp cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
13511 match(If cmp (CmpL (AndL op1 op2) op3));
13512 predicate((n->in(1)->as_Bool()->_test._test == BoolTest::ne
13513 || n->in(1)->as_Bool()->_test._test == BoolTest::eq)
13514 && is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
13515 effect(USE labl);
13516
13517 ins_cost(BRANCH_COST);
13518 format %{ "tb$cmp $op1, $op2, $labl" %}
13519 ins_encode %{
13520 Label* L = $labl$$label;
13521 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13522 int bit = exact_log2($op2$$constant);
13523 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
13524 %}
13525 ins_pipe(pipe_cmp_branch);
13526 %}
13527
13528 instruct far_cmpI_branch_bit(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
13529 match(If cmp (CmpI (AndI op1 op2) op3));
13530 predicate((n->in(1)->as_Bool()->_test._test == BoolTest::ne
13531 || n->in(1)->as_Bool()->_test._test == BoolTest::eq)
13532 && is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
13533 effect(USE labl);
13534
13535 ins_cost(BRANCH_COST);
13536 format %{ "tb$cmp $op1, $op2, $labl" %}
13537 ins_encode %{
13538 Label* L = $labl$$label;
13539 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13540 int bit = exact_log2($op2$$constant);
13541 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
13542 %}
13543 ins_pipe(pipe_cmp_branch);
13544 %}
13545
13546 // Test bits
13547
13548 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
13549 match(Set cr (CmpL (AndL op1 op2) op3));
13550 predicate(Assembler::operand_valid_for_logical_immediate
13551 (/*is_32*/false, n->in(1)->in(2)->get_long()));
13552
13553 ins_cost(INSN_COST);
13554 format %{ "tst $op1, $op2 # long" %}
13555 ins_encode %{
13556 __ tst($op1$$Register, $op2$$constant);
13557 %}
13558 ins_pipe(ialu_reg_reg);
13559 %}
13560
13561 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
13562 match(Set cr (CmpI (AndI op1 op2) op3));
13563 predicate(Assembler::operand_valid_for_logical_immediate
13564 (/*is_32*/true, n->in(1)->in(2)->get_int()));
13565
13566 ins_cost(INSN_COST);
13567 format %{ "tst $op1, $op2 # int" %}
13568 ins_encode %{
13569 __ tstw($op1$$Register, $op2$$constant);
13570 %}
13571 ins_pipe(ialu_reg_reg);
13572 %}
13573
13574 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
13575 match(Set cr (CmpL (AndL op1 op2) op3));
13576
13577 ins_cost(INSN_COST);
13578 format %{ "tst $op1, $op2 # long" %}
13579 ins_encode %{
13580 __ tst($op1$$Register, $op2$$Register);
13581 %}
13582 ins_pipe(ialu_reg_reg);
13583 %}
13584
13585 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
13586 match(Set cr (CmpI (AndI op1 op2) op3));
13587
13588 ins_cost(INSN_COST);
13589 format %{ "tstw $op1, $op2 # int" %}
13590 ins_encode %{
13591 __ tstw($op1$$Register, $op2$$Register);
13592 %}
13593 ins_pipe(ialu_reg_reg);
13594 %}
13595
13596
13597 // Conditional Far Branch
13598 // Conditional Far Branch Unsigned
13599 // TODO: fixme
13600
13601 // counted loop end branch near
13602 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
13603 %{
13604 match(CountedLoopEnd cmp cr);
13605
13606 effect(USE lbl);
13607
13608 ins_cost(BRANCH_COST);
13609 // short variant.
13610 // ins_short_branch(1);
13611 format %{ "b$cmp $lbl \t// counted loop end" %}
13612
13613 ins_encode(aarch64_enc_br_con(cmp, lbl));
13614
13615 ins_pipe(pipe_branch);
13616 %}
13617
13618 // counted loop end branch near Unsigned
13619 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
13620 %{
13621 match(CountedLoopEnd cmp cr);
13622
13623 effect(USE lbl);
13624
13625 ins_cost(BRANCH_COST);
13626 // short variant.
13627 // ins_short_branch(1);
13628 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
13629
13630 ins_encode(aarch64_enc_br_conU(cmp, lbl));
13631
13632 ins_pipe(pipe_branch);
13633 %}
13634
13635 // counted loop end branch far
13636 // counted loop end branch far unsigned
13637 // TODO: fixme
13638
13639 // ============================================================================
13640 // inlined locking and unlocking
13641
13642 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
13643 %{
13644 match(Set cr (FastLock object box));
13645 effect(TEMP tmp, TEMP tmp2);
13646
13647 // TODO
13648 // identify correct cost
13649 ins_cost(5 * INSN_COST);
13650 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
13651
13652 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
13653
13654 ins_pipe(pipe_serial);
13655 %}
13656
13657 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
13658 %{
13659 match(Set cr (FastUnlock object box));
13660 effect(TEMP tmp, TEMP tmp2);
13661
13662 ins_cost(5 * INSN_COST);
13663 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
13664
13665 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
13666
13667 ins_pipe(pipe_serial);
13668 %}
13669
13670
13671 // ============================================================================
13672 // Safepoint Instructions
13673
13674 // TODO
13675 // provide a near and far version of this code
13676
13677 instruct safePoint(rFlagsReg cr, iRegP poll)
13678 %{
13679 match(SafePoint poll);
13680 effect(KILL cr);
13681
13682 format %{
13683 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
13684 %}
13685 ins_encode %{
13686 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
13687 %}
13688 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
13689 %}
13690
13691
13692 // ============================================================================
13693 // Procedure Call/Return Instructions
13694
13695 // Call Java Static Instruction
13696
13697 instruct CallStaticJavaDirect(method meth)
13698 %{
13699 match(CallStaticJava);
13700
13701 effect(USE meth);
13702
13703 predicate(!((CallStaticJavaNode*)n)->is_method_handle_invoke());
13704
13705 ins_cost(CALL_COST);
13706
13707 format %{ "call,static $meth \t// ==> " %}
13708
13709 ins_encode( aarch64_enc_java_static_call(meth),
13710 aarch64_enc_call_epilog );
13711
13712 ins_pipe(pipe_class_call);
13713 %}
13714
13715 // TO HERE
13716
13717 // Call Java Static Instruction (method handle version)
13718
13719 instruct CallStaticJavaDirectHandle(method meth, iRegP_FP reg_mh_save)
13720 %{
13721 match(CallStaticJava);
13722
13723 effect(USE meth);
13724
13725 predicate(((CallStaticJavaNode*)n)->is_method_handle_invoke());
13726
13727 ins_cost(CALL_COST);
13728
13729 format %{ "call,static $meth \t// (methodhandle) ==> " %}
13730
13731 ins_encode( aarch64_enc_java_handle_call(meth),
13732 aarch64_enc_call_epilog );
13733
13734 ins_pipe(pipe_class_call);
13735 %}
13736
13737 // Call Java Dynamic Instruction
13738 instruct CallDynamicJavaDirect(method meth)
13739 %{
13740 match(CallDynamicJava);
13741
13742 effect(USE meth);
13743
13744 ins_cost(CALL_COST);
13745
13746 format %{ "CALL,dynamic $meth \t// ==> " %}
13747
13748 ins_encode( aarch64_enc_java_dynamic_call(meth),
13749 aarch64_enc_call_epilog );
13750
13751 ins_pipe(pipe_class_call);
13752 %}
13753
13754 // Call Runtime Instruction
13755
13756 instruct CallRuntimeDirect(method meth)
13757 %{
13758 match(CallRuntime);
13759
13760 effect(USE meth);
13761
13762 ins_cost(CALL_COST);
13763
13764 format %{ "CALL, runtime $meth" %}
13765
13766 ins_encode( aarch64_enc_java_to_runtime(meth) );
13767
13768 ins_pipe(pipe_class_call);
13769 %}
13770
13771 // Call Runtime Instruction
13772
13773 instruct CallLeafDirect(method meth)
13774 %{
13775 match(CallLeaf);
13776
13777 effect(USE meth);
13778
13779 ins_cost(CALL_COST);
13780
13781 format %{ "CALL, runtime leaf $meth" %}
13782
13783 ins_encode( aarch64_enc_java_to_runtime(meth) );
13784
13785 ins_pipe(pipe_class_call);
13786 %}
13787
13788 // Call Runtime Instruction
13789
13790 instruct CallLeafNoFPDirect(method meth)
13791 %{
13792 match(CallLeafNoFP);
13793
13794 effect(USE meth);
13795
13796 ins_cost(CALL_COST);
13797
13798 format %{ "CALL, runtime leaf nofp $meth" %}
13799
13800 ins_encode( aarch64_enc_java_to_runtime(meth) );
13801
13802 ins_pipe(pipe_class_call);
13803 %}
13804
13805 // Tail Call; Jump from runtime stub to Java code.
13806 // Also known as an 'interprocedural jump'.
13807 // Target of jump will eventually return to caller.
13808 // TailJump below removes the return address.
13809 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop)
13810 %{
13811 match(TailCall jump_target method_oop);
13812
13813 ins_cost(CALL_COST);
13814
13815 format %{ "br $jump_target\t# $method_oop holds method oop" %}
13816
13817 ins_encode(aarch64_enc_tail_call(jump_target));
13818
13819 ins_pipe(pipe_class_call);
13820 %}
13821
13822 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
13823 %{
13824 match(TailJump jump_target ex_oop);
13825
13826 ins_cost(CALL_COST);
13827
13828 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
13829
13830 ins_encode(aarch64_enc_tail_jmp(jump_target));
13831
13832 ins_pipe(pipe_class_call);
13833 %}
13834
13835 // Create exception oop: created by stack-crawling runtime code.
13836 // Created exception is now available to this handler, and is setup
13837 // just prior to jumping to this handler. No code emitted.
13838 // TODO check
13839 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
13840 instruct CreateException(iRegP_R0 ex_oop)
13841 %{
13842 match(Set ex_oop (CreateEx));
13843
13844 format %{ " -- \t// exception oop; no code emitted" %}
13845
13846 size(0);
13847
13848 ins_encode( /*empty*/ );
13849
13850 ins_pipe(pipe_class_empty);
13851 %}
13852
13853 // Rethrow exception: The exception oop will come in the first
13854 // argument position. Then JUMP (not call) to the rethrow stub code.
13855 instruct RethrowException() %{
13856 match(Rethrow);
13857 ins_cost(CALL_COST);
13858
13859 format %{ "b rethrow_stub" %}
13860
13861 ins_encode( aarch64_enc_rethrow() );
13862
13863 ins_pipe(pipe_class_call);
13864 %}
13865
13866
13867 // Return Instruction
13868 // epilog node loads ret address into lr as part of frame pop
13869 instruct Ret()
13870 %{
13871 match(Return);
13872
13873 format %{ "ret\t// return register" %}
13874
13875 ins_encode( aarch64_enc_ret() );
13876
13877 ins_pipe(pipe_branch);
13878 %}
13879
13880 // Die now.
13881 instruct ShouldNotReachHere() %{
13882 match(Halt);
13883
13884 ins_cost(CALL_COST);
13885 format %{ "ShouldNotReachHere" %}
13886
13887 ins_encode %{
13888 // TODO
13889 // implement proper trap call here
13890 __ brk(999);
13891 %}
13892
13893 ins_pipe(pipe_class_default);
13894 %}
13895
13896 // ============================================================================
13897 // Partial Subtype Check
13898 //
13899 // superklass array for an instance of the superklass. Set a hidden
13900 // internal cache on a hit (cache is checked with exposed code in
13901 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
13902 // encoding ALSO sets flags.
13903
13904 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
13905 %{
13906 match(Set result (PartialSubtypeCheck sub super));
13907 effect(KILL cr, KILL temp);
13908
13909 ins_cost(1100); // slightly larger than the next version
13910 format %{ "partialSubtypeCheck $result, $sub, $super" %}
13911
13912 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
13913
13914 opcode(0x1); // Force zero of result reg on hit
13915
13916 ins_pipe(pipe_class_memory);
13917 %}
13918
13919 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
13920 %{
13921 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
13922 effect(KILL temp, KILL result);
13923
13924 ins_cost(1100); // slightly larger than the next version
13925 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
13926
13927 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
13928
13929 opcode(0x0); // Don't zero result reg on hit
13930
13931 ins_pipe(pipe_class_memory);
13932 %}
13933
13934 instruct string_compare(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
13935 iRegI_R0 result, iRegP_R10 tmp1, rFlagsReg cr)
13936 %{
13937 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
13938 effect(KILL tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
13939
13940 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
13941 ins_encode %{
13942 __ string_compare($str1$$Register, $str2$$Register,
13943 $cnt1$$Register, $cnt2$$Register, $result$$Register,
13944 $tmp1$$Register);
13945 %}
13946 ins_pipe(pipe_class_memory);
13947 %}
13948
13949 instruct string_indexof(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
13950 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
13951 %{
13952 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
13953 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
13954 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
13955 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result" %}
13956
13957 ins_encode %{
13958 __ string_indexof($str1$$Register, $str2$$Register,
13959 $cnt1$$Register, $cnt2$$Register,
13960 $tmp1$$Register, $tmp2$$Register,
13961 $tmp3$$Register, $tmp4$$Register,
13962 -1, $result$$Register);
13963 %}
13964 ins_pipe(pipe_class_memory);
13965 %}
13966
13967 instruct string_indexof_con(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
13968 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
13969 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
13970 %{
13971 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
13972 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
13973 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
13974 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result" %}
13975
13976 ins_encode %{
13977 int icnt2 = (int)$int_cnt2$$constant;
13978 __ string_indexof($str1$$Register, $str2$$Register,
13979 $cnt1$$Register, zr,
13980 $tmp1$$Register, $tmp2$$Register,
13981 $tmp3$$Register, $tmp4$$Register,
13982 icnt2, $result$$Register);
13983 %}
13984 ins_pipe(pipe_class_memory);
13985 %}
13986
13987 instruct string_equals(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
13988 iRegI_R0 result, iRegP_R10 tmp, rFlagsReg cr)
13989 %{
13990 match(Set result (StrEquals (Binary str1 str2) cnt));
13991 effect(KILL tmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
13992
13993 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp" %}
13994 ins_encode %{
13995 __ string_equals($str1$$Register, $str2$$Register,
13996 $cnt$$Register, $result$$Register,
13997 $tmp$$Register);
13998 %}
13999 ins_pipe(pipe_class_memory);
14000 %}
14001
14002 instruct array_equals(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
14003 iRegP_R10 tmp, rFlagsReg cr)
14004 %{
14005 match(Set result (AryEq ary1 ary2));
14006 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, KILL cr);
14007
14008 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
14009 ins_encode %{
14010 __ char_arrays_equals($ary1$$Register, $ary2$$Register,
14011 $result$$Register, $tmp$$Register);
14012 %}
14013 ins_pipe(pipe_class_memory);
14014 %}
14015
14016 // encode char[] to byte[] in ISO_8859_1
14017 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
14018 vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
14019 vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
14020 iRegI_R0 result, rFlagsReg cr)
14021 %{
14022 match(Set result (EncodeISOArray src (Binary dst len)));
14023 effect(USE_KILL src, USE_KILL dst, USE_KILL len,
14024 KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
14025
14026 format %{ "Encode array $src,$dst,$len -> $result" %}
14027 ins_encode %{
14028 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
14029 $result$$Register, $Vtmp1$$FloatRegister, $Vtmp2$$FloatRegister,
14030 $Vtmp3$$FloatRegister, $Vtmp4$$FloatRegister);
14031 %}
14032 ins_pipe( pipe_class_memory );
14033 %}
14034
14035 // ============================================================================
14036 // This name is KNOWN by the ADLC and cannot be changed.
14037 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
14038 // for this guy.
14039 instruct tlsLoadP(thread_RegP dst)
14040 %{
14041 match(Set dst (ThreadLocal));
14042
14043 ins_cost(0);
14044
14045 format %{ " -- \t// $dst=Thread::current(), empty" %}
14046
14047 size(0);
14048
14049 ins_encode( /*empty*/ );
14050
14051 ins_pipe(pipe_class_empty);
14052 %}
14053
14054 // ====================VECTOR INSTRUCTIONS=====================================
14055
14056 // Load vector (32 bits)
14057 instruct loadV4(vecD dst, vmem4 mem)
14058 %{
14059 predicate(n->as_LoadVector()->memory_size() == 4);
14060 match(Set dst (LoadVector mem));
14061 ins_cost(4 * INSN_COST);
14062 format %{ "ldrs $dst,$mem\t# vector (32 bits)" %}
14063 ins_encode( aarch64_enc_ldrvS(dst, mem) );
14064 ins_pipe(vload_reg_mem64);
14065 %}
14066
14067 // Load vector (64 bits)
14068 instruct loadV8(vecD dst, vmem8 mem)
14069 %{
14070 predicate(n->as_LoadVector()->memory_size() == 8);
14071 match(Set dst (LoadVector mem));
14072 ins_cost(4 * INSN_COST);
14073 format %{ "ldrd $dst,$mem\t# vector (64 bits)" %}
14074 ins_encode( aarch64_enc_ldrvD(dst, mem) );
14075 ins_pipe(vload_reg_mem64);
14076 %}
14077
14078 // Load Vector (128 bits)
14079 instruct loadV16(vecX dst, vmem16 mem)
14080 %{
14081 predicate(n->as_LoadVector()->memory_size() == 16);
14082 match(Set dst (LoadVector mem));
14083 ins_cost(4 * INSN_COST);
14084 format %{ "ldrq $dst,$mem\t# vector (128 bits)" %}
14085 ins_encode( aarch64_enc_ldrvQ(dst, mem) );
14086 ins_pipe(vload_reg_mem128);
14087 %}
14088
14089 // Store Vector (32 bits)
14090 instruct storeV4(vecD src, vmem4 mem)
14091 %{
14092 predicate(n->as_StoreVector()->memory_size() == 4);
14093 match(Set mem (StoreVector mem src));
14094 ins_cost(4 * INSN_COST);
14095 format %{ "strs $mem,$src\t# vector (32 bits)" %}
14096 ins_encode( aarch64_enc_strvS(src, mem) );
14097 ins_pipe(vstore_reg_mem64);
14098 %}
14099
14100 // Store Vector (64 bits)
14101 instruct storeV8(vecD src, vmem8 mem)
14102 %{
14103 predicate(n->as_StoreVector()->memory_size() == 8);
14104 match(Set mem (StoreVector mem src));
14105 ins_cost(4 * INSN_COST);
14106 format %{ "strd $mem,$src\t# vector (64 bits)" %}
14107 ins_encode( aarch64_enc_strvD(src, mem) );
14108 ins_pipe(vstore_reg_mem64);
14109 %}
14110
14111 // Store Vector (128 bits)
14112 instruct storeV16(vecX src, vmem16 mem)
14113 %{
14114 predicate(n->as_StoreVector()->memory_size() == 16);
14115 match(Set mem (StoreVector mem src));
14116 ins_cost(4 * INSN_COST);
14117 format %{ "strq $mem,$src\t# vector (128 bits)" %}
14118 ins_encode( aarch64_enc_strvQ(src, mem) );
14119 ins_pipe(vstore_reg_mem128);
14120 %}
14121
14122 instruct replicate8B(vecD dst, iRegIorL2I src)
14123 %{
14124 predicate(n->as_Vector()->length() == 4 ||
14125 n->as_Vector()->length() == 8);
14126 match(Set dst (ReplicateB src));
14127 ins_cost(INSN_COST);
14128 format %{ "dup $dst, $src\t# vector (8B)" %}
14129 ins_encode %{
14130 __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($src$$reg));
14131 %}
14132 ins_pipe(vdup_reg_reg64);
14133 %}
14134
14135 instruct replicate16B(vecX dst, iRegIorL2I src)
14136 %{
14137 predicate(n->as_Vector()->length() == 16);
14138 match(Set dst (ReplicateB src));
14139 ins_cost(INSN_COST);
14140 format %{ "dup $dst, $src\t# vector (16B)" %}
14141 ins_encode %{
14142 __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($src$$reg));
14143 %}
14144 ins_pipe(vdup_reg_reg128);
14145 %}
14146
14147 instruct replicate8B_imm(vecD dst, immI con)
14148 %{
14149 predicate(n->as_Vector()->length() == 4 ||
14150 n->as_Vector()->length() == 8);
14151 match(Set dst (ReplicateB con));
14152 ins_cost(INSN_COST);
14153 format %{ "movi $dst, $con\t# vector(8B)" %}
14154 ins_encode %{
14155 __ mov(as_FloatRegister($dst$$reg), __ T8B, $con$$constant & 0xff);
14156 %}
14157 ins_pipe(vmovi_reg_imm64);
14158 %}
14159
14160 instruct replicate16B_imm(vecX dst, immI con)
14161 %{
14162 predicate(n->as_Vector()->length() == 16);
14163 match(Set dst (ReplicateB con));
14164 ins_cost(INSN_COST);
14165 format %{ "movi $dst, $con\t# vector(16B)" %}
14166 ins_encode %{
14167 __ mov(as_FloatRegister($dst$$reg), __ T16B, $con$$constant & 0xff);
14168 %}
14169 ins_pipe(vmovi_reg_imm128);
14170 %}
14171
14172 instruct replicate4S(vecD dst, iRegIorL2I src)
14173 %{
14174 predicate(n->as_Vector()->length() == 2 ||
14175 n->as_Vector()->length() == 4);
14176 match(Set dst (ReplicateS src));
14177 ins_cost(INSN_COST);
14178 format %{ "dup $dst, $src\t# vector (4S)" %}
14179 ins_encode %{
14180 __ dup(as_FloatRegister($dst$$reg), __ T4H, as_Register($src$$reg));
14181 %}
14182 ins_pipe(vdup_reg_reg64);
14183 %}
14184
14185 instruct replicate8S(vecX dst, iRegIorL2I src)
14186 %{
14187 predicate(n->as_Vector()->length() == 8);
14188 match(Set dst (ReplicateS src));
14189 ins_cost(INSN_COST);
14190 format %{ "dup $dst, $src\t# vector (8S)" %}
14191 ins_encode %{
14192 __ dup(as_FloatRegister($dst$$reg), __ T8H, as_Register($src$$reg));
14193 %}
14194 ins_pipe(vdup_reg_reg128);
14195 %}
14196
14197 instruct replicate4S_imm(vecD dst, immI con)
14198 %{
14199 predicate(n->as_Vector()->length() == 2 ||
14200 n->as_Vector()->length() == 4);
14201 match(Set dst (ReplicateS con));
14202 ins_cost(INSN_COST);
14203 format %{ "movi $dst, $con\t# vector(4H)" %}
14204 ins_encode %{
14205 __ mov(as_FloatRegister($dst$$reg), __ T4H, $con$$constant & 0xffff);
14206 %}
14207 ins_pipe(vmovi_reg_imm64);
14208 %}
14209
14210 instruct replicate8S_imm(vecX dst, immI con)
14211 %{
14212 predicate(n->as_Vector()->length() == 8);
14213 match(Set dst (ReplicateS con));
14214 ins_cost(INSN_COST);
14215 format %{ "movi $dst, $con\t# vector(8H)" %}
14216 ins_encode %{
14217 __ mov(as_FloatRegister($dst$$reg), __ T8H, $con$$constant & 0xffff);
14218 %}
14219 ins_pipe(vmovi_reg_imm128);
14220 %}
14221
14222 instruct replicate2I(vecD dst, iRegIorL2I src)
14223 %{
14224 predicate(n->as_Vector()->length() == 2);
14225 match(Set dst (ReplicateI src));
14226 ins_cost(INSN_COST);
14227 format %{ "dup $dst, $src\t# vector (2I)" %}
14228 ins_encode %{
14229 __ dup(as_FloatRegister($dst$$reg), __ T2S, as_Register($src$$reg));
14230 %}
14231 ins_pipe(vdup_reg_reg64);
14232 %}
14233
14234 instruct replicate4I(vecX dst, iRegIorL2I src)
14235 %{
14236 predicate(n->as_Vector()->length() == 4);
14237 match(Set dst (ReplicateI src));
14238 ins_cost(INSN_COST);
14239 format %{ "dup $dst, $src\t# vector (4I)" %}
14240 ins_encode %{
14241 __ dup(as_FloatRegister($dst$$reg), __ T4S, as_Register($src$$reg));
14242 %}
14243 ins_pipe(vdup_reg_reg128);
14244 %}
14245
14246 instruct replicate2I_imm(vecD dst, immI con)
14247 %{
14248 predicate(n->as_Vector()->length() == 2);
14249 match(Set dst (ReplicateI con));
14250 ins_cost(INSN_COST);
14251 format %{ "movi $dst, $con\t# vector(2I)" %}
14252 ins_encode %{
14253 __ mov(as_FloatRegister($dst$$reg), __ T2S, $con$$constant);
14254 %}
14255 ins_pipe(vmovi_reg_imm64);
14256 %}
14257
14258 instruct replicate4I_imm(vecX dst, immI con)
14259 %{
14260 predicate(n->as_Vector()->length() == 4);
14261 match(Set dst (ReplicateI con));
14262 ins_cost(INSN_COST);
14263 format %{ "movi $dst, $con\t# vector(4I)" %}
14264 ins_encode %{
14265 __ mov(as_FloatRegister($dst$$reg), __ T4S, $con$$constant);
14266 %}
14267 ins_pipe(vmovi_reg_imm128);
14268 %}
14269
14270 instruct replicate2L(vecX dst, iRegL src)
14271 %{
14272 predicate(n->as_Vector()->length() == 2);
14273 match(Set dst (ReplicateL src));
14274 ins_cost(INSN_COST);
14275 format %{ "dup $dst, $src\t# vector (2L)" %}
14276 ins_encode %{
14277 __ dup(as_FloatRegister($dst$$reg), __ T2D, as_Register($src$$reg));
14278 %}
14279 ins_pipe(vdup_reg_reg128);
14280 %}
14281
14282 instruct replicate2L_zero(vecX dst, immI0 zero)
14283 %{
14284 predicate(n->as_Vector()->length() == 2);
14285 match(Set dst (ReplicateI zero));
14286 ins_cost(INSN_COST);
14287 format %{ "movi $dst, $zero\t# vector(4I)" %}
14288 ins_encode %{
14289 __ eor(as_FloatRegister($dst$$reg), __ T16B,
14290 as_FloatRegister($dst$$reg),
14291 as_FloatRegister($dst$$reg));
14292 %}
14293 ins_pipe(vmovi_reg_imm128);
14294 %}
14295
14296 instruct replicate2F(vecD dst, vRegF src)
14297 %{
14298 predicate(n->as_Vector()->length() == 2);
14299 match(Set dst (ReplicateF src));
14300 ins_cost(INSN_COST);
14301 format %{ "dup $dst, $src\t# vector (2F)" %}
14302 ins_encode %{
14303 __ dup(as_FloatRegister($dst$$reg), __ T2S,
14304 as_FloatRegister($src$$reg));
14305 %}
14306 ins_pipe(vdup_reg_freg64);
14307 %}
14308
14309 instruct replicate4F(vecX dst, vRegF src)
14310 %{
14311 predicate(n->as_Vector()->length() == 4);
14312 match(Set dst (ReplicateF src));
14313 ins_cost(INSN_COST);
14314 format %{ "dup $dst, $src\t# vector (4F)" %}
14315 ins_encode %{
14316 __ dup(as_FloatRegister($dst$$reg), __ T4S,
14317 as_FloatRegister($src$$reg));
14318 %}
14319 ins_pipe(vdup_reg_freg128);
14320 %}
14321
14322 instruct replicate2D(vecX dst, vRegD src)
14323 %{
14324 predicate(n->as_Vector()->length() == 2);
14325 match(Set dst (ReplicateD src));
14326 ins_cost(INSN_COST);
14327 format %{ "dup $dst, $src\t# vector (2D)" %}
14328 ins_encode %{
14329 __ dup(as_FloatRegister($dst$$reg), __ T2D,
14330 as_FloatRegister($src$$reg));
14331 %}
14332 ins_pipe(vdup_reg_dreg128);
14333 %}
14334
14335 // ====================VECTOR ARITHMETIC=======================================
14336
14337 // --------------------------------- ADD --------------------------------------
14338
14339 instruct vadd8B(vecD dst, vecD src1, vecD src2)
14340 %{
14341 predicate(n->as_Vector()->length() == 4 ||
14342 n->as_Vector()->length() == 8);
14343 match(Set dst (AddVB src1 src2));
14344 ins_cost(INSN_COST);
14345 format %{ "addv $dst,$src1,$src2\t# vector (8B)" %}
14346 ins_encode %{
14347 __ addv(as_FloatRegister($dst$$reg), __ T8B,
14348 as_FloatRegister($src1$$reg),
14349 as_FloatRegister($src2$$reg));
14350 %}
14351 ins_pipe(vdop64);
14352 %}
14353
14354 instruct vadd16B(vecX dst, vecX src1, vecX src2)
14355 %{
14356 predicate(n->as_Vector()->length() == 16);
14357 match(Set dst (AddVB src1 src2));
14358 ins_cost(INSN_COST);
14359 format %{ "addv $dst,$src1,$src2\t# vector (16B)" %}
14360 ins_encode %{
14361 __ addv(as_FloatRegister($dst$$reg), __ T16B,
14362 as_FloatRegister($src1$$reg),
14363 as_FloatRegister($src2$$reg));
14364 %}
14365 ins_pipe(vdop128);
14366 %}
14367
14368 instruct vadd4S(vecD dst, vecD src1, vecD src2)
14369 %{
14370 predicate(n->as_Vector()->length() == 2 ||
14371 n->as_Vector()->length() == 4);
14372 match(Set dst (AddVS src1 src2));
14373 ins_cost(INSN_COST);
14374 format %{ "addv $dst,$src1,$src2\t# vector (4H)" %}
14375 ins_encode %{
14376 __ addv(as_FloatRegister($dst$$reg), __ T4H,
14377 as_FloatRegister($src1$$reg),
14378 as_FloatRegister($src2$$reg));
14379 %}
14380 ins_pipe(vdop64);
14381 %}
14382
14383 instruct vadd8S(vecX dst, vecX src1, vecX src2)
14384 %{
14385 predicate(n->as_Vector()->length() == 8);
14386 match(Set dst (AddVS src1 src2));
14387 ins_cost(INSN_COST);
14388 format %{ "addv $dst,$src1,$src2\t# vector (8H)" %}
14389 ins_encode %{
14390 __ addv(as_FloatRegister($dst$$reg), __ T8H,
14391 as_FloatRegister($src1$$reg),
14392 as_FloatRegister($src2$$reg));
14393 %}
14394 ins_pipe(vdop128);
14395 %}
14396
14397 instruct vadd2I(vecD dst, vecD src1, vecD src2)
14398 %{
14399 predicate(n->as_Vector()->length() == 2);
14400 match(Set dst (AddVI src1 src2));
14401 ins_cost(INSN_COST);
14402 format %{ "addv $dst,$src1,$src2\t# vector (2S)" %}
14403 ins_encode %{
14404 __ addv(as_FloatRegister($dst$$reg), __ T2S,
14405 as_FloatRegister($src1$$reg),
14406 as_FloatRegister($src2$$reg));
14407 %}
14408 ins_pipe(vdop64);
14409 %}
14410
14411 instruct vadd4I(vecX dst, vecX src1, vecX src2)
14412 %{
14413 predicate(n->as_Vector()->length() == 4);
14414 match(Set dst (AddVI src1 src2));
14415 ins_cost(INSN_COST);
14416 format %{ "addv $dst,$src1,$src2\t# vector (4S)" %}
14417 ins_encode %{
14418 __ addv(as_FloatRegister($dst$$reg), __ T4S,
14419 as_FloatRegister($src1$$reg),
14420 as_FloatRegister($src2$$reg));
14421 %}
14422 ins_pipe(vdop128);
14423 %}
14424
14425 instruct vadd2L(vecX dst, vecX src1, vecX src2)
14426 %{
14427 predicate(n->as_Vector()->length() == 2);
14428 match(Set dst (AddVL src1 src2));
14429 ins_cost(INSN_COST);
14430 format %{ "addv $dst,$src1,$src2\t# vector (2L)" %}
14431 ins_encode %{
14432 __ addv(as_FloatRegister($dst$$reg), __ T2D,
14433 as_FloatRegister($src1$$reg),
14434 as_FloatRegister($src2$$reg));
14435 %}
14436 ins_pipe(vdop128);
14437 %}
14438
14439 instruct vadd2F(vecD dst, vecD src1, vecD src2)
14440 %{
14441 predicate(n->as_Vector()->length() == 2);
14442 match(Set dst (AddVF src1 src2));
14443 ins_cost(INSN_COST);
14444 format %{ "fadd $dst,$src1,$src2\t# vector (2S)" %}
14445 ins_encode %{
14446 __ fadd(as_FloatRegister($dst$$reg), __ T2S,
14447 as_FloatRegister($src1$$reg),
14448 as_FloatRegister($src2$$reg));
14449 %}
14450 ins_pipe(vdop_fp64);
14451 %}
14452
14453 instruct vadd4F(vecX dst, vecX src1, vecX src2)
14454 %{
14455 predicate(n->as_Vector()->length() == 4);
14456 match(Set dst (AddVF src1 src2));
14457 ins_cost(INSN_COST);
14458 format %{ "fadd $dst,$src1,$src2\t# vector (4S)" %}
14459 ins_encode %{
14460 __ fadd(as_FloatRegister($dst$$reg), __ T4S,
14461 as_FloatRegister($src1$$reg),
14462 as_FloatRegister($src2$$reg));
14463 %}
14464 ins_pipe(vdop_fp128);
14465 %}
14466
14467 instruct vadd2D(vecX dst, vecX src1, vecX src2)
14468 %{
14469 match(Set dst (AddVD src1 src2));
14470 ins_cost(INSN_COST);
14471 format %{ "fadd $dst,$src1,$src2\t# vector (2D)" %}
14472 ins_encode %{
14473 __ fadd(as_FloatRegister($dst$$reg), __ T2D,
14474 as_FloatRegister($src1$$reg),
14475 as_FloatRegister($src2$$reg));
14476 %}
14477 ins_pipe(vdop_fp128);
14478 %}
14479
14480 // --------------------------------- SUB --------------------------------------
14481
14482 instruct vsub8B(vecD dst, vecD src1, vecD src2)
14483 %{
14484 predicate(n->as_Vector()->length() == 4 ||
14485 n->as_Vector()->length() == 8);
14486 match(Set dst (SubVB src1 src2));
14487 ins_cost(INSN_COST);
14488 format %{ "subv $dst,$src1,$src2\t# vector (8B)" %}
14489 ins_encode %{
14490 __ subv(as_FloatRegister($dst$$reg), __ T8B,
14491 as_FloatRegister($src1$$reg),
14492 as_FloatRegister($src2$$reg));
14493 %}
14494 ins_pipe(vdop64);
14495 %}
14496
14497 instruct vsub16B(vecX dst, vecX src1, vecX src2)
14498 %{
14499 predicate(n->as_Vector()->length() == 16);
14500 match(Set dst (SubVB src1 src2));
14501 ins_cost(INSN_COST);
14502 format %{ "subv $dst,$src1,$src2\t# vector (16B)" %}
14503 ins_encode %{
14504 __ subv(as_FloatRegister($dst$$reg), __ T16B,
14505 as_FloatRegister($src1$$reg),
14506 as_FloatRegister($src2$$reg));
14507 %}
14508 ins_pipe(vdop128);
14509 %}
14510
14511 instruct vsub4S(vecD dst, vecD src1, vecD src2)
14512 %{
14513 predicate(n->as_Vector()->length() == 2 ||
14514 n->as_Vector()->length() == 4);
14515 match(Set dst (SubVS src1 src2));
14516 ins_cost(INSN_COST);
14517 format %{ "subv $dst,$src1,$src2\t# vector (4H)" %}
14518 ins_encode %{
14519 __ subv(as_FloatRegister($dst$$reg), __ T4H,
14520 as_FloatRegister($src1$$reg),
14521 as_FloatRegister($src2$$reg));
14522 %}
14523 ins_pipe(vdop64);
14524 %}
14525
14526 instruct vsub8S(vecX dst, vecX src1, vecX src2)
14527 %{
14528 predicate(n->as_Vector()->length() == 8);
14529 match(Set dst (SubVS src1 src2));
14530 ins_cost(INSN_COST);
14531 format %{ "subv $dst,$src1,$src2\t# vector (8H)" %}
14532 ins_encode %{
14533 __ subv(as_FloatRegister($dst$$reg), __ T8H,
14534 as_FloatRegister($src1$$reg),
14535 as_FloatRegister($src2$$reg));
14536 %}
14537 ins_pipe(vdop128);
14538 %}
14539
14540 instruct vsub2I(vecD dst, vecD src1, vecD src2)
14541 %{
14542 predicate(n->as_Vector()->length() == 2);
14543 match(Set dst (SubVI src1 src2));
14544 ins_cost(INSN_COST);
14545 format %{ "subv $dst,$src1,$src2\t# vector (2S)" %}
14546 ins_encode %{
14547 __ subv(as_FloatRegister($dst$$reg), __ T2S,
14548 as_FloatRegister($src1$$reg),
14549 as_FloatRegister($src2$$reg));
14550 %}
14551 ins_pipe(vdop64);
14552 %}
14553
14554 instruct vsub4I(vecX dst, vecX src1, vecX src2)
14555 %{
14556 predicate(n->as_Vector()->length() == 4);
14557 match(Set dst (SubVI src1 src2));
14558 ins_cost(INSN_COST);
14559 format %{ "subv $dst,$src1,$src2\t# vector (4S)" %}
14560 ins_encode %{
14561 __ subv(as_FloatRegister($dst$$reg), __ T4S,
14562 as_FloatRegister($src1$$reg),
14563 as_FloatRegister($src2$$reg));
14564 %}
14565 ins_pipe(vdop128);
14566 %}
14567
14568 instruct vsub2L(vecX dst, vecX src1, vecX src2)
14569 %{
14570 predicate(n->as_Vector()->length() == 2);
14571 match(Set dst (SubVL src1 src2));
14572 ins_cost(INSN_COST);
14573 format %{ "subv $dst,$src1,$src2\t# vector (2L)" %}
14574 ins_encode %{
14575 __ subv(as_FloatRegister($dst$$reg), __ T2D,
14576 as_FloatRegister($src1$$reg),
14577 as_FloatRegister($src2$$reg));
14578 %}
14579 ins_pipe(vdop128);
14580 %}
14581
14582 instruct vsub2F(vecD dst, vecD src1, vecD src2)
14583 %{
14584 predicate(n->as_Vector()->length() == 2);
14585 match(Set dst (SubVF src1 src2));
14586 ins_cost(INSN_COST);
14587 format %{ "fsub $dst,$src1,$src2\t# vector (2S)" %}
14588 ins_encode %{
14589 __ fsub(as_FloatRegister($dst$$reg), __ T2S,
14590 as_FloatRegister($src1$$reg),
14591 as_FloatRegister($src2$$reg));
14592 %}
14593 ins_pipe(vdop_fp64);
14594 %}
14595
14596 instruct vsub4F(vecX dst, vecX src1, vecX src2)
14597 %{
14598 predicate(n->as_Vector()->length() == 4);
14599 match(Set dst (SubVF src1 src2));
14600 ins_cost(INSN_COST);
14601 format %{ "fsub $dst,$src1,$src2\t# vector (4S)" %}
14602 ins_encode %{
14603 __ fsub(as_FloatRegister($dst$$reg), __ T4S,
14604 as_FloatRegister($src1$$reg),
14605 as_FloatRegister($src2$$reg));
14606 %}
14607 ins_pipe(vdop_fp128);
14608 %}
14609
14610 instruct vsub2D(vecX dst, vecX src1, vecX src2)
14611 %{
14612 predicate(n->as_Vector()->length() == 2);
14613 match(Set dst (SubVD src1 src2));
14614 ins_cost(INSN_COST);
14615 format %{ "fsub $dst,$src1,$src2\t# vector (2D)" %}
14616 ins_encode %{
14617 __ fsub(as_FloatRegister($dst$$reg), __ T2D,
14618 as_FloatRegister($src1$$reg),
14619 as_FloatRegister($src2$$reg));
14620 %}
14621 ins_pipe(vdop_fp128);
14622 %}
14623
14624 // --------------------------------- MUL --------------------------------------
14625
14626 instruct vmul4S(vecD dst, vecD src1, vecD src2)
14627 %{
14628 predicate(n->as_Vector()->length() == 2 ||
14629 n->as_Vector()->length() == 4);
14630 match(Set dst (MulVS src1 src2));
14631 ins_cost(INSN_COST);
14632 format %{ "mulv $dst,$src1,$src2\t# vector (4H)" %}
14633 ins_encode %{
14634 __ mulv(as_FloatRegister($dst$$reg), __ T4H,
14635 as_FloatRegister($src1$$reg),
14636 as_FloatRegister($src2$$reg));
14637 %}
14638 ins_pipe(vmul64);
14639 %}
14640
14641 instruct vmul8S(vecX dst, vecX src1, vecX src2)
14642 %{
14643 predicate(n->as_Vector()->length() == 8);
14644 match(Set dst (MulVS src1 src2));
14645 ins_cost(INSN_COST);
14646 format %{ "mulv $dst,$src1,$src2\t# vector (8H)" %}
14647 ins_encode %{
14648 __ mulv(as_FloatRegister($dst$$reg), __ T8H,
14649 as_FloatRegister($src1$$reg),
14650 as_FloatRegister($src2$$reg));
14651 %}
14652 ins_pipe(vmul128);
14653 %}
14654
14655 instruct vmul2I(vecD dst, vecD src1, vecD src2)
14656 %{
14657 predicate(n->as_Vector()->length() == 2);
14658 match(Set dst (MulVI src1 src2));
14659 ins_cost(INSN_COST);
14660 format %{ "mulv $dst,$src1,$src2\t# vector (2S)" %}
14661 ins_encode %{
14662 __ mulv(as_FloatRegister($dst$$reg), __ T2S,
14663 as_FloatRegister($src1$$reg),
14664 as_FloatRegister($src2$$reg));
14665 %}
14666 ins_pipe(vmul64);
14667 %}
14668
14669 instruct vmul4I(vecX dst, vecX src1, vecX src2)
14670 %{
14671 predicate(n->as_Vector()->length() == 4);
14672 match(Set dst (MulVI src1 src2));
14673 ins_cost(INSN_COST);
14674 format %{ "mulv $dst,$src1,$src2\t# vector (4S)" %}
14675 ins_encode %{
14676 __ mulv(as_FloatRegister($dst$$reg), __ T4S,
14677 as_FloatRegister($src1$$reg),
14678 as_FloatRegister($src2$$reg));
14679 %}
14680 ins_pipe(vmul128);
14681 %}
14682
14683 instruct vmul2F(vecD dst, vecD src1, vecD src2)
14684 %{
14685 predicate(n->as_Vector()->length() == 2);
14686 match(Set dst (MulVF src1 src2));
14687 ins_cost(INSN_COST);
14688 format %{ "fmul $dst,$src1,$src2\t# vector (2S)" %}
14689 ins_encode %{
14690 __ fmul(as_FloatRegister($dst$$reg), __ T2S,
14691 as_FloatRegister($src1$$reg),
14692 as_FloatRegister($src2$$reg));
14693 %}
14694 ins_pipe(vmuldiv_fp64);
14695 %}
14696
14697 instruct vmul4F(vecX dst, vecX src1, vecX src2)
14698 %{
14699 predicate(n->as_Vector()->length() == 4);
14700 match(Set dst (MulVF src1 src2));
14701 ins_cost(INSN_COST);
14702 format %{ "fmul $dst,$src1,$src2\t# vector (4S)" %}
14703 ins_encode %{
14704 __ fmul(as_FloatRegister($dst$$reg), __ T4S,
14705 as_FloatRegister($src1$$reg),
14706 as_FloatRegister($src2$$reg));
14707 %}
14708 ins_pipe(vmuldiv_fp128);
14709 %}
14710
14711 instruct vmul2D(vecX dst, vecX src1, vecX src2)
14712 %{
14713 predicate(n->as_Vector()->length() == 2);
14714 match(Set dst (MulVD src1 src2));
14715 ins_cost(INSN_COST);
14716 format %{ "fmul $dst,$src1,$src2\t# vector (2D)" %}
14717 ins_encode %{
14718 __ fmul(as_FloatRegister($dst$$reg), __ T2D,
14719 as_FloatRegister($src1$$reg),
14720 as_FloatRegister($src2$$reg));
14721 %}
14722 ins_pipe(vmuldiv_fp128);
14723 %}
14724
14725 // --------------------------------- MLA --------------------------------------
14726
14727 instruct vmla4S(vecD dst, vecD src1, vecD src2)
14728 %{
14729 predicate(n->as_Vector()->length() == 2 ||
14730 n->as_Vector()->length() == 4);
14731 match(Set dst (AddVS dst (MulVS src1 src2)));
14732 ins_cost(INSN_COST);
14733 format %{ "mlav $dst,$src1,$src2\t# vector (4H)" %}
14734 ins_encode %{
14735 __ mlav(as_FloatRegister($dst$$reg), __ T4H,
14736 as_FloatRegister($src1$$reg),
14737 as_FloatRegister($src2$$reg));
14738 %}
14739 ins_pipe(vmla64);
14740 %}
14741
14742 instruct vmla8S(vecX dst, vecX src1, vecX src2)
14743 %{
14744 predicate(n->as_Vector()->length() == 8);
14745 match(Set dst (AddVS dst (MulVS src1 src2)));
14746 ins_cost(INSN_COST);
14747 format %{ "mlav $dst,$src1,$src2\t# vector (8H)" %}
14748 ins_encode %{
14749 __ mlav(as_FloatRegister($dst$$reg), __ T8H,
14750 as_FloatRegister($src1$$reg),
14751 as_FloatRegister($src2$$reg));
14752 %}
14753 ins_pipe(vmla128);
14754 %}
14755
14756 instruct vmla2I(vecD dst, vecD src1, vecD src2)
14757 %{
14758 predicate(n->as_Vector()->length() == 2);
14759 match(Set dst (AddVI dst (MulVI src1 src2)));
14760 ins_cost(INSN_COST);
14761 format %{ "mlav $dst,$src1,$src2\t# vector (2S)" %}
14762 ins_encode %{
14763 __ mlav(as_FloatRegister($dst$$reg), __ T2S,
14764 as_FloatRegister($src1$$reg),
14765 as_FloatRegister($src2$$reg));
14766 %}
14767 ins_pipe(vmla64);
14768 %}
14769
14770 instruct vmla4I(vecX dst, vecX src1, vecX src2)
14771 %{
14772 predicate(n->as_Vector()->length() == 4);
14773 match(Set dst (AddVI dst (MulVI src1 src2)));
14774 ins_cost(INSN_COST);
14775 format %{ "mlav $dst,$src1,$src2\t# vector (4S)" %}
14776 ins_encode %{
14777 __ mlav(as_FloatRegister($dst$$reg), __ T4S,
14778 as_FloatRegister($src1$$reg),
14779 as_FloatRegister($src2$$reg));
14780 %}
14781 ins_pipe(vmla128);
14782 %}
14783
14784 // --------------------------------- MLS --------------------------------------
14785
14786 instruct vmls4S(vecD dst, vecD src1, vecD src2)
14787 %{
14788 predicate(n->as_Vector()->length() == 2 ||
14789 n->as_Vector()->length() == 4);
14790 match(Set dst (SubVS dst (MulVS src1 src2)));
14791 ins_cost(INSN_COST);
14792 format %{ "mlsv $dst,$src1,$src2\t# vector (4H)" %}
14793 ins_encode %{
14794 __ mlsv(as_FloatRegister($dst$$reg), __ T4H,
14795 as_FloatRegister($src1$$reg),
14796 as_FloatRegister($src2$$reg));
14797 %}
14798 ins_pipe(vmla64);
14799 %}
14800
14801 instruct vmls8S(vecX dst, vecX src1, vecX src2)
14802 %{
14803 predicate(n->as_Vector()->length() == 8);
14804 match(Set dst (SubVS dst (MulVS src1 src2)));
14805 ins_cost(INSN_COST);
14806 format %{ "mlsv $dst,$src1,$src2\t# vector (8H)" %}
14807 ins_encode %{
14808 __ mlsv(as_FloatRegister($dst$$reg), __ T8H,
14809 as_FloatRegister($src1$$reg),
14810 as_FloatRegister($src2$$reg));
14811 %}
14812 ins_pipe(vmla128);
14813 %}
14814
14815 instruct vmls2I(vecD dst, vecD src1, vecD src2)
14816 %{
14817 predicate(n->as_Vector()->length() == 2);
14818 match(Set dst (SubVI dst (MulVI src1 src2)));
14819 ins_cost(INSN_COST);
14820 format %{ "mlsv $dst,$src1,$src2\t# vector (2S)" %}
14821 ins_encode %{
14822 __ mlsv(as_FloatRegister($dst$$reg), __ T2S,
14823 as_FloatRegister($src1$$reg),
14824 as_FloatRegister($src2$$reg));
14825 %}
14826 ins_pipe(vmla64);
14827 %}
14828
14829 instruct vmls4I(vecX dst, vecX src1, vecX src2)
14830 %{
14831 predicate(n->as_Vector()->length() == 4);
14832 match(Set dst (SubVI dst (MulVI src1 src2)));
14833 ins_cost(INSN_COST);
14834 format %{ "mlsv $dst,$src1,$src2\t# vector (4S)" %}
14835 ins_encode %{
14836 __ mlsv(as_FloatRegister($dst$$reg), __ T4S,
14837 as_FloatRegister($src1$$reg),
14838 as_FloatRegister($src2$$reg));
14839 %}
14840 ins_pipe(vmla128);
14841 %}
14842
14843 // --------------------------------- DIV --------------------------------------
14844
14845 instruct vdiv2F(vecD dst, vecD src1, vecD src2)
14846 %{
14847 predicate(n->as_Vector()->length() == 2);
14848 match(Set dst (DivVF src1 src2));
14849 ins_cost(INSN_COST);
14850 format %{ "fdiv $dst,$src1,$src2\t# vector (2S)" %}
14851 ins_encode %{
14852 __ fdiv(as_FloatRegister($dst$$reg), __ T2S,
14853 as_FloatRegister($src1$$reg),
14854 as_FloatRegister($src2$$reg));
14855 %}
14856 ins_pipe(vmuldiv_fp64);
14857 %}
14858
14859 instruct vdiv4F(vecX dst, vecX src1, vecX src2)
14860 %{
14861 predicate(n->as_Vector()->length() == 4);
14862 match(Set dst (DivVF src1 src2));
14863 ins_cost(INSN_COST);
14864 format %{ "fdiv $dst,$src1,$src2\t# vector (4S)" %}
14865 ins_encode %{
14866 __ fdiv(as_FloatRegister($dst$$reg), __ T4S,
14867 as_FloatRegister($src1$$reg),
14868 as_FloatRegister($src2$$reg));
14869 %}
14870 ins_pipe(vmuldiv_fp128);
14871 %}
14872
14873 instruct vdiv2D(vecX dst, vecX src1, vecX src2)
14874 %{
14875 predicate(n->as_Vector()->length() == 2);
14876 match(Set dst (DivVD src1 src2));
14877 ins_cost(INSN_COST);
14878 format %{ "fdiv $dst,$src1,$src2\t# vector (2D)" %}
14879 ins_encode %{
14880 __ fdiv(as_FloatRegister($dst$$reg), __ T2D,
14881 as_FloatRegister($src1$$reg),
14882 as_FloatRegister($src2$$reg));
14883 %}
14884 ins_pipe(vmuldiv_fp128);
14885 %}
14886
14887 // --------------------------------- AND --------------------------------------
14888
14889 instruct vand8B(vecD dst, vecD src1, vecD src2)
14890 %{
14891 predicate(n->as_Vector()->length_in_bytes() == 4 ||
14892 n->as_Vector()->length_in_bytes() == 8);
14893 match(Set dst (AndV src1 src2));
14894 ins_cost(INSN_COST);
14895 format %{ "and $dst,$src1,$src2\t# vector (8B)" %}
14896 ins_encode %{
14897 __ andr(as_FloatRegister($dst$$reg), __ T8B,
14898 as_FloatRegister($src1$$reg),
14899 as_FloatRegister($src2$$reg));
14900 %}
14901 ins_pipe(vlogical64);
14902 %}
14903
14904 instruct vand16B(vecX dst, vecX src1, vecX src2)
14905 %{
14906 predicate(n->as_Vector()->length_in_bytes() == 16);
14907 match(Set dst (AndV src1 src2));
14908 ins_cost(INSN_COST);
14909 format %{ "and $dst,$src1,$src2\t# vector (16B)" %}
14910 ins_encode %{
14911 __ andr(as_FloatRegister($dst$$reg), __ T16B,
14912 as_FloatRegister($src1$$reg),
14913 as_FloatRegister($src2$$reg));
14914 %}
14915 ins_pipe(vlogical128);
14916 %}
14917
14918 // --------------------------------- OR ---------------------------------------
14919
14920 instruct vor8B(vecD dst, vecD src1, vecD src2)
14921 %{
14922 predicate(n->as_Vector()->length_in_bytes() == 4 ||
14923 n->as_Vector()->length_in_bytes() == 8);
14924 match(Set dst (OrV src1 src2));
14925 ins_cost(INSN_COST);
14926 format %{ "and $dst,$src1,$src2\t# vector (8B)" %}
14927 ins_encode %{
14928 __ orr(as_FloatRegister($dst$$reg), __ T8B,
14929 as_FloatRegister($src1$$reg),
14930 as_FloatRegister($src2$$reg));
14931 %}
14932 ins_pipe(vlogical64);
14933 %}
14934
14935 instruct vor16B(vecX dst, vecX src1, vecX src2)
14936 %{
14937 predicate(n->as_Vector()->length_in_bytes() == 16);
14938 match(Set dst (OrV src1 src2));
14939 ins_cost(INSN_COST);
14940 format %{ "orr $dst,$src1,$src2\t# vector (16B)" %}
14941 ins_encode %{
14942 __ orr(as_FloatRegister($dst$$reg), __ T16B,
14943 as_FloatRegister($src1$$reg),
14944 as_FloatRegister($src2$$reg));
14945 %}
14946 ins_pipe(vlogical128);
14947 %}
14948
14949 // --------------------------------- XOR --------------------------------------
14950
14951 instruct vxor8B(vecD dst, vecD src1, vecD src2)
14952 %{
14953 predicate(n->as_Vector()->length_in_bytes() == 4 ||
14954 n->as_Vector()->length_in_bytes() == 8);
14955 match(Set dst (XorV src1 src2));
14956 ins_cost(INSN_COST);
14957 format %{ "xor $dst,$src1,$src2\t# vector (8B)" %}
14958 ins_encode %{
14959 __ eor(as_FloatRegister($dst$$reg), __ T8B,
14960 as_FloatRegister($src1$$reg),
14961 as_FloatRegister($src2$$reg));
14962 %}
14963 ins_pipe(vlogical64);
14964 %}
14965
14966 instruct vxor16B(vecX dst, vecX src1, vecX src2)
14967 %{
14968 predicate(n->as_Vector()->length_in_bytes() == 16);
14969 match(Set dst (XorV src1 src2));
14970 ins_cost(INSN_COST);
14971 format %{ "xor $dst,$src1,$src2\t# vector (16B)" %}
14972 ins_encode %{
14973 __ eor(as_FloatRegister($dst$$reg), __ T16B,
14974 as_FloatRegister($src1$$reg),
14975 as_FloatRegister($src2$$reg));
14976 %}
14977 ins_pipe(vlogical128);
14978 %}
14979
14980 // ------------------------------ Shift ---------------------------------------
14981 instruct vshiftcnt8B(vecD dst, iRegIorL2I cnt) %{
14982 predicate(n->as_Vector()->length_in_bytes() == 8);
14983 match(Set dst (LShiftCntV cnt));
14984 match(Set dst (RShiftCntV cnt));
14985 format %{ "dup $dst, $cnt\t# shift count vector (8B)" %}
14986 ins_encode %{
14987 __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($cnt$$reg));
14988 %}
14989 ins_pipe(vdup_reg_reg64);
14990 %}
14991
14992 instruct vshiftcnt16B(vecX dst, iRegIorL2I cnt) %{
14993 predicate(n->as_Vector()->length_in_bytes() == 16);
14994 match(Set dst (LShiftCntV cnt));
14995 match(Set dst (RShiftCntV cnt));
14996 format %{ "dup $dst, $cnt\t# shift count vector (16B)" %}
14997 ins_encode %{
14998 __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($cnt$$reg));
14999 %}
15000 ins_pipe(vdup_reg_reg128);
15001 %}
15002
15003 instruct vsll8B(vecD dst, vecD src, vecD shift) %{
15004 predicate(n->as_Vector()->length() == 4 ||
15005 n->as_Vector()->length() == 8);
15006 match(Set dst (LShiftVB src shift));
15007 ins_cost(INSN_COST);
15008 format %{ "sshl $dst,$src,$shift\t# vector (8B)" %}
15009 ins_encode %{
15010 __ sshl(as_FloatRegister($dst$$reg), __ T8B,
15011 as_FloatRegister($src$$reg),
15012 as_FloatRegister($shift$$reg));
15013 %}
15014 ins_pipe(vshift64);
15015 %}
15016
15017 instruct vsll16B(vecX dst, vecX src, vecX shift) %{
15018 predicate(n->as_Vector()->length() == 16);
15019 match(Set dst (LShiftVB src shift));
15020 ins_cost(INSN_COST);
15021 format %{ "sshl $dst,$src,$shift\t# vector (16B)" %}
15022 ins_encode %{
15023 __ sshl(as_FloatRegister($dst$$reg), __ T16B,
15024 as_FloatRegister($src$$reg),
15025 as_FloatRegister($shift$$reg));
15026 %}
15027 ins_pipe(vshift128);
15028 %}
15029
15030 // Right shifts with vector shift count on aarch64 SIMD are implemented
15031 // as left shift by negative shift count.
15032 // There are two cases for vector shift count.
15033 //
15034 // Case 1: The vector shift count is from replication.
15035 // | |
15036 // LoadVector RShiftCntV
15037 // | /
15038 // RShiftVI
15039 // Note: In inner loop, multiple neg instructions are used, which can be
15040 // moved to outer loop and merge into one neg instruction.
15041 //
15042 // Case 2: The vector shift count is from loading.
15043 // This case isn't supported by middle-end now. But it's supported by
15044 // panama/vectorIntrinsics(JEP 338: Vector API).
15045 // | |
15046 // LoadVector LoadVector
15047 // | /
15048 // RShiftVI
15049 //
15050
15051 instruct vsra8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
15052 predicate(n->as_Vector()->length() == 4 ||
15053 n->as_Vector()->length() == 8);
15054 match(Set dst (RShiftVB src shift));
15055 ins_cost(INSN_COST);
15056 effect(TEMP tmp);
15057 format %{ "negr $tmp,$shift\t"
15058 "sshl $dst,$src,$tmp\t# vector (8B)" %}
15059 ins_encode %{
15060 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
15061 as_FloatRegister($shift$$reg));
15062 __ sshl(as_FloatRegister($dst$$reg), __ T8B,
15063 as_FloatRegister($src$$reg),
15064 as_FloatRegister($tmp$$reg));
15065 %}
15066 ins_pipe(vshift64);
15067 %}
15068
15069 instruct vsra16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
15070 predicate(n->as_Vector()->length() == 16);
15071 match(Set dst (RShiftVB src shift));
15072 ins_cost(INSN_COST);
15073 effect(TEMP tmp);
15074 format %{ "negr $tmp,$shift\t"
15075 "sshl $dst,$src,$tmp\t# vector (16B)" %}
15076 ins_encode %{
15077 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15078 as_FloatRegister($shift$$reg));
15079 __ sshl(as_FloatRegister($dst$$reg), __ T16B,
15080 as_FloatRegister($src$$reg),
15081 as_FloatRegister($tmp$$reg));
15082 %}
15083 ins_pipe(vshift128);
15084 %}
15085
15086 instruct vsrl8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
15087 predicate(n->as_Vector()->length() == 4 ||
15088 n->as_Vector()->length() == 8);
15089 match(Set dst (URShiftVB src shift));
15090 ins_cost(INSN_COST);
15091 effect(TEMP tmp);
15092 format %{ "negr $tmp,$shift\t"
15093 "ushl $dst,$src,$tmp\t# vector (8B)" %}
15094 ins_encode %{
15095 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
15096 as_FloatRegister($shift$$reg));
15097 __ ushl(as_FloatRegister($dst$$reg), __ T8B,
15098 as_FloatRegister($src$$reg),
15099 as_FloatRegister($tmp$$reg));
15100 %}
15101 ins_pipe(vshift64);
15102 %}
15103
15104 instruct vsrl16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
15105 predicate(n->as_Vector()->length() == 16);
15106 match(Set dst (URShiftVB src shift));
15107 ins_cost(INSN_COST);
15108 effect(TEMP tmp);
15109 format %{ "negr $tmp,$shift\t"
15110 "ushl $dst,$src,$tmp\t# vector (16B)" %}
15111 ins_encode %{
15112 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15113 as_FloatRegister($shift$$reg));
15114 __ ushl(as_FloatRegister($dst$$reg), __ T16B,
15115 as_FloatRegister($src$$reg),
15116 as_FloatRegister($tmp$$reg));
15117 %}
15118 ins_pipe(vshift128);
15119 %}
15120
15121 instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
15122 predicate(n->as_Vector()->length() == 4 ||
15123 n->as_Vector()->length() == 8);
15124 match(Set dst (LShiftVB src shift));
15125 ins_cost(INSN_COST);
15126 format %{ "shl $dst, $src, $shift\t# vector (8B)" %}
15127 ins_encode %{
15128 int sh = (int)$shift$$constant & 31;
15129 if (sh >= 8) {
15130 __ eor(as_FloatRegister($dst$$reg), __ T8B,
15131 as_FloatRegister($src$$reg),
15132 as_FloatRegister($src$$reg));
15133 } else {
15134 __ shl(as_FloatRegister($dst$$reg), __ T8B,
15135 as_FloatRegister($src$$reg), sh);
15136 }
15137 %}
15138 ins_pipe(vshift64_imm);
15139 %}
15140
15141 instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
15142 predicate(n->as_Vector()->length() == 16);
15143 match(Set dst (LShiftVB src shift));
15144 ins_cost(INSN_COST);
15145 format %{ "shl $dst, $src, $shift\t# vector (16B)" %}
15146 ins_encode %{
15147 int sh = (int)$shift$$constant & 31;
15148 if (sh >= 8) {
15149 __ eor(as_FloatRegister($dst$$reg), __ T16B,
15150 as_FloatRegister($src$$reg),
15151 as_FloatRegister($src$$reg));
15152 } else {
15153 __ shl(as_FloatRegister($dst$$reg), __ T16B,
15154 as_FloatRegister($src$$reg), sh);
15155 }
15156 %}
15157 ins_pipe(vshift128_imm);
15158 %}
15159
15160 instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{
15161 predicate(n->as_Vector()->length() == 4 ||
15162 n->as_Vector()->length() == 8);
15163 match(Set dst (RShiftVB src shift));
15164 ins_cost(INSN_COST);
15165 format %{ "sshr $dst, $src, $shift\t# vector (8B)" %}
15166 ins_encode %{
15167 int sh = (int)$shift$$constant & 31;
15168 if (sh >= 8) sh = 7;
15169 sh = -sh & 7;
15170 __ sshr(as_FloatRegister($dst$$reg), __ T8B,
15171 as_FloatRegister($src$$reg), sh);
15172 %}
15173 ins_pipe(vshift64_imm);
15174 %}
15175
15176 instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
15177 predicate(n->as_Vector()->length() == 16);
15178 match(Set dst (RShiftVB src shift));
15179 ins_cost(INSN_COST);
15180 format %{ "sshr $dst, $src, $shift\t# vector (16B)" %}
15181 ins_encode %{
15182 int sh = (int)$shift$$constant & 31;
15183 if (sh >= 8) sh = 7;
15184 sh = -sh & 7;
15185 __ sshr(as_FloatRegister($dst$$reg), __ T16B,
15186 as_FloatRegister($src$$reg), sh);
15187 %}
15188 ins_pipe(vshift128_imm);
15189 %}
15190
15191 instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{
15192 predicate(n->as_Vector()->length() == 4 ||
15193 n->as_Vector()->length() == 8);
15194 match(Set dst (URShiftVB src shift));
15195 ins_cost(INSN_COST);
15196 format %{ "ushr $dst, $src, $shift\t# vector (8B)" %}
15197 ins_encode %{
15198 int sh = (int)$shift$$constant & 31;
15199 if (sh >= 8) {
15200 __ eor(as_FloatRegister($dst$$reg), __ T8B,
15201 as_FloatRegister($src$$reg),
15202 as_FloatRegister($src$$reg));
15203 } else {
15204 __ ushr(as_FloatRegister($dst$$reg), __ T8B,
15205 as_FloatRegister($src$$reg), -sh & 7);
15206 }
15207 %}
15208 ins_pipe(vshift64_imm);
15209 %}
15210
15211 instruct vsrl16B_imm(vecX dst, vecX src, immI shift) %{
15212 predicate(n->as_Vector()->length() == 16);
15213 match(Set dst (URShiftVB src shift));
15214 ins_cost(INSN_COST);
15215 format %{ "ushr $dst, $src, $shift\t# vector (16B)" %}
15216 ins_encode %{
15217 int sh = (int)$shift$$constant & 31;
15218 if (sh >= 8) {
15219 __ eor(as_FloatRegister($dst$$reg), __ T16B,
15220 as_FloatRegister($src$$reg),
15221 as_FloatRegister($src$$reg));
15222 } else {
15223 __ ushr(as_FloatRegister($dst$$reg), __ T16B,
15224 as_FloatRegister($src$$reg), -sh & 7);
15225 }
15226 %}
15227 ins_pipe(vshift128_imm);
15228 %}
15229
15230 instruct vsll4S(vecD dst, vecD src, vecD shift) %{
15231 predicate(n->as_Vector()->length() == 2 ||
15232 n->as_Vector()->length() == 4);
15233 match(Set dst (LShiftVS src shift));
15234 ins_cost(INSN_COST);
15235 format %{ "sshl $dst,$src,$shift\t# vector (4H)" %}
15236 ins_encode %{
15237 __ sshl(as_FloatRegister($dst$$reg), __ T4H,
15238 as_FloatRegister($src$$reg),
15239 as_FloatRegister($shift$$reg));
15240 %}
15241 ins_pipe(vshift64);
15242 %}
15243
15244 instruct vsll8S(vecX dst, vecX src, vecX shift) %{
15245 predicate(n->as_Vector()->length() == 8);
15246 match(Set dst (LShiftVS src shift));
15247 ins_cost(INSN_COST);
15248 format %{ "sshl $dst,$src,$shift\t# vector (8H)" %}
15249 ins_encode %{
15250 __ sshl(as_FloatRegister($dst$$reg), __ T8H,
15251 as_FloatRegister($src$$reg),
15252 as_FloatRegister($shift$$reg));
15253 %}
15254 ins_pipe(vshift128);
15255 %}
15256
15257 instruct vsra4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
15258 predicate(n->as_Vector()->length() == 2 ||
15259 n->as_Vector()->length() == 4);
15260 match(Set dst (RShiftVS src shift));
15261 ins_cost(INSN_COST);
15262 effect(TEMP tmp);
15263 format %{ "negr $tmp,$shift\t"
15264 "sshl $dst,$src,$tmp\t# vector (4H)" %}
15265 ins_encode %{
15266 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
15267 as_FloatRegister($shift$$reg));
15268 __ sshl(as_FloatRegister($dst$$reg), __ T4H,
15269 as_FloatRegister($src$$reg),
15270 as_FloatRegister($tmp$$reg));
15271 %}
15272 ins_pipe(vshift64);
15273 %}
15274
15275 instruct vsra8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
15276 predicate(n->as_Vector()->length() == 8);
15277 match(Set dst (RShiftVS src shift));
15278 ins_cost(INSN_COST);
15279 effect(TEMP tmp);
15280 format %{ "negr $tmp,$shift\t"
15281 "sshl $dst,$src,$tmp\t# vector (8H)" %}
15282 ins_encode %{
15283 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15284 as_FloatRegister($shift$$reg));
15285 __ sshl(as_FloatRegister($dst$$reg), __ T8H,
15286 as_FloatRegister($src$$reg),
15287 as_FloatRegister($tmp$$reg));
15288 %}
15289 ins_pipe(vshift128);
15290 %}
15291
15292 instruct vsrl4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
15293 predicate(n->as_Vector()->length() == 2 ||
15294 n->as_Vector()->length() == 4);
15295 match(Set dst (URShiftVS src shift));
15296 ins_cost(INSN_COST);
15297 effect(TEMP tmp);
15298 format %{ "negr $tmp,$shift\t"
15299 "ushl $dst,$src,$tmp\t# vector (4H)" %}
15300 ins_encode %{
15301 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
15302 as_FloatRegister($shift$$reg));
15303 __ ushl(as_FloatRegister($dst$$reg), __ T4H,
15304 as_FloatRegister($src$$reg),
15305 as_FloatRegister($tmp$$reg));
15306 %}
15307 ins_pipe(vshift64);
15308 %}
15309
15310 instruct vsrl8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
15311 predicate(n->as_Vector()->length() == 8);
15312 match(Set dst (URShiftVS src shift));
15313 ins_cost(INSN_COST);
15314 effect(TEMP tmp);
15315 format %{ "negr $tmp,$shift\t"
15316 "ushl $dst,$src,$tmp\t# vector (8H)" %}
15317 ins_encode %{
15318 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15319 as_FloatRegister($shift$$reg));
15320 __ ushl(as_FloatRegister($dst$$reg), __ T8H,
15321 as_FloatRegister($src$$reg),
15322 as_FloatRegister($tmp$$reg));
15323 %}
15324 ins_pipe(vshift128);
15325 %}
15326
15327 instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
15328 predicate(n->as_Vector()->length() == 2 ||
15329 n->as_Vector()->length() == 4);
15330 match(Set dst (LShiftVS src shift));
15331 ins_cost(INSN_COST);
15332 format %{ "shl $dst, $src, $shift\t# vector (4H)" %}
15333 ins_encode %{
15334 int sh = (int)$shift$$constant & 31;
15335 if (sh >= 16) {
15336 __ eor(as_FloatRegister($dst$$reg), __ T8B,
15337 as_FloatRegister($src$$reg),
15338 as_FloatRegister($src$$reg));
15339 } else {
15340 __ shl(as_FloatRegister($dst$$reg), __ T4H,
15341 as_FloatRegister($src$$reg), sh);
15342 }
15343 %}
15344 ins_pipe(vshift64_imm);
15345 %}
15346
15347 instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
15348 predicate(n->as_Vector()->length() == 8);
15349 match(Set dst (LShiftVS src shift));
15350 ins_cost(INSN_COST);
15351 format %{ "shl $dst, $src, $shift\t# vector (8H)" %}
15352 ins_encode %{
15353 int sh = (int)$shift$$constant & 31;
15354 if (sh >= 16) {
15355 __ eor(as_FloatRegister($dst$$reg), __ T16B,
15356 as_FloatRegister($src$$reg),
15357 as_FloatRegister($src$$reg));
15358 } else {
15359 __ shl(as_FloatRegister($dst$$reg), __ T8H,
15360 as_FloatRegister($src$$reg), sh);
15361 }
15362 %}
15363 ins_pipe(vshift128_imm);
15364 %}
15365
15366 instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{
15367 predicate(n->as_Vector()->length() == 2 ||
15368 n->as_Vector()->length() == 4);
15369 match(Set dst (RShiftVS src shift));
15370 ins_cost(INSN_COST);
15371 format %{ "sshr $dst, $src, $shift\t# vector (4H)" %}
15372 ins_encode %{
15373 int sh = (int)$shift$$constant & 31;
15374 if (sh >= 16) sh = 15;
15375 sh = -sh & 15;
15376 __ sshr(as_FloatRegister($dst$$reg), __ T4H,
15377 as_FloatRegister($src$$reg), sh);
15378 %}
15379 ins_pipe(vshift64_imm);
15380 %}
15381
15382 instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
15383 predicate(n->as_Vector()->length() == 8);
15384 match(Set dst (RShiftVS src shift));
15385 ins_cost(INSN_COST);
15386 format %{ "sshr $dst, $src, $shift\t# vector (8H)" %}
15387 ins_encode %{
15388 int sh = (int)$shift$$constant & 31;
15389 if (sh >= 16) sh = 15;
15390 sh = -sh & 15;
15391 __ sshr(as_FloatRegister($dst$$reg), __ T8H,
15392 as_FloatRegister($src$$reg), sh);
15393 %}
15394 ins_pipe(vshift128_imm);
15395 %}
15396
15397 instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{
15398 predicate(n->as_Vector()->length() == 2 ||
15399 n->as_Vector()->length() == 4);
15400 match(Set dst (URShiftVS src shift));
15401 ins_cost(INSN_COST);
15402 format %{ "ushr $dst, $src, $shift\t# vector (4H)" %}
15403 ins_encode %{
15404 int sh = (int)$shift$$constant & 31;
15405 if (sh >= 16) {
15406 __ eor(as_FloatRegister($dst$$reg), __ T8B,
15407 as_FloatRegister($src$$reg),
15408 as_FloatRegister($src$$reg));
15409 } else {
15410 __ ushr(as_FloatRegister($dst$$reg), __ T4H,
15411 as_FloatRegister($src$$reg), -sh & 15);
15412 }
15413 %}
15414 ins_pipe(vshift64_imm);
15415 %}
15416
15417 instruct vsrl8S_imm(vecX dst, vecX src, immI shift) %{
15418 predicate(n->as_Vector()->length() == 8);
15419 match(Set dst (URShiftVS src shift));
15420 ins_cost(INSN_COST);
15421 format %{ "ushr $dst, $src, $shift\t# vector (8H)" %}
15422 ins_encode %{
15423 int sh = (int)$shift$$constant & 31;
15424 if (sh >= 16) {
15425 __ eor(as_FloatRegister($dst$$reg), __ T16B,
15426 as_FloatRegister($src$$reg),
15427 as_FloatRegister($src$$reg));
15428 } else {
15429 __ ushr(as_FloatRegister($dst$$reg), __ T8H,
15430 as_FloatRegister($src$$reg), -sh & 15);
15431 }
15432 %}
15433 ins_pipe(vshift128_imm);
15434 %}
15435
15436 instruct vsll2I(vecD dst, vecD src, vecD shift) %{
15437 predicate(n->as_Vector()->length() == 2);
15438 match(Set dst (LShiftVI src shift));
15439 ins_cost(INSN_COST);
15440 format %{ "sshl $dst,$src,$shift\t# vector (2S)" %}
15441 ins_encode %{
15442 __ sshl(as_FloatRegister($dst$$reg), __ T2S,
15443 as_FloatRegister($src$$reg),
15444 as_FloatRegister($shift$$reg));
15445 %}
15446 ins_pipe(vshift64);
15447 %}
15448
15449 instruct vsll4I(vecX dst, vecX src, vecX shift) %{
15450 predicate(n->as_Vector()->length() == 4);
15451 match(Set dst (LShiftVI src shift));
15452 ins_cost(INSN_COST);
15453 format %{ "sshl $dst,$src,$shift\t# vector (4S)" %}
15454 ins_encode %{
15455 __ sshl(as_FloatRegister($dst$$reg), __ T4S,
15456 as_FloatRegister($src$$reg),
15457 as_FloatRegister($shift$$reg));
15458 %}
15459 ins_pipe(vshift128);
15460 %}
15461
15462 instruct vsra2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
15463 predicate(n->as_Vector()->length() == 2);
15464 match(Set dst (RShiftVI src shift));
15465 ins_cost(INSN_COST);
15466 effect(TEMP tmp);
15467 format %{ "negr $tmp,$shift\t"
15468 "sshl $dst,$src,$tmp\t# vector (2S)" %}
15469 ins_encode %{
15470 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
15471 as_FloatRegister($shift$$reg));
15472 __ sshl(as_FloatRegister($dst$$reg), __ T2S,
15473 as_FloatRegister($src$$reg),
15474 as_FloatRegister($tmp$$reg));
15475 %}
15476 ins_pipe(vshift64);
15477 %}
15478
15479 instruct vsra4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
15480 predicate(n->as_Vector()->length() == 4);
15481 match(Set dst (RShiftVI src shift));
15482 ins_cost(INSN_COST);
15483 effect(TEMP tmp);
15484 format %{ "negr $tmp,$shift\t"
15485 "sshl $dst,$src,$tmp\t# vector (4S)" %}
15486 ins_encode %{
15487 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15488 as_FloatRegister($shift$$reg));
15489 __ sshl(as_FloatRegister($dst$$reg), __ T4S,
15490 as_FloatRegister($src$$reg),
15491 as_FloatRegister($tmp$$reg));
15492 %}
15493 ins_pipe(vshift128);
15494 %}
15495
15496 instruct vsrl2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
15497 predicate(n->as_Vector()->length() == 2);
15498 match(Set dst (URShiftVI src shift));
15499 ins_cost(INSN_COST);
15500 effect(TEMP tmp);
15501 format %{ "negr $tmp,$shift\t"
15502 "ushl $dst,$src,$tmp\t# vector (2S)" %}
15503 ins_encode %{
15504 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
15505 as_FloatRegister($shift$$reg));
15506 __ ushl(as_FloatRegister($dst$$reg), __ T2S,
15507 as_FloatRegister($src$$reg),
15508 as_FloatRegister($tmp$$reg));
15509 %}
15510 ins_pipe(vshift64);
15511 %}
15512
15513 instruct vsrl4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
15514 predicate(n->as_Vector()->length() == 4);
15515 match(Set dst (URShiftVI src shift));
15516 ins_cost(INSN_COST);
15517 effect(TEMP tmp);
15518 format %{ "negr $tmp,$shift\t"
15519 "ushl $dst,$src,$tmp\t# vector (4S)" %}
15520 ins_encode %{
15521 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15522 as_FloatRegister($shift$$reg));
15523 __ ushl(as_FloatRegister($dst$$reg), __ T4S,
15524 as_FloatRegister($src$$reg),
15525 as_FloatRegister($tmp$$reg));
15526 %}
15527 ins_pipe(vshift128);
15528 %}
15529
15530 instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
15531 predicate(n->as_Vector()->length() == 2);
15532 match(Set dst (LShiftVI src shift));
15533 ins_cost(INSN_COST);
15534 format %{ "shl $dst, $src, $shift\t# vector (2S)" %}
15535 ins_encode %{
15536 __ shl(as_FloatRegister($dst$$reg), __ T2S,
15537 as_FloatRegister($src$$reg),
15538 (int)$shift$$constant & 31);
15539 %}
15540 ins_pipe(vshift64_imm);
15541 %}
15542
15543 instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{
15544 predicate(n->as_Vector()->length() == 4);
15545 match(Set dst (LShiftVI src shift));
15546 ins_cost(INSN_COST);
15547 format %{ "shl $dst, $src, $shift\t# vector (4S)" %}
15548 ins_encode %{
15549 __ shl(as_FloatRegister($dst$$reg), __ T4S,
15550 as_FloatRegister($src$$reg),
15551 (int)$shift$$constant & 31);
15552 %}
15553 ins_pipe(vshift128_imm);
15554 %}
15555
15556 instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{
15557 predicate(n->as_Vector()->length() == 2);
15558 match(Set dst (RShiftVI src shift));
15559 ins_cost(INSN_COST);
15560 format %{ "sshr $dst, $src, $shift\t# vector (2S)" %}
15561 ins_encode %{
15562 __ sshr(as_FloatRegister($dst$$reg), __ T2S,
15563 as_FloatRegister($src$$reg),
15564 -(int)$shift$$constant & 31);
15565 %}
15566 ins_pipe(vshift64_imm);
15567 %}
15568
15569 instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{
15570 predicate(n->as_Vector()->length() == 4);
15571 match(Set dst (RShiftVI src shift));
15572 ins_cost(INSN_COST);
15573 format %{ "sshr $dst, $src, $shift\t# vector (4S)" %}
15574 ins_encode %{
15575 __ sshr(as_FloatRegister($dst$$reg), __ T4S,
15576 as_FloatRegister($src$$reg),
15577 -(int)$shift$$constant & 31);
15578 %}
15579 ins_pipe(vshift128_imm);
15580 %}
15581
15582 instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{
15583 predicate(n->as_Vector()->length() == 2);
15584 match(Set dst (URShiftVI src shift));
15585 ins_cost(INSN_COST);
15586 format %{ "ushr $dst, $src, $shift\t# vector (2S)" %}
15587 ins_encode %{
15588 __ ushr(as_FloatRegister($dst$$reg), __ T2S,
15589 as_FloatRegister($src$$reg),
15590 -(int)$shift$$constant & 31);
15591 %}
15592 ins_pipe(vshift64_imm);
15593 %}
15594
15595 instruct vsrl4I_imm(vecX dst, vecX src, immI shift) %{
15596 predicate(n->as_Vector()->length() == 4);
15597 match(Set dst (URShiftVI src shift));
15598 ins_cost(INSN_COST);
15599 format %{ "ushr $dst, $src, $shift\t# vector (4S)" %}
15600 ins_encode %{
15601 __ ushr(as_FloatRegister($dst$$reg), __ T4S,
15602 as_FloatRegister($src$$reg),
15603 -(int)$shift$$constant & 31);
15604 %}
15605 ins_pipe(vshift128_imm);
15606 %}
15607
15608 instruct vsll2L(vecX dst, vecX src, vecX shift) %{
15609 predicate(n->as_Vector()->length() == 2);
15610 match(Set dst (LShiftVL src shift));
15611 ins_cost(INSN_COST);
15612 format %{ "sshl $dst,$src,$shift\t# vector (2D)" %}
15613 ins_encode %{
15614 __ sshl(as_FloatRegister($dst$$reg), __ T2D,
15615 as_FloatRegister($src$$reg),
15616 as_FloatRegister($shift$$reg));
15617 %}
15618 ins_pipe(vshift128);
15619 %}
15620
15621 instruct vsra2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
15622 predicate(n->as_Vector()->length() == 2);
15623 match(Set dst (RShiftVL src shift));
15624 ins_cost(INSN_COST);
15625 effect(TEMP tmp);
15626 format %{ "negr $tmp,$shift\t"
15627 "sshl $dst,$src,$tmp\t# vector (2D)" %}
15628 ins_encode %{
15629 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15630 as_FloatRegister($shift$$reg));
15631 __ sshl(as_FloatRegister($dst$$reg), __ T2D,
15632 as_FloatRegister($src$$reg),
15633 as_FloatRegister($tmp$$reg));
15634 %}
15635 ins_pipe(vshift128);
15636 %}
15637
15638 instruct vsrl2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
15639 predicate(n->as_Vector()->length() == 2);
15640 match(Set dst (URShiftVL src shift));
15641 ins_cost(INSN_COST);
15642 effect(TEMP tmp);
15643 format %{ "negr $tmp,$shift\t"
15644 "ushl $dst,$src,$tmp\t# vector (2D)" %}
15645 ins_encode %{
15646 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15647 as_FloatRegister($shift$$reg));
15648 __ ushl(as_FloatRegister($dst$$reg), __ T2D,
15649 as_FloatRegister($src$$reg),
15650 as_FloatRegister($tmp$$reg));
15651 %}
15652 ins_pipe(vshift128);
15653 %}
15654
15655 instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
15656 predicate(n->as_Vector()->length() == 2);
15657 match(Set dst (LShiftVL src shift));
15658 ins_cost(INSN_COST);
15659 format %{ "shl $dst, $src, $shift\t# vector (2D)" %}
15660 ins_encode %{
15661 __ shl(as_FloatRegister($dst$$reg), __ T2D,
15662 as_FloatRegister($src$$reg),
15663 (int)$shift$$constant & 63);
15664 %}
15665 ins_pipe(vshift128_imm);
15666 %}
15667
15668 instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{
15669 predicate(n->as_Vector()->length() == 2);
15670 match(Set dst (RShiftVL src shift));
15671 ins_cost(INSN_COST);
15672 format %{ "sshr $dst, $src, $shift\t# vector (2D)" %}
15673 ins_encode %{
15674 __ sshr(as_FloatRegister($dst$$reg), __ T2D,
15675 as_FloatRegister($src$$reg),
15676 -(int)$shift$$constant & 63);
15677 %}
15678 ins_pipe(vshift128_imm);
15679 %}
15680
15681 instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{
15682 predicate(n->as_Vector()->length() == 2);
15683 match(Set dst (URShiftVL src shift));
15684 ins_cost(INSN_COST);
15685 format %{ "ushr $dst, $src, $shift\t# vector (2D)" %}
15686 ins_encode %{
15687 __ ushr(as_FloatRegister($dst$$reg), __ T2D,
15688 as_FloatRegister($src$$reg),
15689 -(int)$shift$$constant & 63);
15690 %}
15691 ins_pipe(vshift128_imm);
15692 %}
15693
15694 //----------PEEPHOLE RULES-----------------------------------------------------
15695 // These must follow all instruction definitions as they use the names
15696 // defined in the instructions definitions.
15697 //
15698 // peepmatch ( root_instr_name [preceding_instruction]* );
15699 //
15700 // peepconstraint %{
15701 // (instruction_number.operand_name relational_op instruction_number.operand_name
15702 // [, ...] );
15703 // // instruction numbers are zero-based using left to right order in peepmatch
15704 //
15705 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
15706 // // provide an instruction_number.operand_name for each operand that appears
15707 // // in the replacement instruction's match rule
15708 //
15709 // ---------VM FLAGS---------------------------------------------------------
15710 //
15711 // All peephole optimizations can be turned off using -XX:-OptoPeephole
15712 //
15713 // Each peephole rule is given an identifying number starting with zero and
15714 // increasing by one in the order seen by the parser. An individual peephole
15715 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
15716 // on the command-line.
15717 //
15718 // ---------CURRENT LIMITATIONS----------------------------------------------
15719 //
15720 // Only match adjacent instructions in same basic block
15721 // Only equality constraints
15722 // Only constraints between operands, not (0.dest_reg == RAX_enc)
15723 // Only one replacement instruction
15724 //
15725 // ---------EXAMPLE----------------------------------------------------------
15726 //
15727 // // pertinent parts of existing instructions in architecture description
15728 // instruct movI(iRegINoSp dst, iRegI src)
15729 // %{
15730 // match(Set dst (CopyI src));
15731 // %}
15732 //
15733 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
15734 // %{
15735 // match(Set dst (AddI dst src));
15736 // effect(KILL cr);
15737 // %}
15738 //
15739 // // Change (inc mov) to lea
15740 // peephole %{
15741 // // increment preceeded by register-register move
15742 // peepmatch ( incI_iReg movI );
15743 // // require that the destination register of the increment
15744 // // match the destination register of the move
15745 // peepconstraint ( 0.dst == 1.dst );
15746 // // construct a replacement instruction that sets
15747 // // the destination to ( move's source register + one )
15748 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
15749 // %}
15750 //
15751
15752 // Implementation no longer uses movX instructions since
15753 // machine-independent system no longer uses CopyX nodes.
15754 //
15755 // peephole
15756 // %{
15757 // peepmatch (incI_iReg movI);
15758 // peepconstraint (0.dst == 1.dst);
15759 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
15760 // %}
15761
15762 // peephole
15763 // %{
15764 // peepmatch (decI_iReg movI);
15765 // peepconstraint (0.dst == 1.dst);
15766 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
15767 // %}
15768
15769 // peephole
15770 // %{
15771 // peepmatch (addI_iReg_imm movI);
15772 // peepconstraint (0.dst == 1.dst);
15773 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
15774 // %}
15775
15776 // peephole
15777 // %{
15778 // peepmatch (incL_iReg movL);
15779 // peepconstraint (0.dst == 1.dst);
15780 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
15781 // %}
15782
15783 // peephole
15784 // %{
15785 // peepmatch (decL_iReg movL);
15786 // peepconstraint (0.dst == 1.dst);
15787 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
15788 // %}
15789
15790 // peephole
15791 // %{
15792 // peepmatch (addL_iReg_imm movL);
15793 // peepconstraint (0.dst == 1.dst);
15794 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
15795 // %}
15796
15797 // peephole
15798 // %{
15799 // peepmatch (addP_iReg_imm movP);
15800 // peepconstraint (0.dst == 1.dst);
15801 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
15802 // %}
15803
15804 // // Change load of spilled value to only a spill
15805 // instruct storeI(memory mem, iRegI src)
15806 // %{
15807 // match(Set mem (StoreI mem src));
15808 // %}
15809 //
15810 // instruct loadI(iRegINoSp dst, memory mem)
15811 // %{
15812 // match(Set dst (LoadI mem));
15813 // %}
15814 //
15815
15816 //----------SMARTSPILL RULES---------------------------------------------------
15817 // These must follow all instruction definitions as they use the names
15818 // defined in the instructions definitions.
15819
15820 // Local Variables:
15821 // mode: c++
15822 // End: