1 //
2 // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2019, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // archtecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 invisible to the allocator (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
98 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
99 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
100 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
101 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
102 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
103 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
104 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
105 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
106 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
107 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
108 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
109 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
110 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
111 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
112 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
113 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18->as_VMReg() );
114 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
115 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
116 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
117 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
118 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
119 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
120 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
121 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
122 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
123 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
124 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
125 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
126 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
127 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
128 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
129 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
130 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
131 reg_def R27 ( NS, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
132 reg_def R27_H ( NS, SOE, Op_RegI, 27, r27->as_VMReg()->next());
133 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
134 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
135 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
136 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
137 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
138 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
139 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
140 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
141
142 // ----------------------------
143 // Float/Double Registers
144 // ----------------------------
145
146 // Double Registers
147
148 // The rules of ADL require that double registers be defined in pairs.
149 // Each pair must be two 32-bit values, but not necessarily a pair of
150 // single float registers. In each pair, ADLC-assigned register numbers
151 // must be adjacent, with the lower number even. Finally, when the
152 // CPU stores such a register pair to memory, the word associated with
153 // the lower ADLC-assigned number must be stored to the lower address.
154
155 // AArch64 has 32 floating-point registers. Each can store a vector of
156 // single or double precision floating-point values up to 8 * 32
157 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
158 // use the first float or double element of the vector.
159
160 // for Java use float registers v0-v15 are always save on call whereas
161 // the platform ABI treats v8-v15 as callee save). float registers
162 // v16-v31 are SOC as per the platform spec
163
164 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
165 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
166 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
167 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
168
169 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
170 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
171 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
172 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
173
174 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
175 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
176 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
177 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
178
179 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
180 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
181 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
182 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
183
184 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
185 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
186 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
187 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
188
189 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
190 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
191 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
192 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
193
194 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
195 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
196 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
197 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
198
199 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
200 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
201 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
202 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
203
204 reg_def V8 ( SOC, SOC, Op_RegF, 8, v8->as_VMReg() );
205 reg_def V8_H ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next() );
206 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
207 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
208
209 reg_def V9 ( SOC, SOC, Op_RegF, 9, v9->as_VMReg() );
210 reg_def V9_H ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next() );
211 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
212 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
213
214 reg_def V10 ( SOC, SOC, Op_RegF, 10, v10->as_VMReg() );
215 reg_def V10_H( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next() );
216 reg_def V10_J( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2));
217 reg_def V10_K( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3));
218
219 reg_def V11 ( SOC, SOC, Op_RegF, 11, v11->as_VMReg() );
220 reg_def V11_H( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next() );
221 reg_def V11_J( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2));
222 reg_def V11_K( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3));
223
224 reg_def V12 ( SOC, SOC, Op_RegF, 12, v12->as_VMReg() );
225 reg_def V12_H( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next() );
226 reg_def V12_J( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2));
227 reg_def V12_K( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3));
228
229 reg_def V13 ( SOC, SOC, Op_RegF, 13, v13->as_VMReg() );
230 reg_def V13_H( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next() );
231 reg_def V13_J( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2));
232 reg_def V13_K( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3));
233
234 reg_def V14 ( SOC, SOC, Op_RegF, 14, v14->as_VMReg() );
235 reg_def V14_H( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next() );
236 reg_def V14_J( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2));
237 reg_def V14_K( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3));
238
239 reg_def V15 ( SOC, SOC, Op_RegF, 15, v15->as_VMReg() );
240 reg_def V15_H( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next() );
241 reg_def V15_J( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2));
242 reg_def V15_K( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3));
243
244 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
245 reg_def V16_H( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
246 reg_def V16_J( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2));
247 reg_def V16_K( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3));
248
249 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
250 reg_def V17_H( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
251 reg_def V17_J( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2));
252 reg_def V17_K( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3));
253
254 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
255 reg_def V18_H( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
256 reg_def V18_J( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2));
257 reg_def V18_K( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3));
258
259 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
260 reg_def V19_H( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
261 reg_def V19_J( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2));
262 reg_def V19_K( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3));
263
264 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
265 reg_def V20_H( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
266 reg_def V20_J( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2));
267 reg_def V20_K( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3));
268
269 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
270 reg_def V21_H( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
271 reg_def V21_J( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2));
272 reg_def V21_K( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3));
273
274 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
275 reg_def V22_H( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
276 reg_def V22_J( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2));
277 reg_def V22_K( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3));
278
279 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
280 reg_def V23_H( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
281 reg_def V23_J( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2));
282 reg_def V23_K( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3));
283
284 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
285 reg_def V24_H( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
286 reg_def V24_J( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2));
287 reg_def V24_K( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3));
288
289 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
290 reg_def V25_H( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
291 reg_def V25_J( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2));
292 reg_def V25_K( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3));
293
294 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
295 reg_def V26_H( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
296 reg_def V26_J( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2));
297 reg_def V26_K( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3));
298
299 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
300 reg_def V27_H( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
301 reg_def V27_J( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2));
302 reg_def V27_K( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3));
303
304 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
305 reg_def V28_H( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
306 reg_def V28_J( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2));
307 reg_def V28_K( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3));
308
309 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
310 reg_def V29_H( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
311 reg_def V29_J( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2));
312 reg_def V29_K( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3));
313
314 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
315 reg_def V30_H( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
316 reg_def V30_J( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2));
317 reg_def V30_K( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3));
318
319 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
320 reg_def V31_H( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
321 reg_def V31_J( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2));
322 reg_def V31_K( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3));
323
324 // ----------------------------
325 // Special Registers
326 // ----------------------------
327
328 // the AArch64 CSPR status flag register is not directly acessible as
329 // instruction operand. the FPSR status flag register is a system
330 // register which can be written/read using MSR/MRS but again does not
331 // appear as an operand (a code identifying the FSPR occurs as an
332 // immediate value in the instruction).
333
334 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
335
336
337 // Specify priority of register selection within phases of register
338 // allocation. Highest priority is first. A useful heuristic is to
339 // give registers a low priority when they are required by machine
340 // instructions, like EAX and EDX on I486, and choose no-save registers
341 // before save-on-call, & save-on-call before save-on-entry. Registers
342 // which participate in fixed calling sequences should come last.
343 // Registers which are used as pairs must fall on an even boundary.
344
345 alloc_class chunk0(
346 // volatiles
347 R10, R10_H,
348 R11, R11_H,
349 R12, R12_H,
350 R13, R13_H,
351 R14, R14_H,
352 R15, R15_H,
353 R16, R16_H,
354 R17, R17_H,
355 R18, R18_H,
356
357 // arg registers
358 R0, R0_H,
359 R1, R1_H,
360 R2, R2_H,
361 R3, R3_H,
362 R4, R4_H,
363 R5, R5_H,
364 R6, R6_H,
365 R7, R7_H,
366
367 // non-volatiles
368 R19, R19_H,
369 R20, R20_H,
370 R21, R21_H,
371 R22, R22_H,
372 R23, R23_H,
373 R24, R24_H,
374 R25, R25_H,
375 R26, R26_H,
376
377 // non-allocatable registers
378
379 R27, R27_H, // heapbase
380 R28, R28_H, // thread
381 R29, R29_H, // fp
382 R30, R30_H, // lr
383 R31, R31_H, // sp
384 );
385
386 alloc_class chunk1(
387
388 // no save
389 V16, V16_H, V16_J, V16_K,
390 V17, V17_H, V17_J, V17_K,
391 V18, V18_H, V18_J, V18_K,
392 V19, V19_H, V19_J, V19_K,
393 V20, V20_H, V20_J, V20_K,
394 V21, V21_H, V21_J, V21_K,
395 V22, V22_H, V22_J, V22_K,
396 V23, V23_H, V23_J, V23_K,
397 V24, V24_H, V24_J, V24_K,
398 V25, V25_H, V25_J, V25_K,
399 V26, V26_H, V26_J, V26_K,
400 V27, V27_H, V27_J, V27_K,
401 V28, V28_H, V28_J, V28_K,
402 V29, V29_H, V29_J, V29_K,
403 V30, V30_H, V30_J, V30_K,
404 V31, V31_H, V31_J, V31_K,
405
406 // arg registers
407 V0, V0_H, V0_J, V0_K,
408 V1, V1_H, V1_J, V1_K,
409 V2, V2_H, V2_J, V2_K,
410 V3, V3_H, V3_J, V3_K,
411 V4, V4_H, V4_J, V4_K,
412 V5, V5_H, V5_J, V5_K,
413 V6, V6_H, V6_J, V6_K,
414 V7, V7_H, V7_J, V7_K,
415
416 // non-volatiles
417 V8, V8_H, V8_J, V8_K,
418 V9, V9_H, V9_J, V9_K,
419 V10, V10_H, V10_J, V10_K,
420 V11, V11_H, V11_J, V11_K,
421 V12, V12_H, V12_J, V12_K,
422 V13, V13_H, V13_J, V13_K,
423 V14, V14_H, V14_J, V14_K,
424 V15, V15_H, V15_J, V15_K,
425 );
426
427 alloc_class chunk2(RFLAGS);
428
429 //----------Architecture Description Register Classes--------------------------
430 // Several register classes are automatically defined based upon information in
431 // this architecture description.
432 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
433 // 2) reg_class compiler_method_oop_reg ( /* as def'd in frame section */ )
434 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
435 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
436 //
437
438 // Class for all 32 bit integer registers -- excludes SP which will
439 // never be used as an integer register
440 reg_class any_reg32(
441 R0,
442 R1,
443 R2,
444 R3,
445 R4,
446 R5,
447 R6,
448 R7,
449 R10,
450 R11,
451 R12,
452 R13,
453 R14,
454 R15,
455 R16,
456 R17,
457 R18,
458 R19,
459 R20,
460 R21,
461 R22,
462 R23,
463 R24,
464 R25,
465 R26,
466 R27,
467 R28,
468 R29,
469 R30
470 );
471
472 // Singleton class for R0 int register
473 reg_class int_r0_reg(R0);
474
475 // Singleton class for R2 int register
476 reg_class int_r2_reg(R2);
477
478 // Singleton class for R3 int register
479 reg_class int_r3_reg(R3);
480
481 // Singleton class for R4 int register
482 reg_class int_r4_reg(R4);
483
484 // Class for all long integer registers (including RSP)
485 reg_class any_reg(
486 R0, R0_H,
487 R1, R1_H,
488 R2, R2_H,
489 R3, R3_H,
490 R4, R4_H,
491 R5, R5_H,
492 R6, R6_H,
493 R7, R7_H,
494 R10, R10_H,
495 R11, R11_H,
496 R12, R12_H,
497 R13, R13_H,
498 R14, R14_H,
499 R15, R15_H,
500 R16, R16_H,
501 R17, R17_H,
502 R18, R18_H,
503 R19, R19_H,
504 R20, R20_H,
505 R21, R21_H,
506 R22, R22_H,
507 R23, R23_H,
508 R24, R24_H,
509 R25, R25_H,
510 R26, R26_H,
511 R27, R27_H,
512 R28, R28_H,
513 R29, R29_H,
514 R30, R30_H,
515 R31, R31_H
516 );
517
518 // Class for all non-special integer registers
519 reg_class no_special_reg32_no_fp(
520 R0,
521 R1,
522 R2,
523 R3,
524 R4,
525 R5,
526 R6,
527 R7,
528 R10,
529 R11,
530 R12, // rmethod
531 R13,
532 R14,
533 R15,
534 R16,
535 R17,
536 R18,
537 R19,
538 R20,
539 R21,
540 R22,
541 R23,
542 R24,
543 R25,
544 R26
545 /* R27, */ // heapbase
546 /* R28, */ // thread
547 /* R29, */ // fp
548 /* R30, */ // lr
549 /* R31 */ // sp
550 );
551
552 reg_class no_special_reg32_with_fp(
553 R0,
554 R1,
555 R2,
556 R3,
557 R4,
558 R5,
559 R6,
560 R7,
561 R10,
562 R11,
563 R12, // rmethod
564 R13,
565 R14,
566 R15,
567 R16,
568 R17,
569 R18,
570 R19,
571 R20,
572 R21,
573 R22,
574 R23,
575 R24,
576 R25,
577 R26
578 /* R27, */ // heapbase
579 /* R28, */ // thread
580 R29, // fp
581 /* R30, */ // lr
582 /* R31 */ // sp
583 );
584
585 reg_class_dynamic no_special_reg32(no_special_reg32_no_fp, no_special_reg32_with_fp, %{ PreserveFramePointer %});
586
587 // Class for all non-special long integer registers
588 reg_class no_special_reg_no_fp(
589 R0, R0_H,
590 R1, R1_H,
591 R2, R2_H,
592 R3, R3_H,
593 R4, R4_H,
594 R5, R5_H,
595 R6, R6_H,
596 R7, R7_H,
597 R10, R10_H,
598 R11, R11_H,
599 R12, R12_H, // rmethod
600 R13, R13_H,
601 R14, R14_H,
602 R15, R15_H,
603 R16, R16_H,
604 R17, R17_H,
605 R18, R18_H,
606 R19, R19_H,
607 R20, R20_H,
608 R21, R21_H,
609 R22, R22_H,
610 R23, R23_H,
611 R24, R24_H,
612 R25, R25_H,
613 R26, R26_H,
614 /* R27, R27_H, */ // heapbase
615 /* R28, R28_H, */ // thread
616 /* R29, R29_H, */ // fp
617 /* R30, R30_H, */ // lr
618 /* R31, R31_H */ // sp
619 );
620
621 reg_class no_special_reg_with_fp(
622 R0, R0_H,
623 R1, R1_H,
624 R2, R2_H,
625 R3, R3_H,
626 R4, R4_H,
627 R5, R5_H,
628 R6, R6_H,
629 R7, R7_H,
630 R10, R10_H,
631 R11, R11_H,
632 R12, R12_H, // rmethod
633 R13, R13_H,
634 R14, R14_H,
635 R15, R15_H,
636 R16, R16_H,
637 R17, R17_H,
638 R18, R18_H,
639 R19, R19_H,
640 R20, R20_H,
641 R21, R21_H,
642 R22, R22_H,
643 R23, R23_H,
644 R24, R24_H,
645 R25, R25_H,
646 R26, R26_H,
647 /* R27, R27_H, */ // heapbase
648 /* R28, R28_H, */ // thread
649 R29, R29_H, // fp
650 /* R30, R30_H, */ // lr
651 /* R31, R31_H */ // sp
652 );
653
654 reg_class_dynamic no_special_reg(no_special_reg_no_fp, no_special_reg_with_fp, %{ PreserveFramePointer %});
655
656 // Class for 64 bit register r0
657 reg_class r0_reg(
658 R0, R0_H
659 );
660
661 // Class for 64 bit register r1
662 reg_class r1_reg(
663 R1, R1_H
664 );
665
666 // Class for 64 bit register r2
667 reg_class r2_reg(
668 R2, R2_H
669 );
670
671 // Class for 64 bit register r3
672 reg_class r3_reg(
673 R3, R3_H
674 );
675
676 // Class for 64 bit register r4
677 reg_class r4_reg(
678 R4, R4_H
679 );
680
681 // Class for 64 bit register r5
682 reg_class r5_reg(
683 R5, R5_H
684 );
685
686 // Class for 64 bit register r10
687 reg_class r10_reg(
688 R10, R10_H
689 );
690
691 // Class for 64 bit register r11
692 reg_class r11_reg(
693 R11, R11_H
694 );
695
696 // Class for method register
697 reg_class method_reg(
698 R12, R12_H
699 );
700
701 // Class for heapbase register
702 reg_class heapbase_reg(
703 R27, R27_H
704 );
705
706 // Class for thread register
707 reg_class thread_reg(
708 R28, R28_H
709 );
710
711 // Class for frame pointer register
712 reg_class fp_reg(
713 R29, R29_H
714 );
715
716 // Class for link register
717 reg_class lr_reg(
718 R30, R30_H
719 );
720
721 // Class for long sp register
722 reg_class sp_reg(
723 R31, R31_H
724 );
725
726 // Class for all pointer registers
727 reg_class ptr_reg(
728 R0, R0_H,
729 R1, R1_H,
730 R2, R2_H,
731 R3, R3_H,
732 R4, R4_H,
733 R5, R5_H,
734 R6, R6_H,
735 R7, R7_H,
736 R10, R10_H,
737 R11, R11_H,
738 R12, R12_H,
739 R13, R13_H,
740 R14, R14_H,
741 R15, R15_H,
742 R16, R16_H,
743 R17, R17_H,
744 R18, R18_H,
745 R19, R19_H,
746 R20, R20_H,
747 R21, R21_H,
748 R22, R22_H,
749 R23, R23_H,
750 R24, R24_H,
751 R25, R25_H,
752 R26, R26_H,
753 R27, R27_H,
754 R28, R28_H,
755 R29, R29_H,
756 R30, R30_H,
757 R31, R31_H
758 );
759
760 // Class for all non_special pointer registers
761 reg_class no_special_ptr_reg(
762 R0, R0_H,
763 R1, R1_H,
764 R2, R2_H,
765 R3, R3_H,
766 R4, R4_H,
767 R5, R5_H,
768 R6, R6_H,
769 R7, R7_H,
770 R10, R10_H,
771 R11, R11_H,
772 R12, R12_H,
773 R13, R13_H,
774 R14, R14_H,
775 R15, R15_H,
776 R16, R16_H,
777 R17, R17_H,
778 R18, R18_H,
779 R19, R19_H,
780 R20, R20_H,
781 R21, R21_H,
782 R22, R22_H,
783 R23, R23_H,
784 R24, R24_H,
785 R25, R25_H,
786 R26, R26_H,
787 /* R27, R27_H, */ // heapbase
788 /* R28, R28_H, */ // thread
789 /* R29, R29_H, */ // fp
790 /* R30, R30_H, */ // lr
791 /* R31, R31_H */ // sp
792 );
793
794 // Class for all float registers
795 reg_class float_reg(
796 V0,
797 V1,
798 V2,
799 V3,
800 V4,
801 V5,
802 V6,
803 V7,
804 V8,
805 V9,
806 V10,
807 V11,
808 V12,
809 V13,
810 V14,
811 V15,
812 V16,
813 V17,
814 V18,
815 V19,
816 V20,
817 V21,
818 V22,
819 V23,
820 V24,
821 V25,
822 V26,
823 V27,
824 V28,
825 V29,
826 V30,
827 V31
828 );
829
830 // Double precision float registers have virtual `high halves' that
831 // are needed by the allocator.
832 // Class for all double registers
833 reg_class double_reg(
834 V0, V0_H,
835 V1, V1_H,
836 V2, V2_H,
837 V3, V3_H,
838 V4, V4_H,
839 V5, V5_H,
840 V6, V6_H,
841 V7, V7_H,
842 V8, V8_H,
843 V9, V9_H,
844 V10, V10_H,
845 V11, V11_H,
846 V12, V12_H,
847 V13, V13_H,
848 V14, V14_H,
849 V15, V15_H,
850 V16, V16_H,
851 V17, V17_H,
852 V18, V18_H,
853 V19, V19_H,
854 V20, V20_H,
855 V21, V21_H,
856 V22, V22_H,
857 V23, V23_H,
858 V24, V24_H,
859 V25, V25_H,
860 V26, V26_H,
861 V27, V27_H,
862 V28, V28_H,
863 V29, V29_H,
864 V30, V30_H,
865 V31, V31_H
866 );
867
868 // Class for all 64bit vector registers
869 reg_class vectord_reg(
870 V0, V0_H,
871 V1, V1_H,
872 V2, V2_H,
873 V3, V3_H,
874 V4, V4_H,
875 V5, V5_H,
876 V6, V6_H,
877 V7, V7_H,
878 V8, V8_H,
879 V9, V9_H,
880 V10, V10_H,
881 V11, V11_H,
882 V12, V12_H,
883 V13, V13_H,
884 V14, V14_H,
885 V15, V15_H,
886 V16, V16_H,
887 V17, V17_H,
888 V18, V18_H,
889 V19, V19_H,
890 V20, V20_H,
891 V21, V21_H,
892 V22, V22_H,
893 V23, V23_H,
894 V24, V24_H,
895 V25, V25_H,
896 V26, V26_H,
897 V27, V27_H,
898 V28, V28_H,
899 V29, V29_H,
900 V30, V30_H,
901 V31, V31_H
902 );
903
904 // Class for all 128bit vector registers
905 reg_class vectorx_reg(
906 V0, V0_H, V0_J, V0_K,
907 V1, V1_H, V1_J, V1_K,
908 V2, V2_H, V2_J, V2_K,
909 V3, V3_H, V3_J, V3_K,
910 V4, V4_H, V4_J, V4_K,
911 V5, V5_H, V5_J, V5_K,
912 V6, V6_H, V6_J, V6_K,
913 V7, V7_H, V7_J, V7_K,
914 V8, V8_H, V8_J, V8_K,
915 V9, V9_H, V9_J, V9_K,
916 V10, V10_H, V10_J, V10_K,
917 V11, V11_H, V11_J, V11_K,
918 V12, V12_H, V12_J, V12_K,
919 V13, V13_H, V13_J, V13_K,
920 V14, V14_H, V14_J, V14_K,
921 V15, V15_H, V15_J, V15_K,
922 V16, V16_H, V16_J, V16_K,
923 V17, V17_H, V17_J, V17_K,
924 V18, V18_H, V18_J, V18_K,
925 V19, V19_H, V19_J, V19_K,
926 V20, V20_H, V20_J, V20_K,
927 V21, V21_H, V21_J, V21_K,
928 V22, V22_H, V22_J, V22_K,
929 V23, V23_H, V23_J, V23_K,
930 V24, V24_H, V24_J, V24_K,
931 V25, V25_H, V25_J, V25_K,
932 V26, V26_H, V26_J, V26_K,
933 V27, V27_H, V27_J, V27_K,
934 V28, V28_H, V28_J, V28_K,
935 V29, V29_H, V29_J, V29_K,
936 V30, V30_H, V30_J, V30_K,
937 V31, V31_H, V31_J, V31_K
938 );
939
940 // Class for 128 bit register v0
941 reg_class v0_reg(
942 V0, V0_H
943 );
944
945 // Class for 128 bit register v1
946 reg_class v1_reg(
947 V1, V1_H
948 );
949
950 // Class for 128 bit register v2
951 reg_class v2_reg(
952 V2, V2_H
953 );
954
955 // Class for 128 bit register v3
956 reg_class v3_reg(
957 V3, V3_H
958 );
959
960 // Singleton class for condition codes
961 reg_class int_flags(RFLAGS);
962
963 %}
964
965 //----------DEFINITION BLOCK---------------------------------------------------
966 // Define name --> value mappings to inform the ADLC of an integer valued name
967 // Current support includes integer values in the range [0, 0x7FFFFFFF]
968 // Format:
969 // int_def <name> ( <int_value>, <expression>);
970 // Generated Code in ad_<arch>.hpp
971 // #define <name> (<expression>)
972 // // value == <int_value>
973 // Generated code in ad_<arch>.cpp adlc_verification()
974 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
975 //
976
977 // we follow the ppc-aix port in using a simple cost model which ranks
978 // register operations as cheap, memory ops as more expensive and
979 // branches as most expensive. the first two have a low as well as a
980 // normal cost. huge cost appears to be a way of saying don't do
981 // something
982
983 definitions %{
984 // The default cost (of a register move instruction).
985 int_def INSN_COST ( 100, 100);
986 int_def BRANCH_COST ( 200, 2 * INSN_COST);
987 int_def CALL_COST ( 200, 2 * INSN_COST);
988 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
989 %}
990
991
992 //----------SOURCE BLOCK-------------------------------------------------------
993 // This is a block of C++ code which provides values, functions, and
994 // definitions necessary in the rest of the architecture description
995
996 source_hpp %{
997
998 #include "asm/macroAssembler.hpp"
999 #include "gc/shared/cardTable.hpp"
1000 #include "gc/shared/cardTableBarrierSet.hpp"
1001 #include "gc/shared/collectedHeap.hpp"
1002 #include "opto/addnode.hpp"
1003
1004 class CallStubImpl {
1005
1006 //--------------------------------------------------------------
1007 //---< Used for optimization in Compile::shorten_branches >---
1008 //--------------------------------------------------------------
1009
1010 public:
1011 // Size of call trampoline stub.
1012 static uint size_call_trampoline() {
1013 return 0; // no call trampolines on this platform
1014 }
1015
1016 // number of relocations needed by a call trampoline stub
1017 static uint reloc_call_trampoline() {
1018 return 0; // no call trampolines on this platform
1019 }
1020 };
1021
1022 class HandlerImpl {
1023
1024 public:
1025
1026 static int emit_exception_handler(CodeBuffer &cbuf);
1027 static int emit_deopt_handler(CodeBuffer& cbuf);
1028
1029 static uint size_exception_handler() {
1030 return MacroAssembler::far_branch_size();
1031 }
1032
1033 static uint size_deopt_handler() {
1034 // count one adr and one far branch instruction
1035 return 4 * NativeInstruction::instruction_size;
1036 }
1037 };
1038
1039 bool is_CAS(int opcode, bool maybe_volatile);
1040
1041 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1042
1043 bool unnecessary_acquire(const Node *barrier);
1044 bool needs_acquiring_load(const Node *load);
1045
1046 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1047
1048 bool unnecessary_release(const Node *barrier);
1049 bool unnecessary_volatile(const Node *barrier);
1050 bool needs_releasing_store(const Node *store);
1051
1052 // predicate controlling translation of CompareAndSwapX
1053 bool needs_acquiring_load_exclusive(const Node *load);
1054
1055 // predicate controlling translation of StoreCM
1056 bool unnecessary_storestore(const Node *storecm);
1057
1058 // predicate controlling addressing modes
1059 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1060 %}
1061
1062 source %{
1063
1064 // Optimizaton of volatile gets and puts
1065 // -------------------------------------
1066 //
1067 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1068 // use to implement volatile reads and writes. For a volatile read
1069 // we simply need
1070 //
1071 // ldar<x>
1072 //
1073 // and for a volatile write we need
1074 //
1075 // stlr<x>
1076 //
1077 // Alternatively, we can implement them by pairing a normal
1078 // load/store with a memory barrier. For a volatile read we need
1079 //
1080 // ldr<x>
1081 // dmb ishld
1082 //
1083 // for a volatile write
1084 //
1085 // dmb ish
1086 // str<x>
1087 // dmb ish
1088 //
1089 // We can also use ldaxr and stlxr to implement compare and swap CAS
1090 // sequences. These are normally translated to an instruction
1091 // sequence like the following
1092 //
1093 // dmb ish
1094 // retry:
1095 // ldxr<x> rval raddr
1096 // cmp rval rold
1097 // b.ne done
1098 // stlxr<x> rval, rnew, rold
1099 // cbnz rval retry
1100 // done:
1101 // cset r0, eq
1102 // dmb ishld
1103 //
1104 // Note that the exclusive store is already using an stlxr
1105 // instruction. That is required to ensure visibility to other
1106 // threads of the exclusive write (assuming it succeeds) before that
1107 // of any subsequent writes.
1108 //
1109 // The following instruction sequence is an improvement on the above
1110 //
1111 // retry:
1112 // ldaxr<x> rval raddr
1113 // cmp rval rold
1114 // b.ne done
1115 // stlxr<x> rval, rnew, rold
1116 // cbnz rval retry
1117 // done:
1118 // cset r0, eq
1119 //
1120 // We don't need the leading dmb ish since the stlxr guarantees
1121 // visibility of prior writes in the case that the swap is
1122 // successful. Crucially we don't have to worry about the case where
1123 // the swap is not successful since no valid program should be
1124 // relying on visibility of prior changes by the attempting thread
1125 // in the case where the CAS fails.
1126 //
1127 // Similarly, we don't need the trailing dmb ishld if we substitute
1128 // an ldaxr instruction since that will provide all the guarantees we
1129 // require regarding observation of changes made by other threads
1130 // before any change to the CAS address observed by the load.
1131 //
1132 // In order to generate the desired instruction sequence we need to
1133 // be able to identify specific 'signature' ideal graph node
1134 // sequences which i) occur as a translation of a volatile reads or
1135 // writes or CAS operations and ii) do not occur through any other
1136 // translation or graph transformation. We can then provide
1137 // alternative aldc matching rules which translate these node
1138 // sequences to the desired machine code sequences. Selection of the
1139 // alternative rules can be implemented by predicates which identify
1140 // the relevant node sequences.
1141 //
1142 // The ideal graph generator translates a volatile read to the node
1143 // sequence
1144 //
1145 // LoadX[mo_acquire]
1146 // MemBarAcquire
1147 //
1148 // As a special case when using the compressed oops optimization we
1149 // may also see this variant
1150 //
1151 // LoadN[mo_acquire]
1152 // DecodeN
1153 // MemBarAcquire
1154 //
1155 // A volatile write is translated to the node sequence
1156 //
1157 // MemBarRelease
1158 // StoreX[mo_release] {CardMark}-optional
1159 // MemBarVolatile
1160 //
1161 // n.b. the above node patterns are generated with a strict
1162 // 'signature' configuration of input and output dependencies (see
1163 // the predicates below for exact details). The card mark may be as
1164 // simple as a few extra nodes or, in a few GC configurations, may
1165 // include more complex control flow between the leading and
1166 // trailing memory barriers. However, whatever the card mark
1167 // configuration these signatures are unique to translated volatile
1168 // reads/stores -- they will not appear as a result of any other
1169 // bytecode translation or inlining nor as a consequence of
1170 // optimizing transforms.
1171 //
1172 // We also want to catch inlined unsafe volatile gets and puts and
1173 // be able to implement them using either ldar<x>/stlr<x> or some
1174 // combination of ldr<x>/stlr<x> and dmb instructions.
1175 //
1176 // Inlined unsafe volatiles puts manifest as a minor variant of the
1177 // normal volatile put node sequence containing an extra cpuorder
1178 // membar
1179 //
1180 // MemBarRelease
1181 // MemBarCPUOrder
1182 // StoreX[mo_release] {CardMark}-optional
1183 // MemBarCPUOrder
1184 // MemBarVolatile
1185 //
1186 // n.b. as an aside, a cpuorder membar is not itself subject to
1187 // matching and translation by adlc rules. However, the rule
1188 // predicates need to detect its presence in order to correctly
1189 // select the desired adlc rules.
1190 //
1191 // Inlined unsafe volatile gets manifest as a slightly different
1192 // node sequence to a normal volatile get because of the
1193 // introduction of some CPUOrder memory barriers to bracket the
1194 // Load. However, but the same basic skeleton of a LoadX feeding a
1195 // MemBarAcquire, possibly thorugh an optional DecodeN, is still
1196 // present
1197 //
1198 // MemBarCPUOrder
1199 // || \\
1200 // MemBarCPUOrder LoadX[mo_acquire]
1201 // || |
1202 // || {DecodeN} optional
1203 // || /
1204 // MemBarAcquire
1205 //
1206 // In this case the acquire membar does not directly depend on the
1207 // load. However, we can be sure that the load is generated from an
1208 // inlined unsafe volatile get if we see it dependent on this unique
1209 // sequence of membar nodes. Similarly, given an acquire membar we
1210 // can know that it was added because of an inlined unsafe volatile
1211 // get if it is fed and feeds a cpuorder membar and if its feed
1212 // membar also feeds an acquiring load.
1213 //
1214 // Finally an inlined (Unsafe) CAS operation is translated to the
1215 // following ideal graph
1216 //
1217 // MemBarRelease
1218 // MemBarCPUOrder
1219 // CompareAndSwapX {CardMark}-optional
1220 // MemBarCPUOrder
1221 // MemBarAcquire
1222 //
1223 // So, where we can identify these volatile read and write
1224 // signatures we can choose to plant either of the above two code
1225 // sequences. For a volatile read we can simply plant a normal
1226 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1227 // also choose to inhibit translation of the MemBarAcquire and
1228 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1229 //
1230 // When we recognise a volatile store signature we can choose to
1231 // plant at a dmb ish as a translation for the MemBarRelease, a
1232 // normal str<x> and then a dmb ish for the MemBarVolatile.
1233 // Alternatively, we can inhibit translation of the MemBarRelease
1234 // and MemBarVolatile and instead plant a simple stlr<x>
1235 // instruction.
1236 //
1237 // when we recognise a CAS signature we can choose to plant a dmb
1238 // ish as a translation for the MemBarRelease, the conventional
1239 // macro-instruction sequence for the CompareAndSwap node (which
1240 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1241 // Alternatively, we can elide generation of the dmb instructions
1242 // and plant the alternative CompareAndSwap macro-instruction
1243 // sequence (which uses ldaxr<x>).
1244 //
1245 // Of course, the above only applies when we see these signature
1246 // configurations. We still want to plant dmb instructions in any
1247 // other cases where we may see a MemBarAcquire, MemBarRelease or
1248 // MemBarVolatile. For example, at the end of a constructor which
1249 // writes final/volatile fields we will see a MemBarRelease
1250 // instruction and this needs a 'dmb ish' lest we risk the
1251 // constructed object being visible without making the
1252 // final/volatile field writes visible.
1253 //
1254 // n.b. the translation rules below which rely on detection of the
1255 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1256 // If we see anything other than the signature configurations we
1257 // always just translate the loads and stores to ldr<x> and str<x>
1258 // and translate acquire, release and volatile membars to the
1259 // relevant dmb instructions.
1260 //
1261
1262 // is_CAS(int opcode, bool maybe_volatile)
1263 //
1264 // return true if opcode is one of the possible CompareAndSwapX
1265 // values otherwise false.
1266
1267 bool is_CAS(int opcode, bool maybe_volatile)
1268 {
1269 switch(opcode) {
1270 // We handle these
1271 case Op_CompareAndSwapI:
1272 case Op_CompareAndSwapL:
1273 case Op_CompareAndSwapP:
1274 case Op_CompareAndSwapN:
1275 case Op_ShenandoahCompareAndSwapP:
1276 case Op_ShenandoahCompareAndSwapN:
1277 case Op_CompareAndSwapB:
1278 case Op_CompareAndSwapS:
1279 case Op_GetAndSetI:
1280 case Op_GetAndSetL:
1281 case Op_GetAndSetP:
1282 case Op_GetAndSetN:
1283 case Op_GetAndAddI:
1284 case Op_GetAndAddL:
1285 return true;
1286 case Op_CompareAndExchangeI:
1287 case Op_CompareAndExchangeN:
1288 case Op_CompareAndExchangeB:
1289 case Op_CompareAndExchangeS:
1290 case Op_CompareAndExchangeL:
1291 case Op_CompareAndExchangeP:
1292 case Op_WeakCompareAndSwapB:
1293 case Op_WeakCompareAndSwapS:
1294 case Op_WeakCompareAndSwapI:
1295 case Op_WeakCompareAndSwapL:
1296 case Op_WeakCompareAndSwapP:
1297 case Op_WeakCompareAndSwapN:
1298 case Op_ShenandoahWeakCompareAndSwapP:
1299 case Op_ShenandoahWeakCompareAndSwapN:
1300 case Op_ShenandoahCompareAndExchangeP:
1301 case Op_ShenandoahCompareAndExchangeN:
1302 return maybe_volatile;
1303 default:
1304 return false;
1305 }
1306 }
1307
1308 // helper to determine the maximum number of Phi nodes we may need to
1309 // traverse when searching from a card mark membar for the merge mem
1310 // feeding a trailing membar or vice versa
1311
1312 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1313
1314 bool unnecessary_acquire(const Node *barrier)
1315 {
1316 assert(barrier->is_MemBar(), "expecting a membar");
1317
1318 if (UseBarriersForVolatile) {
1319 // we need to plant a dmb
1320 return false;
1321 }
1322
1323 MemBarNode* mb = barrier->as_MemBar();
1324
1325 if (mb->trailing_load()) {
1326 return true;
1327 }
1328
1329 if (mb->trailing_load_store()) {
1330 Node* load_store = mb->in(MemBarNode::Precedent);
1331 assert(load_store->is_LoadStore(), "unexpected graph shape");
1332 return is_CAS(load_store->Opcode(), true);
1333 }
1334
1335 return false;
1336 }
1337
1338 bool needs_acquiring_load(const Node *n)
1339 {
1340 assert(n->is_Load(), "expecting a load");
1341 if (UseBarriersForVolatile) {
1342 // we use a normal load and a dmb
1343 return false;
1344 }
1345
1346 LoadNode *ld = n->as_Load();
1347
1348 return ld->is_acquire();
1349 }
1350
1351 bool unnecessary_release(const Node *n)
1352 {
1353 assert((n->is_MemBar() &&
1354 n->Opcode() == Op_MemBarRelease),
1355 "expecting a release membar");
1356
1357 if (UseBarriersForVolatile) {
1358 // we need to plant a dmb
1359 return false;
1360 }
1361
1362 MemBarNode *barrier = n->as_MemBar();
1363 if (!barrier->leading()) {
1364 return false;
1365 } else {
1366 Node* trailing = barrier->trailing_membar();
1367 MemBarNode* trailing_mb = trailing->as_MemBar();
1368 assert(trailing_mb->trailing(), "Not a trailing membar?");
1369 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1370
1371 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1372 if (mem->is_Store()) {
1373 assert(mem->as_Store()->is_release(), "");
1374 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1375 return true;
1376 } else {
1377 assert(mem->is_LoadStore(), "");
1378 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1379 return is_CAS(mem->Opcode(), true);
1380 }
1381 }
1382 return false;
1383 }
1384
1385 bool unnecessary_volatile(const Node *n)
1386 {
1387 // assert n->is_MemBar();
1388 if (UseBarriersForVolatile) {
1389 // we need to plant a dmb
1390 return false;
1391 }
1392
1393 MemBarNode *mbvol = n->as_MemBar();
1394
1395 bool release = mbvol->trailing_store();
1396 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1397 #ifdef ASSERT
1398 if (release) {
1399 Node* leading = mbvol->leading_membar();
1400 assert(leading->Opcode() == Op_MemBarRelease, "");
1401 assert(leading->as_MemBar()->leading_store(), "");
1402 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1403 }
1404 #endif
1405
1406 return release;
1407 }
1408
1409 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1410
1411 bool needs_releasing_store(const Node *n)
1412 {
1413 // assert n->is_Store();
1414 if (UseBarriersForVolatile) {
1415 // we use a normal store and dmb combination
1416 return false;
1417 }
1418
1419 StoreNode *st = n->as_Store();
1420
1421 return st->trailing_membar() != NULL;
1422 }
1423
1424 // predicate controlling translation of CAS
1425 //
1426 // returns true if CAS needs to use an acquiring load otherwise false
1427
1428 bool needs_acquiring_load_exclusive(const Node *n)
1429 {
1430 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1431 if (UseBarriersForVolatile) {
1432 return false;
1433 }
1434
1435 LoadStoreNode* ldst = n->as_LoadStore();
1436 if (is_CAS(n->Opcode(), false)) {
1437 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1438 } else {
1439 return ldst->trailing_membar() != NULL;
1440 }
1441
1442 // so we can just return true here
1443 return true;
1444 }
1445
1446 // predicate controlling translation of StoreCM
1447 //
1448 // returns true if a StoreStore must precede the card write otherwise
1449 // false
1450
1451 bool unnecessary_storestore(const Node *storecm)
1452 {
1453 assert(storecm->Opcode() == Op_StoreCM, "expecting a StoreCM");
1454
1455 // we need to generate a dmb ishst between an object put and the
1456 // associated card mark when we are using CMS without conditional
1457 // card marking
1458
1459 if (UseConcMarkSweepGC && !UseCondCardMark) {
1460 return false;
1461 }
1462
1463 // a storestore is unnecesary in all other cases
1464
1465 return true;
1466 }
1467
1468
1469 #define __ _masm.
1470
1471 // advance declarations for helper functions to convert register
1472 // indices to register objects
1473
1474 // the ad file has to provide implementations of certain methods
1475 // expected by the generic code
1476 //
1477 // REQUIRED FUNCTIONALITY
1478
1479 //=============================================================================
1480
1481 // !!!!! Special hack to get all types of calls to specify the byte offset
1482 // from the start of the call to the point where the return address
1483 // will point.
1484
1485 int MachCallStaticJavaNode::ret_addr_offset()
1486 {
1487 // call should be a simple bl
1488 int off = 4;
1489 return off;
1490 }
1491
1492 int MachCallDynamicJavaNode::ret_addr_offset()
1493 {
1494 return 16; // movz, movk, movk, bl
1495 }
1496
1497 int MachCallRuntimeNode::ret_addr_offset() {
1498 // for generated stubs the call will be
1499 // far_call(addr)
1500 // for real runtime callouts it will be six instructions
1501 // see aarch64_enc_java_to_runtime
1502 // adr(rscratch2, retaddr)
1503 // lea(rscratch1, RuntimeAddress(addr)
1504 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1505 // blrt rscratch1
1506 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1507 if (cb) {
1508 return MacroAssembler::far_branch_size();
1509 } else {
1510 return 6 * NativeInstruction::instruction_size;
1511 }
1512 }
1513
1514 // Indicate if the safepoint node needs the polling page as an input
1515
1516 // the shared code plants the oop data at the start of the generated
1517 // code for the safepoint node and that needs ot be at the load
1518 // instruction itself. so we cannot plant a mov of the safepoint poll
1519 // address followed by a load. setting this to true means the mov is
1520 // scheduled as a prior instruction. that's better for scheduling
1521 // anyway.
1522
1523 bool SafePointNode::needs_polling_address_input()
1524 {
1525 return true;
1526 }
1527
1528 //=============================================================================
1529
1530 #ifndef PRODUCT
1531 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1532 st->print("BREAKPOINT");
1533 }
1534 #endif
1535
1536 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1537 MacroAssembler _masm(&cbuf);
1538 __ brk(0);
1539 }
1540
1541 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1542 return MachNode::size(ra_);
1543 }
1544
1545 //=============================================================================
1546
1547 #ifndef PRODUCT
1548 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1549 st->print("nop \t# %d bytes pad for loops and calls", _count);
1550 }
1551 #endif
1552
1553 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1554 MacroAssembler _masm(&cbuf);
1555 for (int i = 0; i < _count; i++) {
1556 __ nop();
1557 }
1558 }
1559
1560 uint MachNopNode::size(PhaseRegAlloc*) const {
1561 return _count * NativeInstruction::instruction_size;
1562 }
1563
1564 //=============================================================================
1565 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1566
1567 int Compile::ConstantTable::calculate_table_base_offset() const {
1568 return 0; // absolute addressing, no offset
1569 }
1570
1571 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1572 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1573 ShouldNotReachHere();
1574 }
1575
1576 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1577 // Empty encoding
1578 }
1579
1580 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1581 return 0;
1582 }
1583
1584 #ifndef PRODUCT
1585 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1586 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1587 }
1588 #endif
1589
1590 #ifndef PRODUCT
1591 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1592 Compile* C = ra_->C;
1593
1594 int framesize = C->frame_slots() << LogBytesPerInt;
1595
1596 if (C->need_stack_bang(framesize))
1597 st->print("# stack bang size=%d\n\t", framesize);
1598
1599 if (framesize < ((1 << 9) + 2 * wordSize)) {
1600 st->print("sub sp, sp, #%d\n\t", framesize);
1601 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1602 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1603 } else {
1604 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1605 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1606 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1607 st->print("sub sp, sp, rscratch1");
1608 }
1609 }
1610 #endif
1611
1612 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1613 Compile* C = ra_->C;
1614 MacroAssembler _masm(&cbuf);
1615
1616 __ verified_entry(C, 0);
1617 __ bind(*_verified_entry);
1618
1619 C->set_frame_complete(cbuf.insts_size());
1620
1621 if (C->has_mach_constant_base_node()) {
1622 // NOTE: We set the table base offset here because users might be
1623 // emitted before MachConstantBaseNode.
1624 Compile::ConstantTable& constant_table = C->constant_table();
1625 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1626 }
1627 }
1628
1629 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1630 {
1631 return MachNode::size(ra_); // too many variables; just compute it
1632 // the hard way
1633 }
1634
1635 int MachPrologNode::reloc() const
1636 {
1637 return 0;
1638 }
1639
1640 //=============================================================================
1641
1642 #ifndef PRODUCT
1643 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1644 Compile* C = ra_->C;
1645 int framesize = C->frame_slots() << LogBytesPerInt;
1646
1647 st->print("# pop frame %d\n\t",framesize);
1648
1649 if (framesize == 0) {
1650 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1651 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1652 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1653 st->print("add sp, sp, #%d\n\t", framesize);
1654 } else {
1655 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1656 st->print("add sp, sp, rscratch1\n\t");
1657 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1658 }
1659
1660 if (do_polling() && C->is_method_compilation()) {
1661 st->print("# touch polling page\n\t");
1662 st->print("mov rscratch1, #0x%lx\n\t", p2i(os::get_polling_page()));
1663 st->print("ldr zr, [rscratch1]");
1664 }
1665 }
1666 #endif
1667
1668 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1669 Compile* C = ra_->C;
1670 MacroAssembler _masm(&cbuf);
1671 int framesize = C->frame_slots() << LogBytesPerInt;
1672
1673 __ remove_frame(framesize);
1674
1675 if (NotifySimulator) {
1676 __ notify(Assembler::method_reentry);
1677 }
1678
1679 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1680 __ reserved_stack_check();
1681 }
1682
1683 if (do_polling() && C->is_method_compilation()) {
1684 __ read_polling_page(rscratch1, os::get_polling_page(), relocInfo::poll_return_type);
1685 }
1686 }
1687
1688 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1689 // Variable size. Determine dynamically.
1690 return MachNode::size(ra_);
1691 }
1692
1693 int MachEpilogNode::reloc() const {
1694 // Return number of relocatable values contained in this instruction.
1695 return 1; // 1 for polling page.
1696 }
1697
1698 const Pipeline * MachEpilogNode::pipeline() const {
1699 return MachNode::pipeline_class();
1700 }
1701
1702 // This method seems to be obsolete. It is declared in machnode.hpp
1703 // and defined in all *.ad files, but it is never called. Should we
1704 // get rid of it?
1705 int MachEpilogNode::safepoint_offset() const {
1706 assert(do_polling(), "no return for this epilog node");
1707 return 4;
1708 }
1709
1710 //=============================================================================
1711
1712 // Figure out which register class each belongs in: rc_int, rc_float or
1713 // rc_stack.
1714 enum RC { rc_bad, rc_int, rc_float, rc_stack };
1715
1716 static enum RC rc_class(OptoReg::Name reg) {
1717
1718 if (reg == OptoReg::Bad) {
1719 return rc_bad;
1720 }
1721
1722 // we have 30 int registers * 2 halves
1723 // (rscratch1 and rscratch2 are omitted)
1724
1725 if (reg < 60) {
1726 return rc_int;
1727 }
1728
1729 // we have 32 float register * 2 halves
1730 if (reg < 60 + 128) {
1731 return rc_float;
1732 }
1733
1734 // Between float regs & stack is the flags regs.
1735 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1736
1737 return rc_stack;
1738 }
1739
1740 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1741 Compile* C = ra_->C;
1742
1743 // Get registers to move.
1744 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1745 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1746 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1747 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1748
1749 enum RC src_hi_rc = rc_class(src_hi);
1750 enum RC src_lo_rc = rc_class(src_lo);
1751 enum RC dst_hi_rc = rc_class(dst_hi);
1752 enum RC dst_lo_rc = rc_class(dst_lo);
1753
1754 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1755
1756 if (src_hi != OptoReg::Bad) {
1757 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1758 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1759 "expected aligned-adjacent pairs");
1760 }
1761
1762 if (src_lo == dst_lo && src_hi == dst_hi) {
1763 return 0; // Self copy, no move.
1764 }
1765
1766 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1767 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1768 int src_offset = ra_->reg2offset(src_lo);
1769 int dst_offset = ra_->reg2offset(dst_lo);
1770
1771 if (bottom_type()->isa_vect() != NULL) {
1772 uint ireg = ideal_reg();
1773 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1774 if (cbuf) {
1775 MacroAssembler _masm(cbuf);
1776 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1777 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1778 // stack->stack
1779 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1780 if (ireg == Op_VecD) {
1781 __ unspill(rscratch1, true, src_offset);
1782 __ spill(rscratch1, true, dst_offset);
1783 } else {
1784 __ spill_copy128(src_offset, dst_offset);
1785 }
1786 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1787 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1788 ireg == Op_VecD ? __ T8B : __ T16B,
1789 as_FloatRegister(Matcher::_regEncode[src_lo]));
1790 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1791 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1792 ireg == Op_VecD ? __ D : __ Q,
1793 ra_->reg2offset(dst_lo));
1794 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1795 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1796 ireg == Op_VecD ? __ D : __ Q,
1797 ra_->reg2offset(src_lo));
1798 } else {
1799 ShouldNotReachHere();
1800 }
1801 }
1802 } else if (cbuf) {
1803 MacroAssembler _masm(cbuf);
1804 switch (src_lo_rc) {
1805 case rc_int:
1806 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1807 if (is64) {
1808 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1809 as_Register(Matcher::_regEncode[src_lo]));
1810 } else {
1811 MacroAssembler _masm(cbuf);
1812 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1813 as_Register(Matcher::_regEncode[src_lo]));
1814 }
1815 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1816 if (is64) {
1817 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1818 as_Register(Matcher::_regEncode[src_lo]));
1819 } else {
1820 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1821 as_Register(Matcher::_regEncode[src_lo]));
1822 }
1823 } else { // gpr --> stack spill
1824 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1825 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
1826 }
1827 break;
1828 case rc_float:
1829 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
1830 if (is64) {
1831 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
1832 as_FloatRegister(Matcher::_regEncode[src_lo]));
1833 } else {
1834 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
1835 as_FloatRegister(Matcher::_regEncode[src_lo]));
1836 }
1837 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
1838 if (cbuf) {
1839 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1840 as_FloatRegister(Matcher::_regEncode[src_lo]));
1841 } else {
1842 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1843 as_FloatRegister(Matcher::_regEncode[src_lo]));
1844 }
1845 } else { // fpr --> stack spill
1846 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1847 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1848 is64 ? __ D : __ S, dst_offset);
1849 }
1850 break;
1851 case rc_stack:
1852 if (dst_lo_rc == rc_int) { // stack --> gpr load
1853 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
1854 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
1855 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1856 is64 ? __ D : __ S, src_offset);
1857 } else { // stack --> stack copy
1858 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1859 __ unspill(rscratch1, is64, src_offset);
1860 __ spill(rscratch1, is64, dst_offset);
1861 }
1862 break;
1863 default:
1864 assert(false, "bad rc_class for spill");
1865 ShouldNotReachHere();
1866 }
1867 }
1868
1869 if (st) {
1870 st->print("spill ");
1871 if (src_lo_rc == rc_stack) {
1872 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
1873 } else {
1874 st->print("%s -> ", Matcher::regName[src_lo]);
1875 }
1876 if (dst_lo_rc == rc_stack) {
1877 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
1878 } else {
1879 st->print("%s", Matcher::regName[dst_lo]);
1880 }
1881 if (bottom_type()->isa_vect() != NULL) {
1882 st->print("\t# vector spill size = %d", ideal_reg()==Op_VecD ? 64:128);
1883 } else {
1884 st->print("\t# spill size = %d", is64 ? 64:32);
1885 }
1886 }
1887
1888 return 0;
1889
1890 }
1891
1892 #ifndef PRODUCT
1893 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1894 if (!ra_)
1895 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
1896 else
1897 implementation(NULL, ra_, false, st);
1898 }
1899 #endif
1900
1901 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1902 implementation(&cbuf, ra_, false, NULL);
1903 }
1904
1905 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1906 return MachNode::size(ra_);
1907 }
1908
1909 //=============================================================================
1910
1911 #ifndef PRODUCT
1912 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1913 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1914 int reg = ra_->get_reg_first(this);
1915 st->print("add %s, rsp, #%d]\t# box lock",
1916 Matcher::regName[reg], offset);
1917 }
1918 #endif
1919
1920 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1921 MacroAssembler _masm(&cbuf);
1922
1923 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1924 int reg = ra_->get_encode(this);
1925
1926 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
1927 __ add(as_Register(reg), sp, offset);
1928 } else {
1929 ShouldNotReachHere();
1930 }
1931 }
1932
1933 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
1934 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
1935 return 4;
1936 }
1937
1938 //=============================================================================
1939 #ifndef PRODUCT
1940 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1941 {
1942 st->print_cr("# MachVEPNode");
1943 if (!_verified) {
1944 st->print_cr("\t load_class");
1945 } else {
1946 st->print_cr("\t unpack_value_arg");
1947 }
1948 }
1949 #endif
1950
1951 void MachVEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1952 {
1953 MacroAssembler _masm(&cbuf);
1954
1955 if (!_verified) {
1956 Label skip;
1957 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
1958 __ br(Assembler::EQ, skip);
1959 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1960 __ bind(skip);
1961
1962 } else {
1963 // Unpack value type args passed as oop and then jump to
1964 // the verified entry point (skipping the unverified entry).
1965 __ unpack_value_args(ra_->C, _receiver_only);
1966 __ b(*_verified_entry);
1967 }
1968 }
1969
1970
1971 uint MachVEPNode::size(PhaseRegAlloc* ra_) const
1972 {
1973 return MachNode::size(ra_); // too many variables; just compute it the hard way
1974 }
1975
1976
1977 //=============================================================================
1978
1979 #ifndef PRODUCT
1980 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1981 {
1982 st->print_cr("# MachUEPNode");
1983 if (UseCompressedClassPointers) {
1984 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1985 if (Universe::narrow_klass_shift() != 0) {
1986 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1987 }
1988 } else {
1989 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1990 }
1991 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
1992 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
1993 }
1994 #endif
1995
1996 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1997 {
1998 // This is the unverified entry point.
1999 MacroAssembler _masm(&cbuf);
2000
2001 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2002 Label skip;
2003 // TODO
2004 // can we avoid this skip and still use a reloc?
2005 __ br(Assembler::EQ, skip);
2006 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2007 __ bind(skip);
2008 }
2009
2010 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2011 {
2012 return MachNode::size(ra_);
2013 }
2014
2015 // REQUIRED EMIT CODE
2016
2017 //=============================================================================
2018
2019 // Emit exception handler code.
2020 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2021 {
2022 // mov rscratch1 #exception_blob_entry_point
2023 // br rscratch1
2024 // Note that the code buffer's insts_mark is always relative to insts.
2025 // That's why we must use the macroassembler to generate a handler.
2026 MacroAssembler _masm(&cbuf);
2027 address base = __ start_a_stub(size_exception_handler());
2028 if (base == NULL) {
2029 ciEnv::current()->record_failure("CodeCache is full");
2030 return 0; // CodeBuffer::expand failed
2031 }
2032 int offset = __ offset();
2033 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2034 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2035 __ end_a_stub();
2036 return offset;
2037 }
2038
2039 // Emit deopt handler code.
2040 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2041 {
2042 // Note that the code buffer's insts_mark is always relative to insts.
2043 // That's why we must use the macroassembler to generate a handler.
2044 MacroAssembler _masm(&cbuf);
2045 address base = __ start_a_stub(size_deopt_handler());
2046 if (base == NULL) {
2047 ciEnv::current()->record_failure("CodeCache is full");
2048 return 0; // CodeBuffer::expand failed
2049 }
2050 int offset = __ offset();
2051
2052 __ adr(lr, __ pc());
2053 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2054
2055 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
2056 __ end_a_stub();
2057 return offset;
2058 }
2059
2060 // REQUIRED MATCHER CODE
2061
2062 //=============================================================================
2063
2064 const bool Matcher::match_rule_supported(int opcode) {
2065
2066 switch (opcode) {
2067 default:
2068 break;
2069 }
2070
2071 if (!has_match_rule(opcode)) {
2072 return false;
2073 }
2074
2075 return true; // Per default match rules are supported.
2076 }
2077
2078 const bool Matcher::match_rule_supported_vector(int opcode, int vlen) {
2079
2080 // TODO
2081 // identify extra cases that we might want to provide match rules for
2082 // e.g. Op_ vector nodes and other intrinsics while guarding with vlen
2083 bool ret_value = match_rule_supported(opcode);
2084 // Add rules here.
2085
2086 return ret_value; // Per default match rules are supported.
2087 }
2088
2089 const bool Matcher::has_predicated_vectors(void) {
2090 return false;
2091 }
2092
2093 const int Matcher::float_pressure(int default_pressure_threshold) {
2094 return default_pressure_threshold;
2095 }
2096
2097 int Matcher::regnum_to_fpu_offset(int regnum)
2098 {
2099 Unimplemented();
2100 return 0;
2101 }
2102
2103 // Is this branch offset short enough that a short branch can be used?
2104 //
2105 // NOTE: If the platform does not provide any short branch variants, then
2106 // this method should return false for offset 0.
2107 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2108 // The passed offset is relative to address of the branch.
2109
2110 return (-32768 <= offset && offset < 32768);
2111 }
2112
2113 const bool Matcher::isSimpleConstant64(jlong value) {
2114 // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
2115 // Probably always true, even if a temp register is required.
2116 return true;
2117 }
2118
2119 // true just means we have fast l2f conversion
2120 const bool Matcher::convL2FSupported(void) {
2121 return true;
2122 }
2123
2124 // Vector width in bytes.
2125 const int Matcher::vector_width_in_bytes(BasicType bt) {
2126 int size = MIN2(16,(int)MaxVectorSize);
2127 // Minimum 2 values in vector
2128 if (size < 2*type2aelembytes(bt)) size = 0;
2129 // But never < 4
2130 if (size < 4) size = 0;
2131 return size;
2132 }
2133
2134 // Limits on vector size (number of elements) loaded into vector.
2135 const int Matcher::max_vector_size(const BasicType bt) {
2136 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2137 }
2138 const int Matcher::min_vector_size(const BasicType bt) {
2139 // For the moment limit the vector size to 8 bytes
2140 int size = 8 / type2aelembytes(bt);
2141 if (size < 2) size = 2;
2142 return size;
2143 }
2144
2145 // Vector ideal reg.
2146 const uint Matcher::vector_ideal_reg(int len) {
2147 switch(len) {
2148 case 8: return Op_VecD;
2149 case 16: return Op_VecX;
2150 }
2151 ShouldNotReachHere();
2152 return 0;
2153 }
2154
2155 const uint Matcher::vector_shift_count_ideal_reg(int size) {
2156 switch(size) {
2157 case 8: return Op_VecD;
2158 case 16: return Op_VecX;
2159 }
2160 ShouldNotReachHere();
2161 return 0;
2162 }
2163
2164 // AES support not yet implemented
2165 const bool Matcher::pass_original_key_for_aes() {
2166 return false;
2167 }
2168
2169 // aarch64 supports misaligned vectors store/load.
2170 const bool Matcher::misaligned_vectors_ok() {
2171 return true;
2172 }
2173
2174 // false => size gets scaled to BytesPerLong, ok.
2175 const bool Matcher::init_array_count_is_in_bytes = false;
2176
2177 // Use conditional move (CMOVL)
2178 const int Matcher::long_cmove_cost() {
2179 // long cmoves are no more expensive than int cmoves
2180 return 0;
2181 }
2182
2183 const int Matcher::float_cmove_cost() {
2184 // float cmoves are no more expensive than int cmoves
2185 return 0;
2186 }
2187
2188 // Does the CPU require late expand (see block.cpp for description of late expand)?
2189 const bool Matcher::require_postalloc_expand = false;
2190
2191 // Do we need to mask the count passed to shift instructions or does
2192 // the cpu only look at the lower 5/6 bits anyway?
2193 const bool Matcher::need_masked_shift_count = false;
2194
2195 // This affects two different things:
2196 // - how Decode nodes are matched
2197 // - how ImplicitNullCheck opportunities are recognized
2198 // If true, the matcher will try to remove all Decodes and match them
2199 // (as operands) into nodes. NullChecks are not prepared to deal with
2200 // Decodes by final_graph_reshaping().
2201 // If false, final_graph_reshaping() forces the decode behind the Cmp
2202 // for a NullCheck. The matcher matches the Decode node into a register.
2203 // Implicit_null_check optimization moves the Decode along with the
2204 // memory operation back up before the NullCheck.
2205 bool Matcher::narrow_oop_use_complex_address() {
2206 return Universe::narrow_oop_shift() == 0;
2207 }
2208
2209 bool Matcher::narrow_klass_use_complex_address() {
2210 // TODO
2211 // decide whether we need to set this to true
2212 return false;
2213 }
2214
2215 bool Matcher::const_oop_prefer_decode() {
2216 // Prefer ConN+DecodeN over ConP in simple compressed oops mode.
2217 return Universe::narrow_oop_base() == NULL;
2218 }
2219
2220 bool Matcher::const_klass_prefer_decode() {
2221 // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
2222 return Universe::narrow_klass_base() == NULL;
2223 }
2224
2225 // Is it better to copy float constants, or load them directly from
2226 // memory? Intel can load a float constant from a direct address,
2227 // requiring no extra registers. Most RISCs will have to materialize
2228 // an address into a register first, so they would do better to copy
2229 // the constant from stack.
2230 const bool Matcher::rematerialize_float_constants = false;
2231
2232 // If CPU can load and store mis-aligned doubles directly then no
2233 // fixup is needed. Else we split the double into 2 integer pieces
2234 // and move it piece-by-piece. Only happens when passing doubles into
2235 // C code as the Java calling convention forces doubles to be aligned.
2236 const bool Matcher::misaligned_doubles_ok = true;
2237
2238 // No-op on amd64
2239 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
2240 Unimplemented();
2241 }
2242
2243 // Advertise here if the CPU requires explicit rounding operations to
2244 // implement the UseStrictFP mode.
2245 const bool Matcher::strict_fp_requires_explicit_rounding = false;
2246
2247 // Are floats converted to double when stored to stack during
2248 // deoptimization?
2249 bool Matcher::float_in_double() { return false; }
2250
2251 // Do ints take an entire long register or just half?
2252 // The relevant question is how the int is callee-saved:
2253 // the whole long is written but de-opt'ing will have to extract
2254 // the relevant 32 bits.
2255 const bool Matcher::int_in_long = true;
2256
2257 // Return whether or not this register is ever used as an argument.
2258 // This function is used on startup to build the trampoline stubs in
2259 // generateOptoStub. Registers not mentioned will be killed by the VM
2260 // call in the trampoline, and arguments in those registers not be
2261 // available to the callee.
2262 bool Matcher::can_be_java_arg(int reg)
2263 {
2264 return
2265 reg == R0_num || reg == R0_H_num ||
2266 reg == R1_num || reg == R1_H_num ||
2267 reg == R2_num || reg == R2_H_num ||
2268 reg == R3_num || reg == R3_H_num ||
2269 reg == R4_num || reg == R4_H_num ||
2270 reg == R5_num || reg == R5_H_num ||
2271 reg == R6_num || reg == R6_H_num ||
2272 reg == R7_num || reg == R7_H_num ||
2273 reg == V0_num || reg == V0_H_num ||
2274 reg == V1_num || reg == V1_H_num ||
2275 reg == V2_num || reg == V2_H_num ||
2276 reg == V3_num || reg == V3_H_num ||
2277 reg == V4_num || reg == V4_H_num ||
2278 reg == V5_num || reg == V5_H_num ||
2279 reg == V6_num || reg == V6_H_num ||
2280 reg == V7_num || reg == V7_H_num;
2281 }
2282
2283 bool Matcher::is_spillable_arg(int reg)
2284 {
2285 return can_be_java_arg(reg);
2286 }
2287
2288 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2289 return false;
2290 }
2291
2292 RegMask Matcher::divI_proj_mask() {
2293 ShouldNotReachHere();
2294 return RegMask();
2295 }
2296
2297 // Register for MODI projection of divmodI.
2298 RegMask Matcher::modI_proj_mask() {
2299 ShouldNotReachHere();
2300 return RegMask();
2301 }
2302
2303 // Register for DIVL projection of divmodL.
2304 RegMask Matcher::divL_proj_mask() {
2305 ShouldNotReachHere();
2306 return RegMask();
2307 }
2308
2309 // Register for MODL projection of divmodL.
2310 RegMask Matcher::modL_proj_mask() {
2311 ShouldNotReachHere();
2312 return RegMask();
2313 }
2314
2315 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2316 return FP_REG_mask();
2317 }
2318
2319 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2320 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2321 Node* u = addp->fast_out(i);
2322 if (u->is_Mem()) {
2323 int opsize = u->as_Mem()->memory_size();
2324 assert(opsize > 0, "unexpected memory operand size");
2325 if (u->as_Mem()->memory_size() != (1<<shift)) {
2326 return false;
2327 }
2328 }
2329 }
2330 return true;
2331 }
2332
2333 const bool Matcher::convi2l_type_required = false;
2334
2335 // Should the Matcher clone shifts on addressing modes, expecting them
2336 // to be subsumed into complex addressing expressions or compute them
2337 // into registers?
2338 bool Matcher::clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2339 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2340 return true;
2341 }
2342
2343 Node *off = m->in(AddPNode::Offset);
2344 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2345 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2346 // Are there other uses besides address expressions?
2347 !is_visited(off)) {
2348 address_visited.set(off->_idx); // Flag as address_visited
2349 mstack.push(off->in(2), Visit);
2350 Node *conv = off->in(1);
2351 if (conv->Opcode() == Op_ConvI2L &&
2352 // Are there other uses besides address expressions?
2353 !is_visited(conv)) {
2354 address_visited.set(conv->_idx); // Flag as address_visited
2355 mstack.push(conv->in(1), Pre_Visit);
2356 } else {
2357 mstack.push(conv, Pre_Visit);
2358 }
2359 address_visited.test_set(m->_idx); // Flag as address_visited
2360 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2361 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2362 return true;
2363 } else if (off->Opcode() == Op_ConvI2L &&
2364 // Are there other uses besides address expressions?
2365 !is_visited(off)) {
2366 address_visited.test_set(m->_idx); // Flag as address_visited
2367 address_visited.set(off->_idx); // Flag as address_visited
2368 mstack.push(off->in(1), Pre_Visit);
2369 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2370 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2371 return true;
2372 }
2373 return false;
2374 }
2375
2376 void Compile::reshape_address(AddPNode* addp) {
2377 }
2378
2379 // helper for encoding java_to_runtime calls on sim
2380 //
2381 // this is needed to compute the extra arguments required when
2382 // planting a call to the simulator blrt instruction. the TypeFunc
2383 // can be queried to identify the counts for integral, and floating
2384 // arguments and the return type
2385
2386 static void getCallInfo(const TypeFunc *tf, int &gpcnt, int &fpcnt, int &rtype)
2387 {
2388 int gps = 0;
2389 int fps = 0;
2390 const TypeTuple *domain = tf->domain_cc();
2391 int max = domain->cnt();
2392 for (int i = TypeFunc::Parms; i < max; i++) {
2393 const Type *t = domain->field_at(i);
2394 switch(t->basic_type()) {
2395 case T_FLOAT:
2396 case T_DOUBLE:
2397 fps++;
2398 default:
2399 gps++;
2400 }
2401 }
2402 gpcnt = gps;
2403 fpcnt = fps;
2404 BasicType rt = tf->return_type();
2405 switch (rt) {
2406 case T_VOID:
2407 rtype = MacroAssembler::ret_type_void;
2408 break;
2409 default:
2410 rtype = MacroAssembler::ret_type_integral;
2411 break;
2412 case T_FLOAT:
2413 rtype = MacroAssembler::ret_type_float;
2414 break;
2415 case T_DOUBLE:
2416 rtype = MacroAssembler::ret_type_double;
2417 break;
2418 }
2419 }
2420
2421 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2422 MacroAssembler _masm(&cbuf); \
2423 { \
2424 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2425 guarantee(DISP == 0, "mode not permitted for volatile"); \
2426 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2427 __ INSN(REG, as_Register(BASE)); \
2428 }
2429
2430 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2431 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2432 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2433 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2434
2435 // Used for all non-volatile memory accesses. The use of
2436 // $mem->opcode() to discover whether this pattern uses sign-extended
2437 // offsets is something of a kludge.
2438 static void loadStore(MacroAssembler masm, mem_insn insn,
2439 Register reg, int opcode,
2440 Register base, int index, int size, int disp)
2441 {
2442 Address::extend scale;
2443
2444 // Hooboy, this is fugly. We need a way to communicate to the
2445 // encoder that the index needs to be sign extended, so we have to
2446 // enumerate all the cases.
2447 switch (opcode) {
2448 case INDINDEXSCALEDI2L:
2449 case INDINDEXSCALEDI2LN:
2450 case INDINDEXI2L:
2451 case INDINDEXI2LN:
2452 scale = Address::sxtw(size);
2453 break;
2454 default:
2455 scale = Address::lsl(size);
2456 }
2457
2458 if (index == -1) {
2459 (masm.*insn)(reg, Address(base, disp));
2460 } else {
2461 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2462 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2463 }
2464 }
2465
2466 static void loadStore(MacroAssembler masm, mem_float_insn insn,
2467 FloatRegister reg, int opcode,
2468 Register base, int index, int size, int disp)
2469 {
2470 Address::extend scale;
2471
2472 switch (opcode) {
2473 case INDINDEXSCALEDI2L:
2474 case INDINDEXSCALEDI2LN:
2475 scale = Address::sxtw(size);
2476 break;
2477 default:
2478 scale = Address::lsl(size);
2479 }
2480
2481 if (index == -1) {
2482 (masm.*insn)(reg, Address(base, disp));
2483 } else {
2484 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2485 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2486 }
2487 }
2488
2489 static void loadStore(MacroAssembler masm, mem_vector_insn insn,
2490 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2491 int opcode, Register base, int index, int size, int disp)
2492 {
2493 if (index == -1) {
2494 (masm.*insn)(reg, T, Address(base, disp));
2495 } else {
2496 assert(disp == 0, "unsupported address mode");
2497 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2498 }
2499 }
2500
2501 %}
2502
2503
2504
2505 //----------ENCODING BLOCK-----------------------------------------------------
2506 // This block specifies the encoding classes used by the compiler to
2507 // output byte streams. Encoding classes are parameterized macros
2508 // used by Machine Instruction Nodes in order to generate the bit
2509 // encoding of the instruction. Operands specify their base encoding
2510 // interface with the interface keyword. There are currently
2511 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2512 // COND_INTER. REG_INTER causes an operand to generate a function
2513 // which returns its register number when queried. CONST_INTER causes
2514 // an operand to generate a function which returns the value of the
2515 // constant when queried. MEMORY_INTER causes an operand to generate
2516 // four functions which return the Base Register, the Index Register,
2517 // the Scale Value, and the Offset Value of the operand when queried.
2518 // COND_INTER causes an operand to generate six functions which return
2519 // the encoding code (ie - encoding bits for the instruction)
2520 // associated with each basic boolean condition for a conditional
2521 // instruction.
2522 //
2523 // Instructions specify two basic values for encoding. Again, a
2524 // function is available to check if the constant displacement is an
2525 // oop. They use the ins_encode keyword to specify their encoding
2526 // classes (which must be a sequence of enc_class names, and their
2527 // parameters, specified in the encoding block), and they use the
2528 // opcode keyword to specify, in order, their primary, secondary, and
2529 // tertiary opcode. Only the opcode sections which a particular
2530 // instruction needs for encoding need to be specified.
2531 encode %{
2532 // Build emit functions for each basic byte or larger field in the
2533 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2534 // from C++ code in the enc_class source block. Emit functions will
2535 // live in the main source block for now. In future, we can
2536 // generalize this by adding a syntax that specifies the sizes of
2537 // fields in an order, so that the adlc can build the emit functions
2538 // automagically
2539
2540 // catch all for unimplemented encodings
2541 enc_class enc_unimplemented %{
2542 MacroAssembler _masm(&cbuf);
2543 __ unimplemented("C2 catch all");
2544 %}
2545
2546 // BEGIN Non-volatile memory access
2547
2548 enc_class aarch64_enc_ldrsbw(iRegI dst, memory mem) %{
2549 Register dst_reg = as_Register($dst$$reg);
2550 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2551 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2552 %}
2553
2554 enc_class aarch64_enc_ldrsb(iRegI dst, memory mem) %{
2555 Register dst_reg = as_Register($dst$$reg);
2556 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2557 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2558 %}
2559
2560 enc_class aarch64_enc_ldrb(iRegI dst, memory mem) %{
2561 Register dst_reg = as_Register($dst$$reg);
2562 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2563 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2564 %}
2565
2566 enc_class aarch64_enc_ldrb(iRegL dst, memory mem) %{
2567 Register dst_reg = as_Register($dst$$reg);
2568 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2569 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2570 %}
2571
2572 enc_class aarch64_enc_ldrshw(iRegI dst, memory mem) %{
2573 Register dst_reg = as_Register($dst$$reg);
2574 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2575 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2576 %}
2577
2578 enc_class aarch64_enc_ldrsh(iRegI dst, memory mem) %{
2579 Register dst_reg = as_Register($dst$$reg);
2580 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2581 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2582 %}
2583
2584 enc_class aarch64_enc_ldrh(iRegI dst, memory mem) %{
2585 Register dst_reg = as_Register($dst$$reg);
2586 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2587 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2588 %}
2589
2590 enc_class aarch64_enc_ldrh(iRegL dst, memory mem) %{
2591 Register dst_reg = as_Register($dst$$reg);
2592 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2593 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2594 %}
2595
2596 enc_class aarch64_enc_ldrw(iRegI dst, memory mem) %{
2597 Register dst_reg = as_Register($dst$$reg);
2598 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2599 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2600 %}
2601
2602 enc_class aarch64_enc_ldrw(iRegL dst, memory mem) %{
2603 Register dst_reg = as_Register($dst$$reg);
2604 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2605 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2606 %}
2607
2608 enc_class aarch64_enc_ldrsw(iRegL dst, memory mem) %{
2609 Register dst_reg = as_Register($dst$$reg);
2610 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2611 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2612 %}
2613
2614 enc_class aarch64_enc_ldr(iRegL dst, memory mem) %{
2615 Register dst_reg = as_Register($dst$$reg);
2616 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2617 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2618 %}
2619
2620 enc_class aarch64_enc_ldrs(vRegF dst, memory mem) %{
2621 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2622 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2623 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2624 %}
2625
2626 enc_class aarch64_enc_ldrd(vRegD dst, memory mem) %{
2627 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2628 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2629 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2630 %}
2631
2632 enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
2633 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2634 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
2635 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2636 %}
2637
2638 enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
2639 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2640 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
2641 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2642 %}
2643
2644 enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
2645 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2646 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
2647 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2648 %}
2649
2650 enc_class aarch64_enc_strb(iRegI src, memory mem) %{
2651 Register src_reg = as_Register($src$$reg);
2652 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2653 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2654 %}
2655
2656 enc_class aarch64_enc_strb0(memory mem) %{
2657 MacroAssembler _masm(&cbuf);
2658 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2659 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2660 %}
2661
2662 enc_class aarch64_enc_strb0_ordered(memory mem) %{
2663 MacroAssembler _masm(&cbuf);
2664 __ membar(Assembler::StoreStore);
2665 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2666 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2667 %}
2668
2669 enc_class aarch64_enc_strh(iRegI src, memory mem) %{
2670 Register src_reg = as_Register($src$$reg);
2671 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2672 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2673 %}
2674
2675 enc_class aarch64_enc_strh0(memory mem) %{
2676 MacroAssembler _masm(&cbuf);
2677 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2678 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2679 %}
2680
2681 enc_class aarch64_enc_strw(iRegI src, memory mem) %{
2682 Register src_reg = as_Register($src$$reg);
2683 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2684 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2685 %}
2686
2687 enc_class aarch64_enc_strw0(memory mem) %{
2688 MacroAssembler _masm(&cbuf);
2689 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2690 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2691 %}
2692
2693 enc_class aarch64_enc_str(iRegL src, memory mem) %{
2694 Register src_reg = as_Register($src$$reg);
2695 // we sometimes get asked to store the stack pointer into the
2696 // current thread -- we cannot do that directly on AArch64
2697 if (src_reg == r31_sp) {
2698 MacroAssembler _masm(&cbuf);
2699 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2700 __ mov(rscratch2, sp);
2701 src_reg = rscratch2;
2702 }
2703 loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2704 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2705 %}
2706
2707 enc_class aarch64_enc_str0(memory mem) %{
2708 MacroAssembler _masm(&cbuf);
2709 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2710 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2711 %}
2712
2713 enc_class aarch64_enc_strs(vRegF src, memory mem) %{
2714 FloatRegister src_reg = as_FloatRegister($src$$reg);
2715 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2716 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2717 %}
2718
2719 enc_class aarch64_enc_strd(vRegD src, memory mem) %{
2720 FloatRegister src_reg = as_FloatRegister($src$$reg);
2721 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2722 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2723 %}
2724
2725 enc_class aarch64_enc_strvS(vecD src, memory mem) %{
2726 FloatRegister src_reg = as_FloatRegister($src$$reg);
2727 loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
2728 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2729 %}
2730
2731 enc_class aarch64_enc_strvD(vecD src, memory mem) %{
2732 FloatRegister src_reg = as_FloatRegister($src$$reg);
2733 loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
2734 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2735 %}
2736
2737 enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
2738 FloatRegister src_reg = as_FloatRegister($src$$reg);
2739 loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
2740 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2741 %}
2742
2743 // END Non-volatile memory access
2744
2745 // volatile loads and stores
2746
2747 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
2748 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2749 rscratch1, stlrb);
2750 %}
2751
2752 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
2753 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2754 rscratch1, stlrh);
2755 %}
2756
2757 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
2758 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2759 rscratch1, stlrw);
2760 %}
2761
2762
2763 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
2764 Register dst_reg = as_Register($dst$$reg);
2765 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2766 rscratch1, ldarb);
2767 __ sxtbw(dst_reg, dst_reg);
2768 %}
2769
2770 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
2771 Register dst_reg = as_Register($dst$$reg);
2772 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2773 rscratch1, ldarb);
2774 __ sxtb(dst_reg, dst_reg);
2775 %}
2776
2777 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
2778 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2779 rscratch1, ldarb);
2780 %}
2781
2782 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
2783 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2784 rscratch1, ldarb);
2785 %}
2786
2787 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
2788 Register dst_reg = as_Register($dst$$reg);
2789 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2790 rscratch1, ldarh);
2791 __ sxthw(dst_reg, dst_reg);
2792 %}
2793
2794 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
2795 Register dst_reg = as_Register($dst$$reg);
2796 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2797 rscratch1, ldarh);
2798 __ sxth(dst_reg, dst_reg);
2799 %}
2800
2801 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
2802 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2803 rscratch1, ldarh);
2804 %}
2805
2806 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
2807 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2808 rscratch1, ldarh);
2809 %}
2810
2811 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
2812 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2813 rscratch1, ldarw);
2814 %}
2815
2816 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
2817 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2818 rscratch1, ldarw);
2819 %}
2820
2821 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
2822 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2823 rscratch1, ldar);
2824 %}
2825
2826 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
2827 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2828 rscratch1, ldarw);
2829 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
2830 %}
2831
2832 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
2833 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2834 rscratch1, ldar);
2835 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
2836 %}
2837
2838 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
2839 Register src_reg = as_Register($src$$reg);
2840 // we sometimes get asked to store the stack pointer into the
2841 // current thread -- we cannot do that directly on AArch64
2842 if (src_reg == r31_sp) {
2843 MacroAssembler _masm(&cbuf);
2844 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2845 __ mov(rscratch2, sp);
2846 src_reg = rscratch2;
2847 }
2848 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2849 rscratch1, stlr);
2850 %}
2851
2852 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
2853 {
2854 MacroAssembler _masm(&cbuf);
2855 FloatRegister src_reg = as_FloatRegister($src$$reg);
2856 __ fmovs(rscratch2, src_reg);
2857 }
2858 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2859 rscratch1, stlrw);
2860 %}
2861
2862 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
2863 {
2864 MacroAssembler _masm(&cbuf);
2865 FloatRegister src_reg = as_FloatRegister($src$$reg);
2866 __ fmovd(rscratch2, src_reg);
2867 }
2868 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2869 rscratch1, stlr);
2870 %}
2871
2872 // synchronized read/update encodings
2873
2874 enc_class aarch64_enc_ldaxr(iRegL dst, memory mem) %{
2875 MacroAssembler _masm(&cbuf);
2876 Register dst_reg = as_Register($dst$$reg);
2877 Register base = as_Register($mem$$base);
2878 int index = $mem$$index;
2879 int scale = $mem$$scale;
2880 int disp = $mem$$disp;
2881 if (index == -1) {
2882 if (disp != 0) {
2883 __ lea(rscratch1, Address(base, disp));
2884 __ ldaxr(dst_reg, rscratch1);
2885 } else {
2886 // TODO
2887 // should we ever get anything other than this case?
2888 __ ldaxr(dst_reg, base);
2889 }
2890 } else {
2891 Register index_reg = as_Register(index);
2892 if (disp == 0) {
2893 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
2894 __ ldaxr(dst_reg, rscratch1);
2895 } else {
2896 __ lea(rscratch1, Address(base, disp));
2897 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
2898 __ ldaxr(dst_reg, rscratch1);
2899 }
2900 }
2901 %}
2902
2903 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory mem) %{
2904 MacroAssembler _masm(&cbuf);
2905 Register src_reg = as_Register($src$$reg);
2906 Register base = as_Register($mem$$base);
2907 int index = $mem$$index;
2908 int scale = $mem$$scale;
2909 int disp = $mem$$disp;
2910 if (index == -1) {
2911 if (disp != 0) {
2912 __ lea(rscratch2, Address(base, disp));
2913 __ stlxr(rscratch1, src_reg, rscratch2);
2914 } else {
2915 // TODO
2916 // should we ever get anything other than this case?
2917 __ stlxr(rscratch1, src_reg, base);
2918 }
2919 } else {
2920 Register index_reg = as_Register(index);
2921 if (disp == 0) {
2922 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
2923 __ stlxr(rscratch1, src_reg, rscratch2);
2924 } else {
2925 __ lea(rscratch2, Address(base, disp));
2926 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
2927 __ stlxr(rscratch1, src_reg, rscratch2);
2928 }
2929 }
2930 __ cmpw(rscratch1, zr);
2931 %}
2932
2933 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2934 MacroAssembler _masm(&cbuf);
2935 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2936 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2937 Assembler::xword, /*acquire*/ false, /*release*/ true,
2938 /*weak*/ false, noreg);
2939 %}
2940
2941 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2942 MacroAssembler _masm(&cbuf);
2943 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2944 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2945 Assembler::word, /*acquire*/ false, /*release*/ true,
2946 /*weak*/ false, noreg);
2947 %}
2948
2949 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2950 MacroAssembler _masm(&cbuf);
2951 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2952 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2953 Assembler::halfword, /*acquire*/ false, /*release*/ true,
2954 /*weak*/ false, noreg);
2955 %}
2956
2957 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2958 MacroAssembler _masm(&cbuf);
2959 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2960 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2961 Assembler::byte, /*acquire*/ false, /*release*/ true,
2962 /*weak*/ false, noreg);
2963 %}
2964
2965
2966 // The only difference between aarch64_enc_cmpxchg and
2967 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
2968 // CompareAndSwap sequence to serve as a barrier on acquiring a
2969 // lock.
2970 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2971 MacroAssembler _masm(&cbuf);
2972 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2973 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2974 Assembler::xword, /*acquire*/ true, /*release*/ true,
2975 /*weak*/ false, noreg);
2976 %}
2977
2978 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2979 MacroAssembler _masm(&cbuf);
2980 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2981 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2982 Assembler::word, /*acquire*/ true, /*release*/ true,
2983 /*weak*/ false, noreg);
2984 %}
2985
2986 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2987 MacroAssembler _masm(&cbuf);
2988 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2989 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2990 Assembler::halfword, /*acquire*/ true, /*release*/ true,
2991 /*weak*/ false, noreg);
2992 %}
2993
2994 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2995 MacroAssembler _masm(&cbuf);
2996 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2997 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2998 Assembler::byte, /*acquire*/ true, /*release*/ true,
2999 /*weak*/ false, noreg);
3000 %}
3001
3002 // auxiliary used for CompareAndSwapX to set result register
3003 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3004 MacroAssembler _masm(&cbuf);
3005 Register res_reg = as_Register($res$$reg);
3006 __ cset(res_reg, Assembler::EQ);
3007 %}
3008
3009 // prefetch encodings
3010
3011 enc_class aarch64_enc_prefetchw(memory mem) %{
3012 MacroAssembler _masm(&cbuf);
3013 Register base = as_Register($mem$$base);
3014 int index = $mem$$index;
3015 int scale = $mem$$scale;
3016 int disp = $mem$$disp;
3017 if (index == -1) {
3018 __ prfm(Address(base, disp), PSTL1KEEP);
3019 } else {
3020 Register index_reg = as_Register(index);
3021 if (disp == 0) {
3022 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3023 } else {
3024 __ lea(rscratch1, Address(base, disp));
3025 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3026 }
3027 }
3028 %}
3029
3030 /// mov envcodings
3031
3032 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3033 MacroAssembler _masm(&cbuf);
3034 u_int32_t con = (u_int32_t)$src$$constant;
3035 Register dst_reg = as_Register($dst$$reg);
3036 if (con == 0) {
3037 __ movw(dst_reg, zr);
3038 } else {
3039 __ movw(dst_reg, con);
3040 }
3041 %}
3042
3043 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3044 MacroAssembler _masm(&cbuf);
3045 Register dst_reg = as_Register($dst$$reg);
3046 u_int64_t con = (u_int64_t)$src$$constant;
3047 if (con == 0) {
3048 __ mov(dst_reg, zr);
3049 } else {
3050 __ mov(dst_reg, con);
3051 }
3052 %}
3053
3054 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3055 MacroAssembler _masm(&cbuf);
3056 Register dst_reg = as_Register($dst$$reg);
3057 address con = (address)$src$$constant;
3058 if (con == NULL || con == (address)1) {
3059 ShouldNotReachHere();
3060 } else {
3061 relocInfo::relocType rtype = $src->constant_reloc();
3062 if (rtype == relocInfo::oop_type) {
3063 __ movoop(dst_reg, (jobject)con, /*immediate*/true);
3064 } else if (rtype == relocInfo::metadata_type) {
3065 __ mov_metadata(dst_reg, (Metadata*)con);
3066 } else {
3067 assert(rtype == relocInfo::none, "unexpected reloc type");
3068 if (con < (address)(uintptr_t)os::vm_page_size()) {
3069 __ mov(dst_reg, con);
3070 } else {
3071 unsigned long offset;
3072 __ adrp(dst_reg, con, offset);
3073 __ add(dst_reg, dst_reg, offset);
3074 }
3075 }
3076 }
3077 %}
3078
3079 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3080 MacroAssembler _masm(&cbuf);
3081 Register dst_reg = as_Register($dst$$reg);
3082 __ mov(dst_reg, zr);
3083 %}
3084
3085 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3086 MacroAssembler _masm(&cbuf);
3087 Register dst_reg = as_Register($dst$$reg);
3088 __ mov(dst_reg, (u_int64_t)1);
3089 %}
3090
3091 enc_class aarch64_enc_mov_poll_page(iRegP dst, immPollPage src) %{
3092 MacroAssembler _masm(&cbuf);
3093 address page = (address)$src$$constant;
3094 Register dst_reg = as_Register($dst$$reg);
3095 unsigned long off;
3096 __ adrp(dst_reg, Address(page, relocInfo::poll_type), off);
3097 assert(off == 0, "assumed offset == 0");
3098 %}
3099
3100 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3101 MacroAssembler _masm(&cbuf);
3102 __ load_byte_map_base($dst$$Register);
3103 %}
3104
3105 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3106 MacroAssembler _masm(&cbuf);
3107 Register dst_reg = as_Register($dst$$reg);
3108 address con = (address)$src$$constant;
3109 if (con == NULL) {
3110 ShouldNotReachHere();
3111 } else {
3112 relocInfo::relocType rtype = $src->constant_reloc();
3113 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3114 __ set_narrow_oop(dst_reg, (jobject)con);
3115 }
3116 %}
3117
3118 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3119 MacroAssembler _masm(&cbuf);
3120 Register dst_reg = as_Register($dst$$reg);
3121 __ mov(dst_reg, zr);
3122 %}
3123
3124 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3125 MacroAssembler _masm(&cbuf);
3126 Register dst_reg = as_Register($dst$$reg);
3127 address con = (address)$src$$constant;
3128 if (con == NULL) {
3129 ShouldNotReachHere();
3130 } else {
3131 relocInfo::relocType rtype = $src->constant_reloc();
3132 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3133 __ set_narrow_klass(dst_reg, (Klass *)con);
3134 }
3135 %}
3136
3137 // arithmetic encodings
3138
3139 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3140 MacroAssembler _masm(&cbuf);
3141 Register dst_reg = as_Register($dst$$reg);
3142 Register src_reg = as_Register($src1$$reg);
3143 int32_t con = (int32_t)$src2$$constant;
3144 // add has primary == 0, subtract has primary == 1
3145 if ($primary) { con = -con; }
3146 if (con < 0) {
3147 __ subw(dst_reg, src_reg, -con);
3148 } else {
3149 __ addw(dst_reg, src_reg, con);
3150 }
3151 %}
3152
3153 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3154 MacroAssembler _masm(&cbuf);
3155 Register dst_reg = as_Register($dst$$reg);
3156 Register src_reg = as_Register($src1$$reg);
3157 int32_t con = (int32_t)$src2$$constant;
3158 // add has primary == 0, subtract has primary == 1
3159 if ($primary) { con = -con; }
3160 if (con < 0) {
3161 __ sub(dst_reg, src_reg, -con);
3162 } else {
3163 __ add(dst_reg, src_reg, con);
3164 }
3165 %}
3166
3167 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3168 MacroAssembler _masm(&cbuf);
3169 Register dst_reg = as_Register($dst$$reg);
3170 Register src1_reg = as_Register($src1$$reg);
3171 Register src2_reg = as_Register($src2$$reg);
3172 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3173 %}
3174
3175 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3176 MacroAssembler _masm(&cbuf);
3177 Register dst_reg = as_Register($dst$$reg);
3178 Register src1_reg = as_Register($src1$$reg);
3179 Register src2_reg = as_Register($src2$$reg);
3180 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3181 %}
3182
3183 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3184 MacroAssembler _masm(&cbuf);
3185 Register dst_reg = as_Register($dst$$reg);
3186 Register src1_reg = as_Register($src1$$reg);
3187 Register src2_reg = as_Register($src2$$reg);
3188 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3189 %}
3190
3191 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3192 MacroAssembler _masm(&cbuf);
3193 Register dst_reg = as_Register($dst$$reg);
3194 Register src1_reg = as_Register($src1$$reg);
3195 Register src2_reg = as_Register($src2$$reg);
3196 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3197 %}
3198
3199 // compare instruction encodings
3200
3201 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3202 MacroAssembler _masm(&cbuf);
3203 Register reg1 = as_Register($src1$$reg);
3204 Register reg2 = as_Register($src2$$reg);
3205 __ cmpw(reg1, reg2);
3206 %}
3207
3208 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3209 MacroAssembler _masm(&cbuf);
3210 Register reg = as_Register($src1$$reg);
3211 int32_t val = $src2$$constant;
3212 if (val >= 0) {
3213 __ subsw(zr, reg, val);
3214 } else {
3215 __ addsw(zr, reg, -val);
3216 }
3217 %}
3218
3219 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3220 MacroAssembler _masm(&cbuf);
3221 Register reg1 = as_Register($src1$$reg);
3222 u_int32_t val = (u_int32_t)$src2$$constant;
3223 __ movw(rscratch1, val);
3224 __ cmpw(reg1, rscratch1);
3225 %}
3226
3227 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3228 MacroAssembler _masm(&cbuf);
3229 Register reg1 = as_Register($src1$$reg);
3230 Register reg2 = as_Register($src2$$reg);
3231 __ cmp(reg1, reg2);
3232 %}
3233
3234 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3235 MacroAssembler _masm(&cbuf);
3236 Register reg = as_Register($src1$$reg);
3237 int64_t val = $src2$$constant;
3238 if (val >= 0) {
3239 __ subs(zr, reg, val);
3240 } else if (val != -val) {
3241 __ adds(zr, reg, -val);
3242 } else {
3243 // aargh, Long.MIN_VALUE is a special case
3244 __ orr(rscratch1, zr, (u_int64_t)val);
3245 __ subs(zr, reg, rscratch1);
3246 }
3247 %}
3248
3249 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3250 MacroAssembler _masm(&cbuf);
3251 Register reg1 = as_Register($src1$$reg);
3252 u_int64_t val = (u_int64_t)$src2$$constant;
3253 __ mov(rscratch1, val);
3254 __ cmp(reg1, rscratch1);
3255 %}
3256
3257 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3258 MacroAssembler _masm(&cbuf);
3259 Register reg1 = as_Register($src1$$reg);
3260 Register reg2 = as_Register($src2$$reg);
3261 __ cmp(reg1, reg2);
3262 %}
3263
3264 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3265 MacroAssembler _masm(&cbuf);
3266 Register reg1 = as_Register($src1$$reg);
3267 Register reg2 = as_Register($src2$$reg);
3268 __ cmpw(reg1, reg2);
3269 %}
3270
3271 enc_class aarch64_enc_testp(iRegP src) %{
3272 MacroAssembler _masm(&cbuf);
3273 Register reg = as_Register($src$$reg);
3274 __ cmp(reg, zr);
3275 %}
3276
3277 enc_class aarch64_enc_testn(iRegN src) %{
3278 MacroAssembler _masm(&cbuf);
3279 Register reg = as_Register($src$$reg);
3280 __ cmpw(reg, zr);
3281 %}
3282
3283 enc_class aarch64_enc_b(label lbl) %{
3284 MacroAssembler _masm(&cbuf);
3285 Label *L = $lbl$$label;
3286 __ b(*L);
3287 %}
3288
3289 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3290 MacroAssembler _masm(&cbuf);
3291 Label *L = $lbl$$label;
3292 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3293 %}
3294
3295 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3296 MacroAssembler _masm(&cbuf);
3297 Label *L = $lbl$$label;
3298 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3299 %}
3300
3301 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3302 %{
3303 Register sub_reg = as_Register($sub$$reg);
3304 Register super_reg = as_Register($super$$reg);
3305 Register temp_reg = as_Register($temp$$reg);
3306 Register result_reg = as_Register($result$$reg);
3307
3308 Label miss;
3309 MacroAssembler _masm(&cbuf);
3310 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3311 NULL, &miss,
3312 /*set_cond_codes:*/ true);
3313 if ($primary) {
3314 __ mov(result_reg, zr);
3315 }
3316 __ bind(miss);
3317 %}
3318
3319 enc_class aarch64_enc_java_static_call(method meth) %{
3320 MacroAssembler _masm(&cbuf);
3321
3322 address addr = (address)$meth$$method;
3323 address call;
3324 if (!_method) {
3325 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3326 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3327 } else {
3328 int method_index = resolved_method_index(cbuf);
3329 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3330 : static_call_Relocation::spec(method_index);
3331 call = __ trampoline_call(Address(addr, rspec), &cbuf);
3332
3333 // Emit stub for static call
3334 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
3335 if (stub == NULL) {
3336 ciEnv::current()->record_failure("CodeCache is full");
3337 return;
3338 }
3339 }
3340 if (call == NULL) {
3341 ciEnv::current()->record_failure("CodeCache is full");
3342 return;
3343 }
3344 %}
3345
3346 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3347 MacroAssembler _masm(&cbuf);
3348 int method_index = resolved_method_index(cbuf);
3349 address call = __ ic_call((address)$meth$$method, method_index);
3350 if (call == NULL) {
3351 ciEnv::current()->record_failure("CodeCache is full");
3352 return;
3353 }
3354 %}
3355
3356 enc_class aarch64_enc_call_epilog() %{
3357 MacroAssembler _masm(&cbuf);
3358 if (VerifyStackAtCalls) {
3359 // Check that stack depth is unchanged: find majik cookie on stack
3360 __ call_Unimplemented();
3361 }
3362 %}
3363
3364 enc_class aarch64_enc_java_to_runtime(method meth) %{
3365 MacroAssembler _masm(&cbuf);
3366
3367 // some calls to generated routines (arraycopy code) are scheduled
3368 // by C2 as runtime calls. if so we can call them using a br (they
3369 // will be in a reachable segment) otherwise we have to use a blrt
3370 // which loads the absolute address into a register.
3371 address entry = (address)$meth$$method;
3372 CodeBlob *cb = CodeCache::find_blob(entry);
3373 if (cb) {
3374 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3375 if (call == NULL) {
3376 ciEnv::current()->record_failure("CodeCache is full");
3377 return;
3378 }
3379 } else {
3380 int gpcnt;
3381 int fpcnt;
3382 int rtype;
3383 getCallInfo(tf(), gpcnt, fpcnt, rtype);
3384 Label retaddr;
3385 __ adr(rscratch2, retaddr);
3386 __ lea(rscratch1, RuntimeAddress(entry));
3387 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3388 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3389 __ blrt(rscratch1, gpcnt, fpcnt, rtype);
3390 __ bind(retaddr);
3391 __ add(sp, sp, 2 * wordSize);
3392 }
3393 %}
3394
3395 enc_class aarch64_enc_rethrow() %{
3396 MacroAssembler _masm(&cbuf);
3397 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3398 %}
3399
3400 enc_class aarch64_enc_ret() %{
3401 MacroAssembler _masm(&cbuf);
3402 __ ret(lr);
3403 %}
3404
3405 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3406 MacroAssembler _masm(&cbuf);
3407 Register target_reg = as_Register($jump_target$$reg);
3408 __ br(target_reg);
3409 %}
3410
3411 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3412 MacroAssembler _masm(&cbuf);
3413 Register target_reg = as_Register($jump_target$$reg);
3414 // exception oop should be in r0
3415 // ret addr has been popped into lr
3416 // callee expects it in r3
3417 __ mov(r3, lr);
3418 __ br(target_reg);
3419 %}
3420
3421 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3422 MacroAssembler _masm(&cbuf);
3423 Register oop = as_Register($object$$reg);
3424 Register box = as_Register($box$$reg);
3425 Register disp_hdr = as_Register($tmp$$reg);
3426 Register tmp = as_Register($tmp2$$reg);
3427 Label cont;
3428 Label object_has_monitor;
3429 Label cas_failed;
3430
3431 assert_different_registers(oop, box, tmp, disp_hdr);
3432
3433 // Load markOop from object into displaced_header.
3434 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3435
3436 if (UseBiasedLocking && !UseOptoBiasInlining) {
3437 __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
3438 }
3439
3440 // Check for existing monitor
3441 __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3442
3443 // Set tmp to be (markOop of object | UNLOCK_VALUE).
3444 __ orr(tmp, disp_hdr, markOopDesc::unlocked_value);
3445
3446 // Initialize the box. (Must happen before we update the object mark!)
3447 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3448
3449 // Compare object markOop with an unlocked value (tmp) and if
3450 // equal exchange the stack address of our box with object markOop.
3451 // On failure disp_hdr contains the possibly locked markOop.
3452 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3453 /*release*/ true, /*weak*/ false, disp_hdr);
3454 __ br(Assembler::EQ, cont);
3455
3456 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3457
3458 // If the compare-and-exchange succeeded, then we found an unlocked
3459 // object, will have now locked it will continue at label cont
3460
3461 __ bind(cas_failed);
3462 // We did not see an unlocked object so try the fast recursive case.
3463
3464 // Check if the owner is self by comparing the value in the
3465 // markOop of object (disp_hdr) with the stack pointer.
3466 __ mov(rscratch1, sp);
3467 __ sub(disp_hdr, disp_hdr, rscratch1);
3468 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place));
3469 // If condition is true we are cont and hence we can store 0 as the
3470 // displaced header in the box, which indicates that it is a recursive lock.
3471 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3472 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3473
3474 __ b(cont);
3475
3476 // Handle existing monitor.
3477 __ bind(object_has_monitor);
3478
3479 // The object's monitor m is unlocked iff m->owner == NULL,
3480 // otherwise m->owner may contain a thread or a stack address.
3481 //
3482 // Try to CAS m->owner from NULL to current thread.
3483 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
3484 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3485 /*release*/ true, /*weak*/ false, noreg); // Sets flags for result
3486
3487 // Store a non-null value into the box to avoid looking like a re-entrant
3488 // lock. The fast-path monitor unlock code checks for
3489 // markOopDesc::monitor_value so use markOopDesc::unused_mark which has the
3490 // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
3491 __ mov(tmp, (address)markOopDesc::unused_mark());
3492 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3493
3494 __ bind(cont);
3495 // flag == EQ indicates success
3496 // flag == NE indicates failure
3497 %}
3498
3499 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3500 MacroAssembler _masm(&cbuf);
3501 Register oop = as_Register($object$$reg);
3502 Register box = as_Register($box$$reg);
3503 Register disp_hdr = as_Register($tmp$$reg);
3504 Register tmp = as_Register($tmp2$$reg);
3505 Label cont;
3506 Label object_has_monitor;
3507
3508 assert_different_registers(oop, box, tmp, disp_hdr);
3509
3510 if (UseBiasedLocking && !UseOptoBiasInlining) {
3511 __ biased_locking_exit(oop, tmp, cont);
3512 }
3513
3514 // Find the lock address and load the displaced header from the stack.
3515 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3516
3517 // If the displaced header is 0, we have a recursive unlock.
3518 __ cmp(disp_hdr, zr);
3519 __ br(Assembler::EQ, cont);
3520
3521 // Handle existing monitor.
3522 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3523 __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3524
3525 // Check if it is still a light weight lock, this is is true if we
3526 // see the stack address of the basicLock in the markOop of the
3527 // object.
3528
3529 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3530 /*release*/ true, /*weak*/ false, tmp);
3531 __ b(cont);
3532
3533 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3534
3535 // Handle existing monitor.
3536 __ bind(object_has_monitor);
3537 __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor
3538 __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3539 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3540 __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3541 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3542 __ cmp(rscratch1, zr); // Sets flags for result
3543 __ br(Assembler::NE, cont);
3544
3545 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3546 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3547 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3548 __ cmp(rscratch1, zr); // Sets flags for result
3549 __ cbnz(rscratch1, cont);
3550 // need a release store here
3551 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3552 __ stlr(zr, tmp); // set unowned
3553
3554 __ bind(cont);
3555 // flag == EQ indicates success
3556 // flag == NE indicates failure
3557 %}
3558
3559 %}
3560
3561 //----------FRAME--------------------------------------------------------------
3562 // Definition of frame structure and management information.
3563 //
3564 // S T A C K L A Y O U T Allocators stack-slot number
3565 // | (to get allocators register number
3566 // G Owned by | | v add OptoReg::stack0())
3567 // r CALLER | |
3568 // o | +--------+ pad to even-align allocators stack-slot
3569 // w V | pad0 | numbers; owned by CALLER
3570 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3571 // h ^ | in | 5
3572 // | | args | 4 Holes in incoming args owned by SELF
3573 // | | | | 3
3574 // | | +--------+
3575 // V | | old out| Empty on Intel, window on Sparc
3576 // | old |preserve| Must be even aligned.
3577 // | SP-+--------+----> Matcher::_old_SP, even aligned
3578 // | | in | 3 area for Intel ret address
3579 // Owned by |preserve| Empty on Sparc.
3580 // SELF +--------+
3581 // | | pad2 | 2 pad to align old SP
3582 // | +--------+ 1
3583 // | | locks | 0
3584 // | +--------+----> OptoReg::stack0(), even aligned
3585 // | | pad1 | 11 pad to align new SP
3586 // | +--------+
3587 // | | | 10
3588 // | | spills | 9 spills
3589 // V | | 8 (pad0 slot for callee)
3590 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3591 // ^ | out | 7
3592 // | | args | 6 Holes in outgoing args owned by CALLEE
3593 // Owned by +--------+
3594 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3595 // | new |preserve| Must be even-aligned.
3596 // | SP-+--------+----> Matcher::_new_SP, even aligned
3597 // | | |
3598 //
3599 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3600 // known from SELF's arguments and the Java calling convention.
3601 // Region 6-7 is determined per call site.
3602 // Note 2: If the calling convention leaves holes in the incoming argument
3603 // area, those holes are owned by SELF. Holes in the outgoing area
3604 // are owned by the CALLEE. Holes should not be nessecary in the
3605 // incoming area, as the Java calling convention is completely under
3606 // the control of the AD file. Doubles can be sorted and packed to
3607 // avoid holes. Holes in the outgoing arguments may be nessecary for
3608 // varargs C calling conventions.
3609 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3610 // even aligned with pad0 as needed.
3611 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3612 // (the latter is true on Intel but is it false on AArch64?)
3613 // region 6-11 is even aligned; it may be padded out more so that
3614 // the region from SP to FP meets the minimum stack alignment.
3615 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3616 // alignment. Region 11, pad1, may be dynamically extended so that
3617 // SP meets the minimum alignment.
3618
3619 frame %{
3620 // What direction does stack grow in (assumed to be same for C & Java)
3621 stack_direction(TOWARDS_LOW);
3622
3623 // These three registers define part of the calling convention
3624 // between compiled code and the interpreter.
3625
3626 // Inline Cache Register or methodOop for I2C.
3627 inline_cache_reg(R12);
3628
3629 // Method Oop Register when calling interpreter.
3630 interpreter_method_oop_reg(R12);
3631
3632 // Number of stack slots consumed by locking an object
3633 sync_stack_slots(2);
3634
3635 // Compiled code's Frame Pointer
3636 frame_pointer(R31);
3637
3638 // Interpreter stores its frame pointer in a register which is
3639 // stored to the stack by I2CAdaptors.
3640 // I2CAdaptors convert from interpreted java to compiled java.
3641 interpreter_frame_pointer(R29);
3642
3643 // Stack alignment requirement
3644 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3645
3646 // Number of stack slots between incoming argument block and the start of
3647 // a new frame. The PROLOG must add this many slots to the stack. The
3648 // EPILOG must remove this many slots. aarch64 needs two slots for
3649 // return address and fp.
3650 // TODO think this is correct but check
3651 in_preserve_stack_slots(4);
3652
3653 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3654 // for calls to C. Supports the var-args backing area for register parms.
3655 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3656
3657 // The after-PROLOG location of the return address. Location of
3658 // return address specifies a type (REG or STACK) and a number
3659 // representing the register number (i.e. - use a register name) or
3660 // stack slot.
3661 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3662 // Otherwise, it is above the locks and verification slot and alignment word
3663 // TODO this may well be correct but need to check why that - 2 is there
3664 // ppc port uses 0 but we definitely need to allow for fixed_slots
3665 // which folds in the space used for monitors
3666 return_addr(STACK - 2 +
3667 align_up((Compile::current()->in_preserve_stack_slots() +
3668 Compile::current()->fixed_slots()),
3669 stack_alignment_in_slots()));
3670
3671 // Body of function which returns an integer array locating
3672 // arguments either in registers or in stack slots. Passed an array
3673 // of ideal registers called "sig" and a "length" count. Stack-slot
3674 // offsets are based on outgoing arguments, i.e. a CALLER setting up
3675 // arguments for a CALLEE. Incoming stack arguments are
3676 // automatically biased by the preserve_stack_slots field above.
3677
3678 calling_convention
3679 %{
3680 // No difference between ingoing/outgoing just pass false
3681 SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3682 %}
3683
3684 c_calling_convention
3685 %{
3686 // This is obviously always outgoing
3687 (void) SharedRuntime::c_calling_convention(sig_bt, regs, NULL, length);
3688 %}
3689
3690 // Location of compiled Java return values. Same as C for now.
3691 return_value
3692 %{
3693 // TODO do we allow ideal_reg == Op_RegN???
3694 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3695 "only return normal values");
3696
3697 static const int lo[Op_RegL + 1] = { // enum name
3698 0, // Op_Node
3699 0, // Op_Set
3700 R0_num, // Op_RegN
3701 R0_num, // Op_RegI
3702 R0_num, // Op_RegP
3703 V0_num, // Op_RegF
3704 V0_num, // Op_RegD
3705 R0_num // Op_RegL
3706 };
3707
3708 static const int hi[Op_RegL + 1] = { // enum name
3709 0, // Op_Node
3710 0, // Op_Set
3711 OptoReg::Bad, // Op_RegN
3712 OptoReg::Bad, // Op_RegI
3713 R0_H_num, // Op_RegP
3714 OptoReg::Bad, // Op_RegF
3715 V0_H_num, // Op_RegD
3716 R0_H_num // Op_RegL
3717 };
3718
3719 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3720 %}
3721 %}
3722
3723 //----------ATTRIBUTES---------------------------------------------------------
3724 //----------Operand Attributes-------------------------------------------------
3725 op_attrib op_cost(1); // Required cost attribute
3726
3727 //----------Instruction Attributes---------------------------------------------
3728 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3729 ins_attrib ins_size(32); // Required size attribute (in bits)
3730 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3731 // a non-matching short branch variant
3732 // of some long branch?
3733 ins_attrib ins_alignment(4); // Required alignment attribute (must
3734 // be a power of 2) specifies the
3735 // alignment that some part of the
3736 // instruction (not necessarily the
3737 // start) requires. If > 1, a
3738 // compute_padding() function must be
3739 // provided for the instruction
3740
3741 //----------OPERANDS-----------------------------------------------------------
3742 // Operand definitions must precede instruction definitions for correct parsing
3743 // in the ADLC because operands constitute user defined types which are used in
3744 // instruction definitions.
3745
3746 //----------Simple Operands----------------------------------------------------
3747
3748 // Integer operands 32 bit
3749 // 32 bit immediate
3750 operand immI()
3751 %{
3752 match(ConI);
3753
3754 op_cost(0);
3755 format %{ %}
3756 interface(CONST_INTER);
3757 %}
3758
3759 // 32 bit zero
3760 operand immI0()
3761 %{
3762 predicate(n->get_int() == 0);
3763 match(ConI);
3764
3765 op_cost(0);
3766 format %{ %}
3767 interface(CONST_INTER);
3768 %}
3769
3770 // 32 bit unit increment
3771 operand immI_1()
3772 %{
3773 predicate(n->get_int() == 1);
3774 match(ConI);
3775
3776 op_cost(0);
3777 format %{ %}
3778 interface(CONST_INTER);
3779 %}
3780
3781 // 32 bit unit decrement
3782 operand immI_M1()
3783 %{
3784 predicate(n->get_int() == -1);
3785 match(ConI);
3786
3787 op_cost(0);
3788 format %{ %}
3789 interface(CONST_INTER);
3790 %}
3791
3792 // Shift values for add/sub extension shift
3793 operand immIExt()
3794 %{
3795 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3796 match(ConI);
3797
3798 op_cost(0);
3799 format %{ %}
3800 interface(CONST_INTER);
3801 %}
3802
3803 operand immI_le_4()
3804 %{
3805 predicate(n->get_int() <= 4);
3806 match(ConI);
3807
3808 op_cost(0);
3809 format %{ %}
3810 interface(CONST_INTER);
3811 %}
3812
3813 operand immI_31()
3814 %{
3815 predicate(n->get_int() == 31);
3816 match(ConI);
3817
3818 op_cost(0);
3819 format %{ %}
3820 interface(CONST_INTER);
3821 %}
3822
3823 operand immI_8()
3824 %{
3825 predicate(n->get_int() == 8);
3826 match(ConI);
3827
3828 op_cost(0);
3829 format %{ %}
3830 interface(CONST_INTER);
3831 %}
3832
3833 operand immI_16()
3834 %{
3835 predicate(n->get_int() == 16);
3836 match(ConI);
3837
3838 op_cost(0);
3839 format %{ %}
3840 interface(CONST_INTER);
3841 %}
3842
3843 operand immI_24()
3844 %{
3845 predicate(n->get_int() == 24);
3846 match(ConI);
3847
3848 op_cost(0);
3849 format %{ %}
3850 interface(CONST_INTER);
3851 %}
3852
3853 operand immI_32()
3854 %{
3855 predicate(n->get_int() == 32);
3856 match(ConI);
3857
3858 op_cost(0);
3859 format %{ %}
3860 interface(CONST_INTER);
3861 %}
3862
3863 operand immI_48()
3864 %{
3865 predicate(n->get_int() == 48);
3866 match(ConI);
3867
3868 op_cost(0);
3869 format %{ %}
3870 interface(CONST_INTER);
3871 %}
3872
3873 operand immI_56()
3874 %{
3875 predicate(n->get_int() == 56);
3876 match(ConI);
3877
3878 op_cost(0);
3879 format %{ %}
3880 interface(CONST_INTER);
3881 %}
3882
3883 operand immI_63()
3884 %{
3885 predicate(n->get_int() == 63);
3886 match(ConI);
3887
3888 op_cost(0);
3889 format %{ %}
3890 interface(CONST_INTER);
3891 %}
3892
3893 operand immI_64()
3894 %{
3895 predicate(n->get_int() == 64);
3896 match(ConI);
3897
3898 op_cost(0);
3899 format %{ %}
3900 interface(CONST_INTER);
3901 %}
3902
3903 operand immI_255()
3904 %{
3905 predicate(n->get_int() == 255);
3906 match(ConI);
3907
3908 op_cost(0);
3909 format %{ %}
3910 interface(CONST_INTER);
3911 %}
3912
3913 operand immI_65535()
3914 %{
3915 predicate(n->get_int() == 65535);
3916 match(ConI);
3917
3918 op_cost(0);
3919 format %{ %}
3920 interface(CONST_INTER);
3921 %}
3922
3923 operand immL_255()
3924 %{
3925 predicate(n->get_long() == 255L);
3926 match(ConL);
3927
3928 op_cost(0);
3929 format %{ %}
3930 interface(CONST_INTER);
3931 %}
3932
3933 operand immL_65535()
3934 %{
3935 predicate(n->get_long() == 65535L);
3936 match(ConL);
3937
3938 op_cost(0);
3939 format %{ %}
3940 interface(CONST_INTER);
3941 %}
3942
3943 operand immL_4294967295()
3944 %{
3945 predicate(n->get_long() == 4294967295L);
3946 match(ConL);
3947
3948 op_cost(0);
3949 format %{ %}
3950 interface(CONST_INTER);
3951 %}
3952
3953 operand immL_bitmask()
3954 %{
3955 predicate(((n->get_long() & 0xc000000000000000l) == 0)
3956 && is_power_of_2(n->get_long() + 1));
3957 match(ConL);
3958
3959 op_cost(0);
3960 format %{ %}
3961 interface(CONST_INTER);
3962 %}
3963
3964 operand immI_bitmask()
3965 %{
3966 predicate(((n->get_int() & 0xc0000000) == 0)
3967 && is_power_of_2(n->get_int() + 1));
3968 match(ConI);
3969
3970 op_cost(0);
3971 format %{ %}
3972 interface(CONST_INTER);
3973 %}
3974
3975 // Scale values for scaled offset addressing modes (up to long but not quad)
3976 operand immIScale()
3977 %{
3978 predicate(0 <= n->get_int() && (n->get_int() <= 3));
3979 match(ConI);
3980
3981 op_cost(0);
3982 format %{ %}
3983 interface(CONST_INTER);
3984 %}
3985
3986 // 26 bit signed offset -- for pc-relative branches
3987 operand immI26()
3988 %{
3989 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
3990 match(ConI);
3991
3992 op_cost(0);
3993 format %{ %}
3994 interface(CONST_INTER);
3995 %}
3996
3997 // 19 bit signed offset -- for pc-relative loads
3998 operand immI19()
3999 %{
4000 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4001 match(ConI);
4002
4003 op_cost(0);
4004 format %{ %}
4005 interface(CONST_INTER);
4006 %}
4007
4008 // 12 bit unsigned offset -- for base plus immediate loads
4009 operand immIU12()
4010 %{
4011 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4012 match(ConI);
4013
4014 op_cost(0);
4015 format %{ %}
4016 interface(CONST_INTER);
4017 %}
4018
4019 operand immLU12()
4020 %{
4021 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4022 match(ConL);
4023
4024 op_cost(0);
4025 format %{ %}
4026 interface(CONST_INTER);
4027 %}
4028
4029 // Offset for scaled or unscaled immediate loads and stores
4030 operand immIOffset()
4031 %{
4032 predicate(Address::offset_ok_for_immed(n->get_int()));
4033 match(ConI);
4034
4035 op_cost(0);
4036 format %{ %}
4037 interface(CONST_INTER);
4038 %}
4039
4040 operand immIOffset4()
4041 %{
4042 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4043 match(ConI);
4044
4045 op_cost(0);
4046 format %{ %}
4047 interface(CONST_INTER);
4048 %}
4049
4050 operand immIOffset8()
4051 %{
4052 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4053 match(ConI);
4054
4055 op_cost(0);
4056 format %{ %}
4057 interface(CONST_INTER);
4058 %}
4059
4060 operand immIOffset16()
4061 %{
4062 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4063 match(ConI);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 operand immLoffset()
4071 %{
4072 predicate(Address::offset_ok_for_immed(n->get_long()));
4073 match(ConL);
4074
4075 op_cost(0);
4076 format %{ %}
4077 interface(CONST_INTER);
4078 %}
4079
4080 operand immLoffset4()
4081 %{
4082 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4083 match(ConL);
4084
4085 op_cost(0);
4086 format %{ %}
4087 interface(CONST_INTER);
4088 %}
4089
4090 operand immLoffset8()
4091 %{
4092 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4093 match(ConL);
4094
4095 op_cost(0);
4096 format %{ %}
4097 interface(CONST_INTER);
4098 %}
4099
4100 operand immLoffset16()
4101 %{
4102 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4103 match(ConL);
4104
4105 op_cost(0);
4106 format %{ %}
4107 interface(CONST_INTER);
4108 %}
4109
4110 // 32 bit integer valid for add sub immediate
4111 operand immIAddSub()
4112 %{
4113 predicate(Assembler::operand_valid_for_add_sub_immediate((long)n->get_int()));
4114 match(ConI);
4115 op_cost(0);
4116 format %{ %}
4117 interface(CONST_INTER);
4118 %}
4119
4120 // 32 bit unsigned integer valid for logical immediate
4121 // TODO -- check this is right when e.g the mask is 0x80000000
4122 operand immILog()
4123 %{
4124 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (unsigned long)n->get_int()));
4125 match(ConI);
4126
4127 op_cost(0);
4128 format %{ %}
4129 interface(CONST_INTER);
4130 %}
4131
4132 // Integer operands 64 bit
4133 // 64 bit immediate
4134 operand immL()
4135 %{
4136 match(ConL);
4137
4138 op_cost(0);
4139 format %{ %}
4140 interface(CONST_INTER);
4141 %}
4142
4143 // 64 bit zero
4144 operand immL0()
4145 %{
4146 predicate(n->get_long() == 0);
4147 match(ConL);
4148
4149 op_cost(0);
4150 format %{ %}
4151 interface(CONST_INTER);
4152 %}
4153
4154 // 64 bit unit increment
4155 operand immL_1()
4156 %{
4157 predicate(n->get_long() == 1);
4158 match(ConL);
4159
4160 op_cost(0);
4161 format %{ %}
4162 interface(CONST_INTER);
4163 %}
4164
4165 // 64 bit unit decrement
4166 operand immL_M1()
4167 %{
4168 predicate(n->get_long() == -1);
4169 match(ConL);
4170
4171 op_cost(0);
4172 format %{ %}
4173 interface(CONST_INTER);
4174 %}
4175
4176 // 32 bit offset of pc in thread anchor
4177
4178 operand immL_pc_off()
4179 %{
4180 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4181 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4182 match(ConL);
4183
4184 op_cost(0);
4185 format %{ %}
4186 interface(CONST_INTER);
4187 %}
4188
4189 // 64 bit integer valid for add sub immediate
4190 operand immLAddSub()
4191 %{
4192 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4193 match(ConL);
4194 op_cost(0);
4195 format %{ %}
4196 interface(CONST_INTER);
4197 %}
4198
4199 // 64 bit integer valid for logical immediate
4200 operand immLLog()
4201 %{
4202 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (unsigned long)n->get_long()));
4203 match(ConL);
4204 op_cost(0);
4205 format %{ %}
4206 interface(CONST_INTER);
4207 %}
4208
4209 // Long Immediate: low 32-bit mask
4210 operand immL_32bits()
4211 %{
4212 predicate(n->get_long() == 0xFFFFFFFFL);
4213 match(ConL);
4214 op_cost(0);
4215 format %{ %}
4216 interface(CONST_INTER);
4217 %}
4218
4219 // Pointer operands
4220 // Pointer Immediate
4221 operand immP()
4222 %{
4223 match(ConP);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 // NULL Pointer Immediate
4231 operand immP0()
4232 %{
4233 predicate(n->get_ptr() == 0);
4234 match(ConP);
4235
4236 op_cost(0);
4237 format %{ %}
4238 interface(CONST_INTER);
4239 %}
4240
4241 // Pointer Immediate One
4242 // this is used in object initialization (initial object header)
4243 operand immP_1()
4244 %{
4245 predicate(n->get_ptr() == 1);
4246 match(ConP);
4247
4248 op_cost(0);
4249 format %{ %}
4250 interface(CONST_INTER);
4251 %}
4252
4253 // Polling Page Pointer Immediate
4254 operand immPollPage()
4255 %{
4256 predicate((address)n->get_ptr() == os::get_polling_page());
4257 match(ConP);
4258
4259 op_cost(0);
4260 format %{ %}
4261 interface(CONST_INTER);
4262 %}
4263
4264 // Card Table Byte Map Base
4265 operand immByteMapBase()
4266 %{
4267 // Get base of card map
4268 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4269 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4270 match(ConP);
4271
4272 op_cost(0);
4273 format %{ %}
4274 interface(CONST_INTER);
4275 %}
4276
4277 // Pointer Immediate Minus One
4278 // this is used when we want to write the current PC to the thread anchor
4279 operand immP_M1()
4280 %{
4281 predicate(n->get_ptr() == -1);
4282 match(ConP);
4283
4284 op_cost(0);
4285 format %{ %}
4286 interface(CONST_INTER);
4287 %}
4288
4289 // Pointer Immediate Minus Two
4290 // this is used when we want to write the current PC to the thread anchor
4291 operand immP_M2()
4292 %{
4293 predicate(n->get_ptr() == -2);
4294 match(ConP);
4295
4296 op_cost(0);
4297 format %{ %}
4298 interface(CONST_INTER);
4299 %}
4300
4301 // Float and Double operands
4302 // Double Immediate
4303 operand immD()
4304 %{
4305 match(ConD);
4306 op_cost(0);
4307 format %{ %}
4308 interface(CONST_INTER);
4309 %}
4310
4311 // Double Immediate: +0.0d
4312 operand immD0()
4313 %{
4314 predicate(jlong_cast(n->getd()) == 0);
4315 match(ConD);
4316
4317 op_cost(0);
4318 format %{ %}
4319 interface(CONST_INTER);
4320 %}
4321
4322 // constant 'double +0.0'.
4323 operand immDPacked()
4324 %{
4325 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4326 match(ConD);
4327 op_cost(0);
4328 format %{ %}
4329 interface(CONST_INTER);
4330 %}
4331
4332 // Float Immediate
4333 operand immF()
4334 %{
4335 match(ConF);
4336 op_cost(0);
4337 format %{ %}
4338 interface(CONST_INTER);
4339 %}
4340
4341 // Float Immediate: +0.0f.
4342 operand immF0()
4343 %{
4344 predicate(jint_cast(n->getf()) == 0);
4345 match(ConF);
4346
4347 op_cost(0);
4348 format %{ %}
4349 interface(CONST_INTER);
4350 %}
4351
4352 //
4353 operand immFPacked()
4354 %{
4355 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4356 match(ConF);
4357 op_cost(0);
4358 format %{ %}
4359 interface(CONST_INTER);
4360 %}
4361
4362 // Narrow pointer operands
4363 // Narrow Pointer Immediate
4364 operand immN()
4365 %{
4366 match(ConN);
4367
4368 op_cost(0);
4369 format %{ %}
4370 interface(CONST_INTER);
4371 %}
4372
4373 // Narrow NULL Pointer Immediate
4374 operand immN0()
4375 %{
4376 predicate(n->get_narrowcon() == 0);
4377 match(ConN);
4378
4379 op_cost(0);
4380 format %{ %}
4381 interface(CONST_INTER);
4382 %}
4383
4384 operand immNKlass()
4385 %{
4386 match(ConNKlass);
4387
4388 op_cost(0);
4389 format %{ %}
4390 interface(CONST_INTER);
4391 %}
4392
4393 // Integer 32 bit Register Operands
4394 // Integer 32 bitRegister (excludes SP)
4395 operand iRegI()
4396 %{
4397 constraint(ALLOC_IN_RC(any_reg32));
4398 match(RegI);
4399 match(iRegINoSp);
4400 op_cost(0);
4401 format %{ %}
4402 interface(REG_INTER);
4403 %}
4404
4405 // Integer 32 bit Register not Special
4406 operand iRegINoSp()
4407 %{
4408 constraint(ALLOC_IN_RC(no_special_reg32));
4409 match(RegI);
4410 op_cost(0);
4411 format %{ %}
4412 interface(REG_INTER);
4413 %}
4414
4415 // Integer 64 bit Register Operands
4416 // Integer 64 bit Register (includes SP)
4417 operand iRegL()
4418 %{
4419 constraint(ALLOC_IN_RC(any_reg));
4420 match(RegL);
4421 match(iRegLNoSp);
4422 op_cost(0);
4423 format %{ %}
4424 interface(REG_INTER);
4425 %}
4426
4427 // Integer 64 bit Register not Special
4428 operand iRegLNoSp()
4429 %{
4430 constraint(ALLOC_IN_RC(no_special_reg));
4431 match(RegL);
4432 match(iRegL_R0);
4433 format %{ %}
4434 interface(REG_INTER);
4435 %}
4436
4437 // Pointer Register Operands
4438 // Pointer Register
4439 operand iRegP()
4440 %{
4441 constraint(ALLOC_IN_RC(ptr_reg));
4442 match(RegP);
4443 match(iRegPNoSp);
4444 match(iRegP_R0);
4445 //match(iRegP_R2);
4446 //match(iRegP_R4);
4447 //match(iRegP_R5);
4448 match(thread_RegP);
4449 op_cost(0);
4450 format %{ %}
4451 interface(REG_INTER);
4452 %}
4453
4454 // Pointer 64 bit Register not Special
4455 operand iRegPNoSp()
4456 %{
4457 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4458 match(RegP);
4459 // match(iRegP);
4460 // match(iRegP_R0);
4461 // match(iRegP_R2);
4462 // match(iRegP_R4);
4463 // match(iRegP_R5);
4464 // match(thread_RegP);
4465 op_cost(0);
4466 format %{ %}
4467 interface(REG_INTER);
4468 %}
4469
4470 // Pointer 64 bit Register R0 only
4471 operand iRegP_R0()
4472 %{
4473 constraint(ALLOC_IN_RC(r0_reg));
4474 match(RegP);
4475 // match(iRegP);
4476 match(iRegPNoSp);
4477 op_cost(0);
4478 format %{ %}
4479 interface(REG_INTER);
4480 %}
4481
4482 // Pointer 64 bit Register R1 only
4483 operand iRegP_R1()
4484 %{
4485 constraint(ALLOC_IN_RC(r1_reg));
4486 match(RegP);
4487 // match(iRegP);
4488 match(iRegPNoSp);
4489 op_cost(0);
4490 format %{ %}
4491 interface(REG_INTER);
4492 %}
4493
4494 // Pointer 64 bit Register R2 only
4495 operand iRegP_R2()
4496 %{
4497 constraint(ALLOC_IN_RC(r2_reg));
4498 match(RegP);
4499 // match(iRegP);
4500 match(iRegPNoSp);
4501 op_cost(0);
4502 format %{ %}
4503 interface(REG_INTER);
4504 %}
4505
4506 // Pointer 64 bit Register R3 only
4507 operand iRegP_R3()
4508 %{
4509 constraint(ALLOC_IN_RC(r3_reg));
4510 match(RegP);
4511 // match(iRegP);
4512 match(iRegPNoSp);
4513 op_cost(0);
4514 format %{ %}
4515 interface(REG_INTER);
4516 %}
4517
4518 // Pointer 64 bit Register R4 only
4519 operand iRegP_R4()
4520 %{
4521 constraint(ALLOC_IN_RC(r4_reg));
4522 match(RegP);
4523 // match(iRegP);
4524 match(iRegPNoSp);
4525 op_cost(0);
4526 format %{ %}
4527 interface(REG_INTER);
4528 %}
4529
4530 // Pointer 64 bit Register R5 only
4531 operand iRegP_R5()
4532 %{
4533 constraint(ALLOC_IN_RC(r5_reg));
4534 match(RegP);
4535 // match(iRegP);
4536 match(iRegPNoSp);
4537 op_cost(0);
4538 format %{ %}
4539 interface(REG_INTER);
4540 %}
4541
4542 // Pointer 64 bit Register R10 only
4543 operand iRegP_R10()
4544 %{
4545 constraint(ALLOC_IN_RC(r10_reg));
4546 match(RegP);
4547 // match(iRegP);
4548 match(iRegPNoSp);
4549 op_cost(0);
4550 format %{ %}
4551 interface(REG_INTER);
4552 %}
4553
4554 // Long 64 bit Register R0 only
4555 operand iRegL_R0()
4556 %{
4557 constraint(ALLOC_IN_RC(r0_reg));
4558 match(RegL);
4559 match(iRegLNoSp);
4560 op_cost(0);
4561 format %{ %}
4562 interface(REG_INTER);
4563 %}
4564
4565 // Long 64 bit Register R2 only
4566 operand iRegL_R2()
4567 %{
4568 constraint(ALLOC_IN_RC(r2_reg));
4569 match(RegL);
4570 match(iRegLNoSp);
4571 op_cost(0);
4572 format %{ %}
4573 interface(REG_INTER);
4574 %}
4575
4576 // Long 64 bit Register R3 only
4577 operand iRegL_R3()
4578 %{
4579 constraint(ALLOC_IN_RC(r3_reg));
4580 match(RegL);
4581 match(iRegLNoSp);
4582 op_cost(0);
4583 format %{ %}
4584 interface(REG_INTER);
4585 %}
4586
4587 // Long 64 bit Register R11 only
4588 operand iRegL_R11()
4589 %{
4590 constraint(ALLOC_IN_RC(r11_reg));
4591 match(RegL);
4592 match(iRegLNoSp);
4593 op_cost(0);
4594 format %{ %}
4595 interface(REG_INTER);
4596 %}
4597
4598 // Pointer 64 bit Register FP only
4599 operand iRegP_FP()
4600 %{
4601 constraint(ALLOC_IN_RC(fp_reg));
4602 match(RegP);
4603 // match(iRegP);
4604 op_cost(0);
4605 format %{ %}
4606 interface(REG_INTER);
4607 %}
4608
4609 // Register R0 only
4610 operand iRegI_R0()
4611 %{
4612 constraint(ALLOC_IN_RC(int_r0_reg));
4613 match(RegI);
4614 match(iRegINoSp);
4615 op_cost(0);
4616 format %{ %}
4617 interface(REG_INTER);
4618 %}
4619
4620 // Register R2 only
4621 operand iRegI_R2()
4622 %{
4623 constraint(ALLOC_IN_RC(int_r2_reg));
4624 match(RegI);
4625 match(iRegINoSp);
4626 op_cost(0);
4627 format %{ %}
4628 interface(REG_INTER);
4629 %}
4630
4631 // Register R3 only
4632 operand iRegI_R3()
4633 %{
4634 constraint(ALLOC_IN_RC(int_r3_reg));
4635 match(RegI);
4636 match(iRegINoSp);
4637 op_cost(0);
4638 format %{ %}
4639 interface(REG_INTER);
4640 %}
4641
4642
4643 // Register R4 only
4644 operand iRegI_R4()
4645 %{
4646 constraint(ALLOC_IN_RC(int_r4_reg));
4647 match(RegI);
4648 match(iRegINoSp);
4649 op_cost(0);
4650 format %{ %}
4651 interface(REG_INTER);
4652 %}
4653
4654
4655 // Pointer Register Operands
4656 // Narrow Pointer Register
4657 operand iRegN()
4658 %{
4659 constraint(ALLOC_IN_RC(any_reg32));
4660 match(RegN);
4661 match(iRegNNoSp);
4662 op_cost(0);
4663 format %{ %}
4664 interface(REG_INTER);
4665 %}
4666
4667 operand iRegN_R0()
4668 %{
4669 constraint(ALLOC_IN_RC(r0_reg));
4670 match(iRegN);
4671 op_cost(0);
4672 format %{ %}
4673 interface(REG_INTER);
4674 %}
4675
4676 operand iRegN_R2()
4677 %{
4678 constraint(ALLOC_IN_RC(r2_reg));
4679 match(iRegN);
4680 op_cost(0);
4681 format %{ %}
4682 interface(REG_INTER);
4683 %}
4684
4685 operand iRegN_R3()
4686 %{
4687 constraint(ALLOC_IN_RC(r3_reg));
4688 match(iRegN);
4689 op_cost(0);
4690 format %{ %}
4691 interface(REG_INTER);
4692 %}
4693
4694 // Integer 64 bit Register not Special
4695 operand iRegNNoSp()
4696 %{
4697 constraint(ALLOC_IN_RC(no_special_reg32));
4698 match(RegN);
4699 op_cost(0);
4700 format %{ %}
4701 interface(REG_INTER);
4702 %}
4703
4704 // heap base register -- used for encoding immN0
4705
4706 operand iRegIHeapbase()
4707 %{
4708 constraint(ALLOC_IN_RC(heapbase_reg));
4709 match(RegI);
4710 op_cost(0);
4711 format %{ %}
4712 interface(REG_INTER);
4713 %}
4714
4715 // Float Register
4716 // Float register operands
4717 operand vRegF()
4718 %{
4719 constraint(ALLOC_IN_RC(float_reg));
4720 match(RegF);
4721
4722 op_cost(0);
4723 format %{ %}
4724 interface(REG_INTER);
4725 %}
4726
4727 // Double Register
4728 // Double register operands
4729 operand vRegD()
4730 %{
4731 constraint(ALLOC_IN_RC(double_reg));
4732 match(RegD);
4733
4734 op_cost(0);
4735 format %{ %}
4736 interface(REG_INTER);
4737 %}
4738
4739 operand vecD()
4740 %{
4741 constraint(ALLOC_IN_RC(vectord_reg));
4742 match(VecD);
4743
4744 op_cost(0);
4745 format %{ %}
4746 interface(REG_INTER);
4747 %}
4748
4749 operand vecX()
4750 %{
4751 constraint(ALLOC_IN_RC(vectorx_reg));
4752 match(VecX);
4753
4754 op_cost(0);
4755 format %{ %}
4756 interface(REG_INTER);
4757 %}
4758
4759 operand vRegD_V0()
4760 %{
4761 constraint(ALLOC_IN_RC(v0_reg));
4762 match(RegD);
4763 op_cost(0);
4764 format %{ %}
4765 interface(REG_INTER);
4766 %}
4767
4768 operand vRegD_V1()
4769 %{
4770 constraint(ALLOC_IN_RC(v1_reg));
4771 match(RegD);
4772 op_cost(0);
4773 format %{ %}
4774 interface(REG_INTER);
4775 %}
4776
4777 operand vRegD_V2()
4778 %{
4779 constraint(ALLOC_IN_RC(v2_reg));
4780 match(RegD);
4781 op_cost(0);
4782 format %{ %}
4783 interface(REG_INTER);
4784 %}
4785
4786 operand vRegD_V3()
4787 %{
4788 constraint(ALLOC_IN_RC(v3_reg));
4789 match(RegD);
4790 op_cost(0);
4791 format %{ %}
4792 interface(REG_INTER);
4793 %}
4794
4795 // Flags register, used as output of signed compare instructions
4796
4797 // note that on AArch64 we also use this register as the output for
4798 // for floating point compare instructions (CmpF CmpD). this ensures
4799 // that ordered inequality tests use GT, GE, LT or LE none of which
4800 // pass through cases where the result is unordered i.e. one or both
4801 // inputs to the compare is a NaN. this means that the ideal code can
4802 // replace e.g. a GT with an LE and not end up capturing the NaN case
4803 // (where the comparison should always fail). EQ and NE tests are
4804 // always generated in ideal code so that unordered folds into the NE
4805 // case, matching the behaviour of AArch64 NE.
4806 //
4807 // This differs from x86 where the outputs of FP compares use a
4808 // special FP flags registers and where compares based on this
4809 // register are distinguished into ordered inequalities (cmpOpUCF) and
4810 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
4811 // to explicitly handle the unordered case in branches. x86 also has
4812 // to include extra CMoveX rules to accept a cmpOpUCF input.
4813
4814 operand rFlagsReg()
4815 %{
4816 constraint(ALLOC_IN_RC(int_flags));
4817 match(RegFlags);
4818
4819 op_cost(0);
4820 format %{ "RFLAGS" %}
4821 interface(REG_INTER);
4822 %}
4823
4824 // Flags register, used as output of unsigned compare instructions
4825 operand rFlagsRegU()
4826 %{
4827 constraint(ALLOC_IN_RC(int_flags));
4828 match(RegFlags);
4829
4830 op_cost(0);
4831 format %{ "RFLAGSU" %}
4832 interface(REG_INTER);
4833 %}
4834
4835 // Special Registers
4836
4837 // Method Register
4838 operand inline_cache_RegP(iRegP reg)
4839 %{
4840 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
4841 match(reg);
4842 match(iRegPNoSp);
4843 op_cost(0);
4844 format %{ %}
4845 interface(REG_INTER);
4846 %}
4847
4848 operand interpreter_method_oop_RegP(iRegP reg)
4849 %{
4850 constraint(ALLOC_IN_RC(method_reg)); // interpreter_method_oop_reg
4851 match(reg);
4852 match(iRegPNoSp);
4853 op_cost(0);
4854 format %{ %}
4855 interface(REG_INTER);
4856 %}
4857
4858 // Thread Register
4859 operand thread_RegP(iRegP reg)
4860 %{
4861 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
4862 match(reg);
4863 op_cost(0);
4864 format %{ %}
4865 interface(REG_INTER);
4866 %}
4867
4868 operand lr_RegP(iRegP reg)
4869 %{
4870 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
4871 match(reg);
4872 op_cost(0);
4873 format %{ %}
4874 interface(REG_INTER);
4875 %}
4876
4877 //----------Memory Operands----------------------------------------------------
4878
4879 operand indirect(iRegP reg)
4880 %{
4881 constraint(ALLOC_IN_RC(ptr_reg));
4882 match(reg);
4883 op_cost(0);
4884 format %{ "[$reg]" %}
4885 interface(MEMORY_INTER) %{
4886 base($reg);
4887 index(0xffffffff);
4888 scale(0x0);
4889 disp(0x0);
4890 %}
4891 %}
4892
4893 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
4894 %{
4895 constraint(ALLOC_IN_RC(ptr_reg));
4896 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
4897 match(AddP reg (LShiftL (ConvI2L ireg) scale));
4898 op_cost(0);
4899 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
4900 interface(MEMORY_INTER) %{
4901 base($reg);
4902 index($ireg);
4903 scale($scale);
4904 disp(0x0);
4905 %}
4906 %}
4907
4908 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
4909 %{
4910 constraint(ALLOC_IN_RC(ptr_reg));
4911 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
4912 match(AddP reg (LShiftL lreg scale));
4913 op_cost(0);
4914 format %{ "$reg, $lreg lsl($scale)" %}
4915 interface(MEMORY_INTER) %{
4916 base($reg);
4917 index($lreg);
4918 scale($scale);
4919 disp(0x0);
4920 %}
4921 %}
4922
4923 operand indIndexI2L(iRegP reg, iRegI ireg)
4924 %{
4925 constraint(ALLOC_IN_RC(ptr_reg));
4926 match(AddP reg (ConvI2L ireg));
4927 op_cost(0);
4928 format %{ "$reg, $ireg, 0, I2L" %}
4929 interface(MEMORY_INTER) %{
4930 base($reg);
4931 index($ireg);
4932 scale(0x0);
4933 disp(0x0);
4934 %}
4935 %}
4936
4937 operand indIndex(iRegP reg, iRegL lreg)
4938 %{
4939 constraint(ALLOC_IN_RC(ptr_reg));
4940 match(AddP reg lreg);
4941 op_cost(0);
4942 format %{ "$reg, $lreg" %}
4943 interface(MEMORY_INTER) %{
4944 base($reg);
4945 index($lreg);
4946 scale(0x0);
4947 disp(0x0);
4948 %}
4949 %}
4950
4951 operand indOffI(iRegP reg, immIOffset off)
4952 %{
4953 constraint(ALLOC_IN_RC(ptr_reg));
4954 match(AddP reg off);
4955 op_cost(0);
4956 format %{ "[$reg, $off]" %}
4957 interface(MEMORY_INTER) %{
4958 base($reg);
4959 index(0xffffffff);
4960 scale(0x0);
4961 disp($off);
4962 %}
4963 %}
4964
4965 operand indOffI4(iRegP reg, immIOffset4 off)
4966 %{
4967 constraint(ALLOC_IN_RC(ptr_reg));
4968 match(AddP reg off);
4969 op_cost(0);
4970 format %{ "[$reg, $off]" %}
4971 interface(MEMORY_INTER) %{
4972 base($reg);
4973 index(0xffffffff);
4974 scale(0x0);
4975 disp($off);
4976 %}
4977 %}
4978
4979 operand indOffI8(iRegP reg, immIOffset8 off)
4980 %{
4981 constraint(ALLOC_IN_RC(ptr_reg));
4982 match(AddP reg off);
4983 op_cost(0);
4984 format %{ "[$reg, $off]" %}
4985 interface(MEMORY_INTER) %{
4986 base($reg);
4987 index(0xffffffff);
4988 scale(0x0);
4989 disp($off);
4990 %}
4991 %}
4992
4993 operand indOffI16(iRegP reg, immIOffset16 off)
4994 %{
4995 constraint(ALLOC_IN_RC(ptr_reg));
4996 match(AddP reg off);
4997 op_cost(0);
4998 format %{ "[$reg, $off]" %}
4999 interface(MEMORY_INTER) %{
5000 base($reg);
5001 index(0xffffffff);
5002 scale(0x0);
5003 disp($off);
5004 %}
5005 %}
5006
5007 operand indOffL(iRegP reg, immLoffset off)
5008 %{
5009 constraint(ALLOC_IN_RC(ptr_reg));
5010 match(AddP reg off);
5011 op_cost(0);
5012 format %{ "[$reg, $off]" %}
5013 interface(MEMORY_INTER) %{
5014 base($reg);
5015 index(0xffffffff);
5016 scale(0x0);
5017 disp($off);
5018 %}
5019 %}
5020
5021 operand indOffL4(iRegP reg, immLoffset4 off)
5022 %{
5023 constraint(ALLOC_IN_RC(ptr_reg));
5024 match(AddP reg off);
5025 op_cost(0);
5026 format %{ "[$reg, $off]" %}
5027 interface(MEMORY_INTER) %{
5028 base($reg);
5029 index(0xffffffff);
5030 scale(0x0);
5031 disp($off);
5032 %}
5033 %}
5034
5035 operand indOffL8(iRegP reg, immLoffset8 off)
5036 %{
5037 constraint(ALLOC_IN_RC(ptr_reg));
5038 match(AddP reg off);
5039 op_cost(0);
5040 format %{ "[$reg, $off]" %}
5041 interface(MEMORY_INTER) %{
5042 base($reg);
5043 index(0xffffffff);
5044 scale(0x0);
5045 disp($off);
5046 %}
5047 %}
5048
5049 operand indOffL16(iRegP reg, immLoffset16 off)
5050 %{
5051 constraint(ALLOC_IN_RC(ptr_reg));
5052 match(AddP reg off);
5053 op_cost(0);
5054 format %{ "[$reg, $off]" %}
5055 interface(MEMORY_INTER) %{
5056 base($reg);
5057 index(0xffffffff);
5058 scale(0x0);
5059 disp($off);
5060 %}
5061 %}
5062
5063 operand indirectN(iRegN reg)
5064 %{
5065 predicate(Universe::narrow_oop_shift() == 0);
5066 constraint(ALLOC_IN_RC(ptr_reg));
5067 match(DecodeN reg);
5068 op_cost(0);
5069 format %{ "[$reg]\t# narrow" %}
5070 interface(MEMORY_INTER) %{
5071 base($reg);
5072 index(0xffffffff);
5073 scale(0x0);
5074 disp(0x0);
5075 %}
5076 %}
5077
5078 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5079 %{
5080 predicate(Universe::narrow_oop_shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5081 constraint(ALLOC_IN_RC(ptr_reg));
5082 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5083 op_cost(0);
5084 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5085 interface(MEMORY_INTER) %{
5086 base($reg);
5087 index($ireg);
5088 scale($scale);
5089 disp(0x0);
5090 %}
5091 %}
5092
5093 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5094 %{
5095 predicate(Universe::narrow_oop_shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5096 constraint(ALLOC_IN_RC(ptr_reg));
5097 match(AddP (DecodeN reg) (LShiftL lreg scale));
5098 op_cost(0);
5099 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5100 interface(MEMORY_INTER) %{
5101 base($reg);
5102 index($lreg);
5103 scale($scale);
5104 disp(0x0);
5105 %}
5106 %}
5107
5108 operand indIndexI2LN(iRegN reg, iRegI ireg)
5109 %{
5110 predicate(Universe::narrow_oop_shift() == 0);
5111 constraint(ALLOC_IN_RC(ptr_reg));
5112 match(AddP (DecodeN reg) (ConvI2L ireg));
5113 op_cost(0);
5114 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5115 interface(MEMORY_INTER) %{
5116 base($reg);
5117 index($ireg);
5118 scale(0x0);
5119 disp(0x0);
5120 %}
5121 %}
5122
5123 operand indIndexN(iRegN reg, iRegL lreg)
5124 %{
5125 predicate(Universe::narrow_oop_shift() == 0);
5126 constraint(ALLOC_IN_RC(ptr_reg));
5127 match(AddP (DecodeN reg) lreg);
5128 op_cost(0);
5129 format %{ "$reg, $lreg\t# narrow" %}
5130 interface(MEMORY_INTER) %{
5131 base($reg);
5132 index($lreg);
5133 scale(0x0);
5134 disp(0x0);
5135 %}
5136 %}
5137
5138 operand indOffIN(iRegN reg, immIOffset off)
5139 %{
5140 predicate(Universe::narrow_oop_shift() == 0);
5141 constraint(ALLOC_IN_RC(ptr_reg));
5142 match(AddP (DecodeN reg) off);
5143 op_cost(0);
5144 format %{ "[$reg, $off]\t# narrow" %}
5145 interface(MEMORY_INTER) %{
5146 base($reg);
5147 index(0xffffffff);
5148 scale(0x0);
5149 disp($off);
5150 %}
5151 %}
5152
5153 operand indOffLN(iRegN reg, immLoffset off)
5154 %{
5155 predicate(Universe::narrow_oop_shift() == 0);
5156 constraint(ALLOC_IN_RC(ptr_reg));
5157 match(AddP (DecodeN reg) off);
5158 op_cost(0);
5159 format %{ "[$reg, $off]\t# narrow" %}
5160 interface(MEMORY_INTER) %{
5161 base($reg);
5162 index(0xffffffff);
5163 scale(0x0);
5164 disp($off);
5165 %}
5166 %}
5167
5168
5169
5170 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5171 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5172 %{
5173 constraint(ALLOC_IN_RC(ptr_reg));
5174 match(AddP reg off);
5175 op_cost(0);
5176 format %{ "[$reg, $off]" %}
5177 interface(MEMORY_INTER) %{
5178 base($reg);
5179 index(0xffffffff);
5180 scale(0x0);
5181 disp($off);
5182 %}
5183 %}
5184
5185 //----------Special Memory Operands--------------------------------------------
5186 // Stack Slot Operand - This operand is used for loading and storing temporary
5187 // values on the stack where a match requires a value to
5188 // flow through memory.
5189 operand stackSlotP(sRegP reg)
5190 %{
5191 constraint(ALLOC_IN_RC(stack_slots));
5192 op_cost(100);
5193 // No match rule because this operand is only generated in matching
5194 // match(RegP);
5195 format %{ "[$reg]" %}
5196 interface(MEMORY_INTER) %{
5197 base(0x1e); // RSP
5198 index(0x0); // No Index
5199 scale(0x0); // No Scale
5200 disp($reg); // Stack Offset
5201 %}
5202 %}
5203
5204 operand stackSlotI(sRegI reg)
5205 %{
5206 constraint(ALLOC_IN_RC(stack_slots));
5207 // No match rule because this operand is only generated in matching
5208 // match(RegI);
5209 format %{ "[$reg]" %}
5210 interface(MEMORY_INTER) %{
5211 base(0x1e); // RSP
5212 index(0x0); // No Index
5213 scale(0x0); // No Scale
5214 disp($reg); // Stack Offset
5215 %}
5216 %}
5217
5218 operand stackSlotF(sRegF reg)
5219 %{
5220 constraint(ALLOC_IN_RC(stack_slots));
5221 // No match rule because this operand is only generated in matching
5222 // match(RegF);
5223 format %{ "[$reg]" %}
5224 interface(MEMORY_INTER) %{
5225 base(0x1e); // RSP
5226 index(0x0); // No Index
5227 scale(0x0); // No Scale
5228 disp($reg); // Stack Offset
5229 %}
5230 %}
5231
5232 operand stackSlotD(sRegD reg)
5233 %{
5234 constraint(ALLOC_IN_RC(stack_slots));
5235 // No match rule because this operand is only generated in matching
5236 // match(RegD);
5237 format %{ "[$reg]" %}
5238 interface(MEMORY_INTER) %{
5239 base(0x1e); // RSP
5240 index(0x0); // No Index
5241 scale(0x0); // No Scale
5242 disp($reg); // Stack Offset
5243 %}
5244 %}
5245
5246 operand stackSlotL(sRegL reg)
5247 %{
5248 constraint(ALLOC_IN_RC(stack_slots));
5249 // No match rule because this operand is only generated in matching
5250 // match(RegL);
5251 format %{ "[$reg]" %}
5252 interface(MEMORY_INTER) %{
5253 base(0x1e); // RSP
5254 index(0x0); // No Index
5255 scale(0x0); // No Scale
5256 disp($reg); // Stack Offset
5257 %}
5258 %}
5259
5260 // Operands for expressing Control Flow
5261 // NOTE: Label is a predefined operand which should not be redefined in
5262 // the AD file. It is generically handled within the ADLC.
5263
5264 //----------Conditional Branch Operands----------------------------------------
5265 // Comparison Op - This is the operation of the comparison, and is limited to
5266 // the following set of codes:
5267 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5268 //
5269 // Other attributes of the comparison, such as unsignedness, are specified
5270 // by the comparison instruction that sets a condition code flags register.
5271 // That result is represented by a flags operand whose subtype is appropriate
5272 // to the unsignedness (etc.) of the comparison.
5273 //
5274 // Later, the instruction which matches both the Comparison Op (a Bool) and
5275 // the flags (produced by the Cmp) specifies the coding of the comparison op
5276 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5277
5278 // used for signed integral comparisons and fp comparisons
5279
5280 operand cmpOp()
5281 %{
5282 match(Bool);
5283
5284 format %{ "" %}
5285 interface(COND_INTER) %{
5286 equal(0x0, "eq");
5287 not_equal(0x1, "ne");
5288 less(0xb, "lt");
5289 greater_equal(0xa, "ge");
5290 less_equal(0xd, "le");
5291 greater(0xc, "gt");
5292 overflow(0x6, "vs");
5293 no_overflow(0x7, "vc");
5294 %}
5295 %}
5296
5297 // used for unsigned integral comparisons
5298
5299 operand cmpOpU()
5300 %{
5301 match(Bool);
5302
5303 format %{ "" %}
5304 interface(COND_INTER) %{
5305 equal(0x0, "eq");
5306 not_equal(0x1, "ne");
5307 less(0x3, "lo");
5308 greater_equal(0x2, "hs");
5309 less_equal(0x9, "ls");
5310 greater(0x8, "hi");
5311 overflow(0x6, "vs");
5312 no_overflow(0x7, "vc");
5313 %}
5314 %}
5315
5316 // used for certain integral comparisons which can be
5317 // converted to cbxx or tbxx instructions
5318
5319 operand cmpOpEqNe()
5320 %{
5321 match(Bool);
5322 match(CmpOp);
5323 op_cost(0);
5324 predicate(n->as_Bool()->_test._test == BoolTest::ne
5325 || n->as_Bool()->_test._test == BoolTest::eq);
5326
5327 format %{ "" %}
5328 interface(COND_INTER) %{
5329 equal(0x0, "eq");
5330 not_equal(0x1, "ne");
5331 less(0xb, "lt");
5332 greater_equal(0xa, "ge");
5333 less_equal(0xd, "le");
5334 greater(0xc, "gt");
5335 overflow(0x6, "vs");
5336 no_overflow(0x7, "vc");
5337 %}
5338 %}
5339
5340 // used for certain integral comparisons which can be
5341 // converted to cbxx or tbxx instructions
5342
5343 operand cmpOpLtGe()
5344 %{
5345 match(Bool);
5346 match(CmpOp);
5347 op_cost(0);
5348
5349 predicate(n->as_Bool()->_test._test == BoolTest::lt
5350 || n->as_Bool()->_test._test == BoolTest::ge);
5351
5352 format %{ "" %}
5353 interface(COND_INTER) %{
5354 equal(0x0, "eq");
5355 not_equal(0x1, "ne");
5356 less(0xb, "lt");
5357 greater_equal(0xa, "ge");
5358 less_equal(0xd, "le");
5359 greater(0xc, "gt");
5360 overflow(0x6, "vs");
5361 no_overflow(0x7, "vc");
5362 %}
5363 %}
5364
5365 // used for certain unsigned integral comparisons which can be
5366 // converted to cbxx or tbxx instructions
5367
5368 operand cmpOpUEqNeLtGe()
5369 %{
5370 match(Bool);
5371 match(CmpOp);
5372 op_cost(0);
5373
5374 predicate(n->as_Bool()->_test._test == BoolTest::eq
5375 || n->as_Bool()->_test._test == BoolTest::ne
5376 || n->as_Bool()->_test._test == BoolTest::lt
5377 || n->as_Bool()->_test._test == BoolTest::ge);
5378
5379 format %{ "" %}
5380 interface(COND_INTER) %{
5381 equal(0x0, "eq");
5382 not_equal(0x1, "ne");
5383 less(0xb, "lt");
5384 greater_equal(0xa, "ge");
5385 less_equal(0xd, "le");
5386 greater(0xc, "gt");
5387 overflow(0x6, "vs");
5388 no_overflow(0x7, "vc");
5389 %}
5390 %}
5391
5392 // Special operand allowing long args to int ops to be truncated for free
5393
5394 operand iRegL2I(iRegL reg) %{
5395
5396 op_cost(0);
5397
5398 match(ConvL2I reg);
5399
5400 format %{ "l2i($reg)" %}
5401
5402 interface(REG_INTER)
5403 %}
5404
5405 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5406 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5407 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5408
5409 //----------OPERAND CLASSES----------------------------------------------------
5410 // Operand Classes are groups of operands that are used as to simplify
5411 // instruction definitions by not requiring the AD writer to specify
5412 // separate instructions for every form of operand when the
5413 // instruction accepts multiple operand types with the same basic
5414 // encoding and format. The classic case of this is memory operands.
5415
5416 // memory is used to define read/write location for load/store
5417 // instruction defs. we can turn a memory op into an Address
5418
5419 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI, indOffL,
5420 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
5421
5422 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5423 // operations. it allows the src to be either an iRegI or a (ConvL2I
5424 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5425 // can be elided because the 32-bit instruction will just employ the
5426 // lower 32 bits anyway.
5427 //
5428 // n.b. this does not elide all L2I conversions. if the truncated
5429 // value is consumed by more than one operation then the ConvL2I
5430 // cannot be bundled into the consuming nodes so an l2i gets planted
5431 // (actually a movw $dst $src) and the downstream instructions consume
5432 // the result of the l2i as an iRegI input. That's a shame since the
5433 // movw is actually redundant but its not too costly.
5434
5435 opclass iRegIorL2I(iRegI, iRegL2I);
5436
5437 //----------PIPELINE-----------------------------------------------------------
5438 // Rules which define the behavior of the target architectures pipeline.
5439
5440 // For specific pipelines, eg A53, define the stages of that pipeline
5441 //pipe_desc(ISS, EX1, EX2, WR);
5442 #define ISS S0
5443 #define EX1 S1
5444 #define EX2 S2
5445 #define WR S3
5446
5447 // Integer ALU reg operation
5448 pipeline %{
5449
5450 attributes %{
5451 // ARM instructions are of fixed length
5452 fixed_size_instructions; // Fixed size instructions TODO does
5453 max_instructions_per_bundle = 2; // A53 = 2, A57 = 4
5454 // ARM instructions come in 32-bit word units
5455 instruction_unit_size = 4; // An instruction is 4 bytes long
5456 instruction_fetch_unit_size = 64; // The processor fetches one line
5457 instruction_fetch_units = 1; // of 64 bytes
5458
5459 // List of nop instructions
5460 nops( MachNop );
5461 %}
5462
5463 // We don't use an actual pipeline model so don't care about resources
5464 // or description. we do use pipeline classes to introduce fixed
5465 // latencies
5466
5467 //----------RESOURCES----------------------------------------------------------
5468 // Resources are the functional units available to the machine
5469
5470 resources( INS0, INS1, INS01 = INS0 | INS1,
5471 ALU0, ALU1, ALU = ALU0 | ALU1,
5472 MAC,
5473 DIV,
5474 BRANCH,
5475 LDST,
5476 NEON_FP);
5477
5478 //----------PIPELINE DESCRIPTION-----------------------------------------------
5479 // Pipeline Description specifies the stages in the machine's pipeline
5480
5481 // Define the pipeline as a generic 6 stage pipeline
5482 pipe_desc(S0, S1, S2, S3, S4, S5);
5483
5484 //----------PIPELINE CLASSES---------------------------------------------------
5485 // Pipeline Classes describe the stages in which input and output are
5486 // referenced by the hardware pipeline.
5487
5488 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5489 %{
5490 single_instruction;
5491 src1 : S1(read);
5492 src2 : S2(read);
5493 dst : S5(write);
5494 INS01 : ISS;
5495 NEON_FP : S5;
5496 %}
5497
5498 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5499 %{
5500 single_instruction;
5501 src1 : S1(read);
5502 src2 : S2(read);
5503 dst : S5(write);
5504 INS01 : ISS;
5505 NEON_FP : S5;
5506 %}
5507
5508 pipe_class fp_uop_s(vRegF dst, vRegF src)
5509 %{
5510 single_instruction;
5511 src : S1(read);
5512 dst : S5(write);
5513 INS01 : ISS;
5514 NEON_FP : S5;
5515 %}
5516
5517 pipe_class fp_uop_d(vRegD dst, vRegD src)
5518 %{
5519 single_instruction;
5520 src : S1(read);
5521 dst : S5(write);
5522 INS01 : ISS;
5523 NEON_FP : S5;
5524 %}
5525
5526 pipe_class fp_d2f(vRegF dst, vRegD src)
5527 %{
5528 single_instruction;
5529 src : S1(read);
5530 dst : S5(write);
5531 INS01 : ISS;
5532 NEON_FP : S5;
5533 %}
5534
5535 pipe_class fp_f2d(vRegD dst, vRegF src)
5536 %{
5537 single_instruction;
5538 src : S1(read);
5539 dst : S5(write);
5540 INS01 : ISS;
5541 NEON_FP : S5;
5542 %}
5543
5544 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5545 %{
5546 single_instruction;
5547 src : S1(read);
5548 dst : S5(write);
5549 INS01 : ISS;
5550 NEON_FP : S5;
5551 %}
5552
5553 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5554 %{
5555 single_instruction;
5556 src : S1(read);
5557 dst : S5(write);
5558 INS01 : ISS;
5559 NEON_FP : S5;
5560 %}
5561
5562 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5563 %{
5564 single_instruction;
5565 src : S1(read);
5566 dst : S5(write);
5567 INS01 : ISS;
5568 NEON_FP : S5;
5569 %}
5570
5571 pipe_class fp_l2f(vRegF dst, iRegL src)
5572 %{
5573 single_instruction;
5574 src : S1(read);
5575 dst : S5(write);
5576 INS01 : ISS;
5577 NEON_FP : S5;
5578 %}
5579
5580 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5581 %{
5582 single_instruction;
5583 src : S1(read);
5584 dst : S5(write);
5585 INS01 : ISS;
5586 NEON_FP : S5;
5587 %}
5588
5589 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5590 %{
5591 single_instruction;
5592 src : S1(read);
5593 dst : S5(write);
5594 INS01 : ISS;
5595 NEON_FP : S5;
5596 %}
5597
5598 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5599 %{
5600 single_instruction;
5601 src : S1(read);
5602 dst : S5(write);
5603 INS01 : ISS;
5604 NEON_FP : S5;
5605 %}
5606
5607 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5608 %{
5609 single_instruction;
5610 src : S1(read);
5611 dst : S5(write);
5612 INS01 : ISS;
5613 NEON_FP : S5;
5614 %}
5615
5616 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5617 %{
5618 single_instruction;
5619 src1 : S1(read);
5620 src2 : S2(read);
5621 dst : S5(write);
5622 INS0 : ISS;
5623 NEON_FP : S5;
5624 %}
5625
5626 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5627 %{
5628 single_instruction;
5629 src1 : S1(read);
5630 src2 : S2(read);
5631 dst : S5(write);
5632 INS0 : ISS;
5633 NEON_FP : S5;
5634 %}
5635
5636 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5637 %{
5638 single_instruction;
5639 cr : S1(read);
5640 src1 : S1(read);
5641 src2 : S1(read);
5642 dst : S3(write);
5643 INS01 : ISS;
5644 NEON_FP : S3;
5645 %}
5646
5647 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
5648 %{
5649 single_instruction;
5650 cr : S1(read);
5651 src1 : S1(read);
5652 src2 : S1(read);
5653 dst : S3(write);
5654 INS01 : ISS;
5655 NEON_FP : S3;
5656 %}
5657
5658 pipe_class fp_imm_s(vRegF dst)
5659 %{
5660 single_instruction;
5661 dst : S3(write);
5662 INS01 : ISS;
5663 NEON_FP : S3;
5664 %}
5665
5666 pipe_class fp_imm_d(vRegD dst)
5667 %{
5668 single_instruction;
5669 dst : S3(write);
5670 INS01 : ISS;
5671 NEON_FP : S3;
5672 %}
5673
5674 pipe_class fp_load_constant_s(vRegF dst)
5675 %{
5676 single_instruction;
5677 dst : S4(write);
5678 INS01 : ISS;
5679 NEON_FP : S4;
5680 %}
5681
5682 pipe_class fp_load_constant_d(vRegD dst)
5683 %{
5684 single_instruction;
5685 dst : S4(write);
5686 INS01 : ISS;
5687 NEON_FP : S4;
5688 %}
5689
5690 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
5691 %{
5692 single_instruction;
5693 dst : S5(write);
5694 src1 : S1(read);
5695 src2 : S1(read);
5696 INS01 : ISS;
5697 NEON_FP : S5;
5698 %}
5699
5700 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
5701 %{
5702 single_instruction;
5703 dst : S5(write);
5704 src1 : S1(read);
5705 src2 : S1(read);
5706 INS0 : ISS;
5707 NEON_FP : S5;
5708 %}
5709
5710 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
5711 %{
5712 single_instruction;
5713 dst : S5(write);
5714 src1 : S1(read);
5715 src2 : S1(read);
5716 dst : S1(read);
5717 INS01 : ISS;
5718 NEON_FP : S5;
5719 %}
5720
5721 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
5722 %{
5723 single_instruction;
5724 dst : S5(write);
5725 src1 : S1(read);
5726 src2 : S1(read);
5727 dst : S1(read);
5728 INS0 : ISS;
5729 NEON_FP : S5;
5730 %}
5731
5732 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
5733 %{
5734 single_instruction;
5735 dst : S4(write);
5736 src1 : S2(read);
5737 src2 : S2(read);
5738 INS01 : ISS;
5739 NEON_FP : S4;
5740 %}
5741
5742 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
5743 %{
5744 single_instruction;
5745 dst : S4(write);
5746 src1 : S2(read);
5747 src2 : S2(read);
5748 INS0 : ISS;
5749 NEON_FP : S4;
5750 %}
5751
5752 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
5753 %{
5754 single_instruction;
5755 dst : S3(write);
5756 src1 : S2(read);
5757 src2 : S2(read);
5758 INS01 : ISS;
5759 NEON_FP : S3;
5760 %}
5761
5762 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
5763 %{
5764 single_instruction;
5765 dst : S3(write);
5766 src1 : S2(read);
5767 src2 : S2(read);
5768 INS0 : ISS;
5769 NEON_FP : S3;
5770 %}
5771
5772 pipe_class vshift64(vecD dst, vecD src, vecX shift)
5773 %{
5774 single_instruction;
5775 dst : S3(write);
5776 src : S1(read);
5777 shift : S1(read);
5778 INS01 : ISS;
5779 NEON_FP : S3;
5780 %}
5781
5782 pipe_class vshift128(vecX dst, vecX src, vecX shift)
5783 %{
5784 single_instruction;
5785 dst : S3(write);
5786 src : S1(read);
5787 shift : S1(read);
5788 INS0 : ISS;
5789 NEON_FP : S3;
5790 %}
5791
5792 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
5793 %{
5794 single_instruction;
5795 dst : S3(write);
5796 src : S1(read);
5797 INS01 : ISS;
5798 NEON_FP : S3;
5799 %}
5800
5801 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
5802 %{
5803 single_instruction;
5804 dst : S3(write);
5805 src : S1(read);
5806 INS0 : ISS;
5807 NEON_FP : S3;
5808 %}
5809
5810 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
5811 %{
5812 single_instruction;
5813 dst : S5(write);
5814 src1 : S1(read);
5815 src2 : S1(read);
5816 INS01 : ISS;
5817 NEON_FP : S5;
5818 %}
5819
5820 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
5821 %{
5822 single_instruction;
5823 dst : S5(write);
5824 src1 : S1(read);
5825 src2 : S1(read);
5826 INS0 : ISS;
5827 NEON_FP : S5;
5828 %}
5829
5830 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
5831 %{
5832 single_instruction;
5833 dst : S5(write);
5834 src1 : S1(read);
5835 src2 : S1(read);
5836 INS0 : ISS;
5837 NEON_FP : S5;
5838 %}
5839
5840 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
5841 %{
5842 single_instruction;
5843 dst : S5(write);
5844 src1 : S1(read);
5845 src2 : S1(read);
5846 INS0 : ISS;
5847 NEON_FP : S5;
5848 %}
5849
5850 pipe_class vsqrt_fp128(vecX dst, vecX src)
5851 %{
5852 single_instruction;
5853 dst : S5(write);
5854 src : S1(read);
5855 INS0 : ISS;
5856 NEON_FP : S5;
5857 %}
5858
5859 pipe_class vunop_fp64(vecD dst, vecD src)
5860 %{
5861 single_instruction;
5862 dst : S5(write);
5863 src : S1(read);
5864 INS01 : ISS;
5865 NEON_FP : S5;
5866 %}
5867
5868 pipe_class vunop_fp128(vecX dst, vecX src)
5869 %{
5870 single_instruction;
5871 dst : S5(write);
5872 src : S1(read);
5873 INS0 : ISS;
5874 NEON_FP : S5;
5875 %}
5876
5877 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
5878 %{
5879 single_instruction;
5880 dst : S3(write);
5881 src : S1(read);
5882 INS01 : ISS;
5883 NEON_FP : S3;
5884 %}
5885
5886 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
5887 %{
5888 single_instruction;
5889 dst : S3(write);
5890 src : S1(read);
5891 INS01 : ISS;
5892 NEON_FP : S3;
5893 %}
5894
5895 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
5896 %{
5897 single_instruction;
5898 dst : S3(write);
5899 src : S1(read);
5900 INS01 : ISS;
5901 NEON_FP : S3;
5902 %}
5903
5904 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
5905 %{
5906 single_instruction;
5907 dst : S3(write);
5908 src : S1(read);
5909 INS01 : ISS;
5910 NEON_FP : S3;
5911 %}
5912
5913 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
5914 %{
5915 single_instruction;
5916 dst : S3(write);
5917 src : S1(read);
5918 INS01 : ISS;
5919 NEON_FP : S3;
5920 %}
5921
5922 pipe_class vmovi_reg_imm64(vecD dst)
5923 %{
5924 single_instruction;
5925 dst : S3(write);
5926 INS01 : ISS;
5927 NEON_FP : S3;
5928 %}
5929
5930 pipe_class vmovi_reg_imm128(vecX dst)
5931 %{
5932 single_instruction;
5933 dst : S3(write);
5934 INS0 : ISS;
5935 NEON_FP : S3;
5936 %}
5937
5938 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
5939 %{
5940 single_instruction;
5941 dst : S5(write);
5942 mem : ISS(read);
5943 INS01 : ISS;
5944 NEON_FP : S3;
5945 %}
5946
5947 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
5948 %{
5949 single_instruction;
5950 dst : S5(write);
5951 mem : ISS(read);
5952 INS01 : ISS;
5953 NEON_FP : S3;
5954 %}
5955
5956 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
5957 %{
5958 single_instruction;
5959 mem : ISS(read);
5960 src : S2(read);
5961 INS01 : ISS;
5962 NEON_FP : S3;
5963 %}
5964
5965 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
5966 %{
5967 single_instruction;
5968 mem : ISS(read);
5969 src : S2(read);
5970 INS01 : ISS;
5971 NEON_FP : S3;
5972 %}
5973
5974 //------- Integer ALU operations --------------------------
5975
5976 // Integer ALU reg-reg operation
5977 // Operands needed in EX1, result generated in EX2
5978 // Eg. ADD x0, x1, x2
5979 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5980 %{
5981 single_instruction;
5982 dst : EX2(write);
5983 src1 : EX1(read);
5984 src2 : EX1(read);
5985 INS01 : ISS; // Dual issue as instruction 0 or 1
5986 ALU : EX2;
5987 %}
5988
5989 // Integer ALU reg-reg operation with constant shift
5990 // Shifted register must be available in LATE_ISS instead of EX1
5991 // Eg. ADD x0, x1, x2, LSL #2
5992 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
5993 %{
5994 single_instruction;
5995 dst : EX2(write);
5996 src1 : EX1(read);
5997 src2 : ISS(read);
5998 INS01 : ISS;
5999 ALU : EX2;
6000 %}
6001
6002 // Integer ALU reg operation with constant shift
6003 // Eg. LSL x0, x1, #shift
6004 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6005 %{
6006 single_instruction;
6007 dst : EX2(write);
6008 src1 : ISS(read);
6009 INS01 : ISS;
6010 ALU : EX2;
6011 %}
6012
6013 // Integer ALU reg-reg operation with variable shift
6014 // Both operands must be available in LATE_ISS instead of EX1
6015 // Result is available in EX1 instead of EX2
6016 // Eg. LSLV x0, x1, x2
6017 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6018 %{
6019 single_instruction;
6020 dst : EX1(write);
6021 src1 : ISS(read);
6022 src2 : ISS(read);
6023 INS01 : ISS;
6024 ALU : EX1;
6025 %}
6026
6027 // Integer ALU reg-reg operation with extract
6028 // As for _vshift above, but result generated in EX2
6029 // Eg. EXTR x0, x1, x2, #N
6030 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6031 %{
6032 single_instruction;
6033 dst : EX2(write);
6034 src1 : ISS(read);
6035 src2 : ISS(read);
6036 INS1 : ISS; // Can only dual issue as Instruction 1
6037 ALU : EX1;
6038 %}
6039
6040 // Integer ALU reg operation
6041 // Eg. NEG x0, x1
6042 pipe_class ialu_reg(iRegI dst, iRegI src)
6043 %{
6044 single_instruction;
6045 dst : EX2(write);
6046 src : EX1(read);
6047 INS01 : ISS;
6048 ALU : EX2;
6049 %}
6050
6051 // Integer ALU reg mmediate operation
6052 // Eg. ADD x0, x1, #N
6053 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6054 %{
6055 single_instruction;
6056 dst : EX2(write);
6057 src1 : EX1(read);
6058 INS01 : ISS;
6059 ALU : EX2;
6060 %}
6061
6062 // Integer ALU immediate operation (no source operands)
6063 // Eg. MOV x0, #N
6064 pipe_class ialu_imm(iRegI dst)
6065 %{
6066 single_instruction;
6067 dst : EX1(write);
6068 INS01 : ISS;
6069 ALU : EX1;
6070 %}
6071
6072 //------- Compare operation -------------------------------
6073
6074 // Compare reg-reg
6075 // Eg. CMP x0, x1
6076 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6077 %{
6078 single_instruction;
6079 // fixed_latency(16);
6080 cr : EX2(write);
6081 op1 : EX1(read);
6082 op2 : EX1(read);
6083 INS01 : ISS;
6084 ALU : EX2;
6085 %}
6086
6087 // Compare reg-reg
6088 // Eg. CMP x0, #N
6089 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6090 %{
6091 single_instruction;
6092 // fixed_latency(16);
6093 cr : EX2(write);
6094 op1 : EX1(read);
6095 INS01 : ISS;
6096 ALU : EX2;
6097 %}
6098
6099 //------- Conditional instructions ------------------------
6100
6101 // Conditional no operands
6102 // Eg. CSINC x0, zr, zr, <cond>
6103 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6104 %{
6105 single_instruction;
6106 cr : EX1(read);
6107 dst : EX2(write);
6108 INS01 : ISS;
6109 ALU : EX2;
6110 %}
6111
6112 // Conditional 2 operand
6113 // EG. CSEL X0, X1, X2, <cond>
6114 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6115 %{
6116 single_instruction;
6117 cr : EX1(read);
6118 src1 : EX1(read);
6119 src2 : EX1(read);
6120 dst : EX2(write);
6121 INS01 : ISS;
6122 ALU : EX2;
6123 %}
6124
6125 // Conditional 2 operand
6126 // EG. CSEL X0, X1, X2, <cond>
6127 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6128 %{
6129 single_instruction;
6130 cr : EX1(read);
6131 src : EX1(read);
6132 dst : EX2(write);
6133 INS01 : ISS;
6134 ALU : EX2;
6135 %}
6136
6137 //------- Multiply pipeline operations --------------------
6138
6139 // Multiply reg-reg
6140 // Eg. MUL w0, w1, w2
6141 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6142 %{
6143 single_instruction;
6144 dst : WR(write);
6145 src1 : ISS(read);
6146 src2 : ISS(read);
6147 INS01 : ISS;
6148 MAC : WR;
6149 %}
6150
6151 // Multiply accumulate
6152 // Eg. MADD w0, w1, w2, w3
6153 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6154 %{
6155 single_instruction;
6156 dst : WR(write);
6157 src1 : ISS(read);
6158 src2 : ISS(read);
6159 src3 : ISS(read);
6160 INS01 : ISS;
6161 MAC : WR;
6162 %}
6163
6164 // Eg. MUL w0, w1, w2
6165 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6166 %{
6167 single_instruction;
6168 fixed_latency(3); // Maximum latency for 64 bit mul
6169 dst : WR(write);
6170 src1 : ISS(read);
6171 src2 : ISS(read);
6172 INS01 : ISS;
6173 MAC : WR;
6174 %}
6175
6176 // Multiply accumulate
6177 // Eg. MADD w0, w1, w2, w3
6178 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6179 %{
6180 single_instruction;
6181 fixed_latency(3); // Maximum latency for 64 bit mul
6182 dst : WR(write);
6183 src1 : ISS(read);
6184 src2 : ISS(read);
6185 src3 : ISS(read);
6186 INS01 : ISS;
6187 MAC : WR;
6188 %}
6189
6190 //------- Divide pipeline operations --------------------
6191
6192 // Eg. SDIV w0, w1, w2
6193 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6194 %{
6195 single_instruction;
6196 fixed_latency(8); // Maximum latency for 32 bit divide
6197 dst : WR(write);
6198 src1 : ISS(read);
6199 src2 : ISS(read);
6200 INS0 : ISS; // Can only dual issue as instruction 0
6201 DIV : WR;
6202 %}
6203
6204 // Eg. SDIV x0, x1, x2
6205 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6206 %{
6207 single_instruction;
6208 fixed_latency(16); // Maximum latency for 64 bit divide
6209 dst : WR(write);
6210 src1 : ISS(read);
6211 src2 : ISS(read);
6212 INS0 : ISS; // Can only dual issue as instruction 0
6213 DIV : WR;
6214 %}
6215
6216 //------- Load pipeline operations ------------------------
6217
6218 // Load - prefetch
6219 // Eg. PFRM <mem>
6220 pipe_class iload_prefetch(memory mem)
6221 %{
6222 single_instruction;
6223 mem : ISS(read);
6224 INS01 : ISS;
6225 LDST : WR;
6226 %}
6227
6228 // Load - reg, mem
6229 // Eg. LDR x0, <mem>
6230 pipe_class iload_reg_mem(iRegI dst, memory mem)
6231 %{
6232 single_instruction;
6233 dst : WR(write);
6234 mem : ISS(read);
6235 INS01 : ISS;
6236 LDST : WR;
6237 %}
6238
6239 // Load - reg, reg
6240 // Eg. LDR x0, [sp, x1]
6241 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6242 %{
6243 single_instruction;
6244 dst : WR(write);
6245 src : ISS(read);
6246 INS01 : ISS;
6247 LDST : WR;
6248 %}
6249
6250 //------- Store pipeline operations -----------------------
6251
6252 // Store - zr, mem
6253 // Eg. STR zr, <mem>
6254 pipe_class istore_mem(memory mem)
6255 %{
6256 single_instruction;
6257 mem : ISS(read);
6258 INS01 : ISS;
6259 LDST : WR;
6260 %}
6261
6262 // Store - reg, mem
6263 // Eg. STR x0, <mem>
6264 pipe_class istore_reg_mem(iRegI src, memory mem)
6265 %{
6266 single_instruction;
6267 mem : ISS(read);
6268 src : EX2(read);
6269 INS01 : ISS;
6270 LDST : WR;
6271 %}
6272
6273 // Store - reg, reg
6274 // Eg. STR x0, [sp, x1]
6275 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6276 %{
6277 single_instruction;
6278 dst : ISS(read);
6279 src : EX2(read);
6280 INS01 : ISS;
6281 LDST : WR;
6282 %}
6283
6284 //------- Store pipeline operations -----------------------
6285
6286 // Branch
6287 pipe_class pipe_branch()
6288 %{
6289 single_instruction;
6290 INS01 : ISS;
6291 BRANCH : EX1;
6292 %}
6293
6294 // Conditional branch
6295 pipe_class pipe_branch_cond(rFlagsReg cr)
6296 %{
6297 single_instruction;
6298 cr : EX1(read);
6299 INS01 : ISS;
6300 BRANCH : EX1;
6301 %}
6302
6303 // Compare & Branch
6304 // EG. CBZ/CBNZ
6305 pipe_class pipe_cmp_branch(iRegI op1)
6306 %{
6307 single_instruction;
6308 op1 : EX1(read);
6309 INS01 : ISS;
6310 BRANCH : EX1;
6311 %}
6312
6313 //------- Synchronisation operations ----------------------
6314
6315 // Any operation requiring serialization.
6316 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6317 pipe_class pipe_serial()
6318 %{
6319 single_instruction;
6320 force_serialization;
6321 fixed_latency(16);
6322 INS01 : ISS(2); // Cannot dual issue with any other instruction
6323 LDST : WR;
6324 %}
6325
6326 // Generic big/slow expanded idiom - also serialized
6327 pipe_class pipe_slow()
6328 %{
6329 instruction_count(10);
6330 multiple_bundles;
6331 force_serialization;
6332 fixed_latency(16);
6333 INS01 : ISS(2); // Cannot dual issue with any other instruction
6334 LDST : WR;
6335 %}
6336
6337 // Empty pipeline class
6338 pipe_class pipe_class_empty()
6339 %{
6340 single_instruction;
6341 fixed_latency(0);
6342 %}
6343
6344 // Default pipeline class.
6345 pipe_class pipe_class_default()
6346 %{
6347 single_instruction;
6348 fixed_latency(2);
6349 %}
6350
6351 // Pipeline class for compares.
6352 pipe_class pipe_class_compare()
6353 %{
6354 single_instruction;
6355 fixed_latency(16);
6356 %}
6357
6358 // Pipeline class for memory operations.
6359 pipe_class pipe_class_memory()
6360 %{
6361 single_instruction;
6362 fixed_latency(16);
6363 %}
6364
6365 // Pipeline class for call.
6366 pipe_class pipe_class_call()
6367 %{
6368 single_instruction;
6369 fixed_latency(100);
6370 %}
6371
6372 // Define the class for the Nop node.
6373 define %{
6374 MachNop = pipe_class_empty;
6375 %}
6376
6377 %}
6378 //----------INSTRUCTIONS-------------------------------------------------------
6379 //
6380 // match -- States which machine-independent subtree may be replaced
6381 // by this instruction.
6382 // ins_cost -- The estimated cost of this instruction is used by instruction
6383 // selection to identify a minimum cost tree of machine
6384 // instructions that matches a tree of machine-independent
6385 // instructions.
6386 // format -- A string providing the disassembly for this instruction.
6387 // The value of an instruction's operand may be inserted
6388 // by referring to it with a '$' prefix.
6389 // opcode -- Three instruction opcodes may be provided. These are referred
6390 // to within an encode class as $primary, $secondary, and $tertiary
6391 // rrspectively. The primary opcode is commonly used to
6392 // indicate the type of machine instruction, while secondary
6393 // and tertiary are often used for prefix options or addressing
6394 // modes.
6395 // ins_encode -- A list of encode classes with parameters. The encode class
6396 // name must have been defined in an 'enc_class' specification
6397 // in the encode section of the architecture description.
6398
6399 // ============================================================================
6400 // Memory (Load/Store) Instructions
6401
6402 // Load Instructions
6403
6404 // Load Byte (8 bit signed)
6405 instruct loadB(iRegINoSp dst, memory mem)
6406 %{
6407 match(Set dst (LoadB mem));
6408 predicate(!needs_acquiring_load(n));
6409
6410 ins_cost(4 * INSN_COST);
6411 format %{ "ldrsbw $dst, $mem\t# byte" %}
6412
6413 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6414
6415 ins_pipe(iload_reg_mem);
6416 %}
6417
6418 // Load Byte (8 bit signed) into long
6419 instruct loadB2L(iRegLNoSp dst, memory mem)
6420 %{
6421 match(Set dst (ConvI2L (LoadB mem)));
6422 predicate(!needs_acquiring_load(n->in(1)));
6423
6424 ins_cost(4 * INSN_COST);
6425 format %{ "ldrsb $dst, $mem\t# byte" %}
6426
6427 ins_encode(aarch64_enc_ldrsb(dst, mem));
6428
6429 ins_pipe(iload_reg_mem);
6430 %}
6431
6432 // Load Byte (8 bit unsigned)
6433 instruct loadUB(iRegINoSp dst, memory mem)
6434 %{
6435 match(Set dst (LoadUB mem));
6436 predicate(!needs_acquiring_load(n));
6437
6438 ins_cost(4 * INSN_COST);
6439 format %{ "ldrbw $dst, $mem\t# byte" %}
6440
6441 ins_encode(aarch64_enc_ldrb(dst, mem));
6442
6443 ins_pipe(iload_reg_mem);
6444 %}
6445
6446 // Load Byte (8 bit unsigned) into long
6447 instruct loadUB2L(iRegLNoSp dst, memory mem)
6448 %{
6449 match(Set dst (ConvI2L (LoadUB mem)));
6450 predicate(!needs_acquiring_load(n->in(1)));
6451
6452 ins_cost(4 * INSN_COST);
6453 format %{ "ldrb $dst, $mem\t# byte" %}
6454
6455 ins_encode(aarch64_enc_ldrb(dst, mem));
6456
6457 ins_pipe(iload_reg_mem);
6458 %}
6459
6460 // Load Short (16 bit signed)
6461 instruct loadS(iRegINoSp dst, memory mem)
6462 %{
6463 match(Set dst (LoadS mem));
6464 predicate(!needs_acquiring_load(n));
6465
6466 ins_cost(4 * INSN_COST);
6467 format %{ "ldrshw $dst, $mem\t# short" %}
6468
6469 ins_encode(aarch64_enc_ldrshw(dst, mem));
6470
6471 ins_pipe(iload_reg_mem);
6472 %}
6473
6474 // Load Short (16 bit signed) into long
6475 instruct loadS2L(iRegLNoSp dst, memory mem)
6476 %{
6477 match(Set dst (ConvI2L (LoadS mem)));
6478 predicate(!needs_acquiring_load(n->in(1)));
6479
6480 ins_cost(4 * INSN_COST);
6481 format %{ "ldrsh $dst, $mem\t# short" %}
6482
6483 ins_encode(aarch64_enc_ldrsh(dst, mem));
6484
6485 ins_pipe(iload_reg_mem);
6486 %}
6487
6488 // Load Char (16 bit unsigned)
6489 instruct loadUS(iRegINoSp dst, memory mem)
6490 %{
6491 match(Set dst (LoadUS mem));
6492 predicate(!needs_acquiring_load(n));
6493
6494 ins_cost(4 * INSN_COST);
6495 format %{ "ldrh $dst, $mem\t# short" %}
6496
6497 ins_encode(aarch64_enc_ldrh(dst, mem));
6498
6499 ins_pipe(iload_reg_mem);
6500 %}
6501
6502 // Load Short/Char (16 bit unsigned) into long
6503 instruct loadUS2L(iRegLNoSp dst, memory mem)
6504 %{
6505 match(Set dst (ConvI2L (LoadUS mem)));
6506 predicate(!needs_acquiring_load(n->in(1)));
6507
6508 ins_cost(4 * INSN_COST);
6509 format %{ "ldrh $dst, $mem\t# short" %}
6510
6511 ins_encode(aarch64_enc_ldrh(dst, mem));
6512
6513 ins_pipe(iload_reg_mem);
6514 %}
6515
6516 // Load Integer (32 bit signed)
6517 instruct loadI(iRegINoSp dst, memory mem)
6518 %{
6519 match(Set dst (LoadI mem));
6520 predicate(!needs_acquiring_load(n));
6521
6522 ins_cost(4 * INSN_COST);
6523 format %{ "ldrw $dst, $mem\t# int" %}
6524
6525 ins_encode(aarch64_enc_ldrw(dst, mem));
6526
6527 ins_pipe(iload_reg_mem);
6528 %}
6529
6530 // Load Integer (32 bit signed) into long
6531 instruct loadI2L(iRegLNoSp dst, memory mem)
6532 %{
6533 match(Set dst (ConvI2L (LoadI mem)));
6534 predicate(!needs_acquiring_load(n->in(1)));
6535
6536 ins_cost(4 * INSN_COST);
6537 format %{ "ldrsw $dst, $mem\t# int" %}
6538
6539 ins_encode(aarch64_enc_ldrsw(dst, mem));
6540
6541 ins_pipe(iload_reg_mem);
6542 %}
6543
6544 // Load Integer (32 bit unsigned) into long
6545 instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
6546 %{
6547 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6548 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6549
6550 ins_cost(4 * INSN_COST);
6551 format %{ "ldrw $dst, $mem\t# int" %}
6552
6553 ins_encode(aarch64_enc_ldrw(dst, mem));
6554
6555 ins_pipe(iload_reg_mem);
6556 %}
6557
6558 // Load Long (64 bit signed)
6559 instruct loadL(iRegLNoSp dst, memory mem)
6560 %{
6561 match(Set dst (LoadL mem));
6562 predicate(!needs_acquiring_load(n));
6563
6564 ins_cost(4 * INSN_COST);
6565 format %{ "ldr $dst, $mem\t# int" %}
6566
6567 ins_encode(aarch64_enc_ldr(dst, mem));
6568
6569 ins_pipe(iload_reg_mem);
6570 %}
6571
6572 // Load Range
6573 instruct loadRange(iRegINoSp dst, memory mem)
6574 %{
6575 match(Set dst (LoadRange mem));
6576
6577 ins_cost(4 * INSN_COST);
6578 format %{ "ldrw $dst, $mem\t# range" %}
6579
6580 ins_encode(aarch64_enc_ldrw(dst, mem));
6581
6582 ins_pipe(iload_reg_mem);
6583 %}
6584
6585 // Load Pointer
6586 instruct loadP(iRegPNoSp dst, memory mem)
6587 %{
6588 match(Set dst (LoadP mem));
6589 predicate(!needs_acquiring_load(n));
6590
6591 ins_cost(4 * INSN_COST);
6592 format %{ "ldr $dst, $mem\t# ptr" %}
6593
6594 ins_encode(aarch64_enc_ldr(dst, mem));
6595
6596 ins_pipe(iload_reg_mem);
6597 %}
6598
6599 // Load Compressed Pointer
6600 instruct loadN(iRegNNoSp dst, memory mem)
6601 %{
6602 match(Set dst (LoadN mem));
6603 predicate(!needs_acquiring_load(n));
6604
6605 ins_cost(4 * INSN_COST);
6606 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6607
6608 ins_encode(aarch64_enc_ldrw(dst, mem));
6609
6610 ins_pipe(iload_reg_mem);
6611 %}
6612
6613 // Load Klass Pointer
6614 instruct loadKlass(iRegPNoSp dst, memory mem)
6615 %{
6616 match(Set dst (LoadKlass mem));
6617 predicate(!needs_acquiring_load(n));
6618
6619 ins_cost(4 * INSN_COST);
6620 format %{ "ldr $dst, $mem\t# class" %}
6621
6622 ins_encode(aarch64_enc_ldr(dst, mem));
6623
6624 ins_pipe(iload_reg_mem);
6625 %}
6626
6627 // Load Narrow Klass Pointer
6628 instruct loadNKlass(iRegNNoSp dst, memory mem)
6629 %{
6630 match(Set dst (LoadNKlass mem));
6631 predicate(!needs_acquiring_load(n));
6632
6633 ins_cost(4 * INSN_COST);
6634 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6635
6636 ins_encode(aarch64_enc_ldrw(dst, mem));
6637
6638 ins_pipe(iload_reg_mem);
6639 %}
6640
6641 // Load Float
6642 instruct loadF(vRegF dst, memory mem)
6643 %{
6644 match(Set dst (LoadF mem));
6645 predicate(!needs_acquiring_load(n));
6646
6647 ins_cost(4 * INSN_COST);
6648 format %{ "ldrs $dst, $mem\t# float" %}
6649
6650 ins_encode( aarch64_enc_ldrs(dst, mem) );
6651
6652 ins_pipe(pipe_class_memory);
6653 %}
6654
6655 // Load Double
6656 instruct loadD(vRegD dst, memory mem)
6657 %{
6658 match(Set dst (LoadD mem));
6659 predicate(!needs_acquiring_load(n));
6660
6661 ins_cost(4 * INSN_COST);
6662 format %{ "ldrd $dst, $mem\t# double" %}
6663
6664 ins_encode( aarch64_enc_ldrd(dst, mem) );
6665
6666 ins_pipe(pipe_class_memory);
6667 %}
6668
6669
6670 // Load Int Constant
6671 instruct loadConI(iRegINoSp dst, immI src)
6672 %{
6673 match(Set dst src);
6674
6675 ins_cost(INSN_COST);
6676 format %{ "mov $dst, $src\t# int" %}
6677
6678 ins_encode( aarch64_enc_movw_imm(dst, src) );
6679
6680 ins_pipe(ialu_imm);
6681 %}
6682
6683 // Load Long Constant
6684 instruct loadConL(iRegLNoSp dst, immL src)
6685 %{
6686 match(Set dst src);
6687
6688 ins_cost(INSN_COST);
6689 format %{ "mov $dst, $src\t# long" %}
6690
6691 ins_encode( aarch64_enc_mov_imm(dst, src) );
6692
6693 ins_pipe(ialu_imm);
6694 %}
6695
6696 // Load Pointer Constant
6697
6698 instruct loadConP(iRegPNoSp dst, immP con)
6699 %{
6700 match(Set dst con);
6701
6702 ins_cost(INSN_COST * 4);
6703 format %{
6704 "mov $dst, $con\t# ptr\n\t"
6705 %}
6706
6707 ins_encode(aarch64_enc_mov_p(dst, con));
6708
6709 ins_pipe(ialu_imm);
6710 %}
6711
6712 // Load Null Pointer Constant
6713
6714 instruct loadConP0(iRegPNoSp dst, immP0 con)
6715 %{
6716 match(Set dst con);
6717
6718 ins_cost(INSN_COST);
6719 format %{ "mov $dst, $con\t# NULL ptr" %}
6720
6721 ins_encode(aarch64_enc_mov_p0(dst, con));
6722
6723 ins_pipe(ialu_imm);
6724 %}
6725
6726 // Load Pointer Constant One
6727
6728 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6729 %{
6730 match(Set dst con);
6731
6732 ins_cost(INSN_COST);
6733 format %{ "mov $dst, $con\t# NULL ptr" %}
6734
6735 ins_encode(aarch64_enc_mov_p1(dst, con));
6736
6737 ins_pipe(ialu_imm);
6738 %}
6739
6740 // Load Poll Page Constant
6741
6742 instruct loadConPollPage(iRegPNoSp dst, immPollPage con)
6743 %{
6744 match(Set dst con);
6745
6746 ins_cost(INSN_COST);
6747 format %{ "adr $dst, $con\t# Poll Page Ptr" %}
6748
6749 ins_encode(aarch64_enc_mov_poll_page(dst, con));
6750
6751 ins_pipe(ialu_imm);
6752 %}
6753
6754 // Load Byte Map Base Constant
6755
6756 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6757 %{
6758 match(Set dst con);
6759
6760 ins_cost(INSN_COST);
6761 format %{ "adr $dst, $con\t# Byte Map Base" %}
6762
6763 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6764
6765 ins_pipe(ialu_imm);
6766 %}
6767
6768 // Load Narrow Pointer Constant
6769
6770 instruct loadConN(iRegNNoSp dst, immN con)
6771 %{
6772 match(Set dst con);
6773
6774 ins_cost(INSN_COST * 4);
6775 format %{ "mov $dst, $con\t# compressed ptr" %}
6776
6777 ins_encode(aarch64_enc_mov_n(dst, con));
6778
6779 ins_pipe(ialu_imm);
6780 %}
6781
6782 // Load Narrow Null Pointer Constant
6783
6784 instruct loadConN0(iRegNNoSp dst, immN0 con)
6785 %{
6786 match(Set dst con);
6787
6788 ins_cost(INSN_COST);
6789 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
6790
6791 ins_encode(aarch64_enc_mov_n0(dst, con));
6792
6793 ins_pipe(ialu_imm);
6794 %}
6795
6796 // Load Narrow Klass Constant
6797
6798 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6799 %{
6800 match(Set dst con);
6801
6802 ins_cost(INSN_COST);
6803 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6804
6805 ins_encode(aarch64_enc_mov_nk(dst, con));
6806
6807 ins_pipe(ialu_imm);
6808 %}
6809
6810 // Load Packed Float Constant
6811
6812 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6813 match(Set dst con);
6814 ins_cost(INSN_COST * 4);
6815 format %{ "fmovs $dst, $con"%}
6816 ins_encode %{
6817 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6818 %}
6819
6820 ins_pipe(fp_imm_s);
6821 %}
6822
6823 // Load Float Constant
6824
6825 instruct loadConF(vRegF dst, immF con) %{
6826 match(Set dst con);
6827
6828 ins_cost(INSN_COST * 4);
6829
6830 format %{
6831 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6832 %}
6833
6834 ins_encode %{
6835 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6836 %}
6837
6838 ins_pipe(fp_load_constant_s);
6839 %}
6840
6841 // Load Packed Double Constant
6842
6843 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6844 match(Set dst con);
6845 ins_cost(INSN_COST);
6846 format %{ "fmovd $dst, $con"%}
6847 ins_encode %{
6848 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6849 %}
6850
6851 ins_pipe(fp_imm_d);
6852 %}
6853
6854 // Load Double Constant
6855
6856 instruct loadConD(vRegD dst, immD con) %{
6857 match(Set dst con);
6858
6859 ins_cost(INSN_COST * 5);
6860 format %{
6861 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6862 %}
6863
6864 ins_encode %{
6865 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6866 %}
6867
6868 ins_pipe(fp_load_constant_d);
6869 %}
6870
6871 // Store Instructions
6872
6873 // Store CMS card-mark Immediate
6874 instruct storeimmCM0(immI0 zero, memory mem)
6875 %{
6876 match(Set mem (StoreCM mem zero));
6877 predicate(unnecessary_storestore(n));
6878
6879 ins_cost(INSN_COST);
6880 format %{ "storestore (elided)\n\t"
6881 "strb zr, $mem\t# byte" %}
6882
6883 ins_encode(aarch64_enc_strb0(mem));
6884
6885 ins_pipe(istore_mem);
6886 %}
6887
6888 // Store CMS card-mark Immediate with intervening StoreStore
6889 // needed when using CMS with no conditional card marking
6890 instruct storeimmCM0_ordered(immI0 zero, memory mem)
6891 %{
6892 match(Set mem (StoreCM mem zero));
6893
6894 ins_cost(INSN_COST * 2);
6895 format %{ "storestore\n\t"
6896 "dmb ishst"
6897 "\n\tstrb zr, $mem\t# byte" %}
6898
6899 ins_encode(aarch64_enc_strb0_ordered(mem));
6900
6901 ins_pipe(istore_mem);
6902 %}
6903
6904 // Store Byte
6905 instruct storeB(iRegIorL2I src, memory mem)
6906 %{
6907 match(Set mem (StoreB mem src));
6908 predicate(!needs_releasing_store(n));
6909
6910 ins_cost(INSN_COST);
6911 format %{ "strb $src, $mem\t# byte" %}
6912
6913 ins_encode(aarch64_enc_strb(src, mem));
6914
6915 ins_pipe(istore_reg_mem);
6916 %}
6917
6918
6919 instruct storeimmB0(immI0 zero, memory mem)
6920 %{
6921 match(Set mem (StoreB mem zero));
6922 predicate(!needs_releasing_store(n));
6923
6924 ins_cost(INSN_COST);
6925 format %{ "strb rscractch2, $mem\t# byte" %}
6926
6927 ins_encode(aarch64_enc_strb0(mem));
6928
6929 ins_pipe(istore_mem);
6930 %}
6931
6932 // Store Char/Short
6933 instruct storeC(iRegIorL2I src, memory mem)
6934 %{
6935 match(Set mem (StoreC mem src));
6936 predicate(!needs_releasing_store(n));
6937
6938 ins_cost(INSN_COST);
6939 format %{ "strh $src, $mem\t# short" %}
6940
6941 ins_encode(aarch64_enc_strh(src, mem));
6942
6943 ins_pipe(istore_reg_mem);
6944 %}
6945
6946 instruct storeimmC0(immI0 zero, memory mem)
6947 %{
6948 match(Set mem (StoreC mem zero));
6949 predicate(!needs_releasing_store(n));
6950
6951 ins_cost(INSN_COST);
6952 format %{ "strh zr, $mem\t# short" %}
6953
6954 ins_encode(aarch64_enc_strh0(mem));
6955
6956 ins_pipe(istore_mem);
6957 %}
6958
6959 // Store Integer
6960
6961 instruct storeI(iRegIorL2I src, memory mem)
6962 %{
6963 match(Set mem(StoreI mem src));
6964 predicate(!needs_releasing_store(n));
6965
6966 ins_cost(INSN_COST);
6967 format %{ "strw $src, $mem\t# int" %}
6968
6969 ins_encode(aarch64_enc_strw(src, mem));
6970
6971 ins_pipe(istore_reg_mem);
6972 %}
6973
6974 instruct storeimmI0(immI0 zero, memory mem)
6975 %{
6976 match(Set mem(StoreI mem zero));
6977 predicate(!needs_releasing_store(n));
6978
6979 ins_cost(INSN_COST);
6980 format %{ "strw zr, $mem\t# int" %}
6981
6982 ins_encode(aarch64_enc_strw0(mem));
6983
6984 ins_pipe(istore_mem);
6985 %}
6986
6987 // Store Long (64 bit signed)
6988 instruct storeL(iRegL src, memory mem)
6989 %{
6990 match(Set mem (StoreL mem src));
6991 predicate(!needs_releasing_store(n));
6992
6993 ins_cost(INSN_COST);
6994 format %{ "str $src, $mem\t# int" %}
6995
6996 ins_encode(aarch64_enc_str(src, mem));
6997
6998 ins_pipe(istore_reg_mem);
6999 %}
7000
7001 // Store Long (64 bit signed)
7002 instruct storeimmL0(immL0 zero, memory mem)
7003 %{
7004 match(Set mem (StoreL mem zero));
7005 predicate(!needs_releasing_store(n));
7006
7007 ins_cost(INSN_COST);
7008 format %{ "str zr, $mem\t# int" %}
7009
7010 ins_encode(aarch64_enc_str0(mem));
7011
7012 ins_pipe(istore_mem);
7013 %}
7014
7015 // Store Pointer
7016 instruct storeP(iRegP src, memory mem)
7017 %{
7018 match(Set mem (StoreP mem src));
7019 predicate(!needs_releasing_store(n));
7020
7021 ins_cost(INSN_COST);
7022 format %{ "str $src, $mem\t# ptr" %}
7023
7024 ins_encode(aarch64_enc_str(src, mem));
7025
7026 ins_pipe(istore_reg_mem);
7027 %}
7028
7029 // Store Pointer
7030 instruct storeimmP0(immP0 zero, memory mem)
7031 %{
7032 match(Set mem (StoreP mem zero));
7033 predicate(!needs_releasing_store(n));
7034
7035 ins_cost(INSN_COST);
7036 format %{ "str zr, $mem\t# ptr" %}
7037
7038 ins_encode(aarch64_enc_str0(mem));
7039
7040 ins_pipe(istore_mem);
7041 %}
7042
7043 // Store Compressed Pointer
7044 instruct storeN(iRegN src, memory mem)
7045 %{
7046 match(Set mem (StoreN mem src));
7047 predicate(!needs_releasing_store(n));
7048
7049 ins_cost(INSN_COST);
7050 format %{ "strw $src, $mem\t# compressed ptr" %}
7051
7052 ins_encode(aarch64_enc_strw(src, mem));
7053
7054 ins_pipe(istore_reg_mem);
7055 %}
7056
7057 instruct storeImmN0(iRegIHeapbase heapbase, immN0 zero, memory mem)
7058 %{
7059 match(Set mem (StoreN mem zero));
7060 predicate(Universe::narrow_oop_base() == NULL &&
7061 Universe::narrow_klass_base() == NULL &&
7062 (!needs_releasing_store(n)));
7063
7064 ins_cost(INSN_COST);
7065 format %{ "strw rheapbase, $mem\t# compressed ptr (rheapbase==0)" %}
7066
7067 ins_encode(aarch64_enc_strw(heapbase, mem));
7068
7069 ins_pipe(istore_reg_mem);
7070 %}
7071
7072 // Store Float
7073 instruct storeF(vRegF src, memory mem)
7074 %{
7075 match(Set mem (StoreF mem src));
7076 predicate(!needs_releasing_store(n));
7077
7078 ins_cost(INSN_COST);
7079 format %{ "strs $src, $mem\t# float" %}
7080
7081 ins_encode( aarch64_enc_strs(src, mem) );
7082
7083 ins_pipe(pipe_class_memory);
7084 %}
7085
7086 // TODO
7087 // implement storeImmF0 and storeFImmPacked
7088
7089 // Store Double
7090 instruct storeD(vRegD src, memory mem)
7091 %{
7092 match(Set mem (StoreD mem src));
7093 predicate(!needs_releasing_store(n));
7094
7095 ins_cost(INSN_COST);
7096 format %{ "strd $src, $mem\t# double" %}
7097
7098 ins_encode( aarch64_enc_strd(src, mem) );
7099
7100 ins_pipe(pipe_class_memory);
7101 %}
7102
7103 // Store Compressed Klass Pointer
7104 instruct storeNKlass(iRegN src, memory mem)
7105 %{
7106 predicate(!needs_releasing_store(n));
7107 match(Set mem (StoreNKlass mem src));
7108
7109 ins_cost(INSN_COST);
7110 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7111
7112 ins_encode(aarch64_enc_strw(src, mem));
7113
7114 ins_pipe(istore_reg_mem);
7115 %}
7116
7117 // TODO
7118 // implement storeImmD0 and storeDImmPacked
7119
7120 // prefetch instructions
7121 // Must be safe to execute with invalid address (cannot fault).
7122
7123 instruct prefetchalloc( memory mem ) %{
7124 match(PrefetchAllocation mem);
7125
7126 ins_cost(INSN_COST);
7127 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7128
7129 ins_encode( aarch64_enc_prefetchw(mem) );
7130
7131 ins_pipe(iload_prefetch);
7132 %}
7133
7134 // ---------------- volatile loads and stores ----------------
7135
7136 // Load Byte (8 bit signed)
7137 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7138 %{
7139 match(Set dst (LoadB mem));
7140
7141 ins_cost(VOLATILE_REF_COST);
7142 format %{ "ldarsb $dst, $mem\t# byte" %}
7143
7144 ins_encode(aarch64_enc_ldarsb(dst, mem));
7145
7146 ins_pipe(pipe_serial);
7147 %}
7148
7149 // Load Byte (8 bit signed) into long
7150 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7151 %{
7152 match(Set dst (ConvI2L (LoadB mem)));
7153
7154 ins_cost(VOLATILE_REF_COST);
7155 format %{ "ldarsb $dst, $mem\t# byte" %}
7156
7157 ins_encode(aarch64_enc_ldarsb(dst, mem));
7158
7159 ins_pipe(pipe_serial);
7160 %}
7161
7162 // Load Byte (8 bit unsigned)
7163 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7164 %{
7165 match(Set dst (LoadUB mem));
7166
7167 ins_cost(VOLATILE_REF_COST);
7168 format %{ "ldarb $dst, $mem\t# byte" %}
7169
7170 ins_encode(aarch64_enc_ldarb(dst, mem));
7171
7172 ins_pipe(pipe_serial);
7173 %}
7174
7175 // Load Byte (8 bit unsigned) into long
7176 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7177 %{
7178 match(Set dst (ConvI2L (LoadUB mem)));
7179
7180 ins_cost(VOLATILE_REF_COST);
7181 format %{ "ldarb $dst, $mem\t# byte" %}
7182
7183 ins_encode(aarch64_enc_ldarb(dst, mem));
7184
7185 ins_pipe(pipe_serial);
7186 %}
7187
7188 // Load Short (16 bit signed)
7189 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7190 %{
7191 match(Set dst (LoadS mem));
7192
7193 ins_cost(VOLATILE_REF_COST);
7194 format %{ "ldarshw $dst, $mem\t# short" %}
7195
7196 ins_encode(aarch64_enc_ldarshw(dst, mem));
7197
7198 ins_pipe(pipe_serial);
7199 %}
7200
7201 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7202 %{
7203 match(Set dst (LoadUS mem));
7204
7205 ins_cost(VOLATILE_REF_COST);
7206 format %{ "ldarhw $dst, $mem\t# short" %}
7207
7208 ins_encode(aarch64_enc_ldarhw(dst, mem));
7209
7210 ins_pipe(pipe_serial);
7211 %}
7212
7213 // Load Short/Char (16 bit unsigned) into long
7214 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7215 %{
7216 match(Set dst (ConvI2L (LoadUS mem)));
7217
7218 ins_cost(VOLATILE_REF_COST);
7219 format %{ "ldarh $dst, $mem\t# short" %}
7220
7221 ins_encode(aarch64_enc_ldarh(dst, mem));
7222
7223 ins_pipe(pipe_serial);
7224 %}
7225
7226 // Load Short/Char (16 bit signed) into long
7227 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7228 %{
7229 match(Set dst (ConvI2L (LoadS mem)));
7230
7231 ins_cost(VOLATILE_REF_COST);
7232 format %{ "ldarh $dst, $mem\t# short" %}
7233
7234 ins_encode(aarch64_enc_ldarsh(dst, mem));
7235
7236 ins_pipe(pipe_serial);
7237 %}
7238
7239 // Load Integer (32 bit signed)
7240 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7241 %{
7242 match(Set dst (LoadI mem));
7243
7244 ins_cost(VOLATILE_REF_COST);
7245 format %{ "ldarw $dst, $mem\t# int" %}
7246
7247 ins_encode(aarch64_enc_ldarw(dst, mem));
7248
7249 ins_pipe(pipe_serial);
7250 %}
7251
7252 // Load Integer (32 bit unsigned) into long
7253 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7254 %{
7255 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7256
7257 ins_cost(VOLATILE_REF_COST);
7258 format %{ "ldarw $dst, $mem\t# int" %}
7259
7260 ins_encode(aarch64_enc_ldarw(dst, mem));
7261
7262 ins_pipe(pipe_serial);
7263 %}
7264
7265 // Load Long (64 bit signed)
7266 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7267 %{
7268 match(Set dst (LoadL mem));
7269
7270 ins_cost(VOLATILE_REF_COST);
7271 format %{ "ldar $dst, $mem\t# int" %}
7272
7273 ins_encode(aarch64_enc_ldar(dst, mem));
7274
7275 ins_pipe(pipe_serial);
7276 %}
7277
7278 // Load Pointer
7279 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7280 %{
7281 match(Set dst (LoadP mem));
7282
7283 ins_cost(VOLATILE_REF_COST);
7284 format %{ "ldar $dst, $mem\t# ptr" %}
7285
7286 ins_encode(aarch64_enc_ldar(dst, mem));
7287
7288 ins_pipe(pipe_serial);
7289 %}
7290
7291 // Load Compressed Pointer
7292 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7293 %{
7294 match(Set dst (LoadN mem));
7295
7296 ins_cost(VOLATILE_REF_COST);
7297 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7298
7299 ins_encode(aarch64_enc_ldarw(dst, mem));
7300
7301 ins_pipe(pipe_serial);
7302 %}
7303
7304 // Load Float
7305 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7306 %{
7307 match(Set dst (LoadF mem));
7308
7309 ins_cost(VOLATILE_REF_COST);
7310 format %{ "ldars $dst, $mem\t# float" %}
7311
7312 ins_encode( aarch64_enc_fldars(dst, mem) );
7313
7314 ins_pipe(pipe_serial);
7315 %}
7316
7317 // Load Double
7318 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7319 %{
7320 match(Set dst (LoadD mem));
7321
7322 ins_cost(VOLATILE_REF_COST);
7323 format %{ "ldard $dst, $mem\t# double" %}
7324
7325 ins_encode( aarch64_enc_fldard(dst, mem) );
7326
7327 ins_pipe(pipe_serial);
7328 %}
7329
7330 // Store Byte
7331 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7332 %{
7333 match(Set mem (StoreB mem src));
7334
7335 ins_cost(VOLATILE_REF_COST);
7336 format %{ "stlrb $src, $mem\t# byte" %}
7337
7338 ins_encode(aarch64_enc_stlrb(src, mem));
7339
7340 ins_pipe(pipe_class_memory);
7341 %}
7342
7343 // Store Char/Short
7344 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7345 %{
7346 match(Set mem (StoreC mem src));
7347
7348 ins_cost(VOLATILE_REF_COST);
7349 format %{ "stlrh $src, $mem\t# short" %}
7350
7351 ins_encode(aarch64_enc_stlrh(src, mem));
7352
7353 ins_pipe(pipe_class_memory);
7354 %}
7355
7356 // Store Integer
7357
7358 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7359 %{
7360 match(Set mem(StoreI mem src));
7361
7362 ins_cost(VOLATILE_REF_COST);
7363 format %{ "stlrw $src, $mem\t# int" %}
7364
7365 ins_encode(aarch64_enc_stlrw(src, mem));
7366
7367 ins_pipe(pipe_class_memory);
7368 %}
7369
7370 // Store Long (64 bit signed)
7371 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7372 %{
7373 match(Set mem (StoreL mem src));
7374
7375 ins_cost(VOLATILE_REF_COST);
7376 format %{ "stlr $src, $mem\t# int" %}
7377
7378 ins_encode(aarch64_enc_stlr(src, mem));
7379
7380 ins_pipe(pipe_class_memory);
7381 %}
7382
7383 // Store Pointer
7384 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7385 %{
7386 match(Set mem (StoreP mem src));
7387
7388 ins_cost(VOLATILE_REF_COST);
7389 format %{ "stlr $src, $mem\t# ptr" %}
7390
7391 ins_encode(aarch64_enc_stlr(src, mem));
7392
7393 ins_pipe(pipe_class_memory);
7394 %}
7395
7396 // Store Compressed Pointer
7397 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7398 %{
7399 match(Set mem (StoreN mem src));
7400
7401 ins_cost(VOLATILE_REF_COST);
7402 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7403
7404 ins_encode(aarch64_enc_stlrw(src, mem));
7405
7406 ins_pipe(pipe_class_memory);
7407 %}
7408
7409 // Store Float
7410 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7411 %{
7412 match(Set mem (StoreF mem src));
7413
7414 ins_cost(VOLATILE_REF_COST);
7415 format %{ "stlrs $src, $mem\t# float" %}
7416
7417 ins_encode( aarch64_enc_fstlrs(src, mem) );
7418
7419 ins_pipe(pipe_class_memory);
7420 %}
7421
7422 // TODO
7423 // implement storeImmF0 and storeFImmPacked
7424
7425 // Store Double
7426 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7427 %{
7428 match(Set mem (StoreD mem src));
7429
7430 ins_cost(VOLATILE_REF_COST);
7431 format %{ "stlrd $src, $mem\t# double" %}
7432
7433 ins_encode( aarch64_enc_fstlrd(src, mem) );
7434
7435 ins_pipe(pipe_class_memory);
7436 %}
7437
7438 // ---------------- end of volatile loads and stores ----------------
7439
7440 // ============================================================================
7441 // BSWAP Instructions
7442
7443 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7444 match(Set dst (ReverseBytesI src));
7445
7446 ins_cost(INSN_COST);
7447 format %{ "revw $dst, $src" %}
7448
7449 ins_encode %{
7450 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7451 %}
7452
7453 ins_pipe(ialu_reg);
7454 %}
7455
7456 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7457 match(Set dst (ReverseBytesL src));
7458
7459 ins_cost(INSN_COST);
7460 format %{ "rev $dst, $src" %}
7461
7462 ins_encode %{
7463 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7464 %}
7465
7466 ins_pipe(ialu_reg);
7467 %}
7468
7469 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7470 match(Set dst (ReverseBytesUS src));
7471
7472 ins_cost(INSN_COST);
7473 format %{ "rev16w $dst, $src" %}
7474
7475 ins_encode %{
7476 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7477 %}
7478
7479 ins_pipe(ialu_reg);
7480 %}
7481
7482 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7483 match(Set dst (ReverseBytesS src));
7484
7485 ins_cost(INSN_COST);
7486 format %{ "rev16w $dst, $src\n\t"
7487 "sbfmw $dst, $dst, #0, #15" %}
7488
7489 ins_encode %{
7490 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7491 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7492 %}
7493
7494 ins_pipe(ialu_reg);
7495 %}
7496
7497 // ============================================================================
7498 // Zero Count Instructions
7499
7500 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7501 match(Set dst (CountLeadingZerosI src));
7502
7503 ins_cost(INSN_COST);
7504 format %{ "clzw $dst, $src" %}
7505 ins_encode %{
7506 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7507 %}
7508
7509 ins_pipe(ialu_reg);
7510 %}
7511
7512 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7513 match(Set dst (CountLeadingZerosL src));
7514
7515 ins_cost(INSN_COST);
7516 format %{ "clz $dst, $src" %}
7517 ins_encode %{
7518 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7519 %}
7520
7521 ins_pipe(ialu_reg);
7522 %}
7523
7524 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7525 match(Set dst (CountTrailingZerosI src));
7526
7527 ins_cost(INSN_COST * 2);
7528 format %{ "rbitw $dst, $src\n\t"
7529 "clzw $dst, $dst" %}
7530 ins_encode %{
7531 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7532 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7533 %}
7534
7535 ins_pipe(ialu_reg);
7536 %}
7537
7538 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7539 match(Set dst (CountTrailingZerosL src));
7540
7541 ins_cost(INSN_COST * 2);
7542 format %{ "rbit $dst, $src\n\t"
7543 "clz $dst, $dst" %}
7544 ins_encode %{
7545 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7546 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7547 %}
7548
7549 ins_pipe(ialu_reg);
7550 %}
7551
7552 //---------- Population Count Instructions -------------------------------------
7553 //
7554
7555 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7556 predicate(UsePopCountInstruction);
7557 match(Set dst (PopCountI src));
7558 effect(TEMP tmp);
7559 ins_cost(INSN_COST * 13);
7560
7561 format %{ "movw $src, $src\n\t"
7562 "mov $tmp, $src\t# vector (1D)\n\t"
7563 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7564 "addv $tmp, $tmp\t# vector (8B)\n\t"
7565 "mov $dst, $tmp\t# vector (1D)" %}
7566 ins_encode %{
7567 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7568 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
7569 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7570 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7571 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7572 %}
7573
7574 ins_pipe(pipe_class_default);
7575 %}
7576
7577 instruct popCountI_mem(iRegINoSp dst, memory mem, vRegF tmp) %{
7578 predicate(UsePopCountInstruction);
7579 match(Set dst (PopCountI (LoadI mem)));
7580 effect(TEMP tmp);
7581 ins_cost(INSN_COST * 13);
7582
7583 format %{ "ldrs $tmp, $mem\n\t"
7584 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7585 "addv $tmp, $tmp\t# vector (8B)\n\t"
7586 "mov $dst, $tmp\t# vector (1D)" %}
7587 ins_encode %{
7588 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7589 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7590 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
7591 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7592 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7593 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7594 %}
7595
7596 ins_pipe(pipe_class_default);
7597 %}
7598
7599 // Note: Long.bitCount(long) returns an int.
7600 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7601 predicate(UsePopCountInstruction);
7602 match(Set dst (PopCountL src));
7603 effect(TEMP tmp);
7604 ins_cost(INSN_COST * 13);
7605
7606 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7607 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7608 "addv $tmp, $tmp\t# vector (8B)\n\t"
7609 "mov $dst, $tmp\t# vector (1D)" %}
7610 ins_encode %{
7611 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
7612 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7613 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7614 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7615 %}
7616
7617 ins_pipe(pipe_class_default);
7618 %}
7619
7620 instruct popCountL_mem(iRegINoSp dst, memory mem, vRegD tmp) %{
7621 predicate(UsePopCountInstruction);
7622 match(Set dst (PopCountL (LoadL mem)));
7623 effect(TEMP tmp);
7624 ins_cost(INSN_COST * 13);
7625
7626 format %{ "ldrd $tmp, $mem\n\t"
7627 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7628 "addv $tmp, $tmp\t# vector (8B)\n\t"
7629 "mov $dst, $tmp\t# vector (1D)" %}
7630 ins_encode %{
7631 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7632 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7633 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
7634 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7635 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7636 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7637 %}
7638
7639 ins_pipe(pipe_class_default);
7640 %}
7641
7642 // ============================================================================
7643 // MemBar Instruction
7644
7645 instruct load_fence() %{
7646 match(LoadFence);
7647 ins_cost(VOLATILE_REF_COST);
7648
7649 format %{ "load_fence" %}
7650
7651 ins_encode %{
7652 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7653 %}
7654 ins_pipe(pipe_serial);
7655 %}
7656
7657 instruct unnecessary_membar_acquire() %{
7658 predicate(unnecessary_acquire(n));
7659 match(MemBarAcquire);
7660 ins_cost(0);
7661
7662 format %{ "membar_acquire (elided)" %}
7663
7664 ins_encode %{
7665 __ block_comment("membar_acquire (elided)");
7666 %}
7667
7668 ins_pipe(pipe_class_empty);
7669 %}
7670
7671 instruct membar_acquire() %{
7672 match(MemBarAcquire);
7673 ins_cost(VOLATILE_REF_COST);
7674
7675 format %{ "membar_acquire\n\t"
7676 "dmb ish" %}
7677
7678 ins_encode %{
7679 __ block_comment("membar_acquire");
7680 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7681 %}
7682
7683 ins_pipe(pipe_serial);
7684 %}
7685
7686
7687 instruct membar_acquire_lock() %{
7688 match(MemBarAcquireLock);
7689 ins_cost(VOLATILE_REF_COST);
7690
7691 format %{ "membar_acquire_lock (elided)" %}
7692
7693 ins_encode %{
7694 __ block_comment("membar_acquire_lock (elided)");
7695 %}
7696
7697 ins_pipe(pipe_serial);
7698 %}
7699
7700 instruct store_fence() %{
7701 match(StoreFence);
7702 ins_cost(VOLATILE_REF_COST);
7703
7704 format %{ "store_fence" %}
7705
7706 ins_encode %{
7707 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7708 %}
7709 ins_pipe(pipe_serial);
7710 %}
7711
7712 instruct unnecessary_membar_release() %{
7713 predicate(unnecessary_release(n));
7714 match(MemBarRelease);
7715 ins_cost(0);
7716
7717 format %{ "membar_release (elided)" %}
7718
7719 ins_encode %{
7720 __ block_comment("membar_release (elided)");
7721 %}
7722 ins_pipe(pipe_serial);
7723 %}
7724
7725 instruct membar_release() %{
7726 match(MemBarRelease);
7727 ins_cost(VOLATILE_REF_COST);
7728
7729 format %{ "membar_release\n\t"
7730 "dmb ish" %}
7731
7732 ins_encode %{
7733 __ block_comment("membar_release");
7734 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7735 %}
7736 ins_pipe(pipe_serial);
7737 %}
7738
7739 instruct membar_storestore() %{
7740 match(MemBarStoreStore);
7741 ins_cost(VOLATILE_REF_COST);
7742
7743 format %{ "MEMBAR-store-store" %}
7744
7745 ins_encode %{
7746 __ membar(Assembler::StoreStore);
7747 %}
7748 ins_pipe(pipe_serial);
7749 %}
7750
7751 instruct membar_release_lock() %{
7752 match(MemBarReleaseLock);
7753 ins_cost(VOLATILE_REF_COST);
7754
7755 format %{ "membar_release_lock (elided)" %}
7756
7757 ins_encode %{
7758 __ block_comment("membar_release_lock (elided)");
7759 %}
7760
7761 ins_pipe(pipe_serial);
7762 %}
7763
7764 instruct unnecessary_membar_volatile() %{
7765 predicate(unnecessary_volatile(n));
7766 match(MemBarVolatile);
7767 ins_cost(0);
7768
7769 format %{ "membar_volatile (elided)" %}
7770
7771 ins_encode %{
7772 __ block_comment("membar_volatile (elided)");
7773 %}
7774
7775 ins_pipe(pipe_serial);
7776 %}
7777
7778 instruct membar_volatile() %{
7779 match(MemBarVolatile);
7780 ins_cost(VOLATILE_REF_COST*100);
7781
7782 format %{ "membar_volatile\n\t"
7783 "dmb ish"%}
7784
7785 ins_encode %{
7786 __ block_comment("membar_volatile");
7787 __ membar(Assembler::StoreLoad);
7788 %}
7789
7790 ins_pipe(pipe_serial);
7791 %}
7792
7793 // ============================================================================
7794 // Cast/Convert Instructions
7795
7796 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7797 match(Set dst (CastX2P src));
7798
7799 ins_cost(INSN_COST);
7800 format %{ "mov $dst, $src\t# long -> ptr" %}
7801
7802 ins_encode %{
7803 if ($dst$$reg != $src$$reg) {
7804 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7805 }
7806 %}
7807
7808 ins_pipe(ialu_reg);
7809 %}
7810
7811 instruct castN2X(iRegLNoSp dst, iRegN src) %{
7812 match(Set dst (CastP2X src));
7813
7814 ins_cost(INSN_COST);
7815 format %{ "mov $dst, $src\t# ptr -> long" %}
7816
7817 ins_encode %{
7818 if ($dst$$reg != $src$$reg) {
7819 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7820 }
7821 %}
7822
7823 ins_pipe(ialu_reg);
7824 %}
7825
7826 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7827 match(Set dst (CastP2X src));
7828
7829 ins_cost(INSN_COST);
7830 format %{ "mov $dst, $src\t# ptr -> long" %}
7831
7832 ins_encode %{
7833 if ($dst$$reg != $src$$reg) {
7834 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7835 }
7836 %}
7837
7838 ins_pipe(ialu_reg);
7839 %}
7840
7841 // Convert oop into int for vectors alignment masking
7842 instruct convP2I(iRegINoSp dst, iRegP src) %{
7843 match(Set dst (ConvL2I (CastP2X src)));
7844
7845 ins_cost(INSN_COST);
7846 format %{ "movw $dst, $src\t# ptr -> int" %}
7847 ins_encode %{
7848 __ movw($dst$$Register, $src$$Register);
7849 %}
7850
7851 ins_pipe(ialu_reg);
7852 %}
7853
7854 // Convert compressed oop into int for vectors alignment masking
7855 // in case of 32bit oops (heap < 4Gb).
7856 instruct convN2I(iRegINoSp dst, iRegN src)
7857 %{
7858 predicate(Universe::narrow_oop_shift() == 0);
7859 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
7860
7861 ins_cost(INSN_COST);
7862 format %{ "mov dst, $src\t# compressed ptr -> int" %}
7863 ins_encode %{
7864 __ movw($dst$$Register, $src$$Register);
7865 %}
7866
7867 ins_pipe(ialu_reg);
7868 %}
7869
7870
7871 // Convert oop pointer into compressed form
7872 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7873 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
7874 match(Set dst (EncodeP src));
7875 effect(KILL cr);
7876 ins_cost(INSN_COST * 3);
7877 format %{ "encode_heap_oop $dst, $src" %}
7878 ins_encode %{
7879 Register s = $src$$Register;
7880 Register d = $dst$$Register;
7881 __ encode_heap_oop(d, s);
7882 %}
7883 ins_pipe(ialu_reg);
7884 %}
7885
7886 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7887 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
7888 match(Set dst (EncodeP src));
7889 ins_cost(INSN_COST * 3);
7890 format %{ "encode_heap_oop_not_null $dst, $src" %}
7891 ins_encode %{
7892 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
7893 %}
7894 ins_pipe(ialu_reg);
7895 %}
7896
7897 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7898 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
7899 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
7900 match(Set dst (DecodeN src));
7901 ins_cost(INSN_COST * 3);
7902 format %{ "decode_heap_oop $dst, $src" %}
7903 ins_encode %{
7904 Register s = $src$$Register;
7905 Register d = $dst$$Register;
7906 __ decode_heap_oop(d, s);
7907 %}
7908 ins_pipe(ialu_reg);
7909 %}
7910
7911 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7912 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
7913 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
7914 match(Set dst (DecodeN src));
7915 ins_cost(INSN_COST * 3);
7916 format %{ "decode_heap_oop_not_null $dst, $src" %}
7917 ins_encode %{
7918 Register s = $src$$Register;
7919 Register d = $dst$$Register;
7920 __ decode_heap_oop_not_null(d, s);
7921 %}
7922 ins_pipe(ialu_reg);
7923 %}
7924
7925 // n.b. AArch64 implementations of encode_klass_not_null and
7926 // decode_klass_not_null do not modify the flags register so, unlike
7927 // Intel, we don't kill CR as a side effect here
7928
7929 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
7930 match(Set dst (EncodePKlass src));
7931
7932 ins_cost(INSN_COST * 3);
7933 format %{ "encode_klass_not_null $dst,$src" %}
7934
7935 ins_encode %{
7936 Register src_reg = as_Register($src$$reg);
7937 Register dst_reg = as_Register($dst$$reg);
7938 __ encode_klass_not_null(dst_reg, src_reg);
7939 %}
7940
7941 ins_pipe(ialu_reg);
7942 %}
7943
7944 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
7945 match(Set dst (DecodeNKlass src));
7946
7947 ins_cost(INSN_COST * 3);
7948 format %{ "decode_klass_not_null $dst,$src" %}
7949
7950 ins_encode %{
7951 Register src_reg = as_Register($src$$reg);
7952 Register dst_reg = as_Register($dst$$reg);
7953 if (dst_reg != src_reg) {
7954 __ decode_klass_not_null(dst_reg, src_reg);
7955 } else {
7956 __ decode_klass_not_null(dst_reg);
7957 }
7958 %}
7959
7960 ins_pipe(ialu_reg);
7961 %}
7962
7963 instruct checkCastPP(iRegPNoSp dst)
7964 %{
7965 match(Set dst (CheckCastPP dst));
7966
7967 size(0);
7968 format %{ "# checkcastPP of $dst" %}
7969 ins_encode(/* empty encoding */);
7970 ins_pipe(pipe_class_empty);
7971 %}
7972
7973 instruct castPP(iRegPNoSp dst)
7974 %{
7975 match(Set dst (CastPP dst));
7976
7977 size(0);
7978 format %{ "# castPP of $dst" %}
7979 ins_encode(/* empty encoding */);
7980 ins_pipe(pipe_class_empty);
7981 %}
7982
7983 instruct castII(iRegI dst)
7984 %{
7985 match(Set dst (CastII dst));
7986
7987 size(0);
7988 format %{ "# castII of $dst" %}
7989 ins_encode(/* empty encoding */);
7990 ins_cost(0);
7991 ins_pipe(pipe_class_empty);
7992 %}
7993
7994 // ============================================================================
7995 // Atomic operation instructions
7996 //
7997 // Intel and SPARC both implement Ideal Node LoadPLocked and
7998 // Store{PIL}Conditional instructions using a normal load for the
7999 // LoadPLocked and a CAS for the Store{PIL}Conditional.
8000 //
8001 // The ideal code appears only to use LoadPLocked/StorePLocked as a
8002 // pair to lock object allocations from Eden space when not using
8003 // TLABs.
8004 //
8005 // There does not appear to be a Load{IL}Locked Ideal Node and the
8006 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
8007 // and to use StoreIConditional only for 32-bit and StoreLConditional
8008 // only for 64-bit.
8009 //
8010 // We implement LoadPLocked and StorePLocked instructions using,
8011 // respectively the AArch64 hw load-exclusive and store-conditional
8012 // instructions. Whereas we must implement each of
8013 // Store{IL}Conditional using a CAS which employs a pair of
8014 // instructions comprising a load-exclusive followed by a
8015 // store-conditional.
8016
8017
8018 // Locked-load (linked load) of the current heap-top
8019 // used when updating the eden heap top
8020 // implemented using ldaxr on AArch64
8021
8022 instruct loadPLocked(iRegPNoSp dst, indirect mem)
8023 %{
8024 match(Set dst (LoadPLocked mem));
8025
8026 ins_cost(VOLATILE_REF_COST);
8027
8028 format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
8029
8030 ins_encode(aarch64_enc_ldaxr(dst, mem));
8031
8032 ins_pipe(pipe_serial);
8033 %}
8034
8035 // Conditional-store of the updated heap-top.
8036 // Used during allocation of the shared heap.
8037 // Sets flag (EQ) on success.
8038 // implemented using stlxr on AArch64.
8039
8040 instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
8041 %{
8042 match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
8043
8044 ins_cost(VOLATILE_REF_COST);
8045
8046 // TODO
8047 // do we need to do a store-conditional release or can we just use a
8048 // plain store-conditional?
8049
8050 format %{
8051 "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
8052 "cmpw rscratch1, zr\t# EQ on successful write"
8053 %}
8054
8055 ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
8056
8057 ins_pipe(pipe_serial);
8058 %}
8059
8060
8061 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
8062 // when attempting to rebias a lock towards the current thread. We
8063 // must use the acquire form of cmpxchg in order to guarantee acquire
8064 // semantics in this case.
8065 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
8066 %{
8067 match(Set cr (StoreLConditional mem (Binary oldval newval)));
8068
8069 ins_cost(VOLATILE_REF_COST);
8070
8071 format %{
8072 "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8073 "cmpw rscratch1, zr\t# EQ on successful write"
8074 %}
8075
8076 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
8077
8078 ins_pipe(pipe_slow);
8079 %}
8080
8081 // storeIConditional also has acquire semantics, for no better reason
8082 // than matching storeLConditional. At the time of writing this
8083 // comment storeIConditional was not used anywhere by AArch64.
8084 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
8085 %{
8086 match(Set cr (StoreIConditional mem (Binary oldval newval)));
8087
8088 ins_cost(VOLATILE_REF_COST);
8089
8090 format %{
8091 "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8092 "cmpw rscratch1, zr\t# EQ on successful write"
8093 %}
8094
8095 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
8096
8097 ins_pipe(pipe_slow);
8098 %}
8099
8100 // standard CompareAndSwapX when we are using barriers
8101 // these have higher priority than the rules selected by a predicate
8102
8103 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8104 // can't match them
8105
8106 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8107
8108 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8109 ins_cost(2 * VOLATILE_REF_COST);
8110
8111 effect(KILL cr);
8112
8113 format %{
8114 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8115 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8116 %}
8117
8118 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8119 aarch64_enc_cset_eq(res));
8120
8121 ins_pipe(pipe_slow);
8122 %}
8123
8124 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8125
8126 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8127 ins_cost(2 * VOLATILE_REF_COST);
8128
8129 effect(KILL cr);
8130
8131 format %{
8132 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8133 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8134 %}
8135
8136 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8137 aarch64_enc_cset_eq(res));
8138
8139 ins_pipe(pipe_slow);
8140 %}
8141
8142 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8143
8144 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8145 ins_cost(2 * VOLATILE_REF_COST);
8146
8147 effect(KILL cr);
8148
8149 format %{
8150 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8151 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8152 %}
8153
8154 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8155 aarch64_enc_cset_eq(res));
8156
8157 ins_pipe(pipe_slow);
8158 %}
8159
8160 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8161
8162 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8163 ins_cost(2 * VOLATILE_REF_COST);
8164
8165 effect(KILL cr);
8166
8167 format %{
8168 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8169 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8170 %}
8171
8172 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8173 aarch64_enc_cset_eq(res));
8174
8175 ins_pipe(pipe_slow);
8176 %}
8177
8178 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8179
8180 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8181 ins_cost(2 * VOLATILE_REF_COST);
8182
8183 effect(KILL cr);
8184
8185 format %{
8186 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8187 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8188 %}
8189
8190 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8191 aarch64_enc_cset_eq(res));
8192
8193 ins_pipe(pipe_slow);
8194 %}
8195
8196 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8197
8198 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8199 ins_cost(2 * VOLATILE_REF_COST);
8200
8201 effect(KILL cr);
8202
8203 format %{
8204 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8205 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8206 %}
8207
8208 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8209 aarch64_enc_cset_eq(res));
8210
8211 ins_pipe(pipe_slow);
8212 %}
8213
8214 // alternative CompareAndSwapX when we are eliding barriers
8215
8216 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8217
8218 predicate(needs_acquiring_load_exclusive(n));
8219 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8220 ins_cost(VOLATILE_REF_COST);
8221
8222 effect(KILL cr);
8223
8224 format %{
8225 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8226 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8227 %}
8228
8229 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8230 aarch64_enc_cset_eq(res));
8231
8232 ins_pipe(pipe_slow);
8233 %}
8234
8235 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8236
8237 predicate(needs_acquiring_load_exclusive(n));
8238 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8239 ins_cost(VOLATILE_REF_COST);
8240
8241 effect(KILL cr);
8242
8243 format %{
8244 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8245 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8246 %}
8247
8248 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8249 aarch64_enc_cset_eq(res));
8250
8251 ins_pipe(pipe_slow);
8252 %}
8253
8254 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8255
8256 predicate(needs_acquiring_load_exclusive(n));
8257 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8258 ins_cost(VOLATILE_REF_COST);
8259
8260 effect(KILL cr);
8261
8262 format %{
8263 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8264 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8265 %}
8266
8267 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8268 aarch64_enc_cset_eq(res));
8269
8270 ins_pipe(pipe_slow);
8271 %}
8272
8273 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8274
8275 predicate(needs_acquiring_load_exclusive(n));
8276 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8277 ins_cost(VOLATILE_REF_COST);
8278
8279 effect(KILL cr);
8280
8281 format %{
8282 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8283 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8284 %}
8285
8286 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8287 aarch64_enc_cset_eq(res));
8288
8289 ins_pipe(pipe_slow);
8290 %}
8291
8292 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8293
8294 predicate(needs_acquiring_load_exclusive(n));
8295 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8296 ins_cost(VOLATILE_REF_COST);
8297
8298 effect(KILL cr);
8299
8300 format %{
8301 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8302 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8303 %}
8304
8305 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8306 aarch64_enc_cset_eq(res));
8307
8308 ins_pipe(pipe_slow);
8309 %}
8310
8311 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8312
8313 predicate(needs_acquiring_load_exclusive(n));
8314 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8315 ins_cost(VOLATILE_REF_COST);
8316
8317 effect(KILL cr);
8318
8319 format %{
8320 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8321 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8322 %}
8323
8324 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8325 aarch64_enc_cset_eq(res));
8326
8327 ins_pipe(pipe_slow);
8328 %}
8329
8330
8331 // ---------------------------------------------------------------------
8332
8333
8334 // BEGIN This section of the file is automatically generated. Do not edit --------------
8335
8336 // Sundry CAS operations. Note that release is always true,
8337 // regardless of the memory ordering of the CAS. This is because we
8338 // need the volatile case to be sequentially consistent but there is
8339 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8340 // can't check the type of memory ordering here, so we always emit a
8341 // STLXR.
8342
8343 // This section is generated from aarch64_ad_cas.m4
8344
8345
8346
8347 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8348 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8349 ins_cost(2 * VOLATILE_REF_COST);
8350 effect(TEMP_DEF res, KILL cr);
8351 format %{
8352 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8353 %}
8354 ins_encode %{
8355 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8356 Assembler::byte, /*acquire*/ false, /*release*/ true,
8357 /*weak*/ false, $res$$Register);
8358 __ sxtbw($res$$Register, $res$$Register);
8359 %}
8360 ins_pipe(pipe_slow);
8361 %}
8362
8363 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8364 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8365 ins_cost(2 * VOLATILE_REF_COST);
8366 effect(TEMP_DEF res, KILL cr);
8367 format %{
8368 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8369 %}
8370 ins_encode %{
8371 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8372 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8373 /*weak*/ false, $res$$Register);
8374 __ sxthw($res$$Register, $res$$Register);
8375 %}
8376 ins_pipe(pipe_slow);
8377 %}
8378
8379 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8380 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8381 ins_cost(2 * VOLATILE_REF_COST);
8382 effect(TEMP_DEF res, KILL cr);
8383 format %{
8384 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8385 %}
8386 ins_encode %{
8387 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8388 Assembler::word, /*acquire*/ false, /*release*/ true,
8389 /*weak*/ false, $res$$Register);
8390 %}
8391 ins_pipe(pipe_slow);
8392 %}
8393
8394 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8395 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8396 ins_cost(2 * VOLATILE_REF_COST);
8397 effect(TEMP_DEF res, KILL cr);
8398 format %{
8399 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8400 %}
8401 ins_encode %{
8402 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8403 Assembler::xword, /*acquire*/ false, /*release*/ true,
8404 /*weak*/ false, $res$$Register);
8405 %}
8406 ins_pipe(pipe_slow);
8407 %}
8408
8409 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8410 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8411 ins_cost(2 * VOLATILE_REF_COST);
8412 effect(TEMP_DEF res, KILL cr);
8413 format %{
8414 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8415 %}
8416 ins_encode %{
8417 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8418 Assembler::word, /*acquire*/ false, /*release*/ true,
8419 /*weak*/ false, $res$$Register);
8420 %}
8421 ins_pipe(pipe_slow);
8422 %}
8423
8424 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8425 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8426 ins_cost(2 * VOLATILE_REF_COST);
8427 effect(TEMP_DEF res, KILL cr);
8428 format %{
8429 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8430 %}
8431 ins_encode %{
8432 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8433 Assembler::xword, /*acquire*/ false, /*release*/ true,
8434 /*weak*/ false, $res$$Register);
8435 %}
8436 ins_pipe(pipe_slow);
8437 %}
8438
8439 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8440 predicate(needs_acquiring_load_exclusive(n));
8441 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8442 ins_cost(VOLATILE_REF_COST);
8443 effect(TEMP_DEF res, KILL cr);
8444 format %{
8445 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8446 %}
8447 ins_encode %{
8448 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8449 Assembler::byte, /*acquire*/ true, /*release*/ true,
8450 /*weak*/ false, $res$$Register);
8451 __ sxtbw($res$$Register, $res$$Register);
8452 %}
8453 ins_pipe(pipe_slow);
8454 %}
8455
8456 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8457 predicate(needs_acquiring_load_exclusive(n));
8458 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8459 ins_cost(VOLATILE_REF_COST);
8460 effect(TEMP_DEF res, KILL cr);
8461 format %{
8462 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8463 %}
8464 ins_encode %{
8465 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8466 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8467 /*weak*/ false, $res$$Register);
8468 __ sxthw($res$$Register, $res$$Register);
8469 %}
8470 ins_pipe(pipe_slow);
8471 %}
8472
8473
8474 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8475 predicate(needs_acquiring_load_exclusive(n));
8476 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8477 ins_cost(VOLATILE_REF_COST);
8478 effect(TEMP_DEF res, KILL cr);
8479 format %{
8480 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8481 %}
8482 ins_encode %{
8483 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8484 Assembler::word, /*acquire*/ true, /*release*/ true,
8485 /*weak*/ false, $res$$Register);
8486 %}
8487 ins_pipe(pipe_slow);
8488 %}
8489
8490 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8491 predicate(needs_acquiring_load_exclusive(n));
8492 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8493 ins_cost(VOLATILE_REF_COST);
8494 effect(TEMP_DEF res, KILL cr);
8495 format %{
8496 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8497 %}
8498 ins_encode %{
8499 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8500 Assembler::xword, /*acquire*/ true, /*release*/ true,
8501 /*weak*/ false, $res$$Register);
8502 %}
8503 ins_pipe(pipe_slow);
8504 %}
8505
8506
8507 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8508 predicate(needs_acquiring_load_exclusive(n));
8509 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8510 ins_cost(VOLATILE_REF_COST);
8511 effect(TEMP_DEF res, KILL cr);
8512 format %{
8513 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8514 %}
8515 ins_encode %{
8516 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8517 Assembler::word, /*acquire*/ true, /*release*/ true,
8518 /*weak*/ false, $res$$Register);
8519 %}
8520 ins_pipe(pipe_slow);
8521 %}
8522
8523 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8524 predicate(needs_acquiring_load_exclusive(n));
8525 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8526 ins_cost(VOLATILE_REF_COST);
8527 effect(TEMP_DEF res, KILL cr);
8528 format %{
8529 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8530 %}
8531 ins_encode %{
8532 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8533 Assembler::xword, /*acquire*/ true, /*release*/ true,
8534 /*weak*/ false, $res$$Register);
8535 %}
8536 ins_pipe(pipe_slow);
8537 %}
8538
8539 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8540 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8541 ins_cost(2 * VOLATILE_REF_COST);
8542 effect(KILL cr);
8543 format %{
8544 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8545 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8546 %}
8547 ins_encode %{
8548 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8549 Assembler::byte, /*acquire*/ false, /*release*/ true,
8550 /*weak*/ true, noreg);
8551 __ csetw($res$$Register, Assembler::EQ);
8552 %}
8553 ins_pipe(pipe_slow);
8554 %}
8555
8556 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8557 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8558 ins_cost(2 * VOLATILE_REF_COST);
8559 effect(KILL cr);
8560 format %{
8561 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8562 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8563 %}
8564 ins_encode %{
8565 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8566 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8567 /*weak*/ true, noreg);
8568 __ csetw($res$$Register, Assembler::EQ);
8569 %}
8570 ins_pipe(pipe_slow);
8571 %}
8572
8573 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8574 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8575 ins_cost(2 * VOLATILE_REF_COST);
8576 effect(KILL cr);
8577 format %{
8578 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8579 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8580 %}
8581 ins_encode %{
8582 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8583 Assembler::word, /*acquire*/ false, /*release*/ true,
8584 /*weak*/ true, noreg);
8585 __ csetw($res$$Register, Assembler::EQ);
8586 %}
8587 ins_pipe(pipe_slow);
8588 %}
8589
8590 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8591 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8592 ins_cost(2 * VOLATILE_REF_COST);
8593 effect(KILL cr);
8594 format %{
8595 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8596 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8597 %}
8598 ins_encode %{
8599 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8600 Assembler::xword, /*acquire*/ false, /*release*/ true,
8601 /*weak*/ true, noreg);
8602 __ csetw($res$$Register, Assembler::EQ);
8603 %}
8604 ins_pipe(pipe_slow);
8605 %}
8606
8607 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8608 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8609 ins_cost(2 * VOLATILE_REF_COST);
8610 effect(KILL cr);
8611 format %{
8612 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8613 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8614 %}
8615 ins_encode %{
8616 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8617 Assembler::word, /*acquire*/ false, /*release*/ true,
8618 /*weak*/ true, noreg);
8619 __ csetw($res$$Register, Assembler::EQ);
8620 %}
8621 ins_pipe(pipe_slow);
8622 %}
8623
8624 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8625 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8626 ins_cost(2 * VOLATILE_REF_COST);
8627 effect(KILL cr);
8628 format %{
8629 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8630 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8631 %}
8632 ins_encode %{
8633 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8634 Assembler::xword, /*acquire*/ false, /*release*/ true,
8635 /*weak*/ true, noreg);
8636 __ csetw($res$$Register, Assembler::EQ);
8637 %}
8638 ins_pipe(pipe_slow);
8639 %}
8640
8641 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8642 predicate(needs_acquiring_load_exclusive(n));
8643 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8644 ins_cost(VOLATILE_REF_COST);
8645 effect(KILL cr);
8646 format %{
8647 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8648 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8649 %}
8650 ins_encode %{
8651 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8652 Assembler::byte, /*acquire*/ true, /*release*/ true,
8653 /*weak*/ true, noreg);
8654 __ csetw($res$$Register, Assembler::EQ);
8655 %}
8656 ins_pipe(pipe_slow);
8657 %}
8658
8659 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8660 predicate(needs_acquiring_load_exclusive(n));
8661 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8662 ins_cost(VOLATILE_REF_COST);
8663 effect(KILL cr);
8664 format %{
8665 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8666 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8667 %}
8668 ins_encode %{
8669 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8670 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8671 /*weak*/ true, noreg);
8672 __ csetw($res$$Register, Assembler::EQ);
8673 %}
8674 ins_pipe(pipe_slow);
8675 %}
8676
8677 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8678 predicate(needs_acquiring_load_exclusive(n));
8679 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8680 ins_cost(VOLATILE_REF_COST);
8681 effect(KILL cr);
8682 format %{
8683 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8684 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8685 %}
8686 ins_encode %{
8687 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8688 Assembler::word, /*acquire*/ true, /*release*/ true,
8689 /*weak*/ true, noreg);
8690 __ csetw($res$$Register, Assembler::EQ);
8691 %}
8692 ins_pipe(pipe_slow);
8693 %}
8694
8695 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8696 predicate(needs_acquiring_load_exclusive(n));
8697 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8698 ins_cost(VOLATILE_REF_COST);
8699 effect(KILL cr);
8700 format %{
8701 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8702 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8703 %}
8704 ins_encode %{
8705 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8706 Assembler::xword, /*acquire*/ true, /*release*/ true,
8707 /*weak*/ true, noreg);
8708 __ csetw($res$$Register, Assembler::EQ);
8709 %}
8710 ins_pipe(pipe_slow);
8711 %}
8712
8713 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8714 predicate(needs_acquiring_load_exclusive(n));
8715 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8716 ins_cost(VOLATILE_REF_COST);
8717 effect(KILL cr);
8718 format %{
8719 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8720 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8721 %}
8722 ins_encode %{
8723 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8724 Assembler::word, /*acquire*/ true, /*release*/ true,
8725 /*weak*/ true, noreg);
8726 __ csetw($res$$Register, Assembler::EQ);
8727 %}
8728 ins_pipe(pipe_slow);
8729 %}
8730
8731 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8732 predicate(needs_acquiring_load_exclusive(n));
8733 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8734 ins_cost(VOLATILE_REF_COST);
8735 effect(KILL cr);
8736 format %{
8737 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8738 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8739 %}
8740 ins_encode %{
8741 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8742 Assembler::xword, /*acquire*/ true, /*release*/ true,
8743 /*weak*/ true, noreg);
8744 __ csetw($res$$Register, Assembler::EQ);
8745 %}
8746 ins_pipe(pipe_slow);
8747 %}
8748
8749 // END This section of the file is automatically generated. Do not edit --------------
8750 // ---------------------------------------------------------------------
8751
8752 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8753 match(Set prev (GetAndSetI mem newv));
8754 ins_cost(2 * VOLATILE_REF_COST);
8755 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8756 ins_encode %{
8757 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8758 %}
8759 ins_pipe(pipe_serial);
8760 %}
8761
8762 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8763 match(Set prev (GetAndSetL mem newv));
8764 ins_cost(2 * VOLATILE_REF_COST);
8765 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8766 ins_encode %{
8767 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8768 %}
8769 ins_pipe(pipe_serial);
8770 %}
8771
8772 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8773 match(Set prev (GetAndSetN mem newv));
8774 ins_cost(2 * VOLATILE_REF_COST);
8775 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8776 ins_encode %{
8777 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8778 %}
8779 ins_pipe(pipe_serial);
8780 %}
8781
8782 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8783 match(Set prev (GetAndSetP mem newv));
8784 ins_cost(2 * VOLATILE_REF_COST);
8785 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8786 ins_encode %{
8787 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8788 %}
8789 ins_pipe(pipe_serial);
8790 %}
8791
8792 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8793 predicate(needs_acquiring_load_exclusive(n));
8794 match(Set prev (GetAndSetI mem newv));
8795 ins_cost(VOLATILE_REF_COST);
8796 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8797 ins_encode %{
8798 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8799 %}
8800 ins_pipe(pipe_serial);
8801 %}
8802
8803 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8804 predicate(needs_acquiring_load_exclusive(n));
8805 match(Set prev (GetAndSetL mem newv));
8806 ins_cost(VOLATILE_REF_COST);
8807 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8808 ins_encode %{
8809 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8810 %}
8811 ins_pipe(pipe_serial);
8812 %}
8813
8814 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8815 predicate(needs_acquiring_load_exclusive(n));
8816 match(Set prev (GetAndSetN mem newv));
8817 ins_cost(VOLATILE_REF_COST);
8818 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8819 ins_encode %{
8820 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8821 %}
8822 ins_pipe(pipe_serial);
8823 %}
8824
8825 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8826 predicate(needs_acquiring_load_exclusive(n));
8827 match(Set prev (GetAndSetP mem newv));
8828 ins_cost(VOLATILE_REF_COST);
8829 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8830 ins_encode %{
8831 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8832 %}
8833 ins_pipe(pipe_serial);
8834 %}
8835
8836
8837 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
8838 match(Set newval (GetAndAddL mem incr));
8839 ins_cost(2 * VOLATILE_REF_COST + 1);
8840 format %{ "get_and_addL $newval, [$mem], $incr" %}
8841 ins_encode %{
8842 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
8843 %}
8844 ins_pipe(pipe_serial);
8845 %}
8846
8847 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
8848 predicate(n->as_LoadStore()->result_not_used());
8849 match(Set dummy (GetAndAddL mem incr));
8850 ins_cost(2 * VOLATILE_REF_COST);
8851 format %{ "get_and_addL [$mem], $incr" %}
8852 ins_encode %{
8853 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
8854 %}
8855 ins_pipe(pipe_serial);
8856 %}
8857
8858 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
8859 match(Set newval (GetAndAddL mem incr));
8860 ins_cost(2 * VOLATILE_REF_COST + 1);
8861 format %{ "get_and_addL $newval, [$mem], $incr" %}
8862 ins_encode %{
8863 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
8864 %}
8865 ins_pipe(pipe_serial);
8866 %}
8867
8868 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
8869 predicate(n->as_LoadStore()->result_not_used());
8870 match(Set dummy (GetAndAddL mem incr));
8871 ins_cost(2 * VOLATILE_REF_COST);
8872 format %{ "get_and_addL [$mem], $incr" %}
8873 ins_encode %{
8874 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
8875 %}
8876 ins_pipe(pipe_serial);
8877 %}
8878
8879 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
8880 match(Set newval (GetAndAddI mem incr));
8881 ins_cost(2 * VOLATILE_REF_COST + 1);
8882 format %{ "get_and_addI $newval, [$mem], $incr" %}
8883 ins_encode %{
8884 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
8885 %}
8886 ins_pipe(pipe_serial);
8887 %}
8888
8889 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
8890 predicate(n->as_LoadStore()->result_not_used());
8891 match(Set dummy (GetAndAddI mem incr));
8892 ins_cost(2 * VOLATILE_REF_COST);
8893 format %{ "get_and_addI [$mem], $incr" %}
8894 ins_encode %{
8895 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
8896 %}
8897 ins_pipe(pipe_serial);
8898 %}
8899
8900 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
8901 match(Set newval (GetAndAddI mem incr));
8902 ins_cost(2 * VOLATILE_REF_COST + 1);
8903 format %{ "get_and_addI $newval, [$mem], $incr" %}
8904 ins_encode %{
8905 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
8906 %}
8907 ins_pipe(pipe_serial);
8908 %}
8909
8910 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
8911 predicate(n->as_LoadStore()->result_not_used());
8912 match(Set dummy (GetAndAddI mem incr));
8913 ins_cost(2 * VOLATILE_REF_COST);
8914 format %{ "get_and_addI [$mem], $incr" %}
8915 ins_encode %{
8916 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
8917 %}
8918 ins_pipe(pipe_serial);
8919 %}
8920
8921 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
8922 predicate(needs_acquiring_load_exclusive(n));
8923 match(Set newval (GetAndAddL mem incr));
8924 ins_cost(VOLATILE_REF_COST + 1);
8925 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
8926 ins_encode %{
8927 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
8928 %}
8929 ins_pipe(pipe_serial);
8930 %}
8931
8932 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
8933 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8934 match(Set dummy (GetAndAddL mem incr));
8935 ins_cost(VOLATILE_REF_COST);
8936 format %{ "get_and_addL_acq [$mem], $incr" %}
8937 ins_encode %{
8938 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
8939 %}
8940 ins_pipe(pipe_serial);
8941 %}
8942
8943 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
8944 predicate(needs_acquiring_load_exclusive(n));
8945 match(Set newval (GetAndAddL mem incr));
8946 ins_cost(VOLATILE_REF_COST + 1);
8947 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
8948 ins_encode %{
8949 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
8950 %}
8951 ins_pipe(pipe_serial);
8952 %}
8953
8954 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
8955 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8956 match(Set dummy (GetAndAddL mem incr));
8957 ins_cost(VOLATILE_REF_COST);
8958 format %{ "get_and_addL_acq [$mem], $incr" %}
8959 ins_encode %{
8960 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
8961 %}
8962 ins_pipe(pipe_serial);
8963 %}
8964
8965 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
8966 predicate(needs_acquiring_load_exclusive(n));
8967 match(Set newval (GetAndAddI mem incr));
8968 ins_cost(VOLATILE_REF_COST + 1);
8969 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
8970 ins_encode %{
8971 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
8972 %}
8973 ins_pipe(pipe_serial);
8974 %}
8975
8976 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
8977 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8978 match(Set dummy (GetAndAddI mem incr));
8979 ins_cost(VOLATILE_REF_COST);
8980 format %{ "get_and_addI_acq [$mem], $incr" %}
8981 ins_encode %{
8982 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
8983 %}
8984 ins_pipe(pipe_serial);
8985 %}
8986
8987 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
8988 predicate(needs_acquiring_load_exclusive(n));
8989 match(Set newval (GetAndAddI mem incr));
8990 ins_cost(VOLATILE_REF_COST + 1);
8991 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
8992 ins_encode %{
8993 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
8994 %}
8995 ins_pipe(pipe_serial);
8996 %}
8997
8998 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
8999 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9000 match(Set dummy (GetAndAddI mem incr));
9001 ins_cost(VOLATILE_REF_COST);
9002 format %{ "get_and_addI_acq [$mem], $incr" %}
9003 ins_encode %{
9004 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9005 %}
9006 ins_pipe(pipe_serial);
9007 %}
9008
9009 // Manifest a CmpL result in an integer register.
9010 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9011 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9012 %{
9013 match(Set dst (CmpL3 src1 src2));
9014 effect(KILL flags);
9015
9016 ins_cost(INSN_COST * 6);
9017 format %{
9018 "cmp $src1, $src2"
9019 "csetw $dst, ne"
9020 "cnegw $dst, lt"
9021 %}
9022 // format %{ "CmpL3 $dst, $src1, $src2" %}
9023 ins_encode %{
9024 __ cmp($src1$$Register, $src2$$Register);
9025 __ csetw($dst$$Register, Assembler::NE);
9026 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9027 %}
9028
9029 ins_pipe(pipe_class_default);
9030 %}
9031
9032 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9033 %{
9034 match(Set dst (CmpL3 src1 src2));
9035 effect(KILL flags);
9036
9037 ins_cost(INSN_COST * 6);
9038 format %{
9039 "cmp $src1, $src2"
9040 "csetw $dst, ne"
9041 "cnegw $dst, lt"
9042 %}
9043 ins_encode %{
9044 int32_t con = (int32_t)$src2$$constant;
9045 if (con < 0) {
9046 __ adds(zr, $src1$$Register, -con);
9047 } else {
9048 __ subs(zr, $src1$$Register, con);
9049 }
9050 __ csetw($dst$$Register, Assembler::NE);
9051 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9052 %}
9053
9054 ins_pipe(pipe_class_default);
9055 %}
9056
9057 // ============================================================================
9058 // Conditional Move Instructions
9059
9060 // n.b. we have identical rules for both a signed compare op (cmpOp)
9061 // and an unsigned compare op (cmpOpU). it would be nice if we could
9062 // define an op class which merged both inputs and use it to type the
9063 // argument to a single rule. unfortunatelyt his fails because the
9064 // opclass does not live up to the COND_INTER interface of its
9065 // component operands. When the generic code tries to negate the
9066 // operand it ends up running the generci Machoper::negate method
9067 // which throws a ShouldNotHappen. So, we have to provide two flavours
9068 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9069
9070 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9071 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9072
9073 ins_cost(INSN_COST * 2);
9074 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9075
9076 ins_encode %{
9077 __ cselw(as_Register($dst$$reg),
9078 as_Register($src2$$reg),
9079 as_Register($src1$$reg),
9080 (Assembler::Condition)$cmp$$cmpcode);
9081 %}
9082
9083 ins_pipe(icond_reg_reg);
9084 %}
9085
9086 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9087 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9088
9089 ins_cost(INSN_COST * 2);
9090 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9091
9092 ins_encode %{
9093 __ cselw(as_Register($dst$$reg),
9094 as_Register($src2$$reg),
9095 as_Register($src1$$reg),
9096 (Assembler::Condition)$cmp$$cmpcode);
9097 %}
9098
9099 ins_pipe(icond_reg_reg);
9100 %}
9101
9102 // special cases where one arg is zero
9103
9104 // n.b. this is selected in preference to the rule above because it
9105 // avoids loading constant 0 into a source register
9106
9107 // TODO
9108 // we ought only to be able to cull one of these variants as the ideal
9109 // transforms ought always to order the zero consistently (to left/right?)
9110
9111 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9112 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9113
9114 ins_cost(INSN_COST * 2);
9115 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9116
9117 ins_encode %{
9118 __ cselw(as_Register($dst$$reg),
9119 as_Register($src$$reg),
9120 zr,
9121 (Assembler::Condition)$cmp$$cmpcode);
9122 %}
9123
9124 ins_pipe(icond_reg);
9125 %}
9126
9127 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9128 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9129
9130 ins_cost(INSN_COST * 2);
9131 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9132
9133 ins_encode %{
9134 __ cselw(as_Register($dst$$reg),
9135 as_Register($src$$reg),
9136 zr,
9137 (Assembler::Condition)$cmp$$cmpcode);
9138 %}
9139
9140 ins_pipe(icond_reg);
9141 %}
9142
9143 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9144 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9145
9146 ins_cost(INSN_COST * 2);
9147 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9148
9149 ins_encode %{
9150 __ cselw(as_Register($dst$$reg),
9151 zr,
9152 as_Register($src$$reg),
9153 (Assembler::Condition)$cmp$$cmpcode);
9154 %}
9155
9156 ins_pipe(icond_reg);
9157 %}
9158
9159 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9160 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9161
9162 ins_cost(INSN_COST * 2);
9163 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9164
9165 ins_encode %{
9166 __ cselw(as_Register($dst$$reg),
9167 zr,
9168 as_Register($src$$reg),
9169 (Assembler::Condition)$cmp$$cmpcode);
9170 %}
9171
9172 ins_pipe(icond_reg);
9173 %}
9174
9175 // special case for creating a boolean 0 or 1
9176
9177 // n.b. this is selected in preference to the rule above because it
9178 // avoids loading constants 0 and 1 into a source register
9179
9180 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9181 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9182
9183 ins_cost(INSN_COST * 2);
9184 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9185
9186 ins_encode %{
9187 // equivalently
9188 // cset(as_Register($dst$$reg),
9189 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9190 __ csincw(as_Register($dst$$reg),
9191 zr,
9192 zr,
9193 (Assembler::Condition)$cmp$$cmpcode);
9194 %}
9195
9196 ins_pipe(icond_none);
9197 %}
9198
9199 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9200 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9201
9202 ins_cost(INSN_COST * 2);
9203 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9204
9205 ins_encode %{
9206 // equivalently
9207 // cset(as_Register($dst$$reg),
9208 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9209 __ csincw(as_Register($dst$$reg),
9210 zr,
9211 zr,
9212 (Assembler::Condition)$cmp$$cmpcode);
9213 %}
9214
9215 ins_pipe(icond_none);
9216 %}
9217
9218 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9219 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9220
9221 ins_cost(INSN_COST * 2);
9222 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9223
9224 ins_encode %{
9225 __ csel(as_Register($dst$$reg),
9226 as_Register($src2$$reg),
9227 as_Register($src1$$reg),
9228 (Assembler::Condition)$cmp$$cmpcode);
9229 %}
9230
9231 ins_pipe(icond_reg_reg);
9232 %}
9233
9234 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9235 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9236
9237 ins_cost(INSN_COST * 2);
9238 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9239
9240 ins_encode %{
9241 __ csel(as_Register($dst$$reg),
9242 as_Register($src2$$reg),
9243 as_Register($src1$$reg),
9244 (Assembler::Condition)$cmp$$cmpcode);
9245 %}
9246
9247 ins_pipe(icond_reg_reg);
9248 %}
9249
9250 // special cases where one arg is zero
9251
9252 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9253 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9254
9255 ins_cost(INSN_COST * 2);
9256 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9257
9258 ins_encode %{
9259 __ csel(as_Register($dst$$reg),
9260 zr,
9261 as_Register($src$$reg),
9262 (Assembler::Condition)$cmp$$cmpcode);
9263 %}
9264
9265 ins_pipe(icond_reg);
9266 %}
9267
9268 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9269 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9270
9271 ins_cost(INSN_COST * 2);
9272 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9273
9274 ins_encode %{
9275 __ csel(as_Register($dst$$reg),
9276 zr,
9277 as_Register($src$$reg),
9278 (Assembler::Condition)$cmp$$cmpcode);
9279 %}
9280
9281 ins_pipe(icond_reg);
9282 %}
9283
9284 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9285 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9286
9287 ins_cost(INSN_COST * 2);
9288 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9289
9290 ins_encode %{
9291 __ csel(as_Register($dst$$reg),
9292 as_Register($src$$reg),
9293 zr,
9294 (Assembler::Condition)$cmp$$cmpcode);
9295 %}
9296
9297 ins_pipe(icond_reg);
9298 %}
9299
9300 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9301 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9302
9303 ins_cost(INSN_COST * 2);
9304 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9305
9306 ins_encode %{
9307 __ csel(as_Register($dst$$reg),
9308 as_Register($src$$reg),
9309 zr,
9310 (Assembler::Condition)$cmp$$cmpcode);
9311 %}
9312
9313 ins_pipe(icond_reg);
9314 %}
9315
9316 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9317 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9318
9319 ins_cost(INSN_COST * 2);
9320 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9321
9322 ins_encode %{
9323 __ csel(as_Register($dst$$reg),
9324 as_Register($src2$$reg),
9325 as_Register($src1$$reg),
9326 (Assembler::Condition)$cmp$$cmpcode);
9327 %}
9328
9329 ins_pipe(icond_reg_reg);
9330 %}
9331
9332 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9333 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9334
9335 ins_cost(INSN_COST * 2);
9336 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9337
9338 ins_encode %{
9339 __ csel(as_Register($dst$$reg),
9340 as_Register($src2$$reg),
9341 as_Register($src1$$reg),
9342 (Assembler::Condition)$cmp$$cmpcode);
9343 %}
9344
9345 ins_pipe(icond_reg_reg);
9346 %}
9347
9348 // special cases where one arg is zero
9349
9350 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9351 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9352
9353 ins_cost(INSN_COST * 2);
9354 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9355
9356 ins_encode %{
9357 __ csel(as_Register($dst$$reg),
9358 zr,
9359 as_Register($src$$reg),
9360 (Assembler::Condition)$cmp$$cmpcode);
9361 %}
9362
9363 ins_pipe(icond_reg);
9364 %}
9365
9366 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9367 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9368
9369 ins_cost(INSN_COST * 2);
9370 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9371
9372 ins_encode %{
9373 __ csel(as_Register($dst$$reg),
9374 zr,
9375 as_Register($src$$reg),
9376 (Assembler::Condition)$cmp$$cmpcode);
9377 %}
9378
9379 ins_pipe(icond_reg);
9380 %}
9381
9382 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9383 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9384
9385 ins_cost(INSN_COST * 2);
9386 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9387
9388 ins_encode %{
9389 __ csel(as_Register($dst$$reg),
9390 as_Register($src$$reg),
9391 zr,
9392 (Assembler::Condition)$cmp$$cmpcode);
9393 %}
9394
9395 ins_pipe(icond_reg);
9396 %}
9397
9398 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9399 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9400
9401 ins_cost(INSN_COST * 2);
9402 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9403
9404 ins_encode %{
9405 __ csel(as_Register($dst$$reg),
9406 as_Register($src$$reg),
9407 zr,
9408 (Assembler::Condition)$cmp$$cmpcode);
9409 %}
9410
9411 ins_pipe(icond_reg);
9412 %}
9413
9414 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9415 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9416
9417 ins_cost(INSN_COST * 2);
9418 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9419
9420 ins_encode %{
9421 __ cselw(as_Register($dst$$reg),
9422 as_Register($src2$$reg),
9423 as_Register($src1$$reg),
9424 (Assembler::Condition)$cmp$$cmpcode);
9425 %}
9426
9427 ins_pipe(icond_reg_reg);
9428 %}
9429
9430 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9431 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9432
9433 ins_cost(INSN_COST * 2);
9434 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9435
9436 ins_encode %{
9437 __ cselw(as_Register($dst$$reg),
9438 as_Register($src2$$reg),
9439 as_Register($src1$$reg),
9440 (Assembler::Condition)$cmp$$cmpcode);
9441 %}
9442
9443 ins_pipe(icond_reg_reg);
9444 %}
9445
9446 // special cases where one arg is zero
9447
9448 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9449 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9450
9451 ins_cost(INSN_COST * 2);
9452 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9453
9454 ins_encode %{
9455 __ cselw(as_Register($dst$$reg),
9456 zr,
9457 as_Register($src$$reg),
9458 (Assembler::Condition)$cmp$$cmpcode);
9459 %}
9460
9461 ins_pipe(icond_reg);
9462 %}
9463
9464 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9465 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9466
9467 ins_cost(INSN_COST * 2);
9468 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9469
9470 ins_encode %{
9471 __ cselw(as_Register($dst$$reg),
9472 zr,
9473 as_Register($src$$reg),
9474 (Assembler::Condition)$cmp$$cmpcode);
9475 %}
9476
9477 ins_pipe(icond_reg);
9478 %}
9479
9480 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9481 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9482
9483 ins_cost(INSN_COST * 2);
9484 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9485
9486 ins_encode %{
9487 __ cselw(as_Register($dst$$reg),
9488 as_Register($src$$reg),
9489 zr,
9490 (Assembler::Condition)$cmp$$cmpcode);
9491 %}
9492
9493 ins_pipe(icond_reg);
9494 %}
9495
9496 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9497 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9498
9499 ins_cost(INSN_COST * 2);
9500 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9501
9502 ins_encode %{
9503 __ cselw(as_Register($dst$$reg),
9504 as_Register($src$$reg),
9505 zr,
9506 (Assembler::Condition)$cmp$$cmpcode);
9507 %}
9508
9509 ins_pipe(icond_reg);
9510 %}
9511
9512 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9513 %{
9514 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9515
9516 ins_cost(INSN_COST * 3);
9517
9518 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9519 ins_encode %{
9520 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9521 __ fcsels(as_FloatRegister($dst$$reg),
9522 as_FloatRegister($src2$$reg),
9523 as_FloatRegister($src1$$reg),
9524 cond);
9525 %}
9526
9527 ins_pipe(fp_cond_reg_reg_s);
9528 %}
9529
9530 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9531 %{
9532 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9533
9534 ins_cost(INSN_COST * 3);
9535
9536 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9537 ins_encode %{
9538 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9539 __ fcsels(as_FloatRegister($dst$$reg),
9540 as_FloatRegister($src2$$reg),
9541 as_FloatRegister($src1$$reg),
9542 cond);
9543 %}
9544
9545 ins_pipe(fp_cond_reg_reg_s);
9546 %}
9547
9548 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9549 %{
9550 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9551
9552 ins_cost(INSN_COST * 3);
9553
9554 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9555 ins_encode %{
9556 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9557 __ fcseld(as_FloatRegister($dst$$reg),
9558 as_FloatRegister($src2$$reg),
9559 as_FloatRegister($src1$$reg),
9560 cond);
9561 %}
9562
9563 ins_pipe(fp_cond_reg_reg_d);
9564 %}
9565
9566 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9567 %{
9568 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9569
9570 ins_cost(INSN_COST * 3);
9571
9572 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9573 ins_encode %{
9574 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9575 __ fcseld(as_FloatRegister($dst$$reg),
9576 as_FloatRegister($src2$$reg),
9577 as_FloatRegister($src1$$reg),
9578 cond);
9579 %}
9580
9581 ins_pipe(fp_cond_reg_reg_d);
9582 %}
9583
9584 // ============================================================================
9585 // Arithmetic Instructions
9586 //
9587
9588 // Integer Addition
9589
9590 // TODO
9591 // these currently employ operations which do not set CR and hence are
9592 // not flagged as killing CR but we would like to isolate the cases
9593 // where we want to set flags from those where we don't. need to work
9594 // out how to do that.
9595
9596 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9597 match(Set dst (AddI src1 src2));
9598
9599 ins_cost(INSN_COST);
9600 format %{ "addw $dst, $src1, $src2" %}
9601
9602 ins_encode %{
9603 __ addw(as_Register($dst$$reg),
9604 as_Register($src1$$reg),
9605 as_Register($src2$$reg));
9606 %}
9607
9608 ins_pipe(ialu_reg_reg);
9609 %}
9610
9611 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9612 match(Set dst (AddI src1 src2));
9613
9614 ins_cost(INSN_COST);
9615 format %{ "addw $dst, $src1, $src2" %}
9616
9617 // use opcode to indicate that this is an add not a sub
9618 opcode(0x0);
9619
9620 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9621
9622 ins_pipe(ialu_reg_imm);
9623 %}
9624
9625 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9626 match(Set dst (AddI (ConvL2I src1) src2));
9627
9628 ins_cost(INSN_COST);
9629 format %{ "addw $dst, $src1, $src2" %}
9630
9631 // use opcode to indicate that this is an add not a sub
9632 opcode(0x0);
9633
9634 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9635
9636 ins_pipe(ialu_reg_imm);
9637 %}
9638
9639 // Pointer Addition
9640 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
9641 match(Set dst (AddP src1 src2));
9642
9643 ins_cost(INSN_COST);
9644 format %{ "add $dst, $src1, $src2\t# ptr" %}
9645
9646 ins_encode %{
9647 __ add(as_Register($dst$$reg),
9648 as_Register($src1$$reg),
9649 as_Register($src2$$reg));
9650 %}
9651
9652 ins_pipe(ialu_reg_reg);
9653 %}
9654
9655 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
9656 match(Set dst (AddP src1 (ConvI2L src2)));
9657
9658 ins_cost(1.9 * INSN_COST);
9659 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9660
9661 ins_encode %{
9662 __ add(as_Register($dst$$reg),
9663 as_Register($src1$$reg),
9664 as_Register($src2$$reg), ext::sxtw);
9665 %}
9666
9667 ins_pipe(ialu_reg_reg);
9668 %}
9669
9670 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
9671 match(Set dst (AddP src1 (LShiftL src2 scale)));
9672
9673 ins_cost(1.9 * INSN_COST);
9674 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9675
9676 ins_encode %{
9677 __ lea(as_Register($dst$$reg),
9678 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9679 Address::lsl($scale$$constant)));
9680 %}
9681
9682 ins_pipe(ialu_reg_reg_shift);
9683 %}
9684
9685 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
9686 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9687
9688 ins_cost(1.9 * INSN_COST);
9689 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9690
9691 ins_encode %{
9692 __ lea(as_Register($dst$$reg),
9693 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9694 Address::sxtw($scale$$constant)));
9695 %}
9696
9697 ins_pipe(ialu_reg_reg_shift);
9698 %}
9699
9700 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9701 match(Set dst (LShiftL (ConvI2L src) scale));
9702
9703 ins_cost(INSN_COST);
9704 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9705
9706 ins_encode %{
9707 __ sbfiz(as_Register($dst$$reg),
9708 as_Register($src$$reg),
9709 $scale$$constant & 63, MIN(32, (-$scale$$constant) & 63));
9710 %}
9711
9712 ins_pipe(ialu_reg_shift);
9713 %}
9714
9715 // Pointer Immediate Addition
9716 // n.b. this needs to be more expensive than using an indirect memory
9717 // operand
9718 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
9719 match(Set dst (AddP src1 src2));
9720
9721 ins_cost(INSN_COST);
9722 format %{ "add $dst, $src1, $src2\t# ptr" %}
9723
9724 // use opcode to indicate that this is an add not a sub
9725 opcode(0x0);
9726
9727 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9728
9729 ins_pipe(ialu_reg_imm);
9730 %}
9731
9732 // Long Addition
9733 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9734
9735 match(Set dst (AddL src1 src2));
9736
9737 ins_cost(INSN_COST);
9738 format %{ "add $dst, $src1, $src2" %}
9739
9740 ins_encode %{
9741 __ add(as_Register($dst$$reg),
9742 as_Register($src1$$reg),
9743 as_Register($src2$$reg));
9744 %}
9745
9746 ins_pipe(ialu_reg_reg);
9747 %}
9748
9749 // No constant pool entries requiredLong Immediate Addition.
9750 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9751 match(Set dst (AddL src1 src2));
9752
9753 ins_cost(INSN_COST);
9754 format %{ "add $dst, $src1, $src2" %}
9755
9756 // use opcode to indicate that this is an add not a sub
9757 opcode(0x0);
9758
9759 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9760
9761 ins_pipe(ialu_reg_imm);
9762 %}
9763
9764 // Integer Subtraction
9765 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9766 match(Set dst (SubI src1 src2));
9767
9768 ins_cost(INSN_COST);
9769 format %{ "subw $dst, $src1, $src2" %}
9770
9771 ins_encode %{
9772 __ subw(as_Register($dst$$reg),
9773 as_Register($src1$$reg),
9774 as_Register($src2$$reg));
9775 %}
9776
9777 ins_pipe(ialu_reg_reg);
9778 %}
9779
9780 // Immediate Subtraction
9781 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9782 match(Set dst (SubI src1 src2));
9783
9784 ins_cost(INSN_COST);
9785 format %{ "subw $dst, $src1, $src2" %}
9786
9787 // use opcode to indicate that this is a sub not an add
9788 opcode(0x1);
9789
9790 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9791
9792 ins_pipe(ialu_reg_imm);
9793 %}
9794
9795 // Long Subtraction
9796 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9797
9798 match(Set dst (SubL src1 src2));
9799
9800 ins_cost(INSN_COST);
9801 format %{ "sub $dst, $src1, $src2" %}
9802
9803 ins_encode %{
9804 __ sub(as_Register($dst$$reg),
9805 as_Register($src1$$reg),
9806 as_Register($src2$$reg));
9807 %}
9808
9809 ins_pipe(ialu_reg_reg);
9810 %}
9811
9812 // No constant pool entries requiredLong Immediate Subtraction.
9813 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9814 match(Set dst (SubL src1 src2));
9815
9816 ins_cost(INSN_COST);
9817 format %{ "sub$dst, $src1, $src2" %}
9818
9819 // use opcode to indicate that this is a sub not an add
9820 opcode(0x1);
9821
9822 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9823
9824 ins_pipe(ialu_reg_imm);
9825 %}
9826
9827 // Integer Negation (special case for sub)
9828
9829 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9830 match(Set dst (SubI zero src));
9831
9832 ins_cost(INSN_COST);
9833 format %{ "negw $dst, $src\t# int" %}
9834
9835 ins_encode %{
9836 __ negw(as_Register($dst$$reg),
9837 as_Register($src$$reg));
9838 %}
9839
9840 ins_pipe(ialu_reg);
9841 %}
9842
9843 // Long Negation
9844
9845 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9846 match(Set dst (SubL zero src));
9847
9848 ins_cost(INSN_COST);
9849 format %{ "neg $dst, $src\t# long" %}
9850
9851 ins_encode %{
9852 __ neg(as_Register($dst$$reg),
9853 as_Register($src$$reg));
9854 %}
9855
9856 ins_pipe(ialu_reg);
9857 %}
9858
9859 // Integer Multiply
9860
9861 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9862 match(Set dst (MulI src1 src2));
9863
9864 ins_cost(INSN_COST * 3);
9865 format %{ "mulw $dst, $src1, $src2" %}
9866
9867 ins_encode %{
9868 __ mulw(as_Register($dst$$reg),
9869 as_Register($src1$$reg),
9870 as_Register($src2$$reg));
9871 %}
9872
9873 ins_pipe(imul_reg_reg);
9874 %}
9875
9876 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9877 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9878
9879 ins_cost(INSN_COST * 3);
9880 format %{ "smull $dst, $src1, $src2" %}
9881
9882 ins_encode %{
9883 __ smull(as_Register($dst$$reg),
9884 as_Register($src1$$reg),
9885 as_Register($src2$$reg));
9886 %}
9887
9888 ins_pipe(imul_reg_reg);
9889 %}
9890
9891 // Long Multiply
9892
9893 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9894 match(Set dst (MulL src1 src2));
9895
9896 ins_cost(INSN_COST * 5);
9897 format %{ "mul $dst, $src1, $src2" %}
9898
9899 ins_encode %{
9900 __ mul(as_Register($dst$$reg),
9901 as_Register($src1$$reg),
9902 as_Register($src2$$reg));
9903 %}
9904
9905 ins_pipe(lmul_reg_reg);
9906 %}
9907
9908 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9909 %{
9910 match(Set dst (MulHiL src1 src2));
9911
9912 ins_cost(INSN_COST * 7);
9913 format %{ "smulh $dst, $src1, $src2, \t# mulhi" %}
9914
9915 ins_encode %{
9916 __ smulh(as_Register($dst$$reg),
9917 as_Register($src1$$reg),
9918 as_Register($src2$$reg));
9919 %}
9920
9921 ins_pipe(lmul_reg_reg);
9922 %}
9923
9924 // Combined Integer Multiply & Add/Sub
9925
9926 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9927 match(Set dst (AddI src3 (MulI src1 src2)));
9928
9929 ins_cost(INSN_COST * 3);
9930 format %{ "madd $dst, $src1, $src2, $src3" %}
9931
9932 ins_encode %{
9933 __ maddw(as_Register($dst$$reg),
9934 as_Register($src1$$reg),
9935 as_Register($src2$$reg),
9936 as_Register($src3$$reg));
9937 %}
9938
9939 ins_pipe(imac_reg_reg);
9940 %}
9941
9942 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9943 match(Set dst (SubI src3 (MulI src1 src2)));
9944
9945 ins_cost(INSN_COST * 3);
9946 format %{ "msub $dst, $src1, $src2, $src3" %}
9947
9948 ins_encode %{
9949 __ msubw(as_Register($dst$$reg),
9950 as_Register($src1$$reg),
9951 as_Register($src2$$reg),
9952 as_Register($src3$$reg));
9953 %}
9954
9955 ins_pipe(imac_reg_reg);
9956 %}
9957
9958 // Combined Integer Multiply & Neg
9959
9960 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9961 match(Set dst (MulI (SubI zero src1) src2));
9962 match(Set dst (MulI src1 (SubI zero src2)));
9963
9964 ins_cost(INSN_COST * 3);
9965 format %{ "mneg $dst, $src1, $src2" %}
9966
9967 ins_encode %{
9968 __ mnegw(as_Register($dst$$reg),
9969 as_Register($src1$$reg),
9970 as_Register($src2$$reg));
9971 %}
9972
9973 ins_pipe(imac_reg_reg);
9974 %}
9975
9976 // Combined Long Multiply & Add/Sub
9977
9978 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9979 match(Set dst (AddL src3 (MulL src1 src2)));
9980
9981 ins_cost(INSN_COST * 5);
9982 format %{ "madd $dst, $src1, $src2, $src3" %}
9983
9984 ins_encode %{
9985 __ madd(as_Register($dst$$reg),
9986 as_Register($src1$$reg),
9987 as_Register($src2$$reg),
9988 as_Register($src3$$reg));
9989 %}
9990
9991 ins_pipe(lmac_reg_reg);
9992 %}
9993
9994 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9995 match(Set dst (SubL src3 (MulL src1 src2)));
9996
9997 ins_cost(INSN_COST * 5);
9998 format %{ "msub $dst, $src1, $src2, $src3" %}
9999
10000 ins_encode %{
10001 __ msub(as_Register($dst$$reg),
10002 as_Register($src1$$reg),
10003 as_Register($src2$$reg),
10004 as_Register($src3$$reg));
10005 %}
10006
10007 ins_pipe(lmac_reg_reg);
10008 %}
10009
10010 // Combined Long Multiply & Neg
10011
10012 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10013 match(Set dst (MulL (SubL zero src1) src2));
10014 match(Set dst (MulL src1 (SubL zero src2)));
10015
10016 ins_cost(INSN_COST * 5);
10017 format %{ "mneg $dst, $src1, $src2" %}
10018
10019 ins_encode %{
10020 __ mneg(as_Register($dst$$reg),
10021 as_Register($src1$$reg),
10022 as_Register($src2$$reg));
10023 %}
10024
10025 ins_pipe(lmac_reg_reg);
10026 %}
10027
10028 // Integer Divide
10029
10030 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10031 match(Set dst (DivI src1 src2));
10032
10033 ins_cost(INSN_COST * 19);
10034 format %{ "sdivw $dst, $src1, $src2" %}
10035
10036 ins_encode(aarch64_enc_divw(dst, src1, src2));
10037 ins_pipe(idiv_reg_reg);
10038 %}
10039
10040 instruct signExtract(iRegINoSp dst, iRegIorL2I src1, immI_31 div1, immI_31 div2) %{
10041 match(Set dst (URShiftI (RShiftI src1 div1) div2));
10042 ins_cost(INSN_COST);
10043 format %{ "lsrw $dst, $src1, $div1" %}
10044 ins_encode %{
10045 __ lsrw(as_Register($dst$$reg), as_Register($src1$$reg), 31);
10046 %}
10047 ins_pipe(ialu_reg_shift);
10048 %}
10049
10050 instruct div2Round(iRegINoSp dst, iRegIorL2I src, immI_31 div1, immI_31 div2) %{
10051 match(Set dst (AddI src (URShiftI (RShiftI src div1) div2)));
10052 ins_cost(INSN_COST);
10053 format %{ "addw $dst, $src, LSR $div1" %}
10054
10055 ins_encode %{
10056 __ addw(as_Register($dst$$reg),
10057 as_Register($src$$reg),
10058 as_Register($src$$reg),
10059 Assembler::LSR, 31);
10060 %}
10061 ins_pipe(ialu_reg);
10062 %}
10063
10064 // Long Divide
10065
10066 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10067 match(Set dst (DivL src1 src2));
10068
10069 ins_cost(INSN_COST * 35);
10070 format %{ "sdiv $dst, $src1, $src2" %}
10071
10072 ins_encode(aarch64_enc_div(dst, src1, src2));
10073 ins_pipe(ldiv_reg_reg);
10074 %}
10075
10076 instruct signExtractL(iRegLNoSp dst, iRegL src1, immI_63 div1, immI_63 div2) %{
10077 match(Set dst (URShiftL (RShiftL src1 div1) div2));
10078 ins_cost(INSN_COST);
10079 format %{ "lsr $dst, $src1, $div1" %}
10080 ins_encode %{
10081 __ lsr(as_Register($dst$$reg), as_Register($src1$$reg), 63);
10082 %}
10083 ins_pipe(ialu_reg_shift);
10084 %}
10085
10086 instruct div2RoundL(iRegLNoSp dst, iRegL src, immI_63 div1, immI_63 div2) %{
10087 match(Set dst (AddL src (URShiftL (RShiftL src div1) div2)));
10088 ins_cost(INSN_COST);
10089 format %{ "add $dst, $src, $div1" %}
10090
10091 ins_encode %{
10092 __ add(as_Register($dst$$reg),
10093 as_Register($src$$reg),
10094 as_Register($src$$reg),
10095 Assembler::LSR, 63);
10096 %}
10097 ins_pipe(ialu_reg);
10098 %}
10099
10100 // Integer Remainder
10101
10102 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10103 match(Set dst (ModI src1 src2));
10104
10105 ins_cost(INSN_COST * 22);
10106 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10107 "msubw($dst, rscratch1, $src2, $src1" %}
10108
10109 ins_encode(aarch64_enc_modw(dst, src1, src2));
10110 ins_pipe(idiv_reg_reg);
10111 %}
10112
10113 // Long Remainder
10114
10115 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10116 match(Set dst (ModL src1 src2));
10117
10118 ins_cost(INSN_COST * 38);
10119 format %{ "sdiv rscratch1, $src1, $src2\n"
10120 "msub($dst, rscratch1, $src2, $src1" %}
10121
10122 ins_encode(aarch64_enc_mod(dst, src1, src2));
10123 ins_pipe(ldiv_reg_reg);
10124 %}
10125
10126 // Integer Shifts
10127
10128 // Shift Left Register
10129 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10130 match(Set dst (LShiftI src1 src2));
10131
10132 ins_cost(INSN_COST * 2);
10133 format %{ "lslvw $dst, $src1, $src2" %}
10134
10135 ins_encode %{
10136 __ lslvw(as_Register($dst$$reg),
10137 as_Register($src1$$reg),
10138 as_Register($src2$$reg));
10139 %}
10140
10141 ins_pipe(ialu_reg_reg_vshift);
10142 %}
10143
10144 // Shift Left Immediate
10145 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10146 match(Set dst (LShiftI src1 src2));
10147
10148 ins_cost(INSN_COST);
10149 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10150
10151 ins_encode %{
10152 __ lslw(as_Register($dst$$reg),
10153 as_Register($src1$$reg),
10154 $src2$$constant & 0x1f);
10155 %}
10156
10157 ins_pipe(ialu_reg_shift);
10158 %}
10159
10160 // Shift Right Logical Register
10161 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10162 match(Set dst (URShiftI src1 src2));
10163
10164 ins_cost(INSN_COST * 2);
10165 format %{ "lsrvw $dst, $src1, $src2" %}
10166
10167 ins_encode %{
10168 __ lsrvw(as_Register($dst$$reg),
10169 as_Register($src1$$reg),
10170 as_Register($src2$$reg));
10171 %}
10172
10173 ins_pipe(ialu_reg_reg_vshift);
10174 %}
10175
10176 // Shift Right Logical Immediate
10177 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10178 match(Set dst (URShiftI src1 src2));
10179
10180 ins_cost(INSN_COST);
10181 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10182
10183 ins_encode %{
10184 __ lsrw(as_Register($dst$$reg),
10185 as_Register($src1$$reg),
10186 $src2$$constant & 0x1f);
10187 %}
10188
10189 ins_pipe(ialu_reg_shift);
10190 %}
10191
10192 // Shift Right Arithmetic Register
10193 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10194 match(Set dst (RShiftI src1 src2));
10195
10196 ins_cost(INSN_COST * 2);
10197 format %{ "asrvw $dst, $src1, $src2" %}
10198
10199 ins_encode %{
10200 __ asrvw(as_Register($dst$$reg),
10201 as_Register($src1$$reg),
10202 as_Register($src2$$reg));
10203 %}
10204
10205 ins_pipe(ialu_reg_reg_vshift);
10206 %}
10207
10208 // Shift Right Arithmetic Immediate
10209 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10210 match(Set dst (RShiftI src1 src2));
10211
10212 ins_cost(INSN_COST);
10213 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10214
10215 ins_encode %{
10216 __ asrw(as_Register($dst$$reg),
10217 as_Register($src1$$reg),
10218 $src2$$constant & 0x1f);
10219 %}
10220
10221 ins_pipe(ialu_reg_shift);
10222 %}
10223
10224 // Combined Int Mask and Right Shift (using UBFM)
10225 // TODO
10226
10227 // Long Shifts
10228
10229 // Shift Left Register
10230 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10231 match(Set dst (LShiftL src1 src2));
10232
10233 ins_cost(INSN_COST * 2);
10234 format %{ "lslv $dst, $src1, $src2" %}
10235
10236 ins_encode %{
10237 __ lslv(as_Register($dst$$reg),
10238 as_Register($src1$$reg),
10239 as_Register($src2$$reg));
10240 %}
10241
10242 ins_pipe(ialu_reg_reg_vshift);
10243 %}
10244
10245 // Shift Left Immediate
10246 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10247 match(Set dst (LShiftL src1 src2));
10248
10249 ins_cost(INSN_COST);
10250 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10251
10252 ins_encode %{
10253 __ lsl(as_Register($dst$$reg),
10254 as_Register($src1$$reg),
10255 $src2$$constant & 0x3f);
10256 %}
10257
10258 ins_pipe(ialu_reg_shift);
10259 %}
10260
10261 // Shift Right Logical Register
10262 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10263 match(Set dst (URShiftL src1 src2));
10264
10265 ins_cost(INSN_COST * 2);
10266 format %{ "lsrv $dst, $src1, $src2" %}
10267
10268 ins_encode %{
10269 __ lsrv(as_Register($dst$$reg),
10270 as_Register($src1$$reg),
10271 as_Register($src2$$reg));
10272 %}
10273
10274 ins_pipe(ialu_reg_reg_vshift);
10275 %}
10276
10277 // Shift Right Logical Immediate
10278 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10279 match(Set dst (URShiftL src1 src2));
10280
10281 ins_cost(INSN_COST);
10282 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10283
10284 ins_encode %{
10285 __ lsr(as_Register($dst$$reg),
10286 as_Register($src1$$reg),
10287 $src2$$constant & 0x3f);
10288 %}
10289
10290 ins_pipe(ialu_reg_shift);
10291 %}
10292
10293 // A special-case pattern for card table stores.
10294 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10295 match(Set dst (URShiftL (CastP2X src1) src2));
10296
10297 ins_cost(INSN_COST);
10298 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10299
10300 ins_encode %{
10301 __ lsr(as_Register($dst$$reg),
10302 as_Register($src1$$reg),
10303 $src2$$constant & 0x3f);
10304 %}
10305
10306 ins_pipe(ialu_reg_shift);
10307 %}
10308
10309 // Shift Right Arithmetic Register
10310 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10311 match(Set dst (RShiftL src1 src2));
10312
10313 ins_cost(INSN_COST * 2);
10314 format %{ "asrv $dst, $src1, $src2" %}
10315
10316 ins_encode %{
10317 __ asrv(as_Register($dst$$reg),
10318 as_Register($src1$$reg),
10319 as_Register($src2$$reg));
10320 %}
10321
10322 ins_pipe(ialu_reg_reg_vshift);
10323 %}
10324
10325 // Shift Right Arithmetic Immediate
10326 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10327 match(Set dst (RShiftL src1 src2));
10328
10329 ins_cost(INSN_COST);
10330 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10331
10332 ins_encode %{
10333 __ asr(as_Register($dst$$reg),
10334 as_Register($src1$$reg),
10335 $src2$$constant & 0x3f);
10336 %}
10337
10338 ins_pipe(ialu_reg_shift);
10339 %}
10340
10341 // BEGIN This section of the file is automatically generated. Do not edit --------------
10342
10343 instruct regL_not_reg(iRegLNoSp dst,
10344 iRegL src1, immL_M1 m1,
10345 rFlagsReg cr) %{
10346 match(Set dst (XorL src1 m1));
10347 ins_cost(INSN_COST);
10348 format %{ "eon $dst, $src1, zr" %}
10349
10350 ins_encode %{
10351 __ eon(as_Register($dst$$reg),
10352 as_Register($src1$$reg),
10353 zr,
10354 Assembler::LSL, 0);
10355 %}
10356
10357 ins_pipe(ialu_reg);
10358 %}
10359 instruct regI_not_reg(iRegINoSp dst,
10360 iRegIorL2I src1, immI_M1 m1,
10361 rFlagsReg cr) %{
10362 match(Set dst (XorI src1 m1));
10363 ins_cost(INSN_COST);
10364 format %{ "eonw $dst, $src1, zr" %}
10365
10366 ins_encode %{
10367 __ eonw(as_Register($dst$$reg),
10368 as_Register($src1$$reg),
10369 zr,
10370 Assembler::LSL, 0);
10371 %}
10372
10373 ins_pipe(ialu_reg);
10374 %}
10375
10376 instruct AndI_reg_not_reg(iRegINoSp dst,
10377 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10378 rFlagsReg cr) %{
10379 match(Set dst (AndI src1 (XorI src2 m1)));
10380 ins_cost(INSN_COST);
10381 format %{ "bicw $dst, $src1, $src2" %}
10382
10383 ins_encode %{
10384 __ bicw(as_Register($dst$$reg),
10385 as_Register($src1$$reg),
10386 as_Register($src2$$reg),
10387 Assembler::LSL, 0);
10388 %}
10389
10390 ins_pipe(ialu_reg_reg);
10391 %}
10392
10393 instruct AndL_reg_not_reg(iRegLNoSp dst,
10394 iRegL src1, iRegL src2, immL_M1 m1,
10395 rFlagsReg cr) %{
10396 match(Set dst (AndL src1 (XorL src2 m1)));
10397 ins_cost(INSN_COST);
10398 format %{ "bic $dst, $src1, $src2" %}
10399
10400 ins_encode %{
10401 __ bic(as_Register($dst$$reg),
10402 as_Register($src1$$reg),
10403 as_Register($src2$$reg),
10404 Assembler::LSL, 0);
10405 %}
10406
10407 ins_pipe(ialu_reg_reg);
10408 %}
10409
10410 instruct OrI_reg_not_reg(iRegINoSp dst,
10411 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10412 rFlagsReg cr) %{
10413 match(Set dst (OrI src1 (XorI src2 m1)));
10414 ins_cost(INSN_COST);
10415 format %{ "ornw $dst, $src1, $src2" %}
10416
10417 ins_encode %{
10418 __ ornw(as_Register($dst$$reg),
10419 as_Register($src1$$reg),
10420 as_Register($src2$$reg),
10421 Assembler::LSL, 0);
10422 %}
10423
10424 ins_pipe(ialu_reg_reg);
10425 %}
10426
10427 instruct OrL_reg_not_reg(iRegLNoSp dst,
10428 iRegL src1, iRegL src2, immL_M1 m1,
10429 rFlagsReg cr) %{
10430 match(Set dst (OrL src1 (XorL src2 m1)));
10431 ins_cost(INSN_COST);
10432 format %{ "orn $dst, $src1, $src2" %}
10433
10434 ins_encode %{
10435 __ orn(as_Register($dst$$reg),
10436 as_Register($src1$$reg),
10437 as_Register($src2$$reg),
10438 Assembler::LSL, 0);
10439 %}
10440
10441 ins_pipe(ialu_reg_reg);
10442 %}
10443
10444 instruct XorI_reg_not_reg(iRegINoSp dst,
10445 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10446 rFlagsReg cr) %{
10447 match(Set dst (XorI m1 (XorI src2 src1)));
10448 ins_cost(INSN_COST);
10449 format %{ "eonw $dst, $src1, $src2" %}
10450
10451 ins_encode %{
10452 __ eonw(as_Register($dst$$reg),
10453 as_Register($src1$$reg),
10454 as_Register($src2$$reg),
10455 Assembler::LSL, 0);
10456 %}
10457
10458 ins_pipe(ialu_reg_reg);
10459 %}
10460
10461 instruct XorL_reg_not_reg(iRegLNoSp dst,
10462 iRegL src1, iRegL src2, immL_M1 m1,
10463 rFlagsReg cr) %{
10464 match(Set dst (XorL m1 (XorL src2 src1)));
10465 ins_cost(INSN_COST);
10466 format %{ "eon $dst, $src1, $src2" %}
10467
10468 ins_encode %{
10469 __ eon(as_Register($dst$$reg),
10470 as_Register($src1$$reg),
10471 as_Register($src2$$reg),
10472 Assembler::LSL, 0);
10473 %}
10474
10475 ins_pipe(ialu_reg_reg);
10476 %}
10477
10478 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10479 iRegIorL2I src1, iRegIorL2I src2,
10480 immI src3, immI_M1 src4, rFlagsReg cr) %{
10481 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10482 ins_cost(1.9 * INSN_COST);
10483 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10484
10485 ins_encode %{
10486 __ bicw(as_Register($dst$$reg),
10487 as_Register($src1$$reg),
10488 as_Register($src2$$reg),
10489 Assembler::LSR,
10490 $src3$$constant & 0x1f);
10491 %}
10492
10493 ins_pipe(ialu_reg_reg_shift);
10494 %}
10495
10496 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10497 iRegL src1, iRegL src2,
10498 immI src3, immL_M1 src4, rFlagsReg cr) %{
10499 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10500 ins_cost(1.9 * INSN_COST);
10501 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10502
10503 ins_encode %{
10504 __ bic(as_Register($dst$$reg),
10505 as_Register($src1$$reg),
10506 as_Register($src2$$reg),
10507 Assembler::LSR,
10508 $src3$$constant & 0x3f);
10509 %}
10510
10511 ins_pipe(ialu_reg_reg_shift);
10512 %}
10513
10514 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10515 iRegIorL2I src1, iRegIorL2I src2,
10516 immI src3, immI_M1 src4, rFlagsReg cr) %{
10517 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10518 ins_cost(1.9 * INSN_COST);
10519 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10520
10521 ins_encode %{
10522 __ bicw(as_Register($dst$$reg),
10523 as_Register($src1$$reg),
10524 as_Register($src2$$reg),
10525 Assembler::ASR,
10526 $src3$$constant & 0x1f);
10527 %}
10528
10529 ins_pipe(ialu_reg_reg_shift);
10530 %}
10531
10532 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10533 iRegL src1, iRegL src2,
10534 immI src3, immL_M1 src4, rFlagsReg cr) %{
10535 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10536 ins_cost(1.9 * INSN_COST);
10537 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10538
10539 ins_encode %{
10540 __ bic(as_Register($dst$$reg),
10541 as_Register($src1$$reg),
10542 as_Register($src2$$reg),
10543 Assembler::ASR,
10544 $src3$$constant & 0x3f);
10545 %}
10546
10547 ins_pipe(ialu_reg_reg_shift);
10548 %}
10549
10550 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10551 iRegIorL2I src1, iRegIorL2I src2,
10552 immI src3, immI_M1 src4, rFlagsReg cr) %{
10553 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10554 ins_cost(1.9 * INSN_COST);
10555 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10556
10557 ins_encode %{
10558 __ bicw(as_Register($dst$$reg),
10559 as_Register($src1$$reg),
10560 as_Register($src2$$reg),
10561 Assembler::LSL,
10562 $src3$$constant & 0x1f);
10563 %}
10564
10565 ins_pipe(ialu_reg_reg_shift);
10566 %}
10567
10568 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10569 iRegL src1, iRegL src2,
10570 immI src3, immL_M1 src4, rFlagsReg cr) %{
10571 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10572 ins_cost(1.9 * INSN_COST);
10573 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10574
10575 ins_encode %{
10576 __ bic(as_Register($dst$$reg),
10577 as_Register($src1$$reg),
10578 as_Register($src2$$reg),
10579 Assembler::LSL,
10580 $src3$$constant & 0x3f);
10581 %}
10582
10583 ins_pipe(ialu_reg_reg_shift);
10584 %}
10585
10586 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10587 iRegIorL2I src1, iRegIorL2I src2,
10588 immI src3, immI_M1 src4, rFlagsReg cr) %{
10589 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10590 ins_cost(1.9 * INSN_COST);
10591 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10592
10593 ins_encode %{
10594 __ eonw(as_Register($dst$$reg),
10595 as_Register($src1$$reg),
10596 as_Register($src2$$reg),
10597 Assembler::LSR,
10598 $src3$$constant & 0x1f);
10599 %}
10600
10601 ins_pipe(ialu_reg_reg_shift);
10602 %}
10603
10604 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10605 iRegL src1, iRegL src2,
10606 immI src3, immL_M1 src4, rFlagsReg cr) %{
10607 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10608 ins_cost(1.9 * INSN_COST);
10609 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10610
10611 ins_encode %{
10612 __ eon(as_Register($dst$$reg),
10613 as_Register($src1$$reg),
10614 as_Register($src2$$reg),
10615 Assembler::LSR,
10616 $src3$$constant & 0x3f);
10617 %}
10618
10619 ins_pipe(ialu_reg_reg_shift);
10620 %}
10621
10622 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10623 iRegIorL2I src1, iRegIorL2I src2,
10624 immI src3, immI_M1 src4, rFlagsReg cr) %{
10625 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10626 ins_cost(1.9 * INSN_COST);
10627 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10628
10629 ins_encode %{
10630 __ eonw(as_Register($dst$$reg),
10631 as_Register($src1$$reg),
10632 as_Register($src2$$reg),
10633 Assembler::ASR,
10634 $src3$$constant & 0x1f);
10635 %}
10636
10637 ins_pipe(ialu_reg_reg_shift);
10638 %}
10639
10640 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10641 iRegL src1, iRegL src2,
10642 immI src3, immL_M1 src4, rFlagsReg cr) %{
10643 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10644 ins_cost(1.9 * INSN_COST);
10645 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10646
10647 ins_encode %{
10648 __ eon(as_Register($dst$$reg),
10649 as_Register($src1$$reg),
10650 as_Register($src2$$reg),
10651 Assembler::ASR,
10652 $src3$$constant & 0x3f);
10653 %}
10654
10655 ins_pipe(ialu_reg_reg_shift);
10656 %}
10657
10658 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10659 iRegIorL2I src1, iRegIorL2I src2,
10660 immI src3, immI_M1 src4, rFlagsReg cr) %{
10661 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10662 ins_cost(1.9 * INSN_COST);
10663 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10664
10665 ins_encode %{
10666 __ eonw(as_Register($dst$$reg),
10667 as_Register($src1$$reg),
10668 as_Register($src2$$reg),
10669 Assembler::LSL,
10670 $src3$$constant & 0x1f);
10671 %}
10672
10673 ins_pipe(ialu_reg_reg_shift);
10674 %}
10675
10676 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10677 iRegL src1, iRegL src2,
10678 immI src3, immL_M1 src4, rFlagsReg cr) %{
10679 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10680 ins_cost(1.9 * INSN_COST);
10681 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10682
10683 ins_encode %{
10684 __ eon(as_Register($dst$$reg),
10685 as_Register($src1$$reg),
10686 as_Register($src2$$reg),
10687 Assembler::LSL,
10688 $src3$$constant & 0x3f);
10689 %}
10690
10691 ins_pipe(ialu_reg_reg_shift);
10692 %}
10693
10694 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10695 iRegIorL2I src1, iRegIorL2I src2,
10696 immI src3, immI_M1 src4, rFlagsReg cr) %{
10697 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10698 ins_cost(1.9 * INSN_COST);
10699 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10700
10701 ins_encode %{
10702 __ ornw(as_Register($dst$$reg),
10703 as_Register($src1$$reg),
10704 as_Register($src2$$reg),
10705 Assembler::LSR,
10706 $src3$$constant & 0x1f);
10707 %}
10708
10709 ins_pipe(ialu_reg_reg_shift);
10710 %}
10711
10712 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10713 iRegL src1, iRegL src2,
10714 immI src3, immL_M1 src4, rFlagsReg cr) %{
10715 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10716 ins_cost(1.9 * INSN_COST);
10717 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10718
10719 ins_encode %{
10720 __ orn(as_Register($dst$$reg),
10721 as_Register($src1$$reg),
10722 as_Register($src2$$reg),
10723 Assembler::LSR,
10724 $src3$$constant & 0x3f);
10725 %}
10726
10727 ins_pipe(ialu_reg_reg_shift);
10728 %}
10729
10730 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10731 iRegIorL2I src1, iRegIorL2I src2,
10732 immI src3, immI_M1 src4, rFlagsReg cr) %{
10733 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10734 ins_cost(1.9 * INSN_COST);
10735 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10736
10737 ins_encode %{
10738 __ ornw(as_Register($dst$$reg),
10739 as_Register($src1$$reg),
10740 as_Register($src2$$reg),
10741 Assembler::ASR,
10742 $src3$$constant & 0x1f);
10743 %}
10744
10745 ins_pipe(ialu_reg_reg_shift);
10746 %}
10747
10748 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10749 iRegL src1, iRegL src2,
10750 immI src3, immL_M1 src4, rFlagsReg cr) %{
10751 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10752 ins_cost(1.9 * INSN_COST);
10753 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10754
10755 ins_encode %{
10756 __ orn(as_Register($dst$$reg),
10757 as_Register($src1$$reg),
10758 as_Register($src2$$reg),
10759 Assembler::ASR,
10760 $src3$$constant & 0x3f);
10761 %}
10762
10763 ins_pipe(ialu_reg_reg_shift);
10764 %}
10765
10766 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10767 iRegIorL2I src1, iRegIorL2I src2,
10768 immI src3, immI_M1 src4, rFlagsReg cr) %{
10769 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10770 ins_cost(1.9 * INSN_COST);
10771 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10772
10773 ins_encode %{
10774 __ ornw(as_Register($dst$$reg),
10775 as_Register($src1$$reg),
10776 as_Register($src2$$reg),
10777 Assembler::LSL,
10778 $src3$$constant & 0x1f);
10779 %}
10780
10781 ins_pipe(ialu_reg_reg_shift);
10782 %}
10783
10784 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10785 iRegL src1, iRegL src2,
10786 immI src3, immL_M1 src4, rFlagsReg cr) %{
10787 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10788 ins_cost(1.9 * INSN_COST);
10789 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10790
10791 ins_encode %{
10792 __ orn(as_Register($dst$$reg),
10793 as_Register($src1$$reg),
10794 as_Register($src2$$reg),
10795 Assembler::LSL,
10796 $src3$$constant & 0x3f);
10797 %}
10798
10799 ins_pipe(ialu_reg_reg_shift);
10800 %}
10801
10802 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10803 iRegIorL2I src1, iRegIorL2I src2,
10804 immI src3, rFlagsReg cr) %{
10805 match(Set dst (AndI src1 (URShiftI src2 src3)));
10806
10807 ins_cost(1.9 * INSN_COST);
10808 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10809
10810 ins_encode %{
10811 __ andw(as_Register($dst$$reg),
10812 as_Register($src1$$reg),
10813 as_Register($src2$$reg),
10814 Assembler::LSR,
10815 $src3$$constant & 0x1f);
10816 %}
10817
10818 ins_pipe(ialu_reg_reg_shift);
10819 %}
10820
10821 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10822 iRegL src1, iRegL src2,
10823 immI src3, rFlagsReg cr) %{
10824 match(Set dst (AndL src1 (URShiftL src2 src3)));
10825
10826 ins_cost(1.9 * INSN_COST);
10827 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10828
10829 ins_encode %{
10830 __ andr(as_Register($dst$$reg),
10831 as_Register($src1$$reg),
10832 as_Register($src2$$reg),
10833 Assembler::LSR,
10834 $src3$$constant & 0x3f);
10835 %}
10836
10837 ins_pipe(ialu_reg_reg_shift);
10838 %}
10839
10840 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10841 iRegIorL2I src1, iRegIorL2I src2,
10842 immI src3, rFlagsReg cr) %{
10843 match(Set dst (AndI src1 (RShiftI src2 src3)));
10844
10845 ins_cost(1.9 * INSN_COST);
10846 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10847
10848 ins_encode %{
10849 __ andw(as_Register($dst$$reg),
10850 as_Register($src1$$reg),
10851 as_Register($src2$$reg),
10852 Assembler::ASR,
10853 $src3$$constant & 0x1f);
10854 %}
10855
10856 ins_pipe(ialu_reg_reg_shift);
10857 %}
10858
10859 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10860 iRegL src1, iRegL src2,
10861 immI src3, rFlagsReg cr) %{
10862 match(Set dst (AndL src1 (RShiftL src2 src3)));
10863
10864 ins_cost(1.9 * INSN_COST);
10865 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10866
10867 ins_encode %{
10868 __ andr(as_Register($dst$$reg),
10869 as_Register($src1$$reg),
10870 as_Register($src2$$reg),
10871 Assembler::ASR,
10872 $src3$$constant & 0x3f);
10873 %}
10874
10875 ins_pipe(ialu_reg_reg_shift);
10876 %}
10877
10878 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10879 iRegIorL2I src1, iRegIorL2I src2,
10880 immI src3, rFlagsReg cr) %{
10881 match(Set dst (AndI src1 (LShiftI src2 src3)));
10882
10883 ins_cost(1.9 * INSN_COST);
10884 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10885
10886 ins_encode %{
10887 __ andw(as_Register($dst$$reg),
10888 as_Register($src1$$reg),
10889 as_Register($src2$$reg),
10890 Assembler::LSL,
10891 $src3$$constant & 0x1f);
10892 %}
10893
10894 ins_pipe(ialu_reg_reg_shift);
10895 %}
10896
10897 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10898 iRegL src1, iRegL src2,
10899 immI src3, rFlagsReg cr) %{
10900 match(Set dst (AndL src1 (LShiftL src2 src3)));
10901
10902 ins_cost(1.9 * INSN_COST);
10903 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10904
10905 ins_encode %{
10906 __ andr(as_Register($dst$$reg),
10907 as_Register($src1$$reg),
10908 as_Register($src2$$reg),
10909 Assembler::LSL,
10910 $src3$$constant & 0x3f);
10911 %}
10912
10913 ins_pipe(ialu_reg_reg_shift);
10914 %}
10915
10916 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10917 iRegIorL2I src1, iRegIorL2I src2,
10918 immI src3, rFlagsReg cr) %{
10919 match(Set dst (XorI src1 (URShiftI src2 src3)));
10920
10921 ins_cost(1.9 * INSN_COST);
10922 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10923
10924 ins_encode %{
10925 __ eorw(as_Register($dst$$reg),
10926 as_Register($src1$$reg),
10927 as_Register($src2$$reg),
10928 Assembler::LSR,
10929 $src3$$constant & 0x1f);
10930 %}
10931
10932 ins_pipe(ialu_reg_reg_shift);
10933 %}
10934
10935 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10936 iRegL src1, iRegL src2,
10937 immI src3, rFlagsReg cr) %{
10938 match(Set dst (XorL src1 (URShiftL src2 src3)));
10939
10940 ins_cost(1.9 * INSN_COST);
10941 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10942
10943 ins_encode %{
10944 __ eor(as_Register($dst$$reg),
10945 as_Register($src1$$reg),
10946 as_Register($src2$$reg),
10947 Assembler::LSR,
10948 $src3$$constant & 0x3f);
10949 %}
10950
10951 ins_pipe(ialu_reg_reg_shift);
10952 %}
10953
10954 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10955 iRegIorL2I src1, iRegIorL2I src2,
10956 immI src3, rFlagsReg cr) %{
10957 match(Set dst (XorI src1 (RShiftI src2 src3)));
10958
10959 ins_cost(1.9 * INSN_COST);
10960 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10961
10962 ins_encode %{
10963 __ eorw(as_Register($dst$$reg),
10964 as_Register($src1$$reg),
10965 as_Register($src2$$reg),
10966 Assembler::ASR,
10967 $src3$$constant & 0x1f);
10968 %}
10969
10970 ins_pipe(ialu_reg_reg_shift);
10971 %}
10972
10973 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10974 iRegL src1, iRegL src2,
10975 immI src3, rFlagsReg cr) %{
10976 match(Set dst (XorL src1 (RShiftL src2 src3)));
10977
10978 ins_cost(1.9 * INSN_COST);
10979 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10980
10981 ins_encode %{
10982 __ eor(as_Register($dst$$reg),
10983 as_Register($src1$$reg),
10984 as_Register($src2$$reg),
10985 Assembler::ASR,
10986 $src3$$constant & 0x3f);
10987 %}
10988
10989 ins_pipe(ialu_reg_reg_shift);
10990 %}
10991
10992 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10993 iRegIorL2I src1, iRegIorL2I src2,
10994 immI src3, rFlagsReg cr) %{
10995 match(Set dst (XorI src1 (LShiftI src2 src3)));
10996
10997 ins_cost(1.9 * INSN_COST);
10998 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10999
11000 ins_encode %{
11001 __ eorw(as_Register($dst$$reg),
11002 as_Register($src1$$reg),
11003 as_Register($src2$$reg),
11004 Assembler::LSL,
11005 $src3$$constant & 0x1f);
11006 %}
11007
11008 ins_pipe(ialu_reg_reg_shift);
11009 %}
11010
11011 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11012 iRegL src1, iRegL src2,
11013 immI src3, rFlagsReg cr) %{
11014 match(Set dst (XorL src1 (LShiftL src2 src3)));
11015
11016 ins_cost(1.9 * INSN_COST);
11017 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11018
11019 ins_encode %{
11020 __ eor(as_Register($dst$$reg),
11021 as_Register($src1$$reg),
11022 as_Register($src2$$reg),
11023 Assembler::LSL,
11024 $src3$$constant & 0x3f);
11025 %}
11026
11027 ins_pipe(ialu_reg_reg_shift);
11028 %}
11029
11030 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11031 iRegIorL2I src1, iRegIorL2I src2,
11032 immI src3, rFlagsReg cr) %{
11033 match(Set dst (OrI src1 (URShiftI src2 src3)));
11034
11035 ins_cost(1.9 * INSN_COST);
11036 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11037
11038 ins_encode %{
11039 __ orrw(as_Register($dst$$reg),
11040 as_Register($src1$$reg),
11041 as_Register($src2$$reg),
11042 Assembler::LSR,
11043 $src3$$constant & 0x1f);
11044 %}
11045
11046 ins_pipe(ialu_reg_reg_shift);
11047 %}
11048
11049 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11050 iRegL src1, iRegL src2,
11051 immI src3, rFlagsReg cr) %{
11052 match(Set dst (OrL src1 (URShiftL src2 src3)));
11053
11054 ins_cost(1.9 * INSN_COST);
11055 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11056
11057 ins_encode %{
11058 __ orr(as_Register($dst$$reg),
11059 as_Register($src1$$reg),
11060 as_Register($src2$$reg),
11061 Assembler::LSR,
11062 $src3$$constant & 0x3f);
11063 %}
11064
11065 ins_pipe(ialu_reg_reg_shift);
11066 %}
11067
11068 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11069 iRegIorL2I src1, iRegIorL2I src2,
11070 immI src3, rFlagsReg cr) %{
11071 match(Set dst (OrI src1 (RShiftI src2 src3)));
11072
11073 ins_cost(1.9 * INSN_COST);
11074 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11075
11076 ins_encode %{
11077 __ orrw(as_Register($dst$$reg),
11078 as_Register($src1$$reg),
11079 as_Register($src2$$reg),
11080 Assembler::ASR,
11081 $src3$$constant & 0x1f);
11082 %}
11083
11084 ins_pipe(ialu_reg_reg_shift);
11085 %}
11086
11087 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11088 iRegL src1, iRegL src2,
11089 immI src3, rFlagsReg cr) %{
11090 match(Set dst (OrL src1 (RShiftL src2 src3)));
11091
11092 ins_cost(1.9 * INSN_COST);
11093 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11094
11095 ins_encode %{
11096 __ orr(as_Register($dst$$reg),
11097 as_Register($src1$$reg),
11098 as_Register($src2$$reg),
11099 Assembler::ASR,
11100 $src3$$constant & 0x3f);
11101 %}
11102
11103 ins_pipe(ialu_reg_reg_shift);
11104 %}
11105
11106 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11107 iRegIorL2I src1, iRegIorL2I src2,
11108 immI src3, rFlagsReg cr) %{
11109 match(Set dst (OrI src1 (LShiftI src2 src3)));
11110
11111 ins_cost(1.9 * INSN_COST);
11112 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11113
11114 ins_encode %{
11115 __ orrw(as_Register($dst$$reg),
11116 as_Register($src1$$reg),
11117 as_Register($src2$$reg),
11118 Assembler::LSL,
11119 $src3$$constant & 0x1f);
11120 %}
11121
11122 ins_pipe(ialu_reg_reg_shift);
11123 %}
11124
11125 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11126 iRegL src1, iRegL src2,
11127 immI src3, rFlagsReg cr) %{
11128 match(Set dst (OrL src1 (LShiftL src2 src3)));
11129
11130 ins_cost(1.9 * INSN_COST);
11131 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11132
11133 ins_encode %{
11134 __ orr(as_Register($dst$$reg),
11135 as_Register($src1$$reg),
11136 as_Register($src2$$reg),
11137 Assembler::LSL,
11138 $src3$$constant & 0x3f);
11139 %}
11140
11141 ins_pipe(ialu_reg_reg_shift);
11142 %}
11143
11144 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11145 iRegIorL2I src1, iRegIorL2I src2,
11146 immI src3, rFlagsReg cr) %{
11147 match(Set dst (AddI src1 (URShiftI src2 src3)));
11148
11149 ins_cost(1.9 * INSN_COST);
11150 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11151
11152 ins_encode %{
11153 __ addw(as_Register($dst$$reg),
11154 as_Register($src1$$reg),
11155 as_Register($src2$$reg),
11156 Assembler::LSR,
11157 $src3$$constant & 0x1f);
11158 %}
11159
11160 ins_pipe(ialu_reg_reg_shift);
11161 %}
11162
11163 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11164 iRegL src1, iRegL src2,
11165 immI src3, rFlagsReg cr) %{
11166 match(Set dst (AddL src1 (URShiftL src2 src3)));
11167
11168 ins_cost(1.9 * INSN_COST);
11169 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11170
11171 ins_encode %{
11172 __ add(as_Register($dst$$reg),
11173 as_Register($src1$$reg),
11174 as_Register($src2$$reg),
11175 Assembler::LSR,
11176 $src3$$constant & 0x3f);
11177 %}
11178
11179 ins_pipe(ialu_reg_reg_shift);
11180 %}
11181
11182 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11183 iRegIorL2I src1, iRegIorL2I src2,
11184 immI src3, rFlagsReg cr) %{
11185 match(Set dst (AddI src1 (RShiftI src2 src3)));
11186
11187 ins_cost(1.9 * INSN_COST);
11188 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11189
11190 ins_encode %{
11191 __ addw(as_Register($dst$$reg),
11192 as_Register($src1$$reg),
11193 as_Register($src2$$reg),
11194 Assembler::ASR,
11195 $src3$$constant & 0x1f);
11196 %}
11197
11198 ins_pipe(ialu_reg_reg_shift);
11199 %}
11200
11201 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11202 iRegL src1, iRegL src2,
11203 immI src3, rFlagsReg cr) %{
11204 match(Set dst (AddL src1 (RShiftL src2 src3)));
11205
11206 ins_cost(1.9 * INSN_COST);
11207 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11208
11209 ins_encode %{
11210 __ add(as_Register($dst$$reg),
11211 as_Register($src1$$reg),
11212 as_Register($src2$$reg),
11213 Assembler::ASR,
11214 $src3$$constant & 0x3f);
11215 %}
11216
11217 ins_pipe(ialu_reg_reg_shift);
11218 %}
11219
11220 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11221 iRegIorL2I src1, iRegIorL2I src2,
11222 immI src3, rFlagsReg cr) %{
11223 match(Set dst (AddI src1 (LShiftI src2 src3)));
11224
11225 ins_cost(1.9 * INSN_COST);
11226 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11227
11228 ins_encode %{
11229 __ addw(as_Register($dst$$reg),
11230 as_Register($src1$$reg),
11231 as_Register($src2$$reg),
11232 Assembler::LSL,
11233 $src3$$constant & 0x1f);
11234 %}
11235
11236 ins_pipe(ialu_reg_reg_shift);
11237 %}
11238
11239 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11240 iRegL src1, iRegL src2,
11241 immI src3, rFlagsReg cr) %{
11242 match(Set dst (AddL src1 (LShiftL src2 src3)));
11243
11244 ins_cost(1.9 * INSN_COST);
11245 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11246
11247 ins_encode %{
11248 __ add(as_Register($dst$$reg),
11249 as_Register($src1$$reg),
11250 as_Register($src2$$reg),
11251 Assembler::LSL,
11252 $src3$$constant & 0x3f);
11253 %}
11254
11255 ins_pipe(ialu_reg_reg_shift);
11256 %}
11257
11258 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11259 iRegIorL2I src1, iRegIorL2I src2,
11260 immI src3, rFlagsReg cr) %{
11261 match(Set dst (SubI src1 (URShiftI src2 src3)));
11262
11263 ins_cost(1.9 * INSN_COST);
11264 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11265
11266 ins_encode %{
11267 __ subw(as_Register($dst$$reg),
11268 as_Register($src1$$reg),
11269 as_Register($src2$$reg),
11270 Assembler::LSR,
11271 $src3$$constant & 0x1f);
11272 %}
11273
11274 ins_pipe(ialu_reg_reg_shift);
11275 %}
11276
11277 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11278 iRegL src1, iRegL src2,
11279 immI src3, rFlagsReg cr) %{
11280 match(Set dst (SubL src1 (URShiftL src2 src3)));
11281
11282 ins_cost(1.9 * INSN_COST);
11283 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11284
11285 ins_encode %{
11286 __ sub(as_Register($dst$$reg),
11287 as_Register($src1$$reg),
11288 as_Register($src2$$reg),
11289 Assembler::LSR,
11290 $src3$$constant & 0x3f);
11291 %}
11292
11293 ins_pipe(ialu_reg_reg_shift);
11294 %}
11295
11296 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11297 iRegIorL2I src1, iRegIorL2I src2,
11298 immI src3, rFlagsReg cr) %{
11299 match(Set dst (SubI src1 (RShiftI src2 src3)));
11300
11301 ins_cost(1.9 * INSN_COST);
11302 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11303
11304 ins_encode %{
11305 __ subw(as_Register($dst$$reg),
11306 as_Register($src1$$reg),
11307 as_Register($src2$$reg),
11308 Assembler::ASR,
11309 $src3$$constant & 0x1f);
11310 %}
11311
11312 ins_pipe(ialu_reg_reg_shift);
11313 %}
11314
11315 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11316 iRegL src1, iRegL src2,
11317 immI src3, rFlagsReg cr) %{
11318 match(Set dst (SubL src1 (RShiftL src2 src3)));
11319
11320 ins_cost(1.9 * INSN_COST);
11321 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11322
11323 ins_encode %{
11324 __ sub(as_Register($dst$$reg),
11325 as_Register($src1$$reg),
11326 as_Register($src2$$reg),
11327 Assembler::ASR,
11328 $src3$$constant & 0x3f);
11329 %}
11330
11331 ins_pipe(ialu_reg_reg_shift);
11332 %}
11333
11334 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11335 iRegIorL2I src1, iRegIorL2I src2,
11336 immI src3, rFlagsReg cr) %{
11337 match(Set dst (SubI src1 (LShiftI src2 src3)));
11338
11339 ins_cost(1.9 * INSN_COST);
11340 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11341
11342 ins_encode %{
11343 __ subw(as_Register($dst$$reg),
11344 as_Register($src1$$reg),
11345 as_Register($src2$$reg),
11346 Assembler::LSL,
11347 $src3$$constant & 0x1f);
11348 %}
11349
11350 ins_pipe(ialu_reg_reg_shift);
11351 %}
11352
11353 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11354 iRegL src1, iRegL src2,
11355 immI src3, rFlagsReg cr) %{
11356 match(Set dst (SubL src1 (LShiftL src2 src3)));
11357
11358 ins_cost(1.9 * INSN_COST);
11359 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11360
11361 ins_encode %{
11362 __ sub(as_Register($dst$$reg),
11363 as_Register($src1$$reg),
11364 as_Register($src2$$reg),
11365 Assembler::LSL,
11366 $src3$$constant & 0x3f);
11367 %}
11368
11369 ins_pipe(ialu_reg_reg_shift);
11370 %}
11371
11372
11373
11374 // Shift Left followed by Shift Right.
11375 // This idiom is used by the compiler for the i2b bytecode etc.
11376 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11377 %{
11378 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11379 // Make sure we are not going to exceed what sbfm can do.
11380 predicate((unsigned int)n->in(2)->get_int() <= 63
11381 && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
11382
11383 ins_cost(INSN_COST * 2);
11384 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11385 ins_encode %{
11386 int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
11387 int s = 63 - lshift;
11388 int r = (rshift - lshift) & 63;
11389 __ sbfm(as_Register($dst$$reg),
11390 as_Register($src$$reg),
11391 r, s);
11392 %}
11393
11394 ins_pipe(ialu_reg_shift);
11395 %}
11396
11397 // Shift Left followed by Shift Right.
11398 // This idiom is used by the compiler for the i2b bytecode etc.
11399 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11400 %{
11401 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11402 // Make sure we are not going to exceed what sbfmw can do.
11403 predicate((unsigned int)n->in(2)->get_int() <= 31
11404 && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
11405
11406 ins_cost(INSN_COST * 2);
11407 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11408 ins_encode %{
11409 int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
11410 int s = 31 - lshift;
11411 int r = (rshift - lshift) & 31;
11412 __ sbfmw(as_Register($dst$$reg),
11413 as_Register($src$$reg),
11414 r, s);
11415 %}
11416
11417 ins_pipe(ialu_reg_shift);
11418 %}
11419
11420 // Shift Left followed by Shift Right.
11421 // This idiom is used by the compiler for the i2b bytecode etc.
11422 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11423 %{
11424 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11425 // Make sure we are not going to exceed what ubfm can do.
11426 predicate((unsigned int)n->in(2)->get_int() <= 63
11427 && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
11428
11429 ins_cost(INSN_COST * 2);
11430 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11431 ins_encode %{
11432 int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
11433 int s = 63 - lshift;
11434 int r = (rshift - lshift) & 63;
11435 __ ubfm(as_Register($dst$$reg),
11436 as_Register($src$$reg),
11437 r, s);
11438 %}
11439
11440 ins_pipe(ialu_reg_shift);
11441 %}
11442
11443 // Shift Left followed by Shift Right.
11444 // This idiom is used by the compiler for the i2b bytecode etc.
11445 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11446 %{
11447 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11448 // Make sure we are not going to exceed what ubfmw can do.
11449 predicate((unsigned int)n->in(2)->get_int() <= 31
11450 && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
11451
11452 ins_cost(INSN_COST * 2);
11453 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11454 ins_encode %{
11455 int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
11456 int s = 31 - lshift;
11457 int r = (rshift - lshift) & 31;
11458 __ ubfmw(as_Register($dst$$reg),
11459 as_Register($src$$reg),
11460 r, s);
11461 %}
11462
11463 ins_pipe(ialu_reg_shift);
11464 %}
11465 // Bitfield extract with shift & mask
11466
11467 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11468 %{
11469 match(Set dst (AndI (URShiftI src rshift) mask));
11470
11471 ins_cost(INSN_COST);
11472 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11473 ins_encode %{
11474 int rshift = $rshift$$constant;
11475 long mask = $mask$$constant;
11476 int width = exact_log2(mask+1);
11477 __ ubfxw(as_Register($dst$$reg),
11478 as_Register($src$$reg), rshift, width);
11479 %}
11480 ins_pipe(ialu_reg_shift);
11481 %}
11482 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11483 %{
11484 match(Set dst (AndL (URShiftL src rshift) mask));
11485
11486 ins_cost(INSN_COST);
11487 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11488 ins_encode %{
11489 int rshift = $rshift$$constant;
11490 long mask = $mask$$constant;
11491 int width = exact_log2(mask+1);
11492 __ ubfx(as_Register($dst$$reg),
11493 as_Register($src$$reg), rshift, width);
11494 %}
11495 ins_pipe(ialu_reg_shift);
11496 %}
11497
11498 // We can use ubfx when extending an And with a mask when we know mask
11499 // is positive. We know that because immI_bitmask guarantees it.
11500 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11501 %{
11502 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11503
11504 ins_cost(INSN_COST * 2);
11505 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11506 ins_encode %{
11507 int rshift = $rshift$$constant;
11508 long mask = $mask$$constant;
11509 int width = exact_log2(mask+1);
11510 __ ubfx(as_Register($dst$$reg),
11511 as_Register($src$$reg), rshift, width);
11512 %}
11513 ins_pipe(ialu_reg_shift);
11514 %}
11515
11516 // We can use ubfiz when masking by a positive number and then left shifting the result.
11517 // We know that the mask is positive because immI_bitmask guarantees it.
11518 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11519 %{
11520 match(Set dst (LShiftI (AndI src mask) lshift));
11521 predicate((unsigned int)n->in(2)->get_int() <= 31 &&
11522 (exact_log2(n->in(1)->in(2)->get_int()+1) + (unsigned int)n->in(2)->get_int()) <= (31+1));
11523
11524 ins_cost(INSN_COST);
11525 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11526 ins_encode %{
11527 int lshift = $lshift$$constant;
11528 long mask = $mask$$constant;
11529 int width = exact_log2(mask+1);
11530 __ ubfizw(as_Register($dst$$reg),
11531 as_Register($src$$reg), lshift, width);
11532 %}
11533 ins_pipe(ialu_reg_shift);
11534 %}
11535 // We can use ubfiz when masking by a positive number and then left shifting the result.
11536 // We know that the mask is positive because immL_bitmask guarantees it.
11537 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11538 %{
11539 match(Set dst (LShiftL (AndL src mask) lshift));
11540 predicate((unsigned int)n->in(2)->get_int() <= 63 &&
11541 (exact_log2_long(n->in(1)->in(2)->get_long()+1) + (unsigned int)n->in(2)->get_int()) <= (63+1));
11542
11543 ins_cost(INSN_COST);
11544 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11545 ins_encode %{
11546 int lshift = $lshift$$constant;
11547 long mask = $mask$$constant;
11548 int width = exact_log2(mask+1);
11549 __ ubfiz(as_Register($dst$$reg),
11550 as_Register($src$$reg), lshift, width);
11551 %}
11552 ins_pipe(ialu_reg_shift);
11553 %}
11554
11555 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11556 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11557 %{
11558 match(Set dst (LShiftL (ConvI2L(AndI src mask)) lshift));
11559 predicate((unsigned int)n->in(2)->get_int() <= 31 &&
11560 (exact_log2((unsigned int)n->in(1)->in(1)->in(2)->get_int()+1) + (unsigned int)n->in(2)->get_int()) <= 32);
11561
11562 ins_cost(INSN_COST);
11563 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11564 ins_encode %{
11565 int lshift = $lshift$$constant;
11566 long mask = $mask$$constant;
11567 int width = exact_log2(mask+1);
11568 __ ubfiz(as_Register($dst$$reg),
11569 as_Register($src$$reg), lshift, width);
11570 %}
11571 ins_pipe(ialu_reg_shift);
11572 %}
11573
11574 // Rotations
11575
11576 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11577 %{
11578 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11579 predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
11580
11581 ins_cost(INSN_COST);
11582 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11583
11584 ins_encode %{
11585 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11586 $rshift$$constant & 63);
11587 %}
11588 ins_pipe(ialu_reg_reg_extr);
11589 %}
11590
11591 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11592 %{
11593 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11594 predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
11595
11596 ins_cost(INSN_COST);
11597 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11598
11599 ins_encode %{
11600 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11601 $rshift$$constant & 31);
11602 %}
11603 ins_pipe(ialu_reg_reg_extr);
11604 %}
11605
11606 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11607 %{
11608 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11609 predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
11610
11611 ins_cost(INSN_COST);
11612 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11613
11614 ins_encode %{
11615 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11616 $rshift$$constant & 63);
11617 %}
11618 ins_pipe(ialu_reg_reg_extr);
11619 %}
11620
11621 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11622 %{
11623 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11624 predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
11625
11626 ins_cost(INSN_COST);
11627 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11628
11629 ins_encode %{
11630 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11631 $rshift$$constant & 31);
11632 %}
11633 ins_pipe(ialu_reg_reg_extr);
11634 %}
11635
11636
11637 // rol expander
11638
11639 instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
11640 %{
11641 effect(DEF dst, USE src, USE shift);
11642
11643 format %{ "rol $dst, $src, $shift" %}
11644 ins_cost(INSN_COST * 3);
11645 ins_encode %{
11646 __ subw(rscratch1, zr, as_Register($shift$$reg));
11647 __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
11648 rscratch1);
11649 %}
11650 ins_pipe(ialu_reg_reg_vshift);
11651 %}
11652
11653 // rol expander
11654
11655 instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
11656 %{
11657 effect(DEF dst, USE src, USE shift);
11658
11659 format %{ "rol $dst, $src, $shift" %}
11660 ins_cost(INSN_COST * 3);
11661 ins_encode %{
11662 __ subw(rscratch1, zr, as_Register($shift$$reg));
11663 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
11664 rscratch1);
11665 %}
11666 ins_pipe(ialu_reg_reg_vshift);
11667 %}
11668
11669 instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
11670 %{
11671 match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
11672
11673 expand %{
11674 rolL_rReg(dst, src, shift, cr);
11675 %}
11676 %}
11677
11678 instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
11679 %{
11680 match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
11681
11682 expand %{
11683 rolL_rReg(dst, src, shift, cr);
11684 %}
11685 %}
11686
11687 instruct rolI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
11688 %{
11689 match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
11690
11691 expand %{
11692 rolI_rReg(dst, src, shift, cr);
11693 %}
11694 %}
11695
11696 instruct rolI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
11697 %{
11698 match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
11699
11700 expand %{
11701 rolI_rReg(dst, src, shift, cr);
11702 %}
11703 %}
11704
11705 // ror expander
11706
11707 instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
11708 %{
11709 effect(DEF dst, USE src, USE shift);
11710
11711 format %{ "ror $dst, $src, $shift" %}
11712 ins_cost(INSN_COST);
11713 ins_encode %{
11714 __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
11715 as_Register($shift$$reg));
11716 %}
11717 ins_pipe(ialu_reg_reg_vshift);
11718 %}
11719
11720 // ror expander
11721
11722 instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
11723 %{
11724 effect(DEF dst, USE src, USE shift);
11725
11726 format %{ "ror $dst, $src, $shift" %}
11727 ins_cost(INSN_COST);
11728 ins_encode %{
11729 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
11730 as_Register($shift$$reg));
11731 %}
11732 ins_pipe(ialu_reg_reg_vshift);
11733 %}
11734
11735 instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
11736 %{
11737 match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
11738
11739 expand %{
11740 rorL_rReg(dst, src, shift, cr);
11741 %}
11742 %}
11743
11744 instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
11745 %{
11746 match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
11747
11748 expand %{
11749 rorL_rReg(dst, src, shift, cr);
11750 %}
11751 %}
11752
11753 instruct rorI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
11754 %{
11755 match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
11756
11757 expand %{
11758 rorI_rReg(dst, src, shift, cr);
11759 %}
11760 %}
11761
11762 instruct rorI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
11763 %{
11764 match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
11765
11766 expand %{
11767 rorI_rReg(dst, src, shift, cr);
11768 %}
11769 %}
11770
11771 // Add/subtract (extended)
11772
11773 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11774 %{
11775 match(Set dst (AddL src1 (ConvI2L src2)));
11776 ins_cost(INSN_COST);
11777 format %{ "add $dst, $src1, $src2, sxtw" %}
11778
11779 ins_encode %{
11780 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11781 as_Register($src2$$reg), ext::sxtw);
11782 %}
11783 ins_pipe(ialu_reg_reg);
11784 %};
11785
11786 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11787 %{
11788 match(Set dst (SubL src1 (ConvI2L src2)));
11789 ins_cost(INSN_COST);
11790 format %{ "sub $dst, $src1, $src2, sxtw" %}
11791
11792 ins_encode %{
11793 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11794 as_Register($src2$$reg), ext::sxtw);
11795 %}
11796 ins_pipe(ialu_reg_reg);
11797 %};
11798
11799
11800 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11801 %{
11802 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11803 ins_cost(INSN_COST);
11804 format %{ "add $dst, $src1, $src2, sxth" %}
11805
11806 ins_encode %{
11807 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11808 as_Register($src2$$reg), ext::sxth);
11809 %}
11810 ins_pipe(ialu_reg_reg);
11811 %}
11812
11813 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11814 %{
11815 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11816 ins_cost(INSN_COST);
11817 format %{ "add $dst, $src1, $src2, sxtb" %}
11818
11819 ins_encode %{
11820 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11821 as_Register($src2$$reg), ext::sxtb);
11822 %}
11823 ins_pipe(ialu_reg_reg);
11824 %}
11825
11826 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11827 %{
11828 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11829 ins_cost(INSN_COST);
11830 format %{ "add $dst, $src1, $src2, uxtb" %}
11831
11832 ins_encode %{
11833 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11834 as_Register($src2$$reg), ext::uxtb);
11835 %}
11836 ins_pipe(ialu_reg_reg);
11837 %}
11838
11839 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11840 %{
11841 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11842 ins_cost(INSN_COST);
11843 format %{ "add $dst, $src1, $src2, sxth" %}
11844
11845 ins_encode %{
11846 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11847 as_Register($src2$$reg), ext::sxth);
11848 %}
11849 ins_pipe(ialu_reg_reg);
11850 %}
11851
11852 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11853 %{
11854 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11855 ins_cost(INSN_COST);
11856 format %{ "add $dst, $src1, $src2, sxtw" %}
11857
11858 ins_encode %{
11859 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11860 as_Register($src2$$reg), ext::sxtw);
11861 %}
11862 ins_pipe(ialu_reg_reg);
11863 %}
11864
11865 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11866 %{
11867 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11868 ins_cost(INSN_COST);
11869 format %{ "add $dst, $src1, $src2, sxtb" %}
11870
11871 ins_encode %{
11872 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11873 as_Register($src2$$reg), ext::sxtb);
11874 %}
11875 ins_pipe(ialu_reg_reg);
11876 %}
11877
11878 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11879 %{
11880 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11881 ins_cost(INSN_COST);
11882 format %{ "add $dst, $src1, $src2, uxtb" %}
11883
11884 ins_encode %{
11885 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11886 as_Register($src2$$reg), ext::uxtb);
11887 %}
11888 ins_pipe(ialu_reg_reg);
11889 %}
11890
11891
11892 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11893 %{
11894 match(Set dst (AddI src1 (AndI src2 mask)));
11895 ins_cost(INSN_COST);
11896 format %{ "addw $dst, $src1, $src2, uxtb" %}
11897
11898 ins_encode %{
11899 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11900 as_Register($src2$$reg), ext::uxtb);
11901 %}
11902 ins_pipe(ialu_reg_reg);
11903 %}
11904
11905 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11906 %{
11907 match(Set dst (AddI src1 (AndI src2 mask)));
11908 ins_cost(INSN_COST);
11909 format %{ "addw $dst, $src1, $src2, uxth" %}
11910
11911 ins_encode %{
11912 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11913 as_Register($src2$$reg), ext::uxth);
11914 %}
11915 ins_pipe(ialu_reg_reg);
11916 %}
11917
11918 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
11919 %{
11920 match(Set dst (AddL src1 (AndL src2 mask)));
11921 ins_cost(INSN_COST);
11922 format %{ "add $dst, $src1, $src2, uxtb" %}
11923
11924 ins_encode %{
11925 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11926 as_Register($src2$$reg), ext::uxtb);
11927 %}
11928 ins_pipe(ialu_reg_reg);
11929 %}
11930
11931 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
11932 %{
11933 match(Set dst (AddL src1 (AndL src2 mask)));
11934 ins_cost(INSN_COST);
11935 format %{ "add $dst, $src1, $src2, uxth" %}
11936
11937 ins_encode %{
11938 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11939 as_Register($src2$$reg), ext::uxth);
11940 %}
11941 ins_pipe(ialu_reg_reg);
11942 %}
11943
11944 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
11945 %{
11946 match(Set dst (AddL src1 (AndL src2 mask)));
11947 ins_cost(INSN_COST);
11948 format %{ "add $dst, $src1, $src2, uxtw" %}
11949
11950 ins_encode %{
11951 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11952 as_Register($src2$$reg), ext::uxtw);
11953 %}
11954 ins_pipe(ialu_reg_reg);
11955 %}
11956
11957 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11958 %{
11959 match(Set dst (SubI src1 (AndI src2 mask)));
11960 ins_cost(INSN_COST);
11961 format %{ "subw $dst, $src1, $src2, uxtb" %}
11962
11963 ins_encode %{
11964 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
11965 as_Register($src2$$reg), ext::uxtb);
11966 %}
11967 ins_pipe(ialu_reg_reg);
11968 %}
11969
11970 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11971 %{
11972 match(Set dst (SubI src1 (AndI src2 mask)));
11973 ins_cost(INSN_COST);
11974 format %{ "subw $dst, $src1, $src2, uxth" %}
11975
11976 ins_encode %{
11977 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
11978 as_Register($src2$$reg), ext::uxth);
11979 %}
11980 ins_pipe(ialu_reg_reg);
11981 %}
11982
11983 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
11984 %{
11985 match(Set dst (SubL src1 (AndL src2 mask)));
11986 ins_cost(INSN_COST);
11987 format %{ "sub $dst, $src1, $src2, uxtb" %}
11988
11989 ins_encode %{
11990 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11991 as_Register($src2$$reg), ext::uxtb);
11992 %}
11993 ins_pipe(ialu_reg_reg);
11994 %}
11995
11996 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
11997 %{
11998 match(Set dst (SubL src1 (AndL src2 mask)));
11999 ins_cost(INSN_COST);
12000 format %{ "sub $dst, $src1, $src2, uxth" %}
12001
12002 ins_encode %{
12003 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12004 as_Register($src2$$reg), ext::uxth);
12005 %}
12006 ins_pipe(ialu_reg_reg);
12007 %}
12008
12009 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12010 %{
12011 match(Set dst (SubL src1 (AndL src2 mask)));
12012 ins_cost(INSN_COST);
12013 format %{ "sub $dst, $src1, $src2, uxtw" %}
12014
12015 ins_encode %{
12016 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12017 as_Register($src2$$reg), ext::uxtw);
12018 %}
12019 ins_pipe(ialu_reg_reg);
12020 %}
12021
12022
12023 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12024 %{
12025 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12026 ins_cost(1.9 * INSN_COST);
12027 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12028
12029 ins_encode %{
12030 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12031 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12032 %}
12033 ins_pipe(ialu_reg_reg_shift);
12034 %}
12035
12036 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12037 %{
12038 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12039 ins_cost(1.9 * INSN_COST);
12040 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12041
12042 ins_encode %{
12043 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12044 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12045 %}
12046 ins_pipe(ialu_reg_reg_shift);
12047 %}
12048
12049 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12050 %{
12051 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12052 ins_cost(1.9 * INSN_COST);
12053 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12054
12055 ins_encode %{
12056 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12057 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12058 %}
12059 ins_pipe(ialu_reg_reg_shift);
12060 %}
12061
12062 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12063 %{
12064 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12065 ins_cost(1.9 * INSN_COST);
12066 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12067
12068 ins_encode %{
12069 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12070 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12071 %}
12072 ins_pipe(ialu_reg_reg_shift);
12073 %}
12074
12075 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12076 %{
12077 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12078 ins_cost(1.9 * INSN_COST);
12079 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12080
12081 ins_encode %{
12082 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12083 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12084 %}
12085 ins_pipe(ialu_reg_reg_shift);
12086 %}
12087
12088 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12089 %{
12090 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12091 ins_cost(1.9 * INSN_COST);
12092 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12093
12094 ins_encode %{
12095 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12096 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12097 %}
12098 ins_pipe(ialu_reg_reg_shift);
12099 %}
12100
12101 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12102 %{
12103 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12104 ins_cost(1.9 * INSN_COST);
12105 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12106
12107 ins_encode %{
12108 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12109 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12110 %}
12111 ins_pipe(ialu_reg_reg_shift);
12112 %}
12113
12114 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12115 %{
12116 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12117 ins_cost(1.9 * INSN_COST);
12118 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12119
12120 ins_encode %{
12121 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12122 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12123 %}
12124 ins_pipe(ialu_reg_reg_shift);
12125 %}
12126
12127 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12128 %{
12129 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12130 ins_cost(1.9 * INSN_COST);
12131 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12132
12133 ins_encode %{
12134 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12135 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12136 %}
12137 ins_pipe(ialu_reg_reg_shift);
12138 %}
12139
12140 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12141 %{
12142 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12143 ins_cost(1.9 * INSN_COST);
12144 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12145
12146 ins_encode %{
12147 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12148 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12149 %}
12150 ins_pipe(ialu_reg_reg_shift);
12151 %}
12152
12153
12154 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12155 %{
12156 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12157 ins_cost(1.9 * INSN_COST);
12158 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12159
12160 ins_encode %{
12161 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12162 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12163 %}
12164 ins_pipe(ialu_reg_reg_shift);
12165 %};
12166
12167 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12168 %{
12169 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12170 ins_cost(1.9 * INSN_COST);
12171 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12172
12173 ins_encode %{
12174 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12175 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12176 %}
12177 ins_pipe(ialu_reg_reg_shift);
12178 %};
12179
12180
12181 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12182 %{
12183 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12184 ins_cost(1.9 * INSN_COST);
12185 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12186
12187 ins_encode %{
12188 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12189 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12190 %}
12191 ins_pipe(ialu_reg_reg_shift);
12192 %}
12193
12194 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12195 %{
12196 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12197 ins_cost(1.9 * INSN_COST);
12198 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12199
12200 ins_encode %{
12201 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12202 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12203 %}
12204 ins_pipe(ialu_reg_reg_shift);
12205 %}
12206
12207 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12208 %{
12209 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12210 ins_cost(1.9 * INSN_COST);
12211 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12212
12213 ins_encode %{
12214 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12215 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12216 %}
12217 ins_pipe(ialu_reg_reg_shift);
12218 %}
12219
12220 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12221 %{
12222 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12223 ins_cost(1.9 * INSN_COST);
12224 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12225
12226 ins_encode %{
12227 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12228 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12229 %}
12230 ins_pipe(ialu_reg_reg_shift);
12231 %}
12232
12233 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12234 %{
12235 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12236 ins_cost(1.9 * INSN_COST);
12237 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12238
12239 ins_encode %{
12240 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12241 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12242 %}
12243 ins_pipe(ialu_reg_reg_shift);
12244 %}
12245
12246 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12247 %{
12248 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12249 ins_cost(1.9 * INSN_COST);
12250 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12251
12252 ins_encode %{
12253 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12254 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12255 %}
12256 ins_pipe(ialu_reg_reg_shift);
12257 %}
12258
12259 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12260 %{
12261 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12262 ins_cost(1.9 * INSN_COST);
12263 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12264
12265 ins_encode %{
12266 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12267 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12268 %}
12269 ins_pipe(ialu_reg_reg_shift);
12270 %}
12271
12272 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12273 %{
12274 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12275 ins_cost(1.9 * INSN_COST);
12276 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12277
12278 ins_encode %{
12279 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12280 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12281 %}
12282 ins_pipe(ialu_reg_reg_shift);
12283 %}
12284
12285 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12286 %{
12287 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12288 ins_cost(1.9 * INSN_COST);
12289 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12290
12291 ins_encode %{
12292 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12293 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12294 %}
12295 ins_pipe(ialu_reg_reg_shift);
12296 %}
12297
12298 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12299 %{
12300 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12301 ins_cost(1.9 * INSN_COST);
12302 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12303
12304 ins_encode %{
12305 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12306 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12307 %}
12308 ins_pipe(ialu_reg_reg_shift);
12309 %}
12310 // END This section of the file is automatically generated. Do not edit --------------
12311
12312 // ============================================================================
12313 // Floating Point Arithmetic Instructions
12314
12315 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12316 match(Set dst (AddF src1 src2));
12317
12318 ins_cost(INSN_COST * 5);
12319 format %{ "fadds $dst, $src1, $src2" %}
12320
12321 ins_encode %{
12322 __ fadds(as_FloatRegister($dst$$reg),
12323 as_FloatRegister($src1$$reg),
12324 as_FloatRegister($src2$$reg));
12325 %}
12326
12327 ins_pipe(fp_dop_reg_reg_s);
12328 %}
12329
12330 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12331 match(Set dst (AddD src1 src2));
12332
12333 ins_cost(INSN_COST * 5);
12334 format %{ "faddd $dst, $src1, $src2" %}
12335
12336 ins_encode %{
12337 __ faddd(as_FloatRegister($dst$$reg),
12338 as_FloatRegister($src1$$reg),
12339 as_FloatRegister($src2$$reg));
12340 %}
12341
12342 ins_pipe(fp_dop_reg_reg_d);
12343 %}
12344
12345 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12346 match(Set dst (SubF src1 src2));
12347
12348 ins_cost(INSN_COST * 5);
12349 format %{ "fsubs $dst, $src1, $src2" %}
12350
12351 ins_encode %{
12352 __ fsubs(as_FloatRegister($dst$$reg),
12353 as_FloatRegister($src1$$reg),
12354 as_FloatRegister($src2$$reg));
12355 %}
12356
12357 ins_pipe(fp_dop_reg_reg_s);
12358 %}
12359
12360 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12361 match(Set dst (SubD src1 src2));
12362
12363 ins_cost(INSN_COST * 5);
12364 format %{ "fsubd $dst, $src1, $src2" %}
12365
12366 ins_encode %{
12367 __ fsubd(as_FloatRegister($dst$$reg),
12368 as_FloatRegister($src1$$reg),
12369 as_FloatRegister($src2$$reg));
12370 %}
12371
12372 ins_pipe(fp_dop_reg_reg_d);
12373 %}
12374
12375 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12376 match(Set dst (MulF src1 src2));
12377
12378 ins_cost(INSN_COST * 6);
12379 format %{ "fmuls $dst, $src1, $src2" %}
12380
12381 ins_encode %{
12382 __ fmuls(as_FloatRegister($dst$$reg),
12383 as_FloatRegister($src1$$reg),
12384 as_FloatRegister($src2$$reg));
12385 %}
12386
12387 ins_pipe(fp_dop_reg_reg_s);
12388 %}
12389
12390 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12391 match(Set dst (MulD src1 src2));
12392
12393 ins_cost(INSN_COST * 6);
12394 format %{ "fmuld $dst, $src1, $src2" %}
12395
12396 ins_encode %{
12397 __ fmuld(as_FloatRegister($dst$$reg),
12398 as_FloatRegister($src1$$reg),
12399 as_FloatRegister($src2$$reg));
12400 %}
12401
12402 ins_pipe(fp_dop_reg_reg_d);
12403 %}
12404
12405 // src1 * src2 + src3
12406 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12407 predicate(UseFMA);
12408 match(Set dst (FmaF src3 (Binary src1 src2)));
12409
12410 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12411
12412 ins_encode %{
12413 __ fmadds(as_FloatRegister($dst$$reg),
12414 as_FloatRegister($src1$$reg),
12415 as_FloatRegister($src2$$reg),
12416 as_FloatRegister($src3$$reg));
12417 %}
12418
12419 ins_pipe(pipe_class_default);
12420 %}
12421
12422 // src1 * src2 + src3
12423 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12424 predicate(UseFMA);
12425 match(Set dst (FmaD src3 (Binary src1 src2)));
12426
12427 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12428
12429 ins_encode %{
12430 __ fmaddd(as_FloatRegister($dst$$reg),
12431 as_FloatRegister($src1$$reg),
12432 as_FloatRegister($src2$$reg),
12433 as_FloatRegister($src3$$reg));
12434 %}
12435
12436 ins_pipe(pipe_class_default);
12437 %}
12438
12439 // -src1 * src2 + src3
12440 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12441 predicate(UseFMA);
12442 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
12443 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12444
12445 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12446
12447 ins_encode %{
12448 __ fmsubs(as_FloatRegister($dst$$reg),
12449 as_FloatRegister($src1$$reg),
12450 as_FloatRegister($src2$$reg),
12451 as_FloatRegister($src3$$reg));
12452 %}
12453
12454 ins_pipe(pipe_class_default);
12455 %}
12456
12457 // -src1 * src2 + src3
12458 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12459 predicate(UseFMA);
12460 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
12461 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12462
12463 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12464
12465 ins_encode %{
12466 __ fmsubd(as_FloatRegister($dst$$reg),
12467 as_FloatRegister($src1$$reg),
12468 as_FloatRegister($src2$$reg),
12469 as_FloatRegister($src3$$reg));
12470 %}
12471
12472 ins_pipe(pipe_class_default);
12473 %}
12474
12475 // -src1 * src2 - src3
12476 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12477 predicate(UseFMA);
12478 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
12479 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
12480
12481 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
12482
12483 ins_encode %{
12484 __ fnmadds(as_FloatRegister($dst$$reg),
12485 as_FloatRegister($src1$$reg),
12486 as_FloatRegister($src2$$reg),
12487 as_FloatRegister($src3$$reg));
12488 %}
12489
12490 ins_pipe(pipe_class_default);
12491 %}
12492
12493 // -src1 * src2 - src3
12494 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12495 predicate(UseFMA);
12496 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
12497 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
12498
12499 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
12500
12501 ins_encode %{
12502 __ fnmaddd(as_FloatRegister($dst$$reg),
12503 as_FloatRegister($src1$$reg),
12504 as_FloatRegister($src2$$reg),
12505 as_FloatRegister($src3$$reg));
12506 %}
12507
12508 ins_pipe(pipe_class_default);
12509 %}
12510
12511 // src1 * src2 - src3
12512 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
12513 predicate(UseFMA);
12514 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
12515
12516 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
12517
12518 ins_encode %{
12519 __ fnmsubs(as_FloatRegister($dst$$reg),
12520 as_FloatRegister($src1$$reg),
12521 as_FloatRegister($src2$$reg),
12522 as_FloatRegister($src3$$reg));
12523 %}
12524
12525 ins_pipe(pipe_class_default);
12526 %}
12527
12528 // src1 * src2 - src3
12529 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
12530 predicate(UseFMA);
12531 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
12532
12533 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
12534
12535 ins_encode %{
12536 // n.b. insn name should be fnmsubd
12537 __ fnmsub(as_FloatRegister($dst$$reg),
12538 as_FloatRegister($src1$$reg),
12539 as_FloatRegister($src2$$reg),
12540 as_FloatRegister($src3$$reg));
12541 %}
12542
12543 ins_pipe(pipe_class_default);
12544 %}
12545
12546
12547 // Math.max(FF)F
12548 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12549 match(Set dst (MaxF src1 src2));
12550
12551 format %{ "fmaxs $dst, $src1, $src2" %}
12552 ins_encode %{
12553 __ fmaxs(as_FloatRegister($dst$$reg),
12554 as_FloatRegister($src1$$reg),
12555 as_FloatRegister($src2$$reg));
12556 %}
12557
12558 ins_pipe(fp_dop_reg_reg_s);
12559 %}
12560
12561 // Math.min(FF)F
12562 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12563 match(Set dst (MinF src1 src2));
12564
12565 format %{ "fmins $dst, $src1, $src2" %}
12566 ins_encode %{
12567 __ fmins(as_FloatRegister($dst$$reg),
12568 as_FloatRegister($src1$$reg),
12569 as_FloatRegister($src2$$reg));
12570 %}
12571
12572 ins_pipe(fp_dop_reg_reg_s);
12573 %}
12574
12575 // Math.max(DD)D
12576 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12577 match(Set dst (MaxD src1 src2));
12578
12579 format %{ "fmaxd $dst, $src1, $src2" %}
12580 ins_encode %{
12581 __ fmaxd(as_FloatRegister($dst$$reg),
12582 as_FloatRegister($src1$$reg),
12583 as_FloatRegister($src2$$reg));
12584 %}
12585
12586 ins_pipe(fp_dop_reg_reg_d);
12587 %}
12588
12589 // Math.min(DD)D
12590 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12591 match(Set dst (MinD src1 src2));
12592
12593 format %{ "fmind $dst, $src1, $src2" %}
12594 ins_encode %{
12595 __ fmind(as_FloatRegister($dst$$reg),
12596 as_FloatRegister($src1$$reg),
12597 as_FloatRegister($src2$$reg));
12598 %}
12599
12600 ins_pipe(fp_dop_reg_reg_d);
12601 %}
12602
12603
12604 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12605 match(Set dst (DivF src1 src2));
12606
12607 ins_cost(INSN_COST * 18);
12608 format %{ "fdivs $dst, $src1, $src2" %}
12609
12610 ins_encode %{
12611 __ fdivs(as_FloatRegister($dst$$reg),
12612 as_FloatRegister($src1$$reg),
12613 as_FloatRegister($src2$$reg));
12614 %}
12615
12616 ins_pipe(fp_div_s);
12617 %}
12618
12619 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12620 match(Set dst (DivD src1 src2));
12621
12622 ins_cost(INSN_COST * 32);
12623 format %{ "fdivd $dst, $src1, $src2" %}
12624
12625 ins_encode %{
12626 __ fdivd(as_FloatRegister($dst$$reg),
12627 as_FloatRegister($src1$$reg),
12628 as_FloatRegister($src2$$reg));
12629 %}
12630
12631 ins_pipe(fp_div_d);
12632 %}
12633
12634 instruct negF_reg_reg(vRegF dst, vRegF src) %{
12635 match(Set dst (NegF src));
12636
12637 ins_cost(INSN_COST * 3);
12638 format %{ "fneg $dst, $src" %}
12639
12640 ins_encode %{
12641 __ fnegs(as_FloatRegister($dst$$reg),
12642 as_FloatRegister($src$$reg));
12643 %}
12644
12645 ins_pipe(fp_uop_s);
12646 %}
12647
12648 instruct negD_reg_reg(vRegD dst, vRegD src) %{
12649 match(Set dst (NegD src));
12650
12651 ins_cost(INSN_COST * 3);
12652 format %{ "fnegd $dst, $src" %}
12653
12654 ins_encode %{
12655 __ fnegd(as_FloatRegister($dst$$reg),
12656 as_FloatRegister($src$$reg));
12657 %}
12658
12659 ins_pipe(fp_uop_d);
12660 %}
12661
12662 instruct absF_reg(vRegF dst, vRegF src) %{
12663 match(Set dst (AbsF src));
12664
12665 ins_cost(INSN_COST * 3);
12666 format %{ "fabss $dst, $src" %}
12667 ins_encode %{
12668 __ fabss(as_FloatRegister($dst$$reg),
12669 as_FloatRegister($src$$reg));
12670 %}
12671
12672 ins_pipe(fp_uop_s);
12673 %}
12674
12675 instruct absD_reg(vRegD dst, vRegD src) %{
12676 match(Set dst (AbsD src));
12677
12678 ins_cost(INSN_COST * 3);
12679 format %{ "fabsd $dst, $src" %}
12680 ins_encode %{
12681 __ fabsd(as_FloatRegister($dst$$reg),
12682 as_FloatRegister($src$$reg));
12683 %}
12684
12685 ins_pipe(fp_uop_d);
12686 %}
12687
12688 instruct sqrtD_reg(vRegD dst, vRegD src) %{
12689 match(Set dst (SqrtD src));
12690
12691 ins_cost(INSN_COST * 50);
12692 format %{ "fsqrtd $dst, $src" %}
12693 ins_encode %{
12694 __ fsqrtd(as_FloatRegister($dst$$reg),
12695 as_FloatRegister($src$$reg));
12696 %}
12697
12698 ins_pipe(fp_div_s);
12699 %}
12700
12701 instruct sqrtF_reg(vRegF dst, vRegF src) %{
12702 match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
12703
12704 ins_cost(INSN_COST * 50);
12705 format %{ "fsqrts $dst, $src" %}
12706 ins_encode %{
12707 __ fsqrts(as_FloatRegister($dst$$reg),
12708 as_FloatRegister($src$$reg));
12709 %}
12710
12711 ins_pipe(fp_div_d);
12712 %}
12713
12714 // ============================================================================
12715 // Logical Instructions
12716
12717 // Integer Logical Instructions
12718
12719 // And Instructions
12720
12721
12722 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
12723 match(Set dst (AndI src1 src2));
12724
12725 format %{ "andw $dst, $src1, $src2\t# int" %}
12726
12727 ins_cost(INSN_COST);
12728 ins_encode %{
12729 __ andw(as_Register($dst$$reg),
12730 as_Register($src1$$reg),
12731 as_Register($src2$$reg));
12732 %}
12733
12734 ins_pipe(ialu_reg_reg);
12735 %}
12736
12737 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
12738 match(Set dst (AndI src1 src2));
12739
12740 format %{ "andsw $dst, $src1, $src2\t# int" %}
12741
12742 ins_cost(INSN_COST);
12743 ins_encode %{
12744 __ andw(as_Register($dst$$reg),
12745 as_Register($src1$$reg),
12746 (unsigned long)($src2$$constant));
12747 %}
12748
12749 ins_pipe(ialu_reg_imm);
12750 %}
12751
12752 // Or Instructions
12753
12754 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
12755 match(Set dst (OrI src1 src2));
12756
12757 format %{ "orrw $dst, $src1, $src2\t# int" %}
12758
12759 ins_cost(INSN_COST);
12760 ins_encode %{
12761 __ orrw(as_Register($dst$$reg),
12762 as_Register($src1$$reg),
12763 as_Register($src2$$reg));
12764 %}
12765
12766 ins_pipe(ialu_reg_reg);
12767 %}
12768
12769 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
12770 match(Set dst (OrI src1 src2));
12771
12772 format %{ "orrw $dst, $src1, $src2\t# int" %}
12773
12774 ins_cost(INSN_COST);
12775 ins_encode %{
12776 __ orrw(as_Register($dst$$reg),
12777 as_Register($src1$$reg),
12778 (unsigned long)($src2$$constant));
12779 %}
12780
12781 ins_pipe(ialu_reg_imm);
12782 %}
12783
12784 // Xor Instructions
12785
12786 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
12787 match(Set dst (XorI src1 src2));
12788
12789 format %{ "eorw $dst, $src1, $src2\t# int" %}
12790
12791 ins_cost(INSN_COST);
12792 ins_encode %{
12793 __ eorw(as_Register($dst$$reg),
12794 as_Register($src1$$reg),
12795 as_Register($src2$$reg));
12796 %}
12797
12798 ins_pipe(ialu_reg_reg);
12799 %}
12800
12801 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
12802 match(Set dst (XorI src1 src2));
12803
12804 format %{ "eorw $dst, $src1, $src2\t# int" %}
12805
12806 ins_cost(INSN_COST);
12807 ins_encode %{
12808 __ eorw(as_Register($dst$$reg),
12809 as_Register($src1$$reg),
12810 (unsigned long)($src2$$constant));
12811 %}
12812
12813 ins_pipe(ialu_reg_imm);
12814 %}
12815
12816 // Long Logical Instructions
12817 // TODO
12818
12819 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
12820 match(Set dst (AndL src1 src2));
12821
12822 format %{ "and $dst, $src1, $src2\t# int" %}
12823
12824 ins_cost(INSN_COST);
12825 ins_encode %{
12826 __ andr(as_Register($dst$$reg),
12827 as_Register($src1$$reg),
12828 as_Register($src2$$reg));
12829 %}
12830
12831 ins_pipe(ialu_reg_reg);
12832 %}
12833
12834 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
12835 match(Set dst (AndL src1 src2));
12836
12837 format %{ "and $dst, $src1, $src2\t# int" %}
12838
12839 ins_cost(INSN_COST);
12840 ins_encode %{
12841 __ andr(as_Register($dst$$reg),
12842 as_Register($src1$$reg),
12843 (unsigned long)($src2$$constant));
12844 %}
12845
12846 ins_pipe(ialu_reg_imm);
12847 %}
12848
12849 // Or Instructions
12850
12851 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
12852 match(Set dst (OrL src1 src2));
12853
12854 format %{ "orr $dst, $src1, $src2\t# int" %}
12855
12856 ins_cost(INSN_COST);
12857 ins_encode %{
12858 __ orr(as_Register($dst$$reg),
12859 as_Register($src1$$reg),
12860 as_Register($src2$$reg));
12861 %}
12862
12863 ins_pipe(ialu_reg_reg);
12864 %}
12865
12866 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
12867 match(Set dst (OrL src1 src2));
12868
12869 format %{ "orr $dst, $src1, $src2\t# int" %}
12870
12871 ins_cost(INSN_COST);
12872 ins_encode %{
12873 __ orr(as_Register($dst$$reg),
12874 as_Register($src1$$reg),
12875 (unsigned long)($src2$$constant));
12876 %}
12877
12878 ins_pipe(ialu_reg_imm);
12879 %}
12880
12881 // Xor Instructions
12882
12883 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
12884 match(Set dst (XorL src1 src2));
12885
12886 format %{ "eor $dst, $src1, $src2\t# int" %}
12887
12888 ins_cost(INSN_COST);
12889 ins_encode %{
12890 __ eor(as_Register($dst$$reg),
12891 as_Register($src1$$reg),
12892 as_Register($src2$$reg));
12893 %}
12894
12895 ins_pipe(ialu_reg_reg);
12896 %}
12897
12898 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
12899 match(Set dst (XorL src1 src2));
12900
12901 ins_cost(INSN_COST);
12902 format %{ "eor $dst, $src1, $src2\t# int" %}
12903
12904 ins_encode %{
12905 __ eor(as_Register($dst$$reg),
12906 as_Register($src1$$reg),
12907 (unsigned long)($src2$$constant));
12908 %}
12909
12910 ins_pipe(ialu_reg_imm);
12911 %}
12912
12913 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
12914 %{
12915 match(Set dst (ConvI2L src));
12916
12917 ins_cost(INSN_COST);
12918 format %{ "sxtw $dst, $src\t# i2l" %}
12919 ins_encode %{
12920 __ sbfm($dst$$Register, $src$$Register, 0, 31);
12921 %}
12922 ins_pipe(ialu_reg_shift);
12923 %}
12924
12925 // this pattern occurs in bigmath arithmetic
12926 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
12927 %{
12928 match(Set dst (AndL (ConvI2L src) mask));
12929
12930 ins_cost(INSN_COST);
12931 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
12932 ins_encode %{
12933 __ ubfm($dst$$Register, $src$$Register, 0, 31);
12934 %}
12935
12936 ins_pipe(ialu_reg_shift);
12937 %}
12938
12939 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
12940 match(Set dst (ConvL2I src));
12941
12942 ins_cost(INSN_COST);
12943 format %{ "movw $dst, $src \t// l2i" %}
12944
12945 ins_encode %{
12946 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
12947 %}
12948
12949 ins_pipe(ialu_reg);
12950 %}
12951
12952 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
12953 %{
12954 match(Set dst (Conv2B src));
12955 effect(KILL cr);
12956
12957 format %{
12958 "cmpw $src, zr\n\t"
12959 "cset $dst, ne"
12960 %}
12961
12962 ins_encode %{
12963 __ cmpw(as_Register($src$$reg), zr);
12964 __ cset(as_Register($dst$$reg), Assembler::NE);
12965 %}
12966
12967 ins_pipe(ialu_reg);
12968 %}
12969
12970 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
12971 %{
12972 match(Set dst (Conv2B src));
12973 effect(KILL cr);
12974
12975 format %{
12976 "cmp $src, zr\n\t"
12977 "cset $dst, ne"
12978 %}
12979
12980 ins_encode %{
12981 __ cmp(as_Register($src$$reg), zr);
12982 __ cset(as_Register($dst$$reg), Assembler::NE);
12983 %}
12984
12985 ins_pipe(ialu_reg);
12986 %}
12987
12988 instruct convD2F_reg(vRegF dst, vRegD src) %{
12989 match(Set dst (ConvD2F src));
12990
12991 ins_cost(INSN_COST * 5);
12992 format %{ "fcvtd $dst, $src \t// d2f" %}
12993
12994 ins_encode %{
12995 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
12996 %}
12997
12998 ins_pipe(fp_d2f);
12999 %}
13000
13001 instruct convF2D_reg(vRegD dst, vRegF src) %{
13002 match(Set dst (ConvF2D src));
13003
13004 ins_cost(INSN_COST * 5);
13005 format %{ "fcvts $dst, $src \t// f2d" %}
13006
13007 ins_encode %{
13008 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13009 %}
13010
13011 ins_pipe(fp_f2d);
13012 %}
13013
13014 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13015 match(Set dst (ConvF2I src));
13016
13017 ins_cost(INSN_COST * 5);
13018 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13019
13020 ins_encode %{
13021 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13022 %}
13023
13024 ins_pipe(fp_f2i);
13025 %}
13026
13027 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13028 match(Set dst (ConvF2L src));
13029
13030 ins_cost(INSN_COST * 5);
13031 format %{ "fcvtzs $dst, $src \t// f2l" %}
13032
13033 ins_encode %{
13034 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13035 %}
13036
13037 ins_pipe(fp_f2l);
13038 %}
13039
13040 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13041 match(Set dst (ConvI2F src));
13042
13043 ins_cost(INSN_COST * 5);
13044 format %{ "scvtfws $dst, $src \t// i2f" %}
13045
13046 ins_encode %{
13047 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13048 %}
13049
13050 ins_pipe(fp_i2f);
13051 %}
13052
13053 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13054 match(Set dst (ConvL2F src));
13055
13056 ins_cost(INSN_COST * 5);
13057 format %{ "scvtfs $dst, $src \t// l2f" %}
13058
13059 ins_encode %{
13060 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13061 %}
13062
13063 ins_pipe(fp_l2f);
13064 %}
13065
13066 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13067 match(Set dst (ConvD2I src));
13068
13069 ins_cost(INSN_COST * 5);
13070 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13071
13072 ins_encode %{
13073 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13074 %}
13075
13076 ins_pipe(fp_d2i);
13077 %}
13078
13079 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13080 match(Set dst (ConvD2L src));
13081
13082 ins_cost(INSN_COST * 5);
13083 format %{ "fcvtzd $dst, $src \t// d2l" %}
13084
13085 ins_encode %{
13086 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13087 %}
13088
13089 ins_pipe(fp_d2l);
13090 %}
13091
13092 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13093 match(Set dst (ConvI2D src));
13094
13095 ins_cost(INSN_COST * 5);
13096 format %{ "scvtfwd $dst, $src \t// i2d" %}
13097
13098 ins_encode %{
13099 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13100 %}
13101
13102 ins_pipe(fp_i2d);
13103 %}
13104
13105 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13106 match(Set dst (ConvL2D src));
13107
13108 ins_cost(INSN_COST * 5);
13109 format %{ "scvtfd $dst, $src \t// l2d" %}
13110
13111 ins_encode %{
13112 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13113 %}
13114
13115 ins_pipe(fp_l2d);
13116 %}
13117
13118 // stack <-> reg and reg <-> reg shuffles with no conversion
13119
13120 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13121
13122 match(Set dst (MoveF2I src));
13123
13124 effect(DEF dst, USE src);
13125
13126 ins_cost(4 * INSN_COST);
13127
13128 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13129
13130 ins_encode %{
13131 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13132 %}
13133
13134 ins_pipe(iload_reg_reg);
13135
13136 %}
13137
13138 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13139
13140 match(Set dst (MoveI2F src));
13141
13142 effect(DEF dst, USE src);
13143
13144 ins_cost(4 * INSN_COST);
13145
13146 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13147
13148 ins_encode %{
13149 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13150 %}
13151
13152 ins_pipe(pipe_class_memory);
13153
13154 %}
13155
13156 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13157
13158 match(Set dst (MoveD2L src));
13159
13160 effect(DEF dst, USE src);
13161
13162 ins_cost(4 * INSN_COST);
13163
13164 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13165
13166 ins_encode %{
13167 __ ldr($dst$$Register, Address(sp, $src$$disp));
13168 %}
13169
13170 ins_pipe(iload_reg_reg);
13171
13172 %}
13173
13174 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13175
13176 match(Set dst (MoveL2D src));
13177
13178 effect(DEF dst, USE src);
13179
13180 ins_cost(4 * INSN_COST);
13181
13182 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13183
13184 ins_encode %{
13185 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13186 %}
13187
13188 ins_pipe(pipe_class_memory);
13189
13190 %}
13191
13192 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13193
13194 match(Set dst (MoveF2I src));
13195
13196 effect(DEF dst, USE src);
13197
13198 ins_cost(INSN_COST);
13199
13200 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13201
13202 ins_encode %{
13203 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13204 %}
13205
13206 ins_pipe(pipe_class_memory);
13207
13208 %}
13209
13210 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13211
13212 match(Set dst (MoveI2F src));
13213
13214 effect(DEF dst, USE src);
13215
13216 ins_cost(INSN_COST);
13217
13218 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13219
13220 ins_encode %{
13221 __ strw($src$$Register, Address(sp, $dst$$disp));
13222 %}
13223
13224 ins_pipe(istore_reg_reg);
13225
13226 %}
13227
13228 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13229
13230 match(Set dst (MoveD2L src));
13231
13232 effect(DEF dst, USE src);
13233
13234 ins_cost(INSN_COST);
13235
13236 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13237
13238 ins_encode %{
13239 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13240 %}
13241
13242 ins_pipe(pipe_class_memory);
13243
13244 %}
13245
13246 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13247
13248 match(Set dst (MoveL2D src));
13249
13250 effect(DEF dst, USE src);
13251
13252 ins_cost(INSN_COST);
13253
13254 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13255
13256 ins_encode %{
13257 __ str($src$$Register, Address(sp, $dst$$disp));
13258 %}
13259
13260 ins_pipe(istore_reg_reg);
13261
13262 %}
13263
13264 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13265
13266 match(Set dst (MoveF2I src));
13267
13268 effect(DEF dst, USE src);
13269
13270 ins_cost(INSN_COST);
13271
13272 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
13273
13274 ins_encode %{
13275 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
13276 %}
13277
13278 ins_pipe(fp_f2i);
13279
13280 %}
13281
13282 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
13283
13284 match(Set dst (MoveI2F src));
13285
13286 effect(DEF dst, USE src);
13287
13288 ins_cost(INSN_COST);
13289
13290 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
13291
13292 ins_encode %{
13293 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
13294 %}
13295
13296 ins_pipe(fp_i2f);
13297
13298 %}
13299
13300 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13301
13302 match(Set dst (MoveD2L src));
13303
13304 effect(DEF dst, USE src);
13305
13306 ins_cost(INSN_COST);
13307
13308 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
13309
13310 ins_encode %{
13311 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
13312 %}
13313
13314 ins_pipe(fp_d2l);
13315
13316 %}
13317
13318 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
13319
13320 match(Set dst (MoveL2D src));
13321
13322 effect(DEF dst, USE src);
13323
13324 ins_cost(INSN_COST);
13325
13326 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
13327
13328 ins_encode %{
13329 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
13330 %}
13331
13332 ins_pipe(fp_l2d);
13333
13334 %}
13335
13336 // ============================================================================
13337 // clearing of an array
13338
13339 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
13340 %{
13341 match(Set dummy (ClearArray (Binary cnt base) val));
13342 effect(USE_KILL cnt, USE_KILL base);
13343
13344 ins_cost(4 * INSN_COST);
13345 format %{ "ClearArray $cnt, $base, $val" %}
13346
13347 ins_encode %{
13348 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
13349 %}
13350
13351 ins_pipe(pipe_class_memory);
13352 %}
13353
13354 // ============================================================================
13355 // Overflow Math Instructions
13356
13357 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
13358 %{
13359 match(Set cr (OverflowAddI op1 op2));
13360
13361 format %{ "cmnw $op1, $op2\t# overflow check int" %}
13362 ins_cost(INSN_COST);
13363 ins_encode %{
13364 __ cmnw($op1$$Register, $op2$$Register);
13365 %}
13366
13367 ins_pipe(icmp_reg_reg);
13368 %}
13369
13370 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
13371 %{
13372 match(Set cr (OverflowAddI op1 op2));
13373
13374 format %{ "cmnw $op1, $op2\t# overflow check int" %}
13375 ins_cost(INSN_COST);
13376 ins_encode %{
13377 __ cmnw($op1$$Register, $op2$$constant);
13378 %}
13379
13380 ins_pipe(icmp_reg_imm);
13381 %}
13382
13383 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13384 %{
13385 match(Set cr (OverflowAddL op1 op2));
13386
13387 format %{ "cmn $op1, $op2\t# overflow check long" %}
13388 ins_cost(INSN_COST);
13389 ins_encode %{
13390 __ cmn($op1$$Register, $op2$$Register);
13391 %}
13392
13393 ins_pipe(icmp_reg_reg);
13394 %}
13395
13396 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
13397 %{
13398 match(Set cr (OverflowAddL op1 op2));
13399
13400 format %{ "cmn $op1, $op2\t# overflow check long" %}
13401 ins_cost(INSN_COST);
13402 ins_encode %{
13403 __ cmn($op1$$Register, $op2$$constant);
13404 %}
13405
13406 ins_pipe(icmp_reg_imm);
13407 %}
13408
13409 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
13410 %{
13411 match(Set cr (OverflowSubI op1 op2));
13412
13413 format %{ "cmpw $op1, $op2\t# overflow check int" %}
13414 ins_cost(INSN_COST);
13415 ins_encode %{
13416 __ cmpw($op1$$Register, $op2$$Register);
13417 %}
13418
13419 ins_pipe(icmp_reg_reg);
13420 %}
13421
13422 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
13423 %{
13424 match(Set cr (OverflowSubI op1 op2));
13425
13426 format %{ "cmpw $op1, $op2\t# overflow check int" %}
13427 ins_cost(INSN_COST);
13428 ins_encode %{
13429 __ cmpw($op1$$Register, $op2$$constant);
13430 %}
13431
13432 ins_pipe(icmp_reg_imm);
13433 %}
13434
13435 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13436 %{
13437 match(Set cr (OverflowSubL op1 op2));
13438
13439 format %{ "cmp $op1, $op2\t# overflow check long" %}
13440 ins_cost(INSN_COST);
13441 ins_encode %{
13442 __ cmp($op1$$Register, $op2$$Register);
13443 %}
13444
13445 ins_pipe(icmp_reg_reg);
13446 %}
13447
13448 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
13449 %{
13450 match(Set cr (OverflowSubL op1 op2));
13451
13452 format %{ "cmp $op1, $op2\t# overflow check long" %}
13453 ins_cost(INSN_COST);
13454 ins_encode %{
13455 __ subs(zr, $op1$$Register, $op2$$constant);
13456 %}
13457
13458 ins_pipe(icmp_reg_imm);
13459 %}
13460
13461 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
13462 %{
13463 match(Set cr (OverflowSubI zero op1));
13464
13465 format %{ "cmpw zr, $op1\t# overflow check int" %}
13466 ins_cost(INSN_COST);
13467 ins_encode %{
13468 __ cmpw(zr, $op1$$Register);
13469 %}
13470
13471 ins_pipe(icmp_reg_imm);
13472 %}
13473
13474 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
13475 %{
13476 match(Set cr (OverflowSubL zero op1));
13477
13478 format %{ "cmp zr, $op1\t# overflow check long" %}
13479 ins_cost(INSN_COST);
13480 ins_encode %{
13481 __ cmp(zr, $op1$$Register);
13482 %}
13483
13484 ins_pipe(icmp_reg_imm);
13485 %}
13486
13487 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
13488 %{
13489 match(Set cr (OverflowMulI op1 op2));
13490
13491 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
13492 "cmp rscratch1, rscratch1, sxtw\n\t"
13493 "movw rscratch1, #0x80000000\n\t"
13494 "cselw rscratch1, rscratch1, zr, NE\n\t"
13495 "cmpw rscratch1, #1" %}
13496 ins_cost(5 * INSN_COST);
13497 ins_encode %{
13498 __ smull(rscratch1, $op1$$Register, $op2$$Register);
13499 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
13500 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
13501 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
13502 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
13503 %}
13504
13505 ins_pipe(pipe_slow);
13506 %}
13507
13508 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
13509 %{
13510 match(If cmp (OverflowMulI op1 op2));
13511 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
13512 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
13513 effect(USE labl, KILL cr);
13514
13515 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
13516 "cmp rscratch1, rscratch1, sxtw\n\t"
13517 "b$cmp $labl" %}
13518 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
13519 ins_encode %{
13520 Label* L = $labl$$label;
13521 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13522 __ smull(rscratch1, $op1$$Register, $op2$$Register);
13523 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
13524 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
13525 %}
13526
13527 ins_pipe(pipe_serial);
13528 %}
13529
13530 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13531 %{
13532 match(Set cr (OverflowMulL op1 op2));
13533
13534 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
13535 "smulh rscratch2, $op1, $op2\n\t"
13536 "cmp rscratch2, rscratch1, ASR #63\n\t"
13537 "movw rscratch1, #0x80000000\n\t"
13538 "cselw rscratch1, rscratch1, zr, NE\n\t"
13539 "cmpw rscratch1, #1" %}
13540 ins_cost(6 * INSN_COST);
13541 ins_encode %{
13542 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
13543 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
13544 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
13545 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
13546 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
13547 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
13548 %}
13549
13550 ins_pipe(pipe_slow);
13551 %}
13552
13553 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
13554 %{
13555 match(If cmp (OverflowMulL op1 op2));
13556 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
13557 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
13558 effect(USE labl, KILL cr);
13559
13560 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
13561 "smulh rscratch2, $op1, $op2\n\t"
13562 "cmp rscratch2, rscratch1, ASR #63\n\t"
13563 "b$cmp $labl" %}
13564 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
13565 ins_encode %{
13566 Label* L = $labl$$label;
13567 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13568 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
13569 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
13570 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
13571 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
13572 %}
13573
13574 ins_pipe(pipe_serial);
13575 %}
13576
13577 // ============================================================================
13578 // Compare Instructions
13579
13580 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
13581 %{
13582 match(Set cr (CmpI op1 op2));
13583
13584 effect(DEF cr, USE op1, USE op2);
13585
13586 ins_cost(INSN_COST);
13587 format %{ "cmpw $op1, $op2" %}
13588
13589 ins_encode(aarch64_enc_cmpw(op1, op2));
13590
13591 ins_pipe(icmp_reg_reg);
13592 %}
13593
13594 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
13595 %{
13596 match(Set cr (CmpI op1 zero));
13597
13598 effect(DEF cr, USE op1);
13599
13600 ins_cost(INSN_COST);
13601 format %{ "cmpw $op1, 0" %}
13602
13603 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
13604
13605 ins_pipe(icmp_reg_imm);
13606 %}
13607
13608 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
13609 %{
13610 match(Set cr (CmpI op1 op2));
13611
13612 effect(DEF cr, USE op1);
13613
13614 ins_cost(INSN_COST);
13615 format %{ "cmpw $op1, $op2" %}
13616
13617 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
13618
13619 ins_pipe(icmp_reg_imm);
13620 %}
13621
13622 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
13623 %{
13624 match(Set cr (CmpI op1 op2));
13625
13626 effect(DEF cr, USE op1);
13627
13628 ins_cost(INSN_COST * 2);
13629 format %{ "cmpw $op1, $op2" %}
13630
13631 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
13632
13633 ins_pipe(icmp_reg_imm);
13634 %}
13635
13636 // Unsigned compare Instructions; really, same as signed compare
13637 // except it should only be used to feed an If or a CMovI which takes a
13638 // cmpOpU.
13639
13640 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
13641 %{
13642 match(Set cr (CmpU op1 op2));
13643
13644 effect(DEF cr, USE op1, USE op2);
13645
13646 ins_cost(INSN_COST);
13647 format %{ "cmpw $op1, $op2\t# unsigned" %}
13648
13649 ins_encode(aarch64_enc_cmpw(op1, op2));
13650
13651 ins_pipe(icmp_reg_reg);
13652 %}
13653
13654 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
13655 %{
13656 match(Set cr (CmpU op1 zero));
13657
13658 effect(DEF cr, USE op1);
13659
13660 ins_cost(INSN_COST);
13661 format %{ "cmpw $op1, #0\t# unsigned" %}
13662
13663 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
13664
13665 ins_pipe(icmp_reg_imm);
13666 %}
13667
13668 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
13669 %{
13670 match(Set cr (CmpU op1 op2));
13671
13672 effect(DEF cr, USE op1);
13673
13674 ins_cost(INSN_COST);
13675 format %{ "cmpw $op1, $op2\t# unsigned" %}
13676
13677 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
13678
13679 ins_pipe(icmp_reg_imm);
13680 %}
13681
13682 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
13683 %{
13684 match(Set cr (CmpU op1 op2));
13685
13686 effect(DEF cr, USE op1);
13687
13688 ins_cost(INSN_COST * 2);
13689 format %{ "cmpw $op1, $op2\t# unsigned" %}
13690
13691 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
13692
13693 ins_pipe(icmp_reg_imm);
13694 %}
13695
13696 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13697 %{
13698 match(Set cr (CmpL op1 op2));
13699
13700 effect(DEF cr, USE op1, USE op2);
13701
13702 ins_cost(INSN_COST);
13703 format %{ "cmp $op1, $op2" %}
13704
13705 ins_encode(aarch64_enc_cmp(op1, op2));
13706
13707 ins_pipe(icmp_reg_reg);
13708 %}
13709
13710 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
13711 %{
13712 match(Set cr (CmpL op1 zero));
13713
13714 effect(DEF cr, USE op1);
13715
13716 ins_cost(INSN_COST);
13717 format %{ "tst $op1" %}
13718
13719 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
13720
13721 ins_pipe(icmp_reg_imm);
13722 %}
13723
13724 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
13725 %{
13726 match(Set cr (CmpL op1 op2));
13727
13728 effect(DEF cr, USE op1);
13729
13730 ins_cost(INSN_COST);
13731 format %{ "cmp $op1, $op2" %}
13732
13733 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
13734
13735 ins_pipe(icmp_reg_imm);
13736 %}
13737
13738 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
13739 %{
13740 match(Set cr (CmpL op1 op2));
13741
13742 effect(DEF cr, USE op1);
13743
13744 ins_cost(INSN_COST * 2);
13745 format %{ "cmp $op1, $op2" %}
13746
13747 ins_encode(aarch64_enc_cmp_imm(op1, op2));
13748
13749 ins_pipe(icmp_reg_imm);
13750 %}
13751
13752 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
13753 %{
13754 match(Set cr (CmpUL op1 op2));
13755
13756 effect(DEF cr, USE op1, USE op2);
13757
13758 ins_cost(INSN_COST);
13759 format %{ "cmp $op1, $op2" %}
13760
13761 ins_encode(aarch64_enc_cmp(op1, op2));
13762
13763 ins_pipe(icmp_reg_reg);
13764 %}
13765
13766 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
13767 %{
13768 match(Set cr (CmpUL op1 zero));
13769
13770 effect(DEF cr, USE op1);
13771
13772 ins_cost(INSN_COST);
13773 format %{ "tst $op1" %}
13774
13775 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
13776
13777 ins_pipe(icmp_reg_imm);
13778 %}
13779
13780 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
13781 %{
13782 match(Set cr (CmpUL op1 op2));
13783
13784 effect(DEF cr, USE op1);
13785
13786 ins_cost(INSN_COST);
13787 format %{ "cmp $op1, $op2" %}
13788
13789 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
13790
13791 ins_pipe(icmp_reg_imm);
13792 %}
13793
13794 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
13795 %{
13796 match(Set cr (CmpUL op1 op2));
13797
13798 effect(DEF cr, USE op1);
13799
13800 ins_cost(INSN_COST * 2);
13801 format %{ "cmp $op1, $op2" %}
13802
13803 ins_encode(aarch64_enc_cmp_imm(op1, op2));
13804
13805 ins_pipe(icmp_reg_imm);
13806 %}
13807
13808 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
13809 %{
13810 match(Set cr (CmpP op1 op2));
13811
13812 effect(DEF cr, USE op1, USE op2);
13813
13814 ins_cost(INSN_COST);
13815 format %{ "cmp $op1, $op2\t // ptr" %}
13816
13817 ins_encode(aarch64_enc_cmpp(op1, op2));
13818
13819 ins_pipe(icmp_reg_reg);
13820 %}
13821
13822 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
13823 %{
13824 match(Set cr (CmpN op1 op2));
13825
13826 effect(DEF cr, USE op1, USE op2);
13827
13828 ins_cost(INSN_COST);
13829 format %{ "cmp $op1, $op2\t // compressed ptr" %}
13830
13831 ins_encode(aarch64_enc_cmpn(op1, op2));
13832
13833 ins_pipe(icmp_reg_reg);
13834 %}
13835
13836 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
13837 %{
13838 match(Set cr (CmpP op1 zero));
13839
13840 effect(DEF cr, USE op1, USE zero);
13841
13842 ins_cost(INSN_COST);
13843 format %{ "cmp $op1, 0\t // ptr" %}
13844
13845 ins_encode(aarch64_enc_testp(op1));
13846
13847 ins_pipe(icmp_reg_imm);
13848 %}
13849
13850 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
13851 %{
13852 match(Set cr (CmpN op1 zero));
13853
13854 effect(DEF cr, USE op1, USE zero);
13855
13856 ins_cost(INSN_COST);
13857 format %{ "cmp $op1, 0\t // compressed ptr" %}
13858
13859 ins_encode(aarch64_enc_testn(op1));
13860
13861 ins_pipe(icmp_reg_imm);
13862 %}
13863
13864 // FP comparisons
13865 //
13866 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
13867 // using normal cmpOp. See declaration of rFlagsReg for details.
13868
13869 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
13870 %{
13871 match(Set cr (CmpF src1 src2));
13872
13873 ins_cost(3 * INSN_COST);
13874 format %{ "fcmps $src1, $src2" %}
13875
13876 ins_encode %{
13877 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
13878 %}
13879
13880 ins_pipe(pipe_class_compare);
13881 %}
13882
13883 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
13884 %{
13885 match(Set cr (CmpF src1 src2));
13886
13887 ins_cost(3 * INSN_COST);
13888 format %{ "fcmps $src1, 0.0" %}
13889
13890 ins_encode %{
13891 __ fcmps(as_FloatRegister($src1$$reg), 0.0D);
13892 %}
13893
13894 ins_pipe(pipe_class_compare);
13895 %}
13896 // FROM HERE
13897
13898 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
13899 %{
13900 match(Set cr (CmpD src1 src2));
13901
13902 ins_cost(3 * INSN_COST);
13903 format %{ "fcmpd $src1, $src2" %}
13904
13905 ins_encode %{
13906 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
13907 %}
13908
13909 ins_pipe(pipe_class_compare);
13910 %}
13911
13912 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
13913 %{
13914 match(Set cr (CmpD src1 src2));
13915
13916 ins_cost(3 * INSN_COST);
13917 format %{ "fcmpd $src1, 0.0" %}
13918
13919 ins_encode %{
13920 __ fcmpd(as_FloatRegister($src1$$reg), 0.0D);
13921 %}
13922
13923 ins_pipe(pipe_class_compare);
13924 %}
13925
13926 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
13927 %{
13928 match(Set dst (CmpF3 src1 src2));
13929 effect(KILL cr);
13930
13931 ins_cost(5 * INSN_COST);
13932 format %{ "fcmps $src1, $src2\n\t"
13933 "csinvw($dst, zr, zr, eq\n\t"
13934 "csnegw($dst, $dst, $dst, lt)"
13935 %}
13936
13937 ins_encode %{
13938 Label done;
13939 FloatRegister s1 = as_FloatRegister($src1$$reg);
13940 FloatRegister s2 = as_FloatRegister($src2$$reg);
13941 Register d = as_Register($dst$$reg);
13942 __ fcmps(s1, s2);
13943 // installs 0 if EQ else -1
13944 __ csinvw(d, zr, zr, Assembler::EQ);
13945 // keeps -1 if less or unordered else installs 1
13946 __ csnegw(d, d, d, Assembler::LT);
13947 __ bind(done);
13948 %}
13949
13950 ins_pipe(pipe_class_default);
13951
13952 %}
13953
13954 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
13955 %{
13956 match(Set dst (CmpD3 src1 src2));
13957 effect(KILL cr);
13958
13959 ins_cost(5 * INSN_COST);
13960 format %{ "fcmpd $src1, $src2\n\t"
13961 "csinvw($dst, zr, zr, eq\n\t"
13962 "csnegw($dst, $dst, $dst, lt)"
13963 %}
13964
13965 ins_encode %{
13966 Label done;
13967 FloatRegister s1 = as_FloatRegister($src1$$reg);
13968 FloatRegister s2 = as_FloatRegister($src2$$reg);
13969 Register d = as_Register($dst$$reg);
13970 __ fcmpd(s1, s2);
13971 // installs 0 if EQ else -1
13972 __ csinvw(d, zr, zr, Assembler::EQ);
13973 // keeps -1 if less or unordered else installs 1
13974 __ csnegw(d, d, d, Assembler::LT);
13975 __ bind(done);
13976 %}
13977 ins_pipe(pipe_class_default);
13978
13979 %}
13980
13981 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
13982 %{
13983 match(Set dst (CmpF3 src1 zero));
13984 effect(KILL cr);
13985
13986 ins_cost(5 * INSN_COST);
13987 format %{ "fcmps $src1, 0.0\n\t"
13988 "csinvw($dst, zr, zr, eq\n\t"
13989 "csnegw($dst, $dst, $dst, lt)"
13990 %}
13991
13992 ins_encode %{
13993 Label done;
13994 FloatRegister s1 = as_FloatRegister($src1$$reg);
13995 Register d = as_Register($dst$$reg);
13996 __ fcmps(s1, 0.0D);
13997 // installs 0 if EQ else -1
13998 __ csinvw(d, zr, zr, Assembler::EQ);
13999 // keeps -1 if less or unordered else installs 1
14000 __ csnegw(d, d, d, Assembler::LT);
14001 __ bind(done);
14002 %}
14003
14004 ins_pipe(pipe_class_default);
14005
14006 %}
14007
14008 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14009 %{
14010 match(Set dst (CmpD3 src1 zero));
14011 effect(KILL cr);
14012
14013 ins_cost(5 * INSN_COST);
14014 format %{ "fcmpd $src1, 0.0\n\t"
14015 "csinvw($dst, zr, zr, eq\n\t"
14016 "csnegw($dst, $dst, $dst, lt)"
14017 %}
14018
14019 ins_encode %{
14020 Label done;
14021 FloatRegister s1 = as_FloatRegister($src1$$reg);
14022 Register d = as_Register($dst$$reg);
14023 __ fcmpd(s1, 0.0D);
14024 // installs 0 if EQ else -1
14025 __ csinvw(d, zr, zr, Assembler::EQ);
14026 // keeps -1 if less or unordered else installs 1
14027 __ csnegw(d, d, d, Assembler::LT);
14028 __ bind(done);
14029 %}
14030 ins_pipe(pipe_class_default);
14031
14032 %}
14033
14034 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14035 %{
14036 match(Set dst (CmpLTMask p q));
14037 effect(KILL cr);
14038
14039 ins_cost(3 * INSN_COST);
14040
14041 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14042 "csetw $dst, lt\n\t"
14043 "subw $dst, zr, $dst"
14044 %}
14045
14046 ins_encode %{
14047 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14048 __ csetw(as_Register($dst$$reg), Assembler::LT);
14049 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14050 %}
14051
14052 ins_pipe(ialu_reg_reg);
14053 %}
14054
14055 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14056 %{
14057 match(Set dst (CmpLTMask src zero));
14058 effect(KILL cr);
14059
14060 ins_cost(INSN_COST);
14061
14062 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14063
14064 ins_encode %{
14065 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14066 %}
14067
14068 ins_pipe(ialu_reg_shift);
14069 %}
14070
14071 // ============================================================================
14072 // Max and Min
14073
14074 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14075 %{
14076 match(Set dst (MinI src1 src2));
14077
14078 effect(DEF dst, USE src1, USE src2, KILL cr);
14079 size(8);
14080
14081 ins_cost(INSN_COST * 3);
14082 format %{
14083 "cmpw $src1 $src2\t signed int\n\t"
14084 "cselw $dst, $src1, $src2 lt\t"
14085 %}
14086
14087 ins_encode %{
14088 __ cmpw(as_Register($src1$$reg),
14089 as_Register($src2$$reg));
14090 __ cselw(as_Register($dst$$reg),
14091 as_Register($src1$$reg),
14092 as_Register($src2$$reg),
14093 Assembler::LT);
14094 %}
14095
14096 ins_pipe(ialu_reg_reg);
14097 %}
14098 // FROM HERE
14099
14100 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14101 %{
14102 match(Set dst (MaxI src1 src2));
14103
14104 effect(DEF dst, USE src1, USE src2, KILL cr);
14105 size(8);
14106
14107 ins_cost(INSN_COST * 3);
14108 format %{
14109 "cmpw $src1 $src2\t signed int\n\t"
14110 "cselw $dst, $src1, $src2 gt\t"
14111 %}
14112
14113 ins_encode %{
14114 __ cmpw(as_Register($src1$$reg),
14115 as_Register($src2$$reg));
14116 __ cselw(as_Register($dst$$reg),
14117 as_Register($src1$$reg),
14118 as_Register($src2$$reg),
14119 Assembler::GT);
14120 %}
14121
14122 ins_pipe(ialu_reg_reg);
14123 %}
14124
14125 // ============================================================================
14126 // Branch Instructions
14127
14128 // Direct Branch.
14129 instruct branch(label lbl)
14130 %{
14131 match(Goto);
14132
14133 effect(USE lbl);
14134
14135 ins_cost(BRANCH_COST);
14136 format %{ "b $lbl" %}
14137
14138 ins_encode(aarch64_enc_b(lbl));
14139
14140 ins_pipe(pipe_branch);
14141 %}
14142
14143 // Conditional Near Branch
14144 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14145 %{
14146 // Same match rule as `branchConFar'.
14147 match(If cmp cr);
14148
14149 effect(USE lbl);
14150
14151 ins_cost(BRANCH_COST);
14152 // If set to 1 this indicates that the current instruction is a
14153 // short variant of a long branch. This avoids using this
14154 // instruction in first-pass matching. It will then only be used in
14155 // the `Shorten_branches' pass.
14156 // ins_short_branch(1);
14157 format %{ "b$cmp $lbl" %}
14158
14159 ins_encode(aarch64_enc_br_con(cmp, lbl));
14160
14161 ins_pipe(pipe_branch_cond);
14162 %}
14163
14164 // Conditional Near Branch Unsigned
14165 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14166 %{
14167 // Same match rule as `branchConFar'.
14168 match(If cmp cr);
14169
14170 effect(USE lbl);
14171
14172 ins_cost(BRANCH_COST);
14173 // If set to 1 this indicates that the current instruction is a
14174 // short variant of a long branch. This avoids using this
14175 // instruction in first-pass matching. It will then only be used in
14176 // the `Shorten_branches' pass.
14177 // ins_short_branch(1);
14178 format %{ "b$cmp $lbl\t# unsigned" %}
14179
14180 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14181
14182 ins_pipe(pipe_branch_cond);
14183 %}
14184
14185 // Make use of CBZ and CBNZ. These instructions, as well as being
14186 // shorter than (cmp; branch), have the additional benefit of not
14187 // killing the flags.
14188
14189 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14190 match(If cmp (CmpI op1 op2));
14191 effect(USE labl);
14192
14193 ins_cost(BRANCH_COST);
14194 format %{ "cbw$cmp $op1, $labl" %}
14195 ins_encode %{
14196 Label* L = $labl$$label;
14197 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14198 if (cond == Assembler::EQ)
14199 __ cbzw($op1$$Register, *L);
14200 else
14201 __ cbnzw($op1$$Register, *L);
14202 %}
14203 ins_pipe(pipe_cmp_branch);
14204 %}
14205
14206 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14207 match(If cmp (CmpL op1 op2));
14208 effect(USE labl);
14209
14210 ins_cost(BRANCH_COST);
14211 format %{ "cb$cmp $op1, $labl" %}
14212 ins_encode %{
14213 Label* L = $labl$$label;
14214 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14215 if (cond == Assembler::EQ)
14216 __ cbz($op1$$Register, *L);
14217 else
14218 __ cbnz($op1$$Register, *L);
14219 %}
14220 ins_pipe(pipe_cmp_branch);
14221 %}
14222
14223 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14224 match(If cmp (CmpP op1 op2));
14225 effect(USE labl);
14226
14227 ins_cost(BRANCH_COST);
14228 format %{ "cb$cmp $op1, $labl" %}
14229 ins_encode %{
14230 Label* L = $labl$$label;
14231 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14232 if (cond == Assembler::EQ)
14233 __ cbz($op1$$Register, *L);
14234 else
14235 __ cbnz($op1$$Register, *L);
14236 %}
14237 ins_pipe(pipe_cmp_branch);
14238 %}
14239
14240 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14241 match(If cmp (CmpN op1 op2));
14242 effect(USE labl);
14243
14244 ins_cost(BRANCH_COST);
14245 format %{ "cbw$cmp $op1, $labl" %}
14246 ins_encode %{
14247 Label* L = $labl$$label;
14248 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14249 if (cond == Assembler::EQ)
14250 __ cbzw($op1$$Register, *L);
14251 else
14252 __ cbnzw($op1$$Register, *L);
14253 %}
14254 ins_pipe(pipe_cmp_branch);
14255 %}
14256
14257 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
14258 match(If cmp (CmpP (DecodeN oop) zero));
14259 effect(USE labl);
14260
14261 ins_cost(BRANCH_COST);
14262 format %{ "cb$cmp $oop, $labl" %}
14263 ins_encode %{
14264 Label* L = $labl$$label;
14265 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14266 if (cond == Assembler::EQ)
14267 __ cbzw($oop$$Register, *L);
14268 else
14269 __ cbnzw($oop$$Register, *L);
14270 %}
14271 ins_pipe(pipe_cmp_branch);
14272 %}
14273
14274 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
14275 match(If cmp (CmpU op1 op2));
14276 effect(USE labl);
14277
14278 ins_cost(BRANCH_COST);
14279 format %{ "cbw$cmp $op1, $labl" %}
14280 ins_encode %{
14281 Label* L = $labl$$label;
14282 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14283 if (cond == Assembler::EQ || cond == Assembler::LS)
14284 __ cbzw($op1$$Register, *L);
14285 else
14286 __ cbnzw($op1$$Register, *L);
14287 %}
14288 ins_pipe(pipe_cmp_branch);
14289 %}
14290
14291 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
14292 match(If cmp (CmpUL op1 op2));
14293 effect(USE labl);
14294
14295 ins_cost(BRANCH_COST);
14296 format %{ "cb$cmp $op1, $labl" %}
14297 ins_encode %{
14298 Label* L = $labl$$label;
14299 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14300 if (cond == Assembler::EQ || cond == Assembler::LS)
14301 __ cbz($op1$$Register, *L);
14302 else
14303 __ cbnz($op1$$Register, *L);
14304 %}
14305 ins_pipe(pipe_cmp_branch);
14306 %}
14307
14308 // Test bit and Branch
14309
14310 // Patterns for short (< 32KiB) variants
14311 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
14312 match(If cmp (CmpL op1 op2));
14313 effect(USE labl);
14314
14315 ins_cost(BRANCH_COST);
14316 format %{ "cb$cmp $op1, $labl # long" %}
14317 ins_encode %{
14318 Label* L = $labl$$label;
14319 Assembler::Condition cond =
14320 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14321 __ tbr(cond, $op1$$Register, 63, *L);
14322 %}
14323 ins_pipe(pipe_cmp_branch);
14324 ins_short_branch(1);
14325 %}
14326
14327 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
14328 match(If cmp (CmpI op1 op2));
14329 effect(USE labl);
14330
14331 ins_cost(BRANCH_COST);
14332 format %{ "cb$cmp $op1, $labl # int" %}
14333 ins_encode %{
14334 Label* L = $labl$$label;
14335 Assembler::Condition cond =
14336 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14337 __ tbr(cond, $op1$$Register, 31, *L);
14338 %}
14339 ins_pipe(pipe_cmp_branch);
14340 ins_short_branch(1);
14341 %}
14342
14343 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
14344 match(If cmp (CmpL (AndL op1 op2) op3));
14345 predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
14346 effect(USE labl);
14347
14348 ins_cost(BRANCH_COST);
14349 format %{ "tb$cmp $op1, $op2, $labl" %}
14350 ins_encode %{
14351 Label* L = $labl$$label;
14352 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14353 int bit = exact_log2($op2$$constant);
14354 __ tbr(cond, $op1$$Register, bit, *L);
14355 %}
14356 ins_pipe(pipe_cmp_branch);
14357 ins_short_branch(1);
14358 %}
14359
14360 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
14361 match(If cmp (CmpI (AndI op1 op2) op3));
14362 predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
14363 effect(USE labl);
14364
14365 ins_cost(BRANCH_COST);
14366 format %{ "tb$cmp $op1, $op2, $labl" %}
14367 ins_encode %{
14368 Label* L = $labl$$label;
14369 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14370 int bit = exact_log2($op2$$constant);
14371 __ tbr(cond, $op1$$Register, bit, *L);
14372 %}
14373 ins_pipe(pipe_cmp_branch);
14374 ins_short_branch(1);
14375 %}
14376
14377 // And far variants
14378 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
14379 match(If cmp (CmpL op1 op2));
14380 effect(USE labl);
14381
14382 ins_cost(BRANCH_COST);
14383 format %{ "cb$cmp $op1, $labl # long" %}
14384 ins_encode %{
14385 Label* L = $labl$$label;
14386 Assembler::Condition cond =
14387 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14388 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
14389 %}
14390 ins_pipe(pipe_cmp_branch);
14391 %}
14392
14393 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
14394 match(If cmp (CmpI op1 op2));
14395 effect(USE labl);
14396
14397 ins_cost(BRANCH_COST);
14398 format %{ "cb$cmp $op1, $labl # int" %}
14399 ins_encode %{
14400 Label* L = $labl$$label;
14401 Assembler::Condition cond =
14402 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14403 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
14404 %}
14405 ins_pipe(pipe_cmp_branch);
14406 %}
14407
14408 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
14409 match(If cmp (CmpL (AndL op1 op2) op3));
14410 predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
14411 effect(USE labl);
14412
14413 ins_cost(BRANCH_COST);
14414 format %{ "tb$cmp $op1, $op2, $labl" %}
14415 ins_encode %{
14416 Label* L = $labl$$label;
14417 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14418 int bit = exact_log2($op2$$constant);
14419 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
14420 %}
14421 ins_pipe(pipe_cmp_branch);
14422 %}
14423
14424 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
14425 match(If cmp (CmpI (AndI op1 op2) op3));
14426 predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
14427 effect(USE labl);
14428
14429 ins_cost(BRANCH_COST);
14430 format %{ "tb$cmp $op1, $op2, $labl" %}
14431 ins_encode %{
14432 Label* L = $labl$$label;
14433 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14434 int bit = exact_log2($op2$$constant);
14435 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
14436 %}
14437 ins_pipe(pipe_cmp_branch);
14438 %}
14439
14440 // Test bits
14441
14442 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
14443 match(Set cr (CmpL (AndL op1 op2) op3));
14444 predicate(Assembler::operand_valid_for_logical_immediate
14445 (/*is_32*/false, n->in(1)->in(2)->get_long()));
14446
14447 ins_cost(INSN_COST);
14448 format %{ "tst $op1, $op2 # long" %}
14449 ins_encode %{
14450 __ tst($op1$$Register, $op2$$constant);
14451 %}
14452 ins_pipe(ialu_reg_reg);
14453 %}
14454
14455 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
14456 match(Set cr (CmpI (AndI op1 op2) op3));
14457 predicate(Assembler::operand_valid_for_logical_immediate
14458 (/*is_32*/true, n->in(1)->in(2)->get_int()));
14459
14460 ins_cost(INSN_COST);
14461 format %{ "tst $op1, $op2 # int" %}
14462 ins_encode %{
14463 __ tstw($op1$$Register, $op2$$constant);
14464 %}
14465 ins_pipe(ialu_reg_reg);
14466 %}
14467
14468 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
14469 match(Set cr (CmpL (AndL op1 op2) op3));
14470
14471 ins_cost(INSN_COST);
14472 format %{ "tst $op1, $op2 # long" %}
14473 ins_encode %{
14474 __ tst($op1$$Register, $op2$$Register);
14475 %}
14476 ins_pipe(ialu_reg_reg);
14477 %}
14478
14479 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
14480 match(Set cr (CmpI (AndI op1 op2) op3));
14481
14482 ins_cost(INSN_COST);
14483 format %{ "tstw $op1, $op2 # int" %}
14484 ins_encode %{
14485 __ tstw($op1$$Register, $op2$$Register);
14486 %}
14487 ins_pipe(ialu_reg_reg);
14488 %}
14489
14490
14491 // Conditional Far Branch
14492 // Conditional Far Branch Unsigned
14493 // TODO: fixme
14494
14495 // counted loop end branch near
14496 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
14497 %{
14498 match(CountedLoopEnd cmp cr);
14499
14500 effect(USE lbl);
14501
14502 ins_cost(BRANCH_COST);
14503 // short variant.
14504 // ins_short_branch(1);
14505 format %{ "b$cmp $lbl \t// counted loop end" %}
14506
14507 ins_encode(aarch64_enc_br_con(cmp, lbl));
14508
14509 ins_pipe(pipe_branch);
14510 %}
14511
14512 // counted loop end branch near Unsigned
14513 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14514 %{
14515 match(CountedLoopEnd cmp cr);
14516
14517 effect(USE lbl);
14518
14519 ins_cost(BRANCH_COST);
14520 // short variant.
14521 // ins_short_branch(1);
14522 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
14523
14524 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14525
14526 ins_pipe(pipe_branch);
14527 %}
14528
14529 // counted loop end branch far
14530 // counted loop end branch far unsigned
14531 // TODO: fixme
14532
14533 // ============================================================================
14534 // inlined locking and unlocking
14535
14536 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
14537 %{
14538 match(Set cr (FastLock object box));
14539 effect(TEMP tmp, TEMP tmp2);
14540
14541 // TODO
14542 // identify correct cost
14543 ins_cost(5 * INSN_COST);
14544 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
14545
14546 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
14547
14548 ins_pipe(pipe_serial);
14549 %}
14550
14551 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
14552 %{
14553 match(Set cr (FastUnlock object box));
14554 effect(TEMP tmp, TEMP tmp2);
14555
14556 ins_cost(5 * INSN_COST);
14557 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
14558
14559 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
14560
14561 ins_pipe(pipe_serial);
14562 %}
14563
14564
14565 // ============================================================================
14566 // Safepoint Instructions
14567
14568 // TODO
14569 // provide a near and far version of this code
14570
14571 instruct safePoint(iRegP poll)
14572 %{
14573 match(SafePoint poll);
14574
14575 format %{
14576 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
14577 %}
14578 ins_encode %{
14579 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
14580 %}
14581 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
14582 %}
14583
14584
14585 // ============================================================================
14586 // Procedure Call/Return Instructions
14587
14588 // Call Java Static Instruction
14589
14590 instruct CallStaticJavaDirect(method meth)
14591 %{
14592 match(CallStaticJava);
14593
14594 effect(USE meth);
14595
14596 ins_cost(CALL_COST);
14597
14598 format %{ "call,static $meth \t// ==> " %}
14599
14600 ins_encode( aarch64_enc_java_static_call(meth),
14601 aarch64_enc_call_epilog );
14602
14603 ins_pipe(pipe_class_call);
14604 %}
14605
14606 // TO HERE
14607
14608 // Call Java Dynamic Instruction
14609 instruct CallDynamicJavaDirect(method meth)
14610 %{
14611 match(CallDynamicJava);
14612
14613 effect(USE meth);
14614
14615 ins_cost(CALL_COST);
14616
14617 format %{ "CALL,dynamic $meth \t// ==> " %}
14618
14619 ins_encode( aarch64_enc_java_dynamic_call(meth),
14620 aarch64_enc_call_epilog );
14621
14622 ins_pipe(pipe_class_call);
14623 %}
14624
14625 // Call Runtime Instruction
14626
14627 instruct CallRuntimeDirect(method meth)
14628 %{
14629 match(CallRuntime);
14630
14631 effect(USE meth);
14632
14633 ins_cost(CALL_COST);
14634
14635 format %{ "CALL, runtime $meth" %}
14636
14637 ins_encode( aarch64_enc_java_to_runtime(meth) );
14638
14639 ins_pipe(pipe_class_call);
14640 %}
14641
14642 // Call Runtime Instruction
14643
14644 instruct CallLeafDirect(method meth)
14645 %{
14646 match(CallLeaf);
14647
14648 effect(USE meth);
14649
14650 ins_cost(CALL_COST);
14651
14652 format %{ "CALL, runtime leaf $meth" %}
14653
14654 ins_encode( aarch64_enc_java_to_runtime(meth) );
14655
14656 ins_pipe(pipe_class_call);
14657 %}
14658
14659 // Call Runtime Instruction
14660
14661 instruct CallLeafNoFPDirect(method meth)
14662 %{
14663 match(CallLeafNoFP);
14664
14665 effect(USE meth);
14666
14667 ins_cost(CALL_COST);
14668
14669 format %{ "CALL, runtime leaf nofp $meth" %}
14670
14671 ins_encode( aarch64_enc_java_to_runtime(meth) );
14672
14673 ins_pipe(pipe_class_call);
14674 %}
14675
14676 // Tail Call; Jump from runtime stub to Java code.
14677 // Also known as an 'interprocedural jump'.
14678 // Target of jump will eventually return to caller.
14679 // TailJump below removes the return address.
14680 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop)
14681 %{
14682 match(TailCall jump_target method_oop);
14683
14684 ins_cost(CALL_COST);
14685
14686 format %{ "br $jump_target\t# $method_oop holds method oop" %}
14687
14688 ins_encode(aarch64_enc_tail_call(jump_target));
14689
14690 ins_pipe(pipe_class_call);
14691 %}
14692
14693 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
14694 %{
14695 match(TailJump jump_target ex_oop);
14696
14697 ins_cost(CALL_COST);
14698
14699 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
14700
14701 ins_encode(aarch64_enc_tail_jmp(jump_target));
14702
14703 ins_pipe(pipe_class_call);
14704 %}
14705
14706 // Create exception oop: created by stack-crawling runtime code.
14707 // Created exception is now available to this handler, and is setup
14708 // just prior to jumping to this handler. No code emitted.
14709 // TODO check
14710 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
14711 instruct CreateException(iRegP_R0 ex_oop)
14712 %{
14713 match(Set ex_oop (CreateEx));
14714
14715 format %{ " -- \t// exception oop; no code emitted" %}
14716
14717 size(0);
14718
14719 ins_encode( /*empty*/ );
14720
14721 ins_pipe(pipe_class_empty);
14722 %}
14723
14724 // Rethrow exception: The exception oop will come in the first
14725 // argument position. Then JUMP (not call) to the rethrow stub code.
14726 instruct RethrowException() %{
14727 match(Rethrow);
14728 ins_cost(CALL_COST);
14729
14730 format %{ "b rethrow_stub" %}
14731
14732 ins_encode( aarch64_enc_rethrow() );
14733
14734 ins_pipe(pipe_class_call);
14735 %}
14736
14737
14738 // Return Instruction
14739 // epilog node loads ret address into lr as part of frame pop
14740 instruct Ret()
14741 %{
14742 match(Return);
14743
14744 format %{ "ret\t// return register" %}
14745
14746 ins_encode( aarch64_enc_ret() );
14747
14748 ins_pipe(pipe_branch);
14749 %}
14750
14751 // Die now.
14752 instruct ShouldNotReachHere() %{
14753 match(Halt);
14754
14755 ins_cost(CALL_COST);
14756 format %{ "ShouldNotReachHere" %}
14757
14758 ins_encode %{
14759 // +1 so NativeInstruction::is_sigill_zombie_not_entrant() doesn't
14760 // return true
14761 __ dpcs1(0xdead + 1);
14762 %}
14763
14764 ins_pipe(pipe_class_default);
14765 %}
14766
14767 // ============================================================================
14768 // Partial Subtype Check
14769 //
14770 // superklass array for an instance of the superklass. Set a hidden
14771 // internal cache on a hit (cache is checked with exposed code in
14772 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
14773 // encoding ALSO sets flags.
14774
14775 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
14776 %{
14777 match(Set result (PartialSubtypeCheck sub super));
14778 effect(KILL cr, KILL temp);
14779
14780 ins_cost(1100); // slightly larger than the next version
14781 format %{ "partialSubtypeCheck $result, $sub, $super" %}
14782
14783 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
14784
14785 opcode(0x1); // Force zero of result reg on hit
14786
14787 ins_pipe(pipe_class_memory);
14788 %}
14789
14790 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
14791 %{
14792 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
14793 effect(KILL temp, KILL result);
14794
14795 ins_cost(1100); // slightly larger than the next version
14796 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
14797
14798 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
14799
14800 opcode(0x0); // Don't zero result reg on hit
14801
14802 ins_pipe(pipe_class_memory);
14803 %}
14804
14805 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
14806 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
14807 %{
14808 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
14809 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14810 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14811
14812 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
14813 ins_encode %{
14814 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
14815 __ string_compare($str1$$Register, $str2$$Register,
14816 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14817 $tmp1$$Register, $tmp2$$Register,
14818 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
14819 %}
14820 ins_pipe(pipe_class_memory);
14821 %}
14822
14823 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
14824 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
14825 %{
14826 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
14827 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14828 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14829
14830 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
14831 ins_encode %{
14832 __ string_compare($str1$$Register, $str2$$Register,
14833 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14834 $tmp1$$Register, $tmp2$$Register,
14835 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
14836 %}
14837 ins_pipe(pipe_class_memory);
14838 %}
14839
14840 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
14841 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
14842 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
14843 %{
14844 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
14845 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14846 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
14847 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14848
14849 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
14850 ins_encode %{
14851 __ string_compare($str1$$Register, $str2$$Register,
14852 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14853 $tmp1$$Register, $tmp2$$Register,
14854 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
14855 $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
14856 %}
14857 ins_pipe(pipe_class_memory);
14858 %}
14859
14860 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
14861 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
14862 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
14863 %{
14864 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
14865 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14866 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
14867 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14868
14869 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
14870 ins_encode %{
14871 __ string_compare($str1$$Register, $str2$$Register,
14872 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14873 $tmp1$$Register, $tmp2$$Register,
14874 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
14875 $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
14876 %}
14877 ins_pipe(pipe_class_memory);
14878 %}
14879
14880 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
14881 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
14882 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
14883 %{
14884 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
14885 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
14886 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
14887 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
14888 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
14889
14890 ins_encode %{
14891 __ string_indexof($str1$$Register, $str2$$Register,
14892 $cnt1$$Register, $cnt2$$Register,
14893 $tmp1$$Register, $tmp2$$Register,
14894 $tmp3$$Register, $tmp4$$Register,
14895 $tmp5$$Register, $tmp6$$Register,
14896 -1, $result$$Register, StrIntrinsicNode::UU);
14897 %}
14898 ins_pipe(pipe_class_memory);
14899 %}
14900
14901 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
14902 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
14903 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
14904 %{
14905 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
14906 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
14907 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
14908 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
14909 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
14910
14911 ins_encode %{
14912 __ string_indexof($str1$$Register, $str2$$Register,
14913 $cnt1$$Register, $cnt2$$Register,
14914 $tmp1$$Register, $tmp2$$Register,
14915 $tmp3$$Register, $tmp4$$Register,
14916 $tmp5$$Register, $tmp6$$Register,
14917 -1, $result$$Register, StrIntrinsicNode::LL);
14918 %}
14919 ins_pipe(pipe_class_memory);
14920 %}
14921
14922 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
14923 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
14924 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
14925 %{
14926 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
14927 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
14928 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
14929 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
14930 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
14931
14932 ins_encode %{
14933 __ string_indexof($str1$$Register, $str2$$Register,
14934 $cnt1$$Register, $cnt2$$Register,
14935 $tmp1$$Register, $tmp2$$Register,
14936 $tmp3$$Register, $tmp4$$Register,
14937 $tmp5$$Register, $tmp6$$Register,
14938 -1, $result$$Register, StrIntrinsicNode::UL);
14939 %}
14940 ins_pipe(pipe_class_memory);
14941 %}
14942
14943 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
14944 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
14945 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
14946 %{
14947 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
14948 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
14949 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
14950 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
14951 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
14952
14953 ins_encode %{
14954 int icnt2 = (int)$int_cnt2$$constant;
14955 __ string_indexof($str1$$Register, $str2$$Register,
14956 $cnt1$$Register, zr,
14957 $tmp1$$Register, $tmp2$$Register,
14958 $tmp3$$Register, $tmp4$$Register, zr, zr,
14959 icnt2, $result$$Register, StrIntrinsicNode::UU);
14960 %}
14961 ins_pipe(pipe_class_memory);
14962 %}
14963
14964 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
14965 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
14966 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
14967 %{
14968 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
14969 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
14970 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
14971 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
14972 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
14973
14974 ins_encode %{
14975 int icnt2 = (int)$int_cnt2$$constant;
14976 __ string_indexof($str1$$Register, $str2$$Register,
14977 $cnt1$$Register, zr,
14978 $tmp1$$Register, $tmp2$$Register,
14979 $tmp3$$Register, $tmp4$$Register, zr, zr,
14980 icnt2, $result$$Register, StrIntrinsicNode::LL);
14981 %}
14982 ins_pipe(pipe_class_memory);
14983 %}
14984
14985 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
14986 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
14987 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
14988 %{
14989 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
14990 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
14991 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
14992 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
14993 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
14994
14995 ins_encode %{
14996 int icnt2 = (int)$int_cnt2$$constant;
14997 __ string_indexof($str1$$Register, $str2$$Register,
14998 $cnt1$$Register, zr,
14999 $tmp1$$Register, $tmp2$$Register,
15000 $tmp3$$Register, $tmp4$$Register, zr, zr,
15001 icnt2, $result$$Register, StrIntrinsicNode::UL);
15002 %}
15003 ins_pipe(pipe_class_memory);
15004 %}
15005
15006 instruct string_indexofU_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15007 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15008 iRegINoSp tmp3, rFlagsReg cr)
15009 %{
15010 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15011 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15012 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15013
15014 format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15015
15016 ins_encode %{
15017 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15018 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15019 $tmp3$$Register);
15020 %}
15021 ins_pipe(pipe_class_memory);
15022 %}
15023
15024 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15025 iRegI_R0 result, rFlagsReg cr)
15026 %{
15027 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
15028 match(Set result (StrEquals (Binary str1 str2) cnt));
15029 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15030
15031 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15032 ins_encode %{
15033 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15034 __ string_equals($str1$$Register, $str2$$Register,
15035 $result$$Register, $cnt$$Register, 1);
15036 %}
15037 ins_pipe(pipe_class_memory);
15038 %}
15039
15040 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15041 iRegI_R0 result, rFlagsReg cr)
15042 %{
15043 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
15044 match(Set result (StrEquals (Binary str1 str2) cnt));
15045 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15046
15047 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15048 ins_encode %{
15049 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15050 __ string_equals($str1$$Register, $str2$$Register,
15051 $result$$Register, $cnt$$Register, 2);
15052 %}
15053 ins_pipe(pipe_class_memory);
15054 %}
15055
15056 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
15057 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
15058 iRegP_R10 tmp, rFlagsReg cr)
15059 %{
15060 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15061 match(Set result (AryEq ary1 ary2));
15062 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15063
15064 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
15065 ins_encode %{
15066 __ arrays_equals($ary1$$Register, $ary2$$Register,
15067 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15068 $result$$Register, $tmp$$Register, 1);
15069 %}
15070 ins_pipe(pipe_class_memory);
15071 %}
15072
15073 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
15074 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
15075 iRegP_R10 tmp, rFlagsReg cr)
15076 %{
15077 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15078 match(Set result (AryEq ary1 ary2));
15079 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15080
15081 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
15082 ins_encode %{
15083 __ arrays_equals($ary1$$Register, $ary2$$Register,
15084 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15085 $result$$Register, $tmp$$Register, 2);
15086 %}
15087 ins_pipe(pipe_class_memory);
15088 %}
15089
15090 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
15091 %{
15092 match(Set result (HasNegatives ary1 len));
15093 effect(USE_KILL ary1, USE_KILL len, KILL cr);
15094 format %{ "has negatives byte[] $ary1,$len -> $result" %}
15095 ins_encode %{
15096 __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
15097 %}
15098 ins_pipe( pipe_slow );
15099 %}
15100
15101 // fast char[] to byte[] compression
15102 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
15103 vRegD_V0 tmp1, vRegD_V1 tmp2,
15104 vRegD_V2 tmp3, vRegD_V3 tmp4,
15105 iRegI_R0 result, rFlagsReg cr)
15106 %{
15107 match(Set result (StrCompressedCopy src (Binary dst len)));
15108 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15109
15110 format %{ "String Compress $src,$dst -> $result // KILL R1, R2, R3, R4" %}
15111 ins_encode %{
15112 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15113 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
15114 $tmp3$$FloatRegister, $tmp4$$FloatRegister,
15115 $result$$Register);
15116 %}
15117 ins_pipe( pipe_slow );
15118 %}
15119
15120 // fast byte[] to char[] inflation
15121 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
15122 vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
15123 %{
15124 match(Set dummy (StrInflatedCopy src (Binary dst len)));
15125 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15126
15127 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
15128 ins_encode %{
15129 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15130 $tmp1$$FloatRegister, $tmp2$$FloatRegister, $tmp3$$FloatRegister, $tmp4$$Register);
15131 %}
15132 ins_pipe(pipe_class_memory);
15133 %}
15134
15135 // encode char[] to byte[] in ISO_8859_1
15136 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
15137 vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
15138 vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
15139 iRegI_R0 result, rFlagsReg cr)
15140 %{
15141 match(Set result (EncodeISOArray src (Binary dst len)));
15142 effect(USE_KILL src, USE_KILL dst, USE_KILL len,
15143 KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
15144
15145 format %{ "Encode array $src,$dst,$len -> $result" %}
15146 ins_encode %{
15147 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15148 $result$$Register, $Vtmp1$$FloatRegister, $Vtmp2$$FloatRegister,
15149 $Vtmp3$$FloatRegister, $Vtmp4$$FloatRegister);
15150 %}
15151 ins_pipe( pipe_class_memory );
15152 %}
15153
15154 // ============================================================================
15155 // This name is KNOWN by the ADLC and cannot be changed.
15156 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
15157 // for this guy.
15158 instruct tlsLoadP(thread_RegP dst)
15159 %{
15160 match(Set dst (ThreadLocal));
15161
15162 ins_cost(0);
15163
15164 format %{ " -- \t// $dst=Thread::current(), empty" %}
15165
15166 size(0);
15167
15168 ins_encode( /*empty*/ );
15169
15170 ins_pipe(pipe_class_empty);
15171 %}
15172
15173 // ====================VECTOR INSTRUCTIONS=====================================
15174
15175 // Load vector (32 bits)
15176 instruct loadV4(vecD dst, vmem4 mem)
15177 %{
15178 predicate(n->as_LoadVector()->memory_size() == 4);
15179 match(Set dst (LoadVector mem));
15180 ins_cost(4 * INSN_COST);
15181 format %{ "ldrs $dst,$mem\t# vector (32 bits)" %}
15182 ins_encode( aarch64_enc_ldrvS(dst, mem) );
15183 ins_pipe(vload_reg_mem64);
15184 %}
15185
15186 // Load vector (64 bits)
15187 instruct loadV8(vecD dst, vmem8 mem)
15188 %{
15189 predicate(n->as_LoadVector()->memory_size() == 8);
15190 match(Set dst (LoadVector mem));
15191 ins_cost(4 * INSN_COST);
15192 format %{ "ldrd $dst,$mem\t# vector (64 bits)" %}
15193 ins_encode( aarch64_enc_ldrvD(dst, mem) );
15194 ins_pipe(vload_reg_mem64);
15195 %}
15196
15197 // Load Vector (128 bits)
15198 instruct loadV16(vecX dst, vmem16 mem)
15199 %{
15200 predicate(n->as_LoadVector()->memory_size() == 16);
15201 match(Set dst (LoadVector mem));
15202 ins_cost(4 * INSN_COST);
15203 format %{ "ldrq $dst,$mem\t# vector (128 bits)" %}
15204 ins_encode( aarch64_enc_ldrvQ(dst, mem) );
15205 ins_pipe(vload_reg_mem128);
15206 %}
15207
15208 // Store Vector (32 bits)
15209 instruct storeV4(vecD src, vmem4 mem)
15210 %{
15211 predicate(n->as_StoreVector()->memory_size() == 4);
15212 match(Set mem (StoreVector mem src));
15213 ins_cost(4 * INSN_COST);
15214 format %{ "strs $mem,$src\t# vector (32 bits)" %}
15215 ins_encode( aarch64_enc_strvS(src, mem) );
15216 ins_pipe(vstore_reg_mem64);
15217 %}
15218
15219 // Store Vector (64 bits)
15220 instruct storeV8(vecD src, vmem8 mem)
15221 %{
15222 predicate(n->as_StoreVector()->memory_size() == 8);
15223 match(Set mem (StoreVector mem src));
15224 ins_cost(4 * INSN_COST);
15225 format %{ "strd $mem,$src\t# vector (64 bits)" %}
15226 ins_encode( aarch64_enc_strvD(src, mem) );
15227 ins_pipe(vstore_reg_mem64);
15228 %}
15229
15230 // Store Vector (128 bits)
15231 instruct storeV16(vecX src, vmem16 mem)
15232 %{
15233 predicate(n->as_StoreVector()->memory_size() == 16);
15234 match(Set mem (StoreVector mem src));
15235 ins_cost(4 * INSN_COST);
15236 format %{ "strq $mem,$src\t# vector (128 bits)" %}
15237 ins_encode( aarch64_enc_strvQ(src, mem) );
15238 ins_pipe(vstore_reg_mem128);
15239 %}
15240
15241 instruct replicate8B(vecD dst, iRegIorL2I src)
15242 %{
15243 predicate(n->as_Vector()->length() == 4 ||
15244 n->as_Vector()->length() == 8);
15245 match(Set dst (ReplicateB src));
15246 ins_cost(INSN_COST);
15247 format %{ "dup $dst, $src\t# vector (8B)" %}
15248 ins_encode %{
15249 __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($src$$reg));
15250 %}
15251 ins_pipe(vdup_reg_reg64);
15252 %}
15253
15254 instruct replicate16B(vecX dst, iRegIorL2I src)
15255 %{
15256 predicate(n->as_Vector()->length() == 16);
15257 match(Set dst (ReplicateB src));
15258 ins_cost(INSN_COST);
15259 format %{ "dup $dst, $src\t# vector (16B)" %}
15260 ins_encode %{
15261 __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($src$$reg));
15262 %}
15263 ins_pipe(vdup_reg_reg128);
15264 %}
15265
15266 instruct replicate8B_imm(vecD dst, immI con)
15267 %{
15268 predicate(n->as_Vector()->length() == 4 ||
15269 n->as_Vector()->length() == 8);
15270 match(Set dst (ReplicateB con));
15271 ins_cost(INSN_COST);
15272 format %{ "movi $dst, $con\t# vector(8B)" %}
15273 ins_encode %{
15274 __ mov(as_FloatRegister($dst$$reg), __ T8B, $con$$constant & 0xff);
15275 %}
15276 ins_pipe(vmovi_reg_imm64);
15277 %}
15278
15279 instruct replicate16B_imm(vecX dst, immI con)
15280 %{
15281 predicate(n->as_Vector()->length() == 16);
15282 match(Set dst (ReplicateB con));
15283 ins_cost(INSN_COST);
15284 format %{ "movi $dst, $con\t# vector(16B)" %}
15285 ins_encode %{
15286 __ mov(as_FloatRegister($dst$$reg), __ T16B, $con$$constant & 0xff);
15287 %}
15288 ins_pipe(vmovi_reg_imm128);
15289 %}
15290
15291 instruct replicate4S(vecD dst, iRegIorL2I src)
15292 %{
15293 predicate(n->as_Vector()->length() == 2 ||
15294 n->as_Vector()->length() == 4);
15295 match(Set dst (ReplicateS src));
15296 ins_cost(INSN_COST);
15297 format %{ "dup $dst, $src\t# vector (4S)" %}
15298 ins_encode %{
15299 __ dup(as_FloatRegister($dst$$reg), __ T4H, as_Register($src$$reg));
15300 %}
15301 ins_pipe(vdup_reg_reg64);
15302 %}
15303
15304 instruct replicate8S(vecX dst, iRegIorL2I src)
15305 %{
15306 predicate(n->as_Vector()->length() == 8);
15307 match(Set dst (ReplicateS src));
15308 ins_cost(INSN_COST);
15309 format %{ "dup $dst, $src\t# vector (8S)" %}
15310 ins_encode %{
15311 __ dup(as_FloatRegister($dst$$reg), __ T8H, as_Register($src$$reg));
15312 %}
15313 ins_pipe(vdup_reg_reg128);
15314 %}
15315
15316 instruct replicate4S_imm(vecD dst, immI con)
15317 %{
15318 predicate(n->as_Vector()->length() == 2 ||
15319 n->as_Vector()->length() == 4);
15320 match(Set dst (ReplicateS con));
15321 ins_cost(INSN_COST);
15322 format %{ "movi $dst, $con\t# vector(4H)" %}
15323 ins_encode %{
15324 __ mov(as_FloatRegister($dst$$reg), __ T4H, $con$$constant & 0xffff);
15325 %}
15326 ins_pipe(vmovi_reg_imm64);
15327 %}
15328
15329 instruct replicate8S_imm(vecX dst, immI con)
15330 %{
15331 predicate(n->as_Vector()->length() == 8);
15332 match(Set dst (ReplicateS con));
15333 ins_cost(INSN_COST);
15334 format %{ "movi $dst, $con\t# vector(8H)" %}
15335 ins_encode %{
15336 __ mov(as_FloatRegister($dst$$reg), __ T8H, $con$$constant & 0xffff);
15337 %}
15338 ins_pipe(vmovi_reg_imm128);
15339 %}
15340
15341 instruct replicate2I(vecD dst, iRegIorL2I src)
15342 %{
15343 predicate(n->as_Vector()->length() == 2);
15344 match(Set dst (ReplicateI src));
15345 ins_cost(INSN_COST);
15346 format %{ "dup $dst, $src\t# vector (2I)" %}
15347 ins_encode %{
15348 __ dup(as_FloatRegister($dst$$reg), __ T2S, as_Register($src$$reg));
15349 %}
15350 ins_pipe(vdup_reg_reg64);
15351 %}
15352
15353 instruct replicate4I(vecX dst, iRegIorL2I src)
15354 %{
15355 predicate(n->as_Vector()->length() == 4);
15356 match(Set dst (ReplicateI src));
15357 ins_cost(INSN_COST);
15358 format %{ "dup $dst, $src\t# vector (4I)" %}
15359 ins_encode %{
15360 __ dup(as_FloatRegister($dst$$reg), __ T4S, as_Register($src$$reg));
15361 %}
15362 ins_pipe(vdup_reg_reg128);
15363 %}
15364
15365 instruct replicate2I_imm(vecD dst, immI con)
15366 %{
15367 predicate(n->as_Vector()->length() == 2);
15368 match(Set dst (ReplicateI con));
15369 ins_cost(INSN_COST);
15370 format %{ "movi $dst, $con\t# vector(2I)" %}
15371 ins_encode %{
15372 __ mov(as_FloatRegister($dst$$reg), __ T2S, $con$$constant);
15373 %}
15374 ins_pipe(vmovi_reg_imm64);
15375 %}
15376
15377 instruct replicate4I_imm(vecX dst, immI con)
15378 %{
15379 predicate(n->as_Vector()->length() == 4);
15380 match(Set dst (ReplicateI con));
15381 ins_cost(INSN_COST);
15382 format %{ "movi $dst, $con\t# vector(4I)" %}
15383 ins_encode %{
15384 __ mov(as_FloatRegister($dst$$reg), __ T4S, $con$$constant);
15385 %}
15386 ins_pipe(vmovi_reg_imm128);
15387 %}
15388
15389 instruct replicate2L(vecX dst, iRegL src)
15390 %{
15391 predicate(n->as_Vector()->length() == 2);
15392 match(Set dst (ReplicateL src));
15393 ins_cost(INSN_COST);
15394 format %{ "dup $dst, $src\t# vector (2L)" %}
15395 ins_encode %{
15396 __ dup(as_FloatRegister($dst$$reg), __ T2D, as_Register($src$$reg));
15397 %}
15398 ins_pipe(vdup_reg_reg128);
15399 %}
15400
15401 instruct replicate2L_zero(vecX dst, immI0 zero)
15402 %{
15403 predicate(n->as_Vector()->length() == 2);
15404 match(Set dst (ReplicateI zero));
15405 ins_cost(INSN_COST);
15406 format %{ "movi $dst, $zero\t# vector(4I)" %}
15407 ins_encode %{
15408 __ eor(as_FloatRegister($dst$$reg), __ T16B,
15409 as_FloatRegister($dst$$reg),
15410 as_FloatRegister($dst$$reg));
15411 %}
15412 ins_pipe(vmovi_reg_imm128);
15413 %}
15414
15415 instruct replicate2F(vecD dst, vRegF src)
15416 %{
15417 predicate(n->as_Vector()->length() == 2);
15418 match(Set dst (ReplicateF src));
15419 ins_cost(INSN_COST);
15420 format %{ "dup $dst, $src\t# vector (2F)" %}
15421 ins_encode %{
15422 __ dup(as_FloatRegister($dst$$reg), __ T2S,
15423 as_FloatRegister($src$$reg));
15424 %}
15425 ins_pipe(vdup_reg_freg64);
15426 %}
15427
15428 instruct replicate4F(vecX dst, vRegF src)
15429 %{
15430 predicate(n->as_Vector()->length() == 4);
15431 match(Set dst (ReplicateF src));
15432 ins_cost(INSN_COST);
15433 format %{ "dup $dst, $src\t# vector (4F)" %}
15434 ins_encode %{
15435 __ dup(as_FloatRegister($dst$$reg), __ T4S,
15436 as_FloatRegister($src$$reg));
15437 %}
15438 ins_pipe(vdup_reg_freg128);
15439 %}
15440
15441 instruct replicate2D(vecX dst, vRegD src)
15442 %{
15443 predicate(n->as_Vector()->length() == 2);
15444 match(Set dst (ReplicateD src));
15445 ins_cost(INSN_COST);
15446 format %{ "dup $dst, $src\t# vector (2D)" %}
15447 ins_encode %{
15448 __ dup(as_FloatRegister($dst$$reg), __ T2D,
15449 as_FloatRegister($src$$reg));
15450 %}
15451 ins_pipe(vdup_reg_dreg128);
15452 %}
15453
15454 // ====================REDUCTION ARITHMETIC====================================
15455
15456 instruct reduce_add2I(iRegINoSp dst, iRegIorL2I src1, vecD src2, iRegINoSp tmp, iRegINoSp tmp2)
15457 %{
15458 match(Set dst (AddReductionVI src1 src2));
15459 ins_cost(INSN_COST);
15460 effect(TEMP tmp, TEMP tmp2);
15461 format %{ "umov $tmp, $src2, S, 0\n\t"
15462 "umov $tmp2, $src2, S, 1\n\t"
15463 "addw $dst, $src1, $tmp\n\t"
15464 "addw $dst, $dst, $tmp2\t add reduction2i"
15465 %}
15466 ins_encode %{
15467 __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 0);
15468 __ umov($tmp2$$Register, as_FloatRegister($src2$$reg), __ S, 1);
15469 __ addw($dst$$Register, $src1$$Register, $tmp$$Register);
15470 __ addw($dst$$Register, $dst$$Register, $tmp2$$Register);
15471 %}
15472 ins_pipe(pipe_class_default);
15473 %}
15474
15475 instruct reduce_add4I(iRegINoSp dst, iRegIorL2I src1, vecX src2, vecX tmp, iRegINoSp tmp2)
15476 %{
15477 match(Set dst (AddReductionVI src1 src2));
15478 ins_cost(INSN_COST);
15479 effect(TEMP tmp, TEMP tmp2);
15480 format %{ "addv $tmp, T4S, $src2\n\t"
15481 "umov $tmp2, $tmp, S, 0\n\t"
15482 "addw $dst, $tmp2, $src1\t add reduction4i"
15483 %}
15484 ins_encode %{
15485 __ addv(as_FloatRegister($tmp$$reg), __ T4S,
15486 as_FloatRegister($src2$$reg));
15487 __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 0);
15488 __ addw($dst$$Register, $tmp2$$Register, $src1$$Register);
15489 %}
15490 ins_pipe(pipe_class_default);
15491 %}
15492
15493 instruct reduce_mul2I(iRegINoSp dst, iRegIorL2I src1, vecD src2, iRegINoSp tmp)
15494 %{
15495 match(Set dst (MulReductionVI src1 src2));
15496 ins_cost(INSN_COST);
15497 effect(TEMP tmp, TEMP dst);
15498 format %{ "umov $tmp, $src2, S, 0\n\t"
15499 "mul $dst, $tmp, $src1\n\t"
15500 "umov $tmp, $src2, S, 1\n\t"
15501 "mul $dst, $tmp, $dst\t mul reduction2i\n\t"
15502 %}
15503 ins_encode %{
15504 __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 0);
15505 __ mul($dst$$Register, $tmp$$Register, $src1$$Register);
15506 __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 1);
15507 __ mul($dst$$Register, $tmp$$Register, $dst$$Register);
15508 %}
15509 ins_pipe(pipe_class_default);
15510 %}
15511
15512 instruct reduce_mul4I(iRegINoSp dst, iRegIorL2I src1, vecX src2, vecX tmp, iRegINoSp tmp2)
15513 %{
15514 match(Set dst (MulReductionVI src1 src2));
15515 ins_cost(INSN_COST);
15516 effect(TEMP tmp, TEMP tmp2, TEMP dst);
15517 format %{ "ins $tmp, $src2, 0, 1\n\t"
15518 "mul $tmp, $tmp, $src2\n\t"
15519 "umov $tmp2, $tmp, S, 0\n\t"
15520 "mul $dst, $tmp2, $src1\n\t"
15521 "umov $tmp2, $tmp, S, 1\n\t"
15522 "mul $dst, $tmp2, $dst\t mul reduction4i\n\t"
15523 %}
15524 ins_encode %{
15525 __ ins(as_FloatRegister($tmp$$reg), __ D,
15526 as_FloatRegister($src2$$reg), 0, 1);
15527 __ mulv(as_FloatRegister($tmp$$reg), __ T2S,
15528 as_FloatRegister($tmp$$reg), as_FloatRegister($src2$$reg));
15529 __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 0);
15530 __ mul($dst$$Register, $tmp2$$Register, $src1$$Register);
15531 __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 1);
15532 __ mul($dst$$Register, $tmp2$$Register, $dst$$Register);
15533 %}
15534 ins_pipe(pipe_class_default);
15535 %}
15536
15537 instruct reduce_add2F(vRegF dst, vRegF src1, vecD src2, vecD tmp)
15538 %{
15539 match(Set dst (AddReductionVF src1 src2));
15540 ins_cost(INSN_COST);
15541 effect(TEMP tmp, TEMP dst);
15542 format %{ "fadds $dst, $src1, $src2\n\t"
15543 "ins $tmp, S, $src2, 0, 1\n\t"
15544 "fadds $dst, $dst, $tmp\t add reduction2f"
15545 %}
15546 ins_encode %{
15547 __ fadds(as_FloatRegister($dst$$reg),
15548 as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15549 __ ins(as_FloatRegister($tmp$$reg), __ S,
15550 as_FloatRegister($src2$$reg), 0, 1);
15551 __ fadds(as_FloatRegister($dst$$reg),
15552 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15553 %}
15554 ins_pipe(pipe_class_default);
15555 %}
15556
15557 instruct reduce_add4F(vRegF dst, vRegF src1, vecX src2, vecX tmp)
15558 %{
15559 match(Set dst (AddReductionVF src1 src2));
15560 ins_cost(INSN_COST);
15561 effect(TEMP tmp, TEMP dst);
15562 format %{ "fadds $dst, $src1, $src2\n\t"
15563 "ins $tmp, S, $src2, 0, 1\n\t"
15564 "fadds $dst, $dst, $tmp\n\t"
15565 "ins $tmp, S, $src2, 0, 2\n\t"
15566 "fadds $dst, $dst, $tmp\n\t"
15567 "ins $tmp, S, $src2, 0, 3\n\t"
15568 "fadds $dst, $dst, $tmp\t add reduction4f"
15569 %}
15570 ins_encode %{
15571 __ fadds(as_FloatRegister($dst$$reg),
15572 as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15573 __ ins(as_FloatRegister($tmp$$reg), __ S,
15574 as_FloatRegister($src2$$reg), 0, 1);
15575 __ fadds(as_FloatRegister($dst$$reg),
15576 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15577 __ ins(as_FloatRegister($tmp$$reg), __ S,
15578 as_FloatRegister($src2$$reg), 0, 2);
15579 __ fadds(as_FloatRegister($dst$$reg),
15580 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15581 __ ins(as_FloatRegister($tmp$$reg), __ S,
15582 as_FloatRegister($src2$$reg), 0, 3);
15583 __ fadds(as_FloatRegister($dst$$reg),
15584 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15585 %}
15586 ins_pipe(pipe_class_default);
15587 %}
15588
15589 instruct reduce_mul2F(vRegF dst, vRegF src1, vecD src2, vecD tmp)
15590 %{
15591 match(Set dst (MulReductionVF src1 src2));
15592 ins_cost(INSN_COST);
15593 effect(TEMP tmp, TEMP dst);
15594 format %{ "fmuls $dst, $src1, $src2\n\t"
15595 "ins $tmp, S, $src2, 0, 1\n\t"
15596 "fmuls $dst, $dst, $tmp\t add reduction4f"
15597 %}
15598 ins_encode %{
15599 __ fmuls(as_FloatRegister($dst$$reg),
15600 as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15601 __ ins(as_FloatRegister($tmp$$reg), __ S,
15602 as_FloatRegister($src2$$reg), 0, 1);
15603 __ fmuls(as_FloatRegister($dst$$reg),
15604 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15605 %}
15606 ins_pipe(pipe_class_default);
15607 %}
15608
15609 instruct reduce_mul4F(vRegF dst, vRegF src1, vecX src2, vecX tmp)
15610 %{
15611 match(Set dst (MulReductionVF src1 src2));
15612 ins_cost(INSN_COST);
15613 effect(TEMP tmp, TEMP dst);
15614 format %{ "fmuls $dst, $src1, $src2\n\t"
15615 "ins $tmp, S, $src2, 0, 1\n\t"
15616 "fmuls $dst, $dst, $tmp\n\t"
15617 "ins $tmp, S, $src2, 0, 2\n\t"
15618 "fmuls $dst, $dst, $tmp\n\t"
15619 "ins $tmp, S, $src2, 0, 3\n\t"
15620 "fmuls $dst, $dst, $tmp\t add reduction4f"
15621 %}
15622 ins_encode %{
15623 __ fmuls(as_FloatRegister($dst$$reg),
15624 as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15625 __ ins(as_FloatRegister($tmp$$reg), __ S,
15626 as_FloatRegister($src2$$reg), 0, 1);
15627 __ fmuls(as_FloatRegister($dst$$reg),
15628 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15629 __ ins(as_FloatRegister($tmp$$reg), __ S,
15630 as_FloatRegister($src2$$reg), 0, 2);
15631 __ fmuls(as_FloatRegister($dst$$reg),
15632 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15633 __ ins(as_FloatRegister($tmp$$reg), __ S,
15634 as_FloatRegister($src2$$reg), 0, 3);
15635 __ fmuls(as_FloatRegister($dst$$reg),
15636 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15637 %}
15638 ins_pipe(pipe_class_default);
15639 %}
15640
15641 instruct reduce_add2D(vRegD dst, vRegD src1, vecX src2, vecX tmp)
15642 %{
15643 match(Set dst (AddReductionVD src1 src2));
15644 ins_cost(INSN_COST);
15645 effect(TEMP tmp, TEMP dst);
15646 format %{ "faddd $dst, $src1, $src2\n\t"
15647 "ins $tmp, D, $src2, 0, 1\n\t"
15648 "faddd $dst, $dst, $tmp\t add reduction2d"
15649 %}
15650 ins_encode %{
15651 __ faddd(as_FloatRegister($dst$$reg),
15652 as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15653 __ ins(as_FloatRegister($tmp$$reg), __ D,
15654 as_FloatRegister($src2$$reg), 0, 1);
15655 __ faddd(as_FloatRegister($dst$$reg),
15656 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15657 %}
15658 ins_pipe(pipe_class_default);
15659 %}
15660
15661 instruct reduce_mul2D(vRegD dst, vRegD src1, vecX src2, vecX tmp)
15662 %{
15663 match(Set dst (MulReductionVD src1 src2));
15664 ins_cost(INSN_COST);
15665 effect(TEMP tmp, TEMP dst);
15666 format %{ "fmuld $dst, $src1, $src2\n\t"
15667 "ins $tmp, D, $src2, 0, 1\n\t"
15668 "fmuld $dst, $dst, $tmp\t add reduction2d"
15669 %}
15670 ins_encode %{
15671 __ fmuld(as_FloatRegister($dst$$reg),
15672 as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15673 __ ins(as_FloatRegister($tmp$$reg), __ D,
15674 as_FloatRegister($src2$$reg), 0, 1);
15675 __ fmuld(as_FloatRegister($dst$$reg),
15676 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15677 %}
15678 ins_pipe(pipe_class_default);
15679 %}
15680
15681 instruct reduce_max2F(vRegF dst, vRegF src1, vecD src2, vecD tmp) %{
15682 predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15683 match(Set dst (MaxReductionV src1 src2));
15684 ins_cost(INSN_COST);
15685 effect(TEMP_DEF dst, TEMP tmp);
15686 format %{ "fmaxs $dst, $src1, $src2\n\t"
15687 "ins $tmp, S, $src2, 0, 1\n\t"
15688 "fmaxs $dst, $dst, $tmp\t max reduction2F" %}
15689 ins_encode %{
15690 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15691 __ ins(as_FloatRegister($tmp$$reg), __ S, as_FloatRegister($src2$$reg), 0, 1);
15692 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15693 %}
15694 ins_pipe(pipe_class_default);
15695 %}
15696
15697 instruct reduce_max4F(vRegF dst, vRegF src1, vecX src2) %{
15698 predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15699 match(Set dst (MaxReductionV src1 src2));
15700 ins_cost(INSN_COST);
15701 effect(TEMP_DEF dst);
15702 format %{ "fmaxv $dst, T4S, $src2\n\t"
15703 "fmaxs $dst, $dst, $src1\t max reduction4F" %}
15704 ins_encode %{
15705 __ fmaxv(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src2$$reg));
15706 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
15707 %}
15708 ins_pipe(pipe_class_default);
15709 %}
15710
15711 instruct reduce_max2D(vRegD dst, vRegD src1, vecX src2, vecX tmp) %{
15712 predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15713 match(Set dst (MaxReductionV src1 src2));
15714 ins_cost(INSN_COST);
15715 effect(TEMP_DEF dst, TEMP tmp);
15716 format %{ "fmaxd $dst, $src1, $src2\n\t"
15717 "ins $tmp, D, $src2, 0, 1\n\t"
15718 "fmaxd $dst, $dst, $tmp\t max reduction2D" %}
15719 ins_encode %{
15720 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15721 __ ins(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($src2$$reg), 0, 1);
15722 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15723 %}
15724 ins_pipe(pipe_class_default);
15725 %}
15726
15727 instruct reduce_min2F(vRegF dst, vRegF src1, vecD src2, vecD tmp) %{
15728 predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15729 match(Set dst (MinReductionV src1 src2));
15730 ins_cost(INSN_COST);
15731 effect(TEMP_DEF dst, TEMP tmp);
15732 format %{ "fmins $dst, $src1, $src2\n\t"
15733 "ins $tmp, S, $src2, 0, 1\n\t"
15734 "fmins $dst, $dst, $tmp\t min reduction2F" %}
15735 ins_encode %{
15736 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15737 __ ins(as_FloatRegister($tmp$$reg), __ S, as_FloatRegister($src2$$reg), 0, 1);
15738 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15739 %}
15740 ins_pipe(pipe_class_default);
15741 %}
15742
15743 instruct reduce_min4F(vRegF dst, vRegF src1, vecX src2) %{
15744 predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15745 match(Set dst (MinReductionV src1 src2));
15746 ins_cost(INSN_COST);
15747 effect(TEMP_DEF dst);
15748 format %{ "fminv $dst, T4S, $src2\n\t"
15749 "fmins $dst, $dst, $src1\t min reduction4F" %}
15750 ins_encode %{
15751 __ fminv(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src2$$reg));
15752 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
15753 %}
15754 ins_pipe(pipe_class_default);
15755 %}
15756
15757 instruct reduce_min2D(vRegD dst, vRegD src1, vecX src2, vecX tmp) %{
15758 predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15759 match(Set dst (MinReductionV src1 src2));
15760 ins_cost(INSN_COST);
15761 effect(TEMP_DEF dst, TEMP tmp);
15762 format %{ "fmind $dst, $src1, $src2\n\t"
15763 "ins $tmp, D, $src2, 0, 1\n\t"
15764 "fmind $dst, $dst, $tmp\t min reduction2D" %}
15765 ins_encode %{
15766 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15767 __ ins(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($src2$$reg), 0, 1);
15768 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15769 %}
15770 ins_pipe(pipe_class_default);
15771 %}
15772
15773 // ====================VECTOR ARITHMETIC=======================================
15774
15775 // --------------------------------- ADD --------------------------------------
15776
15777 instruct vadd8B(vecD dst, vecD src1, vecD src2)
15778 %{
15779 predicate(n->as_Vector()->length() == 4 ||
15780 n->as_Vector()->length() == 8);
15781 match(Set dst (AddVB src1 src2));
15782 ins_cost(INSN_COST);
15783 format %{ "addv $dst,$src1,$src2\t# vector (8B)" %}
15784 ins_encode %{
15785 __ addv(as_FloatRegister($dst$$reg), __ T8B,
15786 as_FloatRegister($src1$$reg),
15787 as_FloatRegister($src2$$reg));
15788 %}
15789 ins_pipe(vdop64);
15790 %}
15791
15792 instruct vadd16B(vecX dst, vecX src1, vecX src2)
15793 %{
15794 predicate(n->as_Vector()->length() == 16);
15795 match(Set dst (AddVB src1 src2));
15796 ins_cost(INSN_COST);
15797 format %{ "addv $dst,$src1,$src2\t# vector (16B)" %}
15798 ins_encode %{
15799 __ addv(as_FloatRegister($dst$$reg), __ T16B,
15800 as_FloatRegister($src1$$reg),
15801 as_FloatRegister($src2$$reg));
15802 %}
15803 ins_pipe(vdop128);
15804 %}
15805
15806 instruct vadd4S(vecD dst, vecD src1, vecD src2)
15807 %{
15808 predicate(n->as_Vector()->length() == 2 ||
15809 n->as_Vector()->length() == 4);
15810 match(Set dst (AddVS src1 src2));
15811 ins_cost(INSN_COST);
15812 format %{ "addv $dst,$src1,$src2\t# vector (4H)" %}
15813 ins_encode %{
15814 __ addv(as_FloatRegister($dst$$reg), __ T4H,
15815 as_FloatRegister($src1$$reg),
15816 as_FloatRegister($src2$$reg));
15817 %}
15818 ins_pipe(vdop64);
15819 %}
15820
15821 instruct vadd8S(vecX dst, vecX src1, vecX src2)
15822 %{
15823 predicate(n->as_Vector()->length() == 8);
15824 match(Set dst (AddVS src1 src2));
15825 ins_cost(INSN_COST);
15826 format %{ "addv $dst,$src1,$src2\t# vector (8H)" %}
15827 ins_encode %{
15828 __ addv(as_FloatRegister($dst$$reg), __ T8H,
15829 as_FloatRegister($src1$$reg),
15830 as_FloatRegister($src2$$reg));
15831 %}
15832 ins_pipe(vdop128);
15833 %}
15834
15835 instruct vadd2I(vecD dst, vecD src1, vecD src2)
15836 %{
15837 predicate(n->as_Vector()->length() == 2);
15838 match(Set dst (AddVI src1 src2));
15839 ins_cost(INSN_COST);
15840 format %{ "addv $dst,$src1,$src2\t# vector (2S)" %}
15841 ins_encode %{
15842 __ addv(as_FloatRegister($dst$$reg), __ T2S,
15843 as_FloatRegister($src1$$reg),
15844 as_FloatRegister($src2$$reg));
15845 %}
15846 ins_pipe(vdop64);
15847 %}
15848
15849 instruct vadd4I(vecX dst, vecX src1, vecX src2)
15850 %{
15851 predicate(n->as_Vector()->length() == 4);
15852 match(Set dst (AddVI src1 src2));
15853 ins_cost(INSN_COST);
15854 format %{ "addv $dst,$src1,$src2\t# vector (4S)" %}
15855 ins_encode %{
15856 __ addv(as_FloatRegister($dst$$reg), __ T4S,
15857 as_FloatRegister($src1$$reg),
15858 as_FloatRegister($src2$$reg));
15859 %}
15860 ins_pipe(vdop128);
15861 %}
15862
15863 instruct vadd2L(vecX dst, vecX src1, vecX src2)
15864 %{
15865 predicate(n->as_Vector()->length() == 2);
15866 match(Set dst (AddVL src1 src2));
15867 ins_cost(INSN_COST);
15868 format %{ "addv $dst,$src1,$src2\t# vector (2L)" %}
15869 ins_encode %{
15870 __ addv(as_FloatRegister($dst$$reg), __ T2D,
15871 as_FloatRegister($src1$$reg),
15872 as_FloatRegister($src2$$reg));
15873 %}
15874 ins_pipe(vdop128);
15875 %}
15876
15877 instruct vadd2F(vecD dst, vecD src1, vecD src2)
15878 %{
15879 predicate(n->as_Vector()->length() == 2);
15880 match(Set dst (AddVF src1 src2));
15881 ins_cost(INSN_COST);
15882 format %{ "fadd $dst,$src1,$src2\t# vector (2S)" %}
15883 ins_encode %{
15884 __ fadd(as_FloatRegister($dst$$reg), __ T2S,
15885 as_FloatRegister($src1$$reg),
15886 as_FloatRegister($src2$$reg));
15887 %}
15888 ins_pipe(vdop_fp64);
15889 %}
15890
15891 instruct vadd4F(vecX dst, vecX src1, vecX src2)
15892 %{
15893 predicate(n->as_Vector()->length() == 4);
15894 match(Set dst (AddVF src1 src2));
15895 ins_cost(INSN_COST);
15896 format %{ "fadd $dst,$src1,$src2\t# vector (4S)" %}
15897 ins_encode %{
15898 __ fadd(as_FloatRegister($dst$$reg), __ T4S,
15899 as_FloatRegister($src1$$reg),
15900 as_FloatRegister($src2$$reg));
15901 %}
15902 ins_pipe(vdop_fp128);
15903 %}
15904
15905 instruct vadd2D(vecX dst, vecX src1, vecX src2)
15906 %{
15907 match(Set dst (AddVD src1 src2));
15908 ins_cost(INSN_COST);
15909 format %{ "fadd $dst,$src1,$src2\t# vector (2D)" %}
15910 ins_encode %{
15911 __ fadd(as_FloatRegister($dst$$reg), __ T2D,
15912 as_FloatRegister($src1$$reg),
15913 as_FloatRegister($src2$$reg));
15914 %}
15915 ins_pipe(vdop_fp128);
15916 %}
15917
15918 // --------------------------------- SUB --------------------------------------
15919
15920 instruct vsub8B(vecD dst, vecD src1, vecD src2)
15921 %{
15922 predicate(n->as_Vector()->length() == 4 ||
15923 n->as_Vector()->length() == 8);
15924 match(Set dst (SubVB src1 src2));
15925 ins_cost(INSN_COST);
15926 format %{ "subv $dst,$src1,$src2\t# vector (8B)" %}
15927 ins_encode %{
15928 __ subv(as_FloatRegister($dst$$reg), __ T8B,
15929 as_FloatRegister($src1$$reg),
15930 as_FloatRegister($src2$$reg));
15931 %}
15932 ins_pipe(vdop64);
15933 %}
15934
15935 instruct vsub16B(vecX dst, vecX src1, vecX src2)
15936 %{
15937 predicate(n->as_Vector()->length() == 16);
15938 match(Set dst (SubVB src1 src2));
15939 ins_cost(INSN_COST);
15940 format %{ "subv $dst,$src1,$src2\t# vector (16B)" %}
15941 ins_encode %{
15942 __ subv(as_FloatRegister($dst$$reg), __ T16B,
15943 as_FloatRegister($src1$$reg),
15944 as_FloatRegister($src2$$reg));
15945 %}
15946 ins_pipe(vdop128);
15947 %}
15948
15949 instruct vsub4S(vecD dst, vecD src1, vecD src2)
15950 %{
15951 predicate(n->as_Vector()->length() == 2 ||
15952 n->as_Vector()->length() == 4);
15953 match(Set dst (SubVS src1 src2));
15954 ins_cost(INSN_COST);
15955 format %{ "subv $dst,$src1,$src2\t# vector (4H)" %}
15956 ins_encode %{
15957 __ subv(as_FloatRegister($dst$$reg), __ T4H,
15958 as_FloatRegister($src1$$reg),
15959 as_FloatRegister($src2$$reg));
15960 %}
15961 ins_pipe(vdop64);
15962 %}
15963
15964 instruct vsub8S(vecX dst, vecX src1, vecX src2)
15965 %{
15966 predicate(n->as_Vector()->length() == 8);
15967 match(Set dst (SubVS src1 src2));
15968 ins_cost(INSN_COST);
15969 format %{ "subv $dst,$src1,$src2\t# vector (8H)" %}
15970 ins_encode %{
15971 __ subv(as_FloatRegister($dst$$reg), __ T8H,
15972 as_FloatRegister($src1$$reg),
15973 as_FloatRegister($src2$$reg));
15974 %}
15975 ins_pipe(vdop128);
15976 %}
15977
15978 instruct vsub2I(vecD dst, vecD src1, vecD src2)
15979 %{
15980 predicate(n->as_Vector()->length() == 2);
15981 match(Set dst (SubVI src1 src2));
15982 ins_cost(INSN_COST);
15983 format %{ "subv $dst,$src1,$src2\t# vector (2S)" %}
15984 ins_encode %{
15985 __ subv(as_FloatRegister($dst$$reg), __ T2S,
15986 as_FloatRegister($src1$$reg),
15987 as_FloatRegister($src2$$reg));
15988 %}
15989 ins_pipe(vdop64);
15990 %}
15991
15992 instruct vsub4I(vecX dst, vecX src1, vecX src2)
15993 %{
15994 predicate(n->as_Vector()->length() == 4);
15995 match(Set dst (SubVI src1 src2));
15996 ins_cost(INSN_COST);
15997 format %{ "subv $dst,$src1,$src2\t# vector (4S)" %}
15998 ins_encode %{
15999 __ subv(as_FloatRegister($dst$$reg), __ T4S,
16000 as_FloatRegister($src1$$reg),
16001 as_FloatRegister($src2$$reg));
16002 %}
16003 ins_pipe(vdop128);
16004 %}
16005
16006 instruct vsub2L(vecX dst, vecX src1, vecX src2)
16007 %{
16008 predicate(n->as_Vector()->length() == 2);
16009 match(Set dst (SubVL src1 src2));
16010 ins_cost(INSN_COST);
16011 format %{ "subv $dst,$src1,$src2\t# vector (2L)" %}
16012 ins_encode %{
16013 __ subv(as_FloatRegister($dst$$reg), __ T2D,
16014 as_FloatRegister($src1$$reg),
16015 as_FloatRegister($src2$$reg));
16016 %}
16017 ins_pipe(vdop128);
16018 %}
16019
16020 instruct vsub2F(vecD dst, vecD src1, vecD src2)
16021 %{
16022 predicate(n->as_Vector()->length() == 2);
16023 match(Set dst (SubVF src1 src2));
16024 ins_cost(INSN_COST);
16025 format %{ "fsub $dst,$src1,$src2\t# vector (2S)" %}
16026 ins_encode %{
16027 __ fsub(as_FloatRegister($dst$$reg), __ T2S,
16028 as_FloatRegister($src1$$reg),
16029 as_FloatRegister($src2$$reg));
16030 %}
16031 ins_pipe(vdop_fp64);
16032 %}
16033
16034 instruct vsub4F(vecX dst, vecX src1, vecX src2)
16035 %{
16036 predicate(n->as_Vector()->length() == 4);
16037 match(Set dst (SubVF src1 src2));
16038 ins_cost(INSN_COST);
16039 format %{ "fsub $dst,$src1,$src2\t# vector (4S)" %}
16040 ins_encode %{
16041 __ fsub(as_FloatRegister($dst$$reg), __ T4S,
16042 as_FloatRegister($src1$$reg),
16043 as_FloatRegister($src2$$reg));
16044 %}
16045 ins_pipe(vdop_fp128);
16046 %}
16047
16048 instruct vsub2D(vecX dst, vecX src1, vecX src2)
16049 %{
16050 predicate(n->as_Vector()->length() == 2);
16051 match(Set dst (SubVD src1 src2));
16052 ins_cost(INSN_COST);
16053 format %{ "fsub $dst,$src1,$src2\t# vector (2D)" %}
16054 ins_encode %{
16055 __ fsub(as_FloatRegister($dst$$reg), __ T2D,
16056 as_FloatRegister($src1$$reg),
16057 as_FloatRegister($src2$$reg));
16058 %}
16059 ins_pipe(vdop_fp128);
16060 %}
16061
16062 // --------------------------------- MUL --------------------------------------
16063
16064 instruct vmul4S(vecD dst, vecD src1, vecD src2)
16065 %{
16066 predicate(n->as_Vector()->length() == 2 ||
16067 n->as_Vector()->length() == 4);
16068 match(Set dst (MulVS src1 src2));
16069 ins_cost(INSN_COST);
16070 format %{ "mulv $dst,$src1,$src2\t# vector (4H)" %}
16071 ins_encode %{
16072 __ mulv(as_FloatRegister($dst$$reg), __ T4H,
16073 as_FloatRegister($src1$$reg),
16074 as_FloatRegister($src2$$reg));
16075 %}
16076 ins_pipe(vmul64);
16077 %}
16078
16079 instruct vmul8S(vecX dst, vecX src1, vecX src2)
16080 %{
16081 predicate(n->as_Vector()->length() == 8);
16082 match(Set dst (MulVS src1 src2));
16083 ins_cost(INSN_COST);
16084 format %{ "mulv $dst,$src1,$src2\t# vector (8H)" %}
16085 ins_encode %{
16086 __ mulv(as_FloatRegister($dst$$reg), __ T8H,
16087 as_FloatRegister($src1$$reg),
16088 as_FloatRegister($src2$$reg));
16089 %}
16090 ins_pipe(vmul128);
16091 %}
16092
16093 instruct vmul2I(vecD dst, vecD src1, vecD src2)
16094 %{
16095 predicate(n->as_Vector()->length() == 2);
16096 match(Set dst (MulVI src1 src2));
16097 ins_cost(INSN_COST);
16098 format %{ "mulv $dst,$src1,$src2\t# vector (2S)" %}
16099 ins_encode %{
16100 __ mulv(as_FloatRegister($dst$$reg), __ T2S,
16101 as_FloatRegister($src1$$reg),
16102 as_FloatRegister($src2$$reg));
16103 %}
16104 ins_pipe(vmul64);
16105 %}
16106
16107 instruct vmul4I(vecX dst, vecX src1, vecX src2)
16108 %{
16109 predicate(n->as_Vector()->length() == 4);
16110 match(Set dst (MulVI src1 src2));
16111 ins_cost(INSN_COST);
16112 format %{ "mulv $dst,$src1,$src2\t# vector (4S)" %}
16113 ins_encode %{
16114 __ mulv(as_FloatRegister($dst$$reg), __ T4S,
16115 as_FloatRegister($src1$$reg),
16116 as_FloatRegister($src2$$reg));
16117 %}
16118 ins_pipe(vmul128);
16119 %}
16120
16121 instruct vmul2F(vecD dst, vecD src1, vecD src2)
16122 %{
16123 predicate(n->as_Vector()->length() == 2);
16124 match(Set dst (MulVF src1 src2));
16125 ins_cost(INSN_COST);
16126 format %{ "fmul $dst,$src1,$src2\t# vector (2S)" %}
16127 ins_encode %{
16128 __ fmul(as_FloatRegister($dst$$reg), __ T2S,
16129 as_FloatRegister($src1$$reg),
16130 as_FloatRegister($src2$$reg));
16131 %}
16132 ins_pipe(vmuldiv_fp64);
16133 %}
16134
16135 instruct vmul4F(vecX dst, vecX src1, vecX src2)
16136 %{
16137 predicate(n->as_Vector()->length() == 4);
16138 match(Set dst (MulVF src1 src2));
16139 ins_cost(INSN_COST);
16140 format %{ "fmul $dst,$src1,$src2\t# vector (4S)" %}
16141 ins_encode %{
16142 __ fmul(as_FloatRegister($dst$$reg), __ T4S,
16143 as_FloatRegister($src1$$reg),
16144 as_FloatRegister($src2$$reg));
16145 %}
16146 ins_pipe(vmuldiv_fp128);
16147 %}
16148
16149 instruct vmul2D(vecX dst, vecX src1, vecX src2)
16150 %{
16151 predicate(n->as_Vector()->length() == 2);
16152 match(Set dst (MulVD src1 src2));
16153 ins_cost(INSN_COST);
16154 format %{ "fmul $dst,$src1,$src2\t# vector (2D)" %}
16155 ins_encode %{
16156 __ fmul(as_FloatRegister($dst$$reg), __ T2D,
16157 as_FloatRegister($src1$$reg),
16158 as_FloatRegister($src2$$reg));
16159 %}
16160 ins_pipe(vmuldiv_fp128);
16161 %}
16162
16163 // --------------------------------- MLA --------------------------------------
16164
16165 instruct vmla4S(vecD dst, vecD src1, vecD src2)
16166 %{
16167 predicate(n->as_Vector()->length() == 2 ||
16168 n->as_Vector()->length() == 4);
16169 match(Set dst (AddVS dst (MulVS src1 src2)));
16170 ins_cost(INSN_COST);
16171 format %{ "mlav $dst,$src1,$src2\t# vector (4H)" %}
16172 ins_encode %{
16173 __ mlav(as_FloatRegister($dst$$reg), __ T4H,
16174 as_FloatRegister($src1$$reg),
16175 as_FloatRegister($src2$$reg));
16176 %}
16177 ins_pipe(vmla64);
16178 %}
16179
16180 instruct vmla8S(vecX dst, vecX src1, vecX src2)
16181 %{
16182 predicate(n->as_Vector()->length() == 8);
16183 match(Set dst (AddVS dst (MulVS src1 src2)));
16184 ins_cost(INSN_COST);
16185 format %{ "mlav $dst,$src1,$src2\t# vector (8H)" %}
16186 ins_encode %{
16187 __ mlav(as_FloatRegister($dst$$reg), __ T8H,
16188 as_FloatRegister($src1$$reg),
16189 as_FloatRegister($src2$$reg));
16190 %}
16191 ins_pipe(vmla128);
16192 %}
16193
16194 instruct vmla2I(vecD dst, vecD src1, vecD src2)
16195 %{
16196 predicate(n->as_Vector()->length() == 2);
16197 match(Set dst (AddVI dst (MulVI src1 src2)));
16198 ins_cost(INSN_COST);
16199 format %{ "mlav $dst,$src1,$src2\t# vector (2S)" %}
16200 ins_encode %{
16201 __ mlav(as_FloatRegister($dst$$reg), __ T2S,
16202 as_FloatRegister($src1$$reg),
16203 as_FloatRegister($src2$$reg));
16204 %}
16205 ins_pipe(vmla64);
16206 %}
16207
16208 instruct vmla4I(vecX dst, vecX src1, vecX src2)
16209 %{
16210 predicate(n->as_Vector()->length() == 4);
16211 match(Set dst (AddVI dst (MulVI src1 src2)));
16212 ins_cost(INSN_COST);
16213 format %{ "mlav $dst,$src1,$src2\t# vector (4S)" %}
16214 ins_encode %{
16215 __ mlav(as_FloatRegister($dst$$reg), __ T4S,
16216 as_FloatRegister($src1$$reg),
16217 as_FloatRegister($src2$$reg));
16218 %}
16219 ins_pipe(vmla128);
16220 %}
16221
16222 // dst + src1 * src2
16223 instruct vmla2F(vecD dst, vecD src1, vecD src2) %{
16224 predicate(UseFMA && n->as_Vector()->length() == 2);
16225 match(Set dst (FmaVF dst (Binary src1 src2)));
16226 format %{ "fmla $dst,$src1,$src2\t# vector (2S)" %}
16227 ins_cost(INSN_COST);
16228 ins_encode %{
16229 __ fmla(as_FloatRegister($dst$$reg), __ T2S,
16230 as_FloatRegister($src1$$reg),
16231 as_FloatRegister($src2$$reg));
16232 %}
16233 ins_pipe(vmuldiv_fp64);
16234 %}
16235
16236 // dst + src1 * src2
16237 instruct vmla4F(vecX dst, vecX src1, vecX src2) %{
16238 predicate(UseFMA && n->as_Vector()->length() == 4);
16239 match(Set dst (FmaVF dst (Binary src1 src2)));
16240 format %{ "fmla $dst,$src1,$src2\t# vector (4S)" %}
16241 ins_cost(INSN_COST);
16242 ins_encode %{
16243 __ fmla(as_FloatRegister($dst$$reg), __ T4S,
16244 as_FloatRegister($src1$$reg),
16245 as_FloatRegister($src2$$reg));
16246 %}
16247 ins_pipe(vmuldiv_fp128);
16248 %}
16249
16250 // dst + src1 * src2
16251 instruct vmla2D(vecX dst, vecX src1, vecX src2) %{
16252 predicate(UseFMA && n->as_Vector()->length() == 2);
16253 match(Set dst (FmaVD dst (Binary src1 src2)));
16254 format %{ "fmla $dst,$src1,$src2\t# vector (2D)" %}
16255 ins_cost(INSN_COST);
16256 ins_encode %{
16257 __ fmla(as_FloatRegister($dst$$reg), __ T2D,
16258 as_FloatRegister($src1$$reg),
16259 as_FloatRegister($src2$$reg));
16260 %}
16261 ins_pipe(vmuldiv_fp128);
16262 %}
16263
16264 // --------------------------------- MLS --------------------------------------
16265
16266 instruct vmls4S(vecD dst, vecD src1, vecD src2)
16267 %{
16268 predicate(n->as_Vector()->length() == 2 ||
16269 n->as_Vector()->length() == 4);
16270 match(Set dst (SubVS dst (MulVS src1 src2)));
16271 ins_cost(INSN_COST);
16272 format %{ "mlsv $dst,$src1,$src2\t# vector (4H)" %}
16273 ins_encode %{
16274 __ mlsv(as_FloatRegister($dst$$reg), __ T4H,
16275 as_FloatRegister($src1$$reg),
16276 as_FloatRegister($src2$$reg));
16277 %}
16278 ins_pipe(vmla64);
16279 %}
16280
16281 instruct vmls8S(vecX dst, vecX src1, vecX src2)
16282 %{
16283 predicate(n->as_Vector()->length() == 8);
16284 match(Set dst (SubVS dst (MulVS src1 src2)));
16285 ins_cost(INSN_COST);
16286 format %{ "mlsv $dst,$src1,$src2\t# vector (8H)" %}
16287 ins_encode %{
16288 __ mlsv(as_FloatRegister($dst$$reg), __ T8H,
16289 as_FloatRegister($src1$$reg),
16290 as_FloatRegister($src2$$reg));
16291 %}
16292 ins_pipe(vmla128);
16293 %}
16294
16295 instruct vmls2I(vecD dst, vecD src1, vecD src2)
16296 %{
16297 predicate(n->as_Vector()->length() == 2);
16298 match(Set dst (SubVI dst (MulVI src1 src2)));
16299 ins_cost(INSN_COST);
16300 format %{ "mlsv $dst,$src1,$src2\t# vector (2S)" %}
16301 ins_encode %{
16302 __ mlsv(as_FloatRegister($dst$$reg), __ T2S,
16303 as_FloatRegister($src1$$reg),
16304 as_FloatRegister($src2$$reg));
16305 %}
16306 ins_pipe(vmla64);
16307 %}
16308
16309 instruct vmls4I(vecX dst, vecX src1, vecX src2)
16310 %{
16311 predicate(n->as_Vector()->length() == 4);
16312 match(Set dst (SubVI dst (MulVI src1 src2)));
16313 ins_cost(INSN_COST);
16314 format %{ "mlsv $dst,$src1,$src2\t# vector (4S)" %}
16315 ins_encode %{
16316 __ mlsv(as_FloatRegister($dst$$reg), __ T4S,
16317 as_FloatRegister($src1$$reg),
16318 as_FloatRegister($src2$$reg));
16319 %}
16320 ins_pipe(vmla128);
16321 %}
16322
16323 // dst - src1 * src2
16324 instruct vmls2F(vecD dst, vecD src1, vecD src2) %{
16325 predicate(UseFMA && n->as_Vector()->length() == 2);
16326 match(Set dst (FmaVF dst (Binary (NegVF src1) src2)));
16327 match(Set dst (FmaVF dst (Binary src1 (NegVF src2))));
16328 format %{ "fmls $dst,$src1,$src2\t# vector (2S)" %}
16329 ins_cost(INSN_COST);
16330 ins_encode %{
16331 __ fmls(as_FloatRegister($dst$$reg), __ T2S,
16332 as_FloatRegister($src1$$reg),
16333 as_FloatRegister($src2$$reg));
16334 %}
16335 ins_pipe(vmuldiv_fp64);
16336 %}
16337
16338 // dst - src1 * src2
16339 instruct vmls4F(vecX dst, vecX src1, vecX src2) %{
16340 predicate(UseFMA && n->as_Vector()->length() == 4);
16341 match(Set dst (FmaVF dst (Binary (NegVF src1) src2)));
16342 match(Set dst (FmaVF dst (Binary src1 (NegVF src2))));
16343 format %{ "fmls $dst,$src1,$src2\t# vector (4S)" %}
16344 ins_cost(INSN_COST);
16345 ins_encode %{
16346 __ fmls(as_FloatRegister($dst$$reg), __ T4S,
16347 as_FloatRegister($src1$$reg),
16348 as_FloatRegister($src2$$reg));
16349 %}
16350 ins_pipe(vmuldiv_fp128);
16351 %}
16352
16353 // dst - src1 * src2
16354 instruct vmls2D(vecX dst, vecX src1, vecX src2) %{
16355 predicate(UseFMA && n->as_Vector()->length() == 2);
16356 match(Set dst (FmaVD dst (Binary (NegVD src1) src2)));
16357 match(Set dst (FmaVD dst (Binary src1 (NegVD src2))));
16358 format %{ "fmls $dst,$src1,$src2\t# vector (2D)" %}
16359 ins_cost(INSN_COST);
16360 ins_encode %{
16361 __ fmls(as_FloatRegister($dst$$reg), __ T2D,
16362 as_FloatRegister($src1$$reg),
16363 as_FloatRegister($src2$$reg));
16364 %}
16365 ins_pipe(vmuldiv_fp128);
16366 %}
16367
16368 // --------------------------------- DIV --------------------------------------
16369
16370 instruct vdiv2F(vecD dst, vecD src1, vecD src2)
16371 %{
16372 predicate(n->as_Vector()->length() == 2);
16373 match(Set dst (DivVF src1 src2));
16374 ins_cost(INSN_COST);
16375 format %{ "fdiv $dst,$src1,$src2\t# vector (2S)" %}
16376 ins_encode %{
16377 __ fdiv(as_FloatRegister($dst$$reg), __ T2S,
16378 as_FloatRegister($src1$$reg),
16379 as_FloatRegister($src2$$reg));
16380 %}
16381 ins_pipe(vmuldiv_fp64);
16382 %}
16383
16384 instruct vdiv4F(vecX dst, vecX src1, vecX src2)
16385 %{
16386 predicate(n->as_Vector()->length() == 4);
16387 match(Set dst (DivVF src1 src2));
16388 ins_cost(INSN_COST);
16389 format %{ "fdiv $dst,$src1,$src2\t# vector (4S)" %}
16390 ins_encode %{
16391 __ fdiv(as_FloatRegister($dst$$reg), __ T4S,
16392 as_FloatRegister($src1$$reg),
16393 as_FloatRegister($src2$$reg));
16394 %}
16395 ins_pipe(vmuldiv_fp128);
16396 %}
16397
16398 instruct vdiv2D(vecX dst, vecX src1, vecX src2)
16399 %{
16400 predicate(n->as_Vector()->length() == 2);
16401 match(Set dst (DivVD src1 src2));
16402 ins_cost(INSN_COST);
16403 format %{ "fdiv $dst,$src1,$src2\t# vector (2D)" %}
16404 ins_encode %{
16405 __ fdiv(as_FloatRegister($dst$$reg), __ T2D,
16406 as_FloatRegister($src1$$reg),
16407 as_FloatRegister($src2$$reg));
16408 %}
16409 ins_pipe(vmuldiv_fp128);
16410 %}
16411
16412 // --------------------------------- SQRT -------------------------------------
16413
16414 instruct vsqrt2D(vecX dst, vecX src)
16415 %{
16416 predicate(n->as_Vector()->length() == 2);
16417 match(Set dst (SqrtVD src));
16418 format %{ "fsqrt $dst, $src\t# vector (2D)" %}
16419 ins_encode %{
16420 __ fsqrt(as_FloatRegister($dst$$reg), __ T2D,
16421 as_FloatRegister($src$$reg));
16422 %}
16423 ins_pipe(vsqrt_fp128);
16424 %}
16425
16426 // --------------------------------- ABS --------------------------------------
16427
16428 instruct vabs2F(vecD dst, vecD src)
16429 %{
16430 predicate(n->as_Vector()->length() == 2);
16431 match(Set dst (AbsVF src));
16432 ins_cost(INSN_COST * 3);
16433 format %{ "fabs $dst,$src\t# vector (2S)" %}
16434 ins_encode %{
16435 __ fabs(as_FloatRegister($dst$$reg), __ T2S,
16436 as_FloatRegister($src$$reg));
16437 %}
16438 ins_pipe(vunop_fp64);
16439 %}
16440
16441 instruct vabs4F(vecX dst, vecX src)
16442 %{
16443 predicate(n->as_Vector()->length() == 4);
16444 match(Set dst (AbsVF src));
16445 ins_cost(INSN_COST * 3);
16446 format %{ "fabs $dst,$src\t# vector (4S)" %}
16447 ins_encode %{
16448 __ fabs(as_FloatRegister($dst$$reg), __ T4S,
16449 as_FloatRegister($src$$reg));
16450 %}
16451 ins_pipe(vunop_fp128);
16452 %}
16453
16454 instruct vabs2D(vecX dst, vecX src)
16455 %{
16456 predicate(n->as_Vector()->length() == 2);
16457 match(Set dst (AbsVD src));
16458 ins_cost(INSN_COST * 3);
16459 format %{ "fabs $dst,$src\t# vector (2D)" %}
16460 ins_encode %{
16461 __ fabs(as_FloatRegister($dst$$reg), __ T2D,
16462 as_FloatRegister($src$$reg));
16463 %}
16464 ins_pipe(vunop_fp128);
16465 %}
16466
16467 // --------------------------------- NEG --------------------------------------
16468
16469 instruct vneg2F(vecD dst, vecD src)
16470 %{
16471 predicate(n->as_Vector()->length() == 2);
16472 match(Set dst (NegVF src));
16473 ins_cost(INSN_COST * 3);
16474 format %{ "fneg $dst,$src\t# vector (2S)" %}
16475 ins_encode %{
16476 __ fneg(as_FloatRegister($dst$$reg), __ T2S,
16477 as_FloatRegister($src$$reg));
16478 %}
16479 ins_pipe(vunop_fp64);
16480 %}
16481
16482 instruct vneg4F(vecX dst, vecX src)
16483 %{
16484 predicate(n->as_Vector()->length() == 4);
16485 match(Set dst (NegVF src));
16486 ins_cost(INSN_COST * 3);
16487 format %{ "fneg $dst,$src\t# vector (4S)" %}
16488 ins_encode %{
16489 __ fneg(as_FloatRegister($dst$$reg), __ T4S,
16490 as_FloatRegister($src$$reg));
16491 %}
16492 ins_pipe(vunop_fp128);
16493 %}
16494
16495 instruct vneg2D(vecX dst, vecX src)
16496 %{
16497 predicate(n->as_Vector()->length() == 2);
16498 match(Set dst (NegVD src));
16499 ins_cost(INSN_COST * 3);
16500 format %{ "fneg $dst,$src\t# vector (2D)" %}
16501 ins_encode %{
16502 __ fneg(as_FloatRegister($dst$$reg), __ T2D,
16503 as_FloatRegister($src$$reg));
16504 %}
16505 ins_pipe(vunop_fp128);
16506 %}
16507
16508 // --------------------------------- AND --------------------------------------
16509
16510 instruct vand8B(vecD dst, vecD src1, vecD src2)
16511 %{
16512 predicate(n->as_Vector()->length_in_bytes() == 4 ||
16513 n->as_Vector()->length_in_bytes() == 8);
16514 match(Set dst (AndV src1 src2));
16515 ins_cost(INSN_COST);
16516 format %{ "and $dst,$src1,$src2\t# vector (8B)" %}
16517 ins_encode %{
16518 __ andr(as_FloatRegister($dst$$reg), __ T8B,
16519 as_FloatRegister($src1$$reg),
16520 as_FloatRegister($src2$$reg));
16521 %}
16522 ins_pipe(vlogical64);
16523 %}
16524
16525 instruct vand16B(vecX dst, vecX src1, vecX src2)
16526 %{
16527 predicate(n->as_Vector()->length_in_bytes() == 16);
16528 match(Set dst (AndV src1 src2));
16529 ins_cost(INSN_COST);
16530 format %{ "and $dst,$src1,$src2\t# vector (16B)" %}
16531 ins_encode %{
16532 __ andr(as_FloatRegister($dst$$reg), __ T16B,
16533 as_FloatRegister($src1$$reg),
16534 as_FloatRegister($src2$$reg));
16535 %}
16536 ins_pipe(vlogical128);
16537 %}
16538
16539 // --------------------------------- OR ---------------------------------------
16540
16541 instruct vor8B(vecD dst, vecD src1, vecD src2)
16542 %{
16543 predicate(n->as_Vector()->length_in_bytes() == 4 ||
16544 n->as_Vector()->length_in_bytes() == 8);
16545 match(Set dst (OrV src1 src2));
16546 ins_cost(INSN_COST);
16547 format %{ "and $dst,$src1,$src2\t# vector (8B)" %}
16548 ins_encode %{
16549 __ orr(as_FloatRegister($dst$$reg), __ T8B,
16550 as_FloatRegister($src1$$reg),
16551 as_FloatRegister($src2$$reg));
16552 %}
16553 ins_pipe(vlogical64);
16554 %}
16555
16556 instruct vor16B(vecX dst, vecX src1, vecX src2)
16557 %{
16558 predicate(n->as_Vector()->length_in_bytes() == 16);
16559 match(Set dst (OrV src1 src2));
16560 ins_cost(INSN_COST);
16561 format %{ "orr $dst,$src1,$src2\t# vector (16B)" %}
16562 ins_encode %{
16563 __ orr(as_FloatRegister($dst$$reg), __ T16B,
16564 as_FloatRegister($src1$$reg),
16565 as_FloatRegister($src2$$reg));
16566 %}
16567 ins_pipe(vlogical128);
16568 %}
16569
16570 // --------------------------------- XOR --------------------------------------
16571
16572 instruct vxor8B(vecD dst, vecD src1, vecD src2)
16573 %{
16574 predicate(n->as_Vector()->length_in_bytes() == 4 ||
16575 n->as_Vector()->length_in_bytes() == 8);
16576 match(Set dst (XorV src1 src2));
16577 ins_cost(INSN_COST);
16578 format %{ "xor $dst,$src1,$src2\t# vector (8B)" %}
16579 ins_encode %{
16580 __ eor(as_FloatRegister($dst$$reg), __ T8B,
16581 as_FloatRegister($src1$$reg),
16582 as_FloatRegister($src2$$reg));
16583 %}
16584 ins_pipe(vlogical64);
16585 %}
16586
16587 instruct vxor16B(vecX dst, vecX src1, vecX src2)
16588 %{
16589 predicate(n->as_Vector()->length_in_bytes() == 16);
16590 match(Set dst (XorV src1 src2));
16591 ins_cost(INSN_COST);
16592 format %{ "xor $dst,$src1,$src2\t# vector (16B)" %}
16593 ins_encode %{
16594 __ eor(as_FloatRegister($dst$$reg), __ T16B,
16595 as_FloatRegister($src1$$reg),
16596 as_FloatRegister($src2$$reg));
16597 %}
16598 ins_pipe(vlogical128);
16599 %}
16600
16601 // ------------------------------ Shift ---------------------------------------
16602 instruct vshiftcnt8B(vecD dst, iRegIorL2I cnt) %{
16603 predicate(n->as_Vector()->length_in_bytes() == 8);
16604 match(Set dst (LShiftCntV cnt));
16605 match(Set dst (RShiftCntV cnt));
16606 format %{ "dup $dst, $cnt\t# shift count vector (8B)" %}
16607 ins_encode %{
16608 __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($cnt$$reg));
16609 %}
16610 ins_pipe(vdup_reg_reg64);
16611 %}
16612
16613 instruct vshiftcnt16B(vecX dst, iRegIorL2I cnt) %{
16614 predicate(n->as_Vector()->length_in_bytes() == 16);
16615 match(Set dst (LShiftCntV cnt));
16616 match(Set dst (RShiftCntV cnt));
16617 format %{ "dup $dst, $cnt\t# shift count vector (16B)" %}
16618 ins_encode %{
16619 __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($cnt$$reg));
16620 %}
16621 ins_pipe(vdup_reg_reg128);
16622 %}
16623
16624 instruct vsll8B(vecD dst, vecD src, vecD shift) %{
16625 predicate(n->as_Vector()->length() == 4 ||
16626 n->as_Vector()->length() == 8);
16627 match(Set dst (LShiftVB src shift));
16628 ins_cost(INSN_COST);
16629 format %{ "sshl $dst,$src,$shift\t# vector (8B)" %}
16630 ins_encode %{
16631 __ sshl(as_FloatRegister($dst$$reg), __ T8B,
16632 as_FloatRegister($src$$reg),
16633 as_FloatRegister($shift$$reg));
16634 %}
16635 ins_pipe(vshift64);
16636 %}
16637
16638 instruct vsll16B(vecX dst, vecX src, vecX shift) %{
16639 predicate(n->as_Vector()->length() == 16);
16640 match(Set dst (LShiftVB src shift));
16641 ins_cost(INSN_COST);
16642 format %{ "sshl $dst,$src,$shift\t# vector (16B)" %}
16643 ins_encode %{
16644 __ sshl(as_FloatRegister($dst$$reg), __ T16B,
16645 as_FloatRegister($src$$reg),
16646 as_FloatRegister($shift$$reg));
16647 %}
16648 ins_pipe(vshift128);
16649 %}
16650
16651 // Right shifts with vector shift count on aarch64 SIMD are implemented
16652 // as left shift by negative shift count.
16653 // There are two cases for vector shift count.
16654 //
16655 // Case 1: The vector shift count is from replication.
16656 // | |
16657 // LoadVector RShiftCntV
16658 // | /
16659 // RShiftVI
16660 // Note: In inner loop, multiple neg instructions are used, which can be
16661 // moved to outer loop and merge into one neg instruction.
16662 //
16663 // Case 2: The vector shift count is from loading.
16664 // This case isn't supported by middle-end now. But it's supported by
16665 // panama/vectorIntrinsics(JEP 338: Vector API).
16666 // | |
16667 // LoadVector LoadVector
16668 // | /
16669 // RShiftVI
16670 //
16671
16672 instruct vsra8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
16673 predicate(n->as_Vector()->length() == 4 ||
16674 n->as_Vector()->length() == 8);
16675 match(Set dst (RShiftVB src shift));
16676 ins_cost(INSN_COST);
16677 effect(TEMP tmp);
16678 format %{ "negr $tmp,$shift\t"
16679 "sshl $dst,$src,$tmp\t# vector (8B)" %}
16680 ins_encode %{
16681 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
16682 as_FloatRegister($shift$$reg));
16683 __ sshl(as_FloatRegister($dst$$reg), __ T8B,
16684 as_FloatRegister($src$$reg),
16685 as_FloatRegister($tmp$$reg));
16686 %}
16687 ins_pipe(vshift64);
16688 %}
16689
16690 instruct vsra16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
16691 predicate(n->as_Vector()->length() == 16);
16692 match(Set dst (RShiftVB src shift));
16693 ins_cost(INSN_COST);
16694 effect(TEMP tmp);
16695 format %{ "negr $tmp,$shift\t"
16696 "sshl $dst,$src,$tmp\t# vector (16B)" %}
16697 ins_encode %{
16698 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
16699 as_FloatRegister($shift$$reg));
16700 __ sshl(as_FloatRegister($dst$$reg), __ T16B,
16701 as_FloatRegister($src$$reg),
16702 as_FloatRegister($tmp$$reg));
16703 %}
16704 ins_pipe(vshift128);
16705 %}
16706
16707 instruct vsrl8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
16708 predicate(n->as_Vector()->length() == 4 ||
16709 n->as_Vector()->length() == 8);
16710 match(Set dst (URShiftVB src shift));
16711 ins_cost(INSN_COST);
16712 effect(TEMP tmp);
16713 format %{ "negr $tmp,$shift\t"
16714 "ushl $dst,$src,$tmp\t# vector (8B)" %}
16715 ins_encode %{
16716 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
16717 as_FloatRegister($shift$$reg));
16718 __ ushl(as_FloatRegister($dst$$reg), __ T8B,
16719 as_FloatRegister($src$$reg),
16720 as_FloatRegister($tmp$$reg));
16721 %}
16722 ins_pipe(vshift64);
16723 %}
16724
16725 instruct vsrl16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
16726 predicate(n->as_Vector()->length() == 16);
16727 match(Set dst (URShiftVB src shift));
16728 ins_cost(INSN_COST);
16729 effect(TEMP tmp);
16730 format %{ "negr $tmp,$shift\t"
16731 "ushl $dst,$src,$tmp\t# vector (16B)" %}
16732 ins_encode %{
16733 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
16734 as_FloatRegister($shift$$reg));
16735 __ ushl(as_FloatRegister($dst$$reg), __ T16B,
16736 as_FloatRegister($src$$reg),
16737 as_FloatRegister($tmp$$reg));
16738 %}
16739 ins_pipe(vshift128);
16740 %}
16741
16742 instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
16743 predicate(n->as_Vector()->length() == 4 ||
16744 n->as_Vector()->length() == 8);
16745 match(Set dst (LShiftVB src shift));
16746 ins_cost(INSN_COST);
16747 format %{ "shl $dst, $src, $shift\t# vector (8B)" %}
16748 ins_encode %{
16749 int sh = (int)$shift$$constant;
16750 if (sh >= 8) {
16751 __ eor(as_FloatRegister($dst$$reg), __ T8B,
16752 as_FloatRegister($src$$reg),
16753 as_FloatRegister($src$$reg));
16754 } else {
16755 __ shl(as_FloatRegister($dst$$reg), __ T8B,
16756 as_FloatRegister($src$$reg), sh);
16757 }
16758 %}
16759 ins_pipe(vshift64_imm);
16760 %}
16761
16762 instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
16763 predicate(n->as_Vector()->length() == 16);
16764 match(Set dst (LShiftVB src shift));
16765 ins_cost(INSN_COST);
16766 format %{ "shl $dst, $src, $shift\t# vector (16B)" %}
16767 ins_encode %{
16768 int sh = (int)$shift$$constant;
16769 if (sh >= 8) {
16770 __ eor(as_FloatRegister($dst$$reg), __ T16B,
16771 as_FloatRegister($src$$reg),
16772 as_FloatRegister($src$$reg));
16773 } else {
16774 __ shl(as_FloatRegister($dst$$reg), __ T16B,
16775 as_FloatRegister($src$$reg), sh);
16776 }
16777 %}
16778 ins_pipe(vshift128_imm);
16779 %}
16780
16781 instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{
16782 predicate(n->as_Vector()->length() == 4 ||
16783 n->as_Vector()->length() == 8);
16784 match(Set dst (RShiftVB src shift));
16785 ins_cost(INSN_COST);
16786 format %{ "sshr $dst, $src, $shift\t# vector (8B)" %}
16787 ins_encode %{
16788 int sh = (int)$shift$$constant;
16789 if (sh >= 8) sh = 7;
16790 __ sshr(as_FloatRegister($dst$$reg), __ T8B,
16791 as_FloatRegister($src$$reg), sh);
16792 %}
16793 ins_pipe(vshift64_imm);
16794 %}
16795
16796 instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
16797 predicate(n->as_Vector()->length() == 16);
16798 match(Set dst (RShiftVB src shift));
16799 ins_cost(INSN_COST);
16800 format %{ "sshr $dst, $src, $shift\t# vector (16B)" %}
16801 ins_encode %{
16802 int sh = (int)$shift$$constant;
16803 if (sh >= 8) sh = 7;
16804 __ sshr(as_FloatRegister($dst$$reg), __ T16B,
16805 as_FloatRegister($src$$reg), sh);
16806 %}
16807 ins_pipe(vshift128_imm);
16808 %}
16809
16810 instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{
16811 predicate(n->as_Vector()->length() == 4 ||
16812 n->as_Vector()->length() == 8);
16813 match(Set dst (URShiftVB src shift));
16814 ins_cost(INSN_COST);
16815 format %{ "ushr $dst, $src, $shift\t# vector (8B)" %}
16816 ins_encode %{
16817 int sh = (int)$shift$$constant;
16818 if (sh >= 8) {
16819 __ eor(as_FloatRegister($dst$$reg), __ T8B,
16820 as_FloatRegister($src$$reg),
16821 as_FloatRegister($src$$reg));
16822 } else {
16823 __ ushr(as_FloatRegister($dst$$reg), __ T8B,
16824 as_FloatRegister($src$$reg), sh);
16825 }
16826 %}
16827 ins_pipe(vshift64_imm);
16828 %}
16829
16830 instruct vsrl16B_imm(vecX dst, vecX src, immI shift) %{
16831 predicate(n->as_Vector()->length() == 16);
16832 match(Set dst (URShiftVB src shift));
16833 ins_cost(INSN_COST);
16834 format %{ "ushr $dst, $src, $shift\t# vector (16B)" %}
16835 ins_encode %{
16836 int sh = (int)$shift$$constant;
16837 if (sh >= 8) {
16838 __ eor(as_FloatRegister($dst$$reg), __ T16B,
16839 as_FloatRegister($src$$reg),
16840 as_FloatRegister($src$$reg));
16841 } else {
16842 __ ushr(as_FloatRegister($dst$$reg), __ T16B,
16843 as_FloatRegister($src$$reg), sh);
16844 }
16845 %}
16846 ins_pipe(vshift128_imm);
16847 %}
16848
16849 instruct vsll4S(vecD dst, vecD src, vecD shift) %{
16850 predicate(n->as_Vector()->length() == 2 ||
16851 n->as_Vector()->length() == 4);
16852 match(Set dst (LShiftVS src shift));
16853 ins_cost(INSN_COST);
16854 format %{ "sshl $dst,$src,$shift\t# vector (4H)" %}
16855 ins_encode %{
16856 __ sshl(as_FloatRegister($dst$$reg), __ T4H,
16857 as_FloatRegister($src$$reg),
16858 as_FloatRegister($shift$$reg));
16859 %}
16860 ins_pipe(vshift64);
16861 %}
16862
16863 instruct vsll8S(vecX dst, vecX src, vecX shift) %{
16864 predicate(n->as_Vector()->length() == 8);
16865 match(Set dst (LShiftVS src shift));
16866 ins_cost(INSN_COST);
16867 format %{ "sshl $dst,$src,$shift\t# vector (8H)" %}
16868 ins_encode %{
16869 __ sshl(as_FloatRegister($dst$$reg), __ T8H,
16870 as_FloatRegister($src$$reg),
16871 as_FloatRegister($shift$$reg));
16872 %}
16873 ins_pipe(vshift128);
16874 %}
16875
16876 instruct vsra4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
16877 predicate(n->as_Vector()->length() == 2 ||
16878 n->as_Vector()->length() == 4);
16879 match(Set dst (RShiftVS src shift));
16880 ins_cost(INSN_COST);
16881 effect(TEMP tmp);
16882 format %{ "negr $tmp,$shift\t"
16883 "sshl $dst,$src,$tmp\t# vector (4H)" %}
16884 ins_encode %{
16885 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
16886 as_FloatRegister($shift$$reg));
16887 __ sshl(as_FloatRegister($dst$$reg), __ T4H,
16888 as_FloatRegister($src$$reg),
16889 as_FloatRegister($tmp$$reg));
16890 %}
16891 ins_pipe(vshift64);
16892 %}
16893
16894 instruct vsra8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
16895 predicate(n->as_Vector()->length() == 8);
16896 match(Set dst (RShiftVS src shift));
16897 ins_cost(INSN_COST);
16898 effect(TEMP tmp);
16899 format %{ "negr $tmp,$shift\t"
16900 "sshl $dst,$src,$tmp\t# vector (8H)" %}
16901 ins_encode %{
16902 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
16903 as_FloatRegister($shift$$reg));
16904 __ sshl(as_FloatRegister($dst$$reg), __ T8H,
16905 as_FloatRegister($src$$reg),
16906 as_FloatRegister($tmp$$reg));
16907 %}
16908 ins_pipe(vshift128);
16909 %}
16910
16911 instruct vsrl4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
16912 predicate(n->as_Vector()->length() == 2 ||
16913 n->as_Vector()->length() == 4);
16914 match(Set dst (URShiftVS src shift));
16915 ins_cost(INSN_COST);
16916 effect(TEMP tmp);
16917 format %{ "negr $tmp,$shift\t"
16918 "ushl $dst,$src,$tmp\t# vector (4H)" %}
16919 ins_encode %{
16920 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
16921 as_FloatRegister($shift$$reg));
16922 __ ushl(as_FloatRegister($dst$$reg), __ T4H,
16923 as_FloatRegister($src$$reg),
16924 as_FloatRegister($tmp$$reg));
16925 %}
16926 ins_pipe(vshift64);
16927 %}
16928
16929 instruct vsrl8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
16930 predicate(n->as_Vector()->length() == 8);
16931 match(Set dst (URShiftVS src shift));
16932 ins_cost(INSN_COST);
16933 effect(TEMP tmp);
16934 format %{ "negr $tmp,$shift\t"
16935 "ushl $dst,$src,$tmp\t# vector (8H)" %}
16936 ins_encode %{
16937 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
16938 as_FloatRegister($shift$$reg));
16939 __ ushl(as_FloatRegister($dst$$reg), __ T8H,
16940 as_FloatRegister($src$$reg),
16941 as_FloatRegister($tmp$$reg));
16942 %}
16943 ins_pipe(vshift128);
16944 %}
16945
16946 instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
16947 predicate(n->as_Vector()->length() == 2 ||
16948 n->as_Vector()->length() == 4);
16949 match(Set dst (LShiftVS src shift));
16950 ins_cost(INSN_COST);
16951 format %{ "shl $dst, $src, $shift\t# vector (4H)" %}
16952 ins_encode %{
16953 int sh = (int)$shift$$constant;
16954 if (sh >= 16) {
16955 __ eor(as_FloatRegister($dst$$reg), __ T8B,
16956 as_FloatRegister($src$$reg),
16957 as_FloatRegister($src$$reg));
16958 } else {
16959 __ shl(as_FloatRegister($dst$$reg), __ T4H,
16960 as_FloatRegister($src$$reg), sh);
16961 }
16962 %}
16963 ins_pipe(vshift64_imm);
16964 %}
16965
16966 instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
16967 predicate(n->as_Vector()->length() == 8);
16968 match(Set dst (LShiftVS src shift));
16969 ins_cost(INSN_COST);
16970 format %{ "shl $dst, $src, $shift\t# vector (8H)" %}
16971 ins_encode %{
16972 int sh = (int)$shift$$constant;
16973 if (sh >= 16) {
16974 __ eor(as_FloatRegister($dst$$reg), __ T16B,
16975 as_FloatRegister($src$$reg),
16976 as_FloatRegister($src$$reg));
16977 } else {
16978 __ shl(as_FloatRegister($dst$$reg), __ T8H,
16979 as_FloatRegister($src$$reg), sh);
16980 }
16981 %}
16982 ins_pipe(vshift128_imm);
16983 %}
16984
16985 instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{
16986 predicate(n->as_Vector()->length() == 2 ||
16987 n->as_Vector()->length() == 4);
16988 match(Set dst (RShiftVS src shift));
16989 ins_cost(INSN_COST);
16990 format %{ "sshr $dst, $src, $shift\t# vector (4H)" %}
16991 ins_encode %{
16992 int sh = (int)$shift$$constant;
16993 if (sh >= 16) sh = 15;
16994 __ sshr(as_FloatRegister($dst$$reg), __ T4H,
16995 as_FloatRegister($src$$reg), sh);
16996 %}
16997 ins_pipe(vshift64_imm);
16998 %}
16999
17000 instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
17001 predicate(n->as_Vector()->length() == 8);
17002 match(Set dst (RShiftVS src shift));
17003 ins_cost(INSN_COST);
17004 format %{ "sshr $dst, $src, $shift\t# vector (8H)" %}
17005 ins_encode %{
17006 int sh = (int)$shift$$constant;
17007 if (sh >= 16) sh = 15;
17008 __ sshr(as_FloatRegister($dst$$reg), __ T8H,
17009 as_FloatRegister($src$$reg), sh);
17010 %}
17011 ins_pipe(vshift128_imm);
17012 %}
17013
17014 instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{
17015 predicate(n->as_Vector()->length() == 2 ||
17016 n->as_Vector()->length() == 4);
17017 match(Set dst (URShiftVS src shift));
17018 ins_cost(INSN_COST);
17019 format %{ "ushr $dst, $src, $shift\t# vector (4H)" %}
17020 ins_encode %{
17021 int sh = (int)$shift$$constant;
17022 if (sh >= 16) {
17023 __ eor(as_FloatRegister($dst$$reg), __ T8B,
17024 as_FloatRegister($src$$reg),
17025 as_FloatRegister($src$$reg));
17026 } else {
17027 __ ushr(as_FloatRegister($dst$$reg), __ T4H,
17028 as_FloatRegister($src$$reg), sh);
17029 }
17030 %}
17031 ins_pipe(vshift64_imm);
17032 %}
17033
17034 instruct vsrl8S_imm(vecX dst, vecX src, immI shift) %{
17035 predicate(n->as_Vector()->length() == 8);
17036 match(Set dst (URShiftVS src shift));
17037 ins_cost(INSN_COST);
17038 format %{ "ushr $dst, $src, $shift\t# vector (8H)" %}
17039 ins_encode %{
17040 int sh = (int)$shift$$constant;
17041 if (sh >= 16) {
17042 __ eor(as_FloatRegister($dst$$reg), __ T16B,
17043 as_FloatRegister($src$$reg),
17044 as_FloatRegister($src$$reg));
17045 } else {
17046 __ ushr(as_FloatRegister($dst$$reg), __ T8H,
17047 as_FloatRegister($src$$reg), sh);
17048 }
17049 %}
17050 ins_pipe(vshift128_imm);
17051 %}
17052
17053 instruct vsll2I(vecD dst, vecD src, vecD shift) %{
17054 predicate(n->as_Vector()->length() == 2);
17055 match(Set dst (LShiftVI src shift));
17056 ins_cost(INSN_COST);
17057 format %{ "sshl $dst,$src,$shift\t# vector (2S)" %}
17058 ins_encode %{
17059 __ sshl(as_FloatRegister($dst$$reg), __ T2S,
17060 as_FloatRegister($src$$reg),
17061 as_FloatRegister($shift$$reg));
17062 %}
17063 ins_pipe(vshift64);
17064 %}
17065
17066 instruct vsll4I(vecX dst, vecX src, vecX shift) %{
17067 predicate(n->as_Vector()->length() == 4);
17068 match(Set dst (LShiftVI src shift));
17069 ins_cost(INSN_COST);
17070 format %{ "sshl $dst,$src,$shift\t# vector (4S)" %}
17071 ins_encode %{
17072 __ sshl(as_FloatRegister($dst$$reg), __ T4S,
17073 as_FloatRegister($src$$reg),
17074 as_FloatRegister($shift$$reg));
17075 %}
17076 ins_pipe(vshift128);
17077 %}
17078
17079 instruct vsra2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
17080 predicate(n->as_Vector()->length() == 2);
17081 match(Set dst (RShiftVI src shift));
17082 ins_cost(INSN_COST);
17083 effect(TEMP tmp);
17084 format %{ "negr $tmp,$shift\t"
17085 "sshl $dst,$src,$tmp\t# vector (2S)" %}
17086 ins_encode %{
17087 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17088 as_FloatRegister($shift$$reg));
17089 __ sshl(as_FloatRegister($dst$$reg), __ T2S,
17090 as_FloatRegister($src$$reg),
17091 as_FloatRegister($tmp$$reg));
17092 %}
17093 ins_pipe(vshift64);
17094 %}
17095
17096 instruct vsra4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
17097 predicate(n->as_Vector()->length() == 4);
17098 match(Set dst (RShiftVI src shift));
17099 ins_cost(INSN_COST);
17100 effect(TEMP tmp);
17101 format %{ "negr $tmp,$shift\t"
17102 "sshl $dst,$src,$tmp\t# vector (4S)" %}
17103 ins_encode %{
17104 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17105 as_FloatRegister($shift$$reg));
17106 __ sshl(as_FloatRegister($dst$$reg), __ T4S,
17107 as_FloatRegister($src$$reg),
17108 as_FloatRegister($tmp$$reg));
17109 %}
17110 ins_pipe(vshift128);
17111 %}
17112
17113 instruct vsrl2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
17114 predicate(n->as_Vector()->length() == 2);
17115 match(Set dst (URShiftVI src shift));
17116 ins_cost(INSN_COST);
17117 effect(TEMP tmp);
17118 format %{ "negr $tmp,$shift\t"
17119 "ushl $dst,$src,$tmp\t# vector (2S)" %}
17120 ins_encode %{
17121 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17122 as_FloatRegister($shift$$reg));
17123 __ ushl(as_FloatRegister($dst$$reg), __ T2S,
17124 as_FloatRegister($src$$reg),
17125 as_FloatRegister($tmp$$reg));
17126 %}
17127 ins_pipe(vshift64);
17128 %}
17129
17130 instruct vsrl4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
17131 predicate(n->as_Vector()->length() == 4);
17132 match(Set dst (URShiftVI src shift));
17133 ins_cost(INSN_COST);
17134 effect(TEMP tmp);
17135 format %{ "negr $tmp,$shift\t"
17136 "ushl $dst,$src,$tmp\t# vector (4S)" %}
17137 ins_encode %{
17138 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17139 as_FloatRegister($shift$$reg));
17140 __ ushl(as_FloatRegister($dst$$reg), __ T4S,
17141 as_FloatRegister($src$$reg),
17142 as_FloatRegister($tmp$$reg));
17143 %}
17144 ins_pipe(vshift128);
17145 %}
17146
17147 instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
17148 predicate(n->as_Vector()->length() == 2);
17149 match(Set dst (LShiftVI src shift));
17150 ins_cost(INSN_COST);
17151 format %{ "shl $dst, $src, $shift\t# vector (2S)" %}
17152 ins_encode %{
17153 __ shl(as_FloatRegister($dst$$reg), __ T2S,
17154 as_FloatRegister($src$$reg),
17155 (int)$shift$$constant);
17156 %}
17157 ins_pipe(vshift64_imm);
17158 %}
17159
17160 instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{
17161 predicate(n->as_Vector()->length() == 4);
17162 match(Set dst (LShiftVI src shift));
17163 ins_cost(INSN_COST);
17164 format %{ "shl $dst, $src, $shift\t# vector (4S)" %}
17165 ins_encode %{
17166 __ shl(as_FloatRegister($dst$$reg), __ T4S,
17167 as_FloatRegister($src$$reg),
17168 (int)$shift$$constant);
17169 %}
17170 ins_pipe(vshift128_imm);
17171 %}
17172
17173 instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{
17174 predicate(n->as_Vector()->length() == 2);
17175 match(Set dst (RShiftVI src shift));
17176 ins_cost(INSN_COST);
17177 format %{ "sshr $dst, $src, $shift\t# vector (2S)" %}
17178 ins_encode %{
17179 __ sshr(as_FloatRegister($dst$$reg), __ T2S,
17180 as_FloatRegister($src$$reg),
17181 (int)$shift$$constant);
17182 %}
17183 ins_pipe(vshift64_imm);
17184 %}
17185
17186 instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{
17187 predicate(n->as_Vector()->length() == 4);
17188 match(Set dst (RShiftVI src shift));
17189 ins_cost(INSN_COST);
17190 format %{ "sshr $dst, $src, $shift\t# vector (4S)" %}
17191 ins_encode %{
17192 __ sshr(as_FloatRegister($dst$$reg), __ T4S,
17193 as_FloatRegister($src$$reg),
17194 (int)$shift$$constant);
17195 %}
17196 ins_pipe(vshift128_imm);
17197 %}
17198
17199 instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{
17200 predicate(n->as_Vector()->length() == 2);
17201 match(Set dst (URShiftVI src shift));
17202 ins_cost(INSN_COST);
17203 format %{ "ushr $dst, $src, $shift\t# vector (2S)" %}
17204 ins_encode %{
17205 __ ushr(as_FloatRegister($dst$$reg), __ T2S,
17206 as_FloatRegister($src$$reg),
17207 (int)$shift$$constant);
17208 %}
17209 ins_pipe(vshift64_imm);
17210 %}
17211
17212 instruct vsrl4I_imm(vecX dst, vecX src, immI shift) %{
17213 predicate(n->as_Vector()->length() == 4);
17214 match(Set dst (URShiftVI src shift));
17215 ins_cost(INSN_COST);
17216 format %{ "ushr $dst, $src, $shift\t# vector (4S)" %}
17217 ins_encode %{
17218 __ ushr(as_FloatRegister($dst$$reg), __ T4S,
17219 as_FloatRegister($src$$reg),
17220 (int)$shift$$constant);
17221 %}
17222 ins_pipe(vshift128_imm);
17223 %}
17224
17225 instruct vsll2L(vecX dst, vecX src, vecX shift) %{
17226 predicate(n->as_Vector()->length() == 2);
17227 match(Set dst (LShiftVL src shift));
17228 ins_cost(INSN_COST);
17229 format %{ "sshl $dst,$src,$shift\t# vector (2D)" %}
17230 ins_encode %{
17231 __ sshl(as_FloatRegister($dst$$reg), __ T2D,
17232 as_FloatRegister($src$$reg),
17233 as_FloatRegister($shift$$reg));
17234 %}
17235 ins_pipe(vshift128);
17236 %}
17237
17238 instruct vsra2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
17239 predicate(n->as_Vector()->length() == 2);
17240 match(Set dst (RShiftVL src shift));
17241 ins_cost(INSN_COST);
17242 effect(TEMP tmp);
17243 format %{ "negr $tmp,$shift\t"
17244 "sshl $dst,$src,$tmp\t# vector (2D)" %}
17245 ins_encode %{
17246 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17247 as_FloatRegister($shift$$reg));
17248 __ sshl(as_FloatRegister($dst$$reg), __ T2D,
17249 as_FloatRegister($src$$reg),
17250 as_FloatRegister($tmp$$reg));
17251 %}
17252 ins_pipe(vshift128);
17253 %}
17254
17255 instruct vsrl2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
17256 predicate(n->as_Vector()->length() == 2);
17257 match(Set dst (URShiftVL src shift));
17258 ins_cost(INSN_COST);
17259 effect(TEMP tmp);
17260 format %{ "negr $tmp,$shift\t"
17261 "ushl $dst,$src,$tmp\t# vector (2D)" %}
17262 ins_encode %{
17263 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17264 as_FloatRegister($shift$$reg));
17265 __ ushl(as_FloatRegister($dst$$reg), __ T2D,
17266 as_FloatRegister($src$$reg),
17267 as_FloatRegister($tmp$$reg));
17268 %}
17269 ins_pipe(vshift128);
17270 %}
17271
17272 instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
17273 predicate(n->as_Vector()->length() == 2);
17274 match(Set dst (LShiftVL src shift));
17275 ins_cost(INSN_COST);
17276 format %{ "shl $dst, $src, $shift\t# vector (2D)" %}
17277 ins_encode %{
17278 __ shl(as_FloatRegister($dst$$reg), __ T2D,
17279 as_FloatRegister($src$$reg),
17280 (int)$shift$$constant);
17281 %}
17282 ins_pipe(vshift128_imm);
17283 %}
17284
17285 instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{
17286 predicate(n->as_Vector()->length() == 2);
17287 match(Set dst (RShiftVL src shift));
17288 ins_cost(INSN_COST);
17289 format %{ "sshr $dst, $src, $shift\t# vector (2D)" %}
17290 ins_encode %{
17291 __ sshr(as_FloatRegister($dst$$reg), __ T2D,
17292 as_FloatRegister($src$$reg),
17293 (int)$shift$$constant);
17294 %}
17295 ins_pipe(vshift128_imm);
17296 %}
17297
17298 instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{
17299 predicate(n->as_Vector()->length() == 2);
17300 match(Set dst (URShiftVL src shift));
17301 ins_cost(INSN_COST);
17302 format %{ "ushr $dst, $src, $shift\t# vector (2D)" %}
17303 ins_encode %{
17304 __ ushr(as_FloatRegister($dst$$reg), __ T2D,
17305 as_FloatRegister($src$$reg),
17306 (int)$shift$$constant);
17307 %}
17308 ins_pipe(vshift128_imm);
17309 %}
17310
17311 instruct vmax2F(vecD dst, vecD src1, vecD src2)
17312 %{
17313 predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17314 match(Set dst (MaxV src1 src2));
17315 ins_cost(INSN_COST);
17316 format %{ "fmax $dst,$src1,$src2\t# vector (2F)" %}
17317 ins_encode %{
17318 __ fmax(as_FloatRegister($dst$$reg), __ T2S,
17319 as_FloatRegister($src1$$reg),
17320 as_FloatRegister($src2$$reg));
17321 %}
17322 ins_pipe(vdop_fp64);
17323 %}
17324
17325 instruct vmax4F(vecX dst, vecX src1, vecX src2)
17326 %{
17327 predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17328 match(Set dst (MaxV src1 src2));
17329 ins_cost(INSN_COST);
17330 format %{ "fmax $dst,$src1,$src2\t# vector (4S)" %}
17331 ins_encode %{
17332 __ fmax(as_FloatRegister($dst$$reg), __ T4S,
17333 as_FloatRegister($src1$$reg),
17334 as_FloatRegister($src2$$reg));
17335 %}
17336 ins_pipe(vdop_fp128);
17337 %}
17338
17339 instruct vmax2D(vecX dst, vecX src1, vecX src2)
17340 %{
17341 predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
17342 match(Set dst (MaxV src1 src2));
17343 ins_cost(INSN_COST);
17344 format %{ "fmax $dst,$src1,$src2\t# vector (2D)" %}
17345 ins_encode %{
17346 __ fmax(as_FloatRegister($dst$$reg), __ T2D,
17347 as_FloatRegister($src1$$reg),
17348 as_FloatRegister($src2$$reg));
17349 %}
17350 ins_pipe(vdop_fp128);
17351 %}
17352
17353 instruct vmin2F(vecD dst, vecD src1, vecD src2)
17354 %{
17355 predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17356 match(Set dst (MinV src1 src2));
17357 ins_cost(INSN_COST);
17358 format %{ "fmin $dst,$src1,$src2\t# vector (2F)" %}
17359 ins_encode %{
17360 __ fmin(as_FloatRegister($dst$$reg), __ T2S,
17361 as_FloatRegister($src1$$reg),
17362 as_FloatRegister($src2$$reg));
17363 %}
17364 ins_pipe(vdop_fp64);
17365 %}
17366
17367 instruct vmin4F(vecX dst, vecX src1, vecX src2)
17368 %{
17369 predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17370 match(Set dst (MinV src1 src2));
17371 ins_cost(INSN_COST);
17372 format %{ "fmin $dst,$src1,$src2\t# vector (4S)" %}
17373 ins_encode %{
17374 __ fmin(as_FloatRegister($dst$$reg), __ T4S,
17375 as_FloatRegister($src1$$reg),
17376 as_FloatRegister($src2$$reg));
17377 %}
17378 ins_pipe(vdop_fp128);
17379 %}
17380
17381 instruct vmin2D(vecX dst, vecX src1, vecX src2)
17382 %{
17383 predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
17384 match(Set dst (MinV src1 src2));
17385 ins_cost(INSN_COST);
17386 format %{ "fmin $dst,$src1,$src2\t# vector (2D)" %}
17387 ins_encode %{
17388 __ fmin(as_FloatRegister($dst$$reg), __ T2D,
17389 as_FloatRegister($src1$$reg),
17390 as_FloatRegister($src2$$reg));
17391 %}
17392 ins_pipe(vdop_fp128);
17393 %}
17394
17395 //----------PEEPHOLE RULES-----------------------------------------------------
17396 // These must follow all instruction definitions as they use the names
17397 // defined in the instructions definitions.
17398 //
17399 // peepmatch ( root_instr_name [preceding_instruction]* );
17400 //
17401 // peepconstraint %{
17402 // (instruction_number.operand_name relational_op instruction_number.operand_name
17403 // [, ...] );
17404 // // instruction numbers are zero-based using left to right order in peepmatch
17405 //
17406 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17407 // // provide an instruction_number.operand_name for each operand that appears
17408 // // in the replacement instruction's match rule
17409 //
17410 // ---------VM FLAGS---------------------------------------------------------
17411 //
17412 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17413 //
17414 // Each peephole rule is given an identifying number starting with zero and
17415 // increasing by one in the order seen by the parser. An individual peephole
17416 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17417 // on the command-line.
17418 //
17419 // ---------CURRENT LIMITATIONS----------------------------------------------
17420 //
17421 // Only match adjacent instructions in same basic block
17422 // Only equality constraints
17423 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17424 // Only one replacement instruction
17425 //
17426 // ---------EXAMPLE----------------------------------------------------------
17427 //
17428 // // pertinent parts of existing instructions in architecture description
17429 // instruct movI(iRegINoSp dst, iRegI src)
17430 // %{
17431 // match(Set dst (CopyI src));
17432 // %}
17433 //
17434 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17435 // %{
17436 // match(Set dst (AddI dst src));
17437 // effect(KILL cr);
17438 // %}
17439 //
17440 // // Change (inc mov) to lea
17441 // peephole %{
17442 // // increment preceeded by register-register move
17443 // peepmatch ( incI_iReg movI );
17444 // // require that the destination register of the increment
17445 // // match the destination register of the move
17446 // peepconstraint ( 0.dst == 1.dst );
17447 // // construct a replacement instruction that sets
17448 // // the destination to ( move's source register + one )
17449 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17450 // %}
17451 //
17452
17453 // Implementation no longer uses movX instructions since
17454 // machine-independent system no longer uses CopyX nodes.
17455 //
17456 // peephole
17457 // %{
17458 // peepmatch (incI_iReg movI);
17459 // peepconstraint (0.dst == 1.dst);
17460 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17461 // %}
17462
17463 // peephole
17464 // %{
17465 // peepmatch (decI_iReg movI);
17466 // peepconstraint (0.dst == 1.dst);
17467 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17468 // %}
17469
17470 // peephole
17471 // %{
17472 // peepmatch (addI_iReg_imm movI);
17473 // peepconstraint (0.dst == 1.dst);
17474 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17475 // %}
17476
17477 // peephole
17478 // %{
17479 // peepmatch (incL_iReg movL);
17480 // peepconstraint (0.dst == 1.dst);
17481 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17482 // %}
17483
17484 // peephole
17485 // %{
17486 // peepmatch (decL_iReg movL);
17487 // peepconstraint (0.dst == 1.dst);
17488 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17489 // %}
17490
17491 // peephole
17492 // %{
17493 // peepmatch (addL_iReg_imm movL);
17494 // peepconstraint (0.dst == 1.dst);
17495 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17496 // %}
17497
17498 // peephole
17499 // %{
17500 // peepmatch (addP_iReg_imm movP);
17501 // peepconstraint (0.dst == 1.dst);
17502 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17503 // %}
17504
17505 // // Change load of spilled value to only a spill
17506 // instruct storeI(memory mem, iRegI src)
17507 // %{
17508 // match(Set mem (StoreI mem src));
17509 // %}
17510 //
17511 // instruct loadI(iRegINoSp dst, memory mem)
17512 // %{
17513 // match(Set dst (LoadI mem));
17514 // %}
17515 //
17516
17517 //----------SMARTSPILL RULES---------------------------------------------------
17518 // These must follow all instruction definitions as they use the names
17519 // defined in the instructions definitions.
17520
17521 // Local Variables:
17522 // mode: c++
17523 // End: