1 //
2 // Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 // X86 Common Architecture Description File
26
27 //----------REGISTER DEFINITION BLOCK------------------------------------------
28 // This information is used by the matcher and the register allocator to
29 // describe individual registers and classes of registers within the target
30 // archtecture.
31
32 register %{
33 //----------Architecture Description Register Definitions----------------------
34 // General Registers
35 // "reg_def" name ( register save type, C convention save type,
36 // ideal register type, encoding );
37 // Register Save Types:
38 //
39 // NS = No-Save: The register allocator assumes that these registers
40 // can be used without saving upon entry to the method, &
41 // that they do not need to be saved at call sites.
42 //
43 // SOC = Save-On-Call: The register allocator assumes that these registers
44 // can be used without saving upon entry to the method,
45 // but that they must be saved at call sites.
46 //
47 // SOE = Save-On-Entry: The register allocator assumes that these registers
48 // must be saved before using them upon entry to the
49 // method, but they do not need to be saved at call
50 // sites.
51 //
52 // AS = Always-Save: The register allocator assumes that these registers
53 // must be saved before using them upon entry to the
54 // method, & that they must be saved at call sites.
55 //
56 // Ideal Register Type is used to determine how to save & restore a
57 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
58 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
59 //
60 // The encoding number is the actual bit-pattern placed into the opcodes.
61
62 // XMM registers. 512-bit registers or 8 words each, labeled (a)-p.
63 // Word a in each register holds a Float, words ab hold a Double.
64 // The whole registers are used in SSE4.2 version intrinsics,
65 // array copy stubs and superword operations (see UseSSE42Intrinsics,
66 // UseXMMForArrayCopy and UseSuperword flags).
67 // For pre EVEX enabled architectures:
68 // XMM8-XMM15 must be encoded with REX (VEX for UseAVX)
69 // For EVEX enabled architectures:
70 // XMM8-XMM31 must be encoded with REX (EVEX for UseAVX).
71 //
72 // Linux ABI: No register preserved across function calls
73 // XMM0-XMM7 might hold parameters
74 // Windows ABI: XMM6-XMM31 preserved across function calls
75 // XMM0-XMM3 might hold parameters
76
77 reg_def XMM0 ( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg());
78 reg_def XMM0b( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(1));
79 reg_def XMM0c( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(2));
80 reg_def XMM0d( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(3));
81 reg_def XMM0e( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(4));
82 reg_def XMM0f( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(5));
83 reg_def XMM0g( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(6));
84 reg_def XMM0h( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(7));
85 reg_def XMM0i( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(8));
86 reg_def XMM0j( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(9));
87 reg_def XMM0k( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(10));
88 reg_def XMM0l( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(11));
89 reg_def XMM0m( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(12));
90 reg_def XMM0n( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(13));
91 reg_def XMM0o( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(14));
92 reg_def XMM0p( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(15));
93
94 reg_def XMM1 ( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg());
95 reg_def XMM1b( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(1));
96 reg_def XMM1c( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(2));
97 reg_def XMM1d( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(3));
98 reg_def XMM1e( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(4));
99 reg_def XMM1f( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(5));
100 reg_def XMM1g( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(6));
101 reg_def XMM1h( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(7));
102 reg_def XMM1i( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(8));
103 reg_def XMM1j( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(9));
104 reg_def XMM1k( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(10));
105 reg_def XMM1l( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(11));
106 reg_def XMM1m( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(12));
107 reg_def XMM1n( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(13));
108 reg_def XMM1o( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(14));
109 reg_def XMM1p( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(15));
110
111 reg_def XMM2 ( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg());
112 reg_def XMM2b( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(1));
113 reg_def XMM2c( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(2));
114 reg_def XMM2d( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(3));
115 reg_def XMM2e( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(4));
116 reg_def XMM2f( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(5));
117 reg_def XMM2g( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(6));
118 reg_def XMM2h( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(7));
119 reg_def XMM2i( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(8));
120 reg_def XMM2j( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(9));
121 reg_def XMM2k( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(10));
122 reg_def XMM2l( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(11));
123 reg_def XMM2m( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(12));
124 reg_def XMM2n( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(13));
125 reg_def XMM2o( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(14));
126 reg_def XMM2p( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(15));
127
128 reg_def XMM3 ( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg());
129 reg_def XMM3b( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(1));
130 reg_def XMM3c( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(2));
131 reg_def XMM3d( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(3));
132 reg_def XMM3e( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(4));
133 reg_def XMM3f( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(5));
134 reg_def XMM3g( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(6));
135 reg_def XMM3h( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(7));
136 reg_def XMM3i( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(8));
137 reg_def XMM3j( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(9));
138 reg_def XMM3k( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(10));
139 reg_def XMM3l( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(11));
140 reg_def XMM3m( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(12));
141 reg_def XMM3n( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(13));
142 reg_def XMM3o( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(14));
143 reg_def XMM3p( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(15));
144
145 reg_def XMM4 ( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg());
146 reg_def XMM4b( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(1));
147 reg_def XMM4c( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(2));
148 reg_def XMM4d( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(3));
149 reg_def XMM4e( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(4));
150 reg_def XMM4f( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(5));
151 reg_def XMM4g( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(6));
152 reg_def XMM4h( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(7));
153 reg_def XMM4i( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(8));
154 reg_def XMM4j( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(9));
155 reg_def XMM4k( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(10));
156 reg_def XMM4l( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(11));
157 reg_def XMM4m( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(12));
158 reg_def XMM4n( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(13));
159 reg_def XMM4o( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(14));
160 reg_def XMM4p( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(15));
161
162 reg_def XMM5 ( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg());
163 reg_def XMM5b( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(1));
164 reg_def XMM5c( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(2));
165 reg_def XMM5d( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(3));
166 reg_def XMM5e( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(4));
167 reg_def XMM5f( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(5));
168 reg_def XMM5g( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(6));
169 reg_def XMM5h( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(7));
170 reg_def XMM5i( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(8));
171 reg_def XMM5j( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(9));
172 reg_def XMM5k( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(10));
173 reg_def XMM5l( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(11));
174 reg_def XMM5m( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(12));
175 reg_def XMM5n( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(13));
176 reg_def XMM5o( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(14));
177 reg_def XMM5p( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(15));
178
179 #ifdef _WIN64
180
181 reg_def XMM6 ( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg());
182 reg_def XMM6b( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(1));
183 reg_def XMM6c( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(2));
184 reg_def XMM6d( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(3));
185 reg_def XMM6e( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(4));
186 reg_def XMM6f( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(5));
187 reg_def XMM6g( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(6));
188 reg_def XMM6h( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(7));
189 reg_def XMM6i( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(8));
190 reg_def XMM6j( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(9));
191 reg_def XMM6k( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(10));
192 reg_def XMM6l( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(11));
193 reg_def XMM6m( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(12));
194 reg_def XMM6n( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(13));
195 reg_def XMM6o( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(14));
196 reg_def XMM6p( SOC, SOE, Op_RegF, 6, xmm6->as_VMReg()->next(15));
197
198 reg_def XMM7 ( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg());
199 reg_def XMM7b( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(1));
200 reg_def XMM7c( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(2));
201 reg_def XMM7d( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(3));
202 reg_def XMM7e( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(4));
203 reg_def XMM7f( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(5));
204 reg_def XMM7g( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(6));
205 reg_def XMM7h( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(7));
206 reg_def XMM7i( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(8));
207 reg_def XMM7j( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(9));
208 reg_def XMM7k( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(10));
209 reg_def XMM7l( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(11));
210 reg_def XMM7m( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(12));
211 reg_def XMM7n( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(13));
212 reg_def XMM7o( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(14));
213 reg_def XMM7p( SOC, SOE, Op_RegF, 7, xmm7->as_VMReg()->next(15));
214
215 reg_def XMM8 ( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg());
216 reg_def XMM8b( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(1));
217 reg_def XMM8c( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(2));
218 reg_def XMM8d( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(3));
219 reg_def XMM8e( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(4));
220 reg_def XMM8f( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(5));
221 reg_def XMM8g( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(6));
222 reg_def XMM8h( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(7));
223 reg_def XMM8i( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(8));
224 reg_def XMM8j( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(9));
225 reg_def XMM8k( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(10));
226 reg_def XMM8l( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(11));
227 reg_def XMM8m( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(12));
228 reg_def XMM8n( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(13));
229 reg_def XMM8o( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(14));
230 reg_def XMM8p( SOC, SOE, Op_RegF, 8, xmm8->as_VMReg()->next(15));
231
232 reg_def XMM9 ( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg());
233 reg_def XMM9b( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(1));
234 reg_def XMM9c( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(2));
235 reg_def XMM9d( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(3));
236 reg_def XMM9e( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(4));
237 reg_def XMM9f( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(5));
238 reg_def XMM9g( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(6));
239 reg_def XMM9h( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(7));
240 reg_def XMM9i( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(8));
241 reg_def XMM9j( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(9));
242 reg_def XMM9k( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(10));
243 reg_def XMM9l( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(11));
244 reg_def XMM9m( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(12));
245 reg_def XMM9n( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(13));
246 reg_def XMM9o( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(14));
247 reg_def XMM9p( SOC, SOE, Op_RegF, 9, xmm9->as_VMReg()->next(15));
248
249 reg_def XMM10 ( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg());
250 reg_def XMM10b( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(1));
251 reg_def XMM10c( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(2));
252 reg_def XMM10d( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(3));
253 reg_def XMM10e( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(4));
254 reg_def XMM10f( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(5));
255 reg_def XMM10g( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(6));
256 reg_def XMM10h( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(7));
257 reg_def XMM10i( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(8));
258 reg_def XMM10j( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(9));
259 reg_def XMM10k( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(10));
260 reg_def XMM10l( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(11));
261 reg_def XMM10m( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(12));
262 reg_def XMM10n( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(13));
263 reg_def XMM10o( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(14));
264 reg_def XMM10p( SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next(15));
265
266 reg_def XMM11 ( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg());
267 reg_def XMM11b( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(1));
268 reg_def XMM11c( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(2));
269 reg_def XMM11d( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(3));
270 reg_def XMM11e( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(4));
271 reg_def XMM11f( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(5));
272 reg_def XMM11g( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(6));
273 reg_def XMM11h( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(7));
274 reg_def XMM11i( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(8));
275 reg_def XMM11j( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(9));
276 reg_def XMM11k( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(10));
277 reg_def XMM11l( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(11));
278 reg_def XMM11m( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(12));
279 reg_def XMM11n( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(13));
280 reg_def XMM11o( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(14));
281 reg_def XMM11p( SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next(15));
282
283 reg_def XMM12 ( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg());
284 reg_def XMM12b( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(1));
285 reg_def XMM12c( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(2));
286 reg_def XMM12d( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(3));
287 reg_def XMM12e( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(4));
288 reg_def XMM12f( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(5));
289 reg_def XMM12g( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(6));
290 reg_def XMM12h( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(7));
291 reg_def XMM12i( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(8));
292 reg_def XMM12j( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(9));
293 reg_def XMM12k( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(10));
294 reg_def XMM12l( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(11));
295 reg_def XMM12m( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(12));
296 reg_def XMM12n( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(13));
297 reg_def XMM12o( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(14));
298 reg_def XMM12p( SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next(15));
299
300 reg_def XMM13 ( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg());
301 reg_def XMM13b( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(1));
302 reg_def XMM13c( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(2));
303 reg_def XMM13d( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(3));
304 reg_def XMM13e( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(4));
305 reg_def XMM13f( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(5));
306 reg_def XMM13g( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(6));
307 reg_def XMM13h( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(7));
308 reg_def XMM13i( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(8));
309 reg_def XMM13j( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(9));
310 reg_def XMM13k( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(10));
311 reg_def XMM13l( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(11));
312 reg_def XMM13m( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(12));
313 reg_def XMM13n( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(13));
314 reg_def XMM13o( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(14));
315 reg_def XMM13p( SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next(15));
316
317 reg_def XMM14 ( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg());
318 reg_def XMM14b( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(1));
319 reg_def XMM14c( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(2));
320 reg_def XMM14d( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(3));
321 reg_def XMM14e( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(4));
322 reg_def XMM14f( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(5));
323 reg_def XMM14g( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(6));
324 reg_def XMM14h( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(7));
325 reg_def XMM14i( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(8));
326 reg_def XMM14j( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(9));
327 reg_def XMM14k( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(10));
328 reg_def XMM14l( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(11));
329 reg_def XMM14m( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(12));
330 reg_def XMM14n( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(13));
331 reg_def XMM14o( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(14));
332 reg_def XMM14p( SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next(15));
333
334 reg_def XMM15 ( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg());
335 reg_def XMM15b( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(1));
336 reg_def XMM15c( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(2));
337 reg_def XMM15d( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(3));
338 reg_def XMM15e( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(4));
339 reg_def XMM15f( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(5));
340 reg_def XMM15g( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(6));
341 reg_def XMM15h( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(7));
342 reg_def XMM15i( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(8));
343 reg_def XMM15j( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(9));
344 reg_def XMM15k( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(10));
345 reg_def XMM15l( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(11));
346 reg_def XMM15m( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(12));
347 reg_def XMM15n( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(13));
348 reg_def XMM15o( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(14));
349 reg_def XMM15p( SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next(15));
350
351 reg_def XMM16 ( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg());
352 reg_def XMM16b( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(1));
353 reg_def XMM16c( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(2));
354 reg_def XMM16d( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(3));
355 reg_def XMM16e( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(4));
356 reg_def XMM16f( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(5));
357 reg_def XMM16g( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(6));
358 reg_def XMM16h( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(7));
359 reg_def XMM16i( SOC, SOE, Op_RegF, 16, xmm15->as_VMReg()->next(8));
360 reg_def XMM16j( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(9));
361 reg_def XMM16k( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(10));
362 reg_def XMM16l( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(11));
363 reg_def XMM16m( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(12));
364 reg_def XMM16n( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(13));
365 reg_def XMM16o( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(14));
366 reg_def XMM16p( SOC, SOE, Op_RegF, 16, xmm16->as_VMReg()->next(15));
367
368 reg_def XMM17 ( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg());
369 reg_def XMM17b( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(1));
370 reg_def XMM17c( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(2));
371 reg_def XMM17d( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(3));
372 reg_def XMM17e( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(4));
373 reg_def XMM17f( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(5));
374 reg_def XMM17g( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(6));
375 reg_def XMM17h( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(7));
376 reg_def XMM17i( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(8));
377 reg_def XMM17j( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(9));
378 reg_def XMM17k( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(10));
379 reg_def XMM17l( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(11));
380 reg_def XMM17m( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(12));
381 reg_def XMM17n( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(13));
382 reg_def XMM17o( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(14));
383 reg_def XMM17p( SOC, SOE, Op_RegF, 17, xmm17->as_VMReg()->next(15));
384
385 reg_def XMM18 ( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg());
386 reg_def XMM18b( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(1));
387 reg_def XMM18c( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(2));
388 reg_def XMM18d( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(3));
389 reg_def XMM18e( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(4));
390 reg_def XMM18f( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(5));
391 reg_def XMM18g( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(6));
392 reg_def XMM18h( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(7));
393 reg_def XMM18i( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(8));
394 reg_def XMM18j( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(9));
395 reg_def XMM18k( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(10));
396 reg_def XMM18l( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(11));
397 reg_def XMM18m( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(12));
398 reg_def XMM18n( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(13));
399 reg_def XMM18o( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(14));
400 reg_def XMM18p( SOC, SOE, Op_RegF, 18, xmm18->as_VMReg()->next(15));
401
402 reg_def XMM19 ( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg());
403 reg_def XMM19b( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(1));
404 reg_def XMM19c( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(2));
405 reg_def XMM19d( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(3));
406 reg_def XMM19e( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(4));
407 reg_def XMM19f( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(5));
408 reg_def XMM19g( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(6));
409 reg_def XMM19h( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(7));
410 reg_def XMM19i( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(8));
411 reg_def XMM19j( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(9));
412 reg_def XMM19k( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(10));
413 reg_def XMM19l( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(11));
414 reg_def XMM19m( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(12));
415 reg_def XMM19n( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(13));
416 reg_def XMM19o( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(14));
417 reg_def XMM19p( SOC, SOE, Op_RegF, 19, xmm19->as_VMReg()->next(15));
418
419 reg_def XMM20 ( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg());
420 reg_def XMM20b( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(1));
421 reg_def XMM20c( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(2));
422 reg_def XMM20d( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(3));
423 reg_def XMM20e( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(4));
424 reg_def XMM20f( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(5));
425 reg_def XMM20g( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(6));
426 reg_def XMM20h( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(7));
427 reg_def XMM20i( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(8));
428 reg_def XMM20j( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(9));
429 reg_def XMM20k( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(10));
430 reg_def XMM20l( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(11));
431 reg_def XMM20m( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(12));
432 reg_def XMM20n( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(13));
433 reg_def XMM20o( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(14));
434 reg_def XMM20p( SOC, SOE, Op_RegF, 20, xmm20->as_VMReg()->next(15));
435
436 reg_def XMM21 ( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg());
437 reg_def XMM21b( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(1));
438 reg_def XMM21c( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(2));
439 reg_def XMM21d( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(3));
440 reg_def XMM21e( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(4));
441 reg_def XMM21f( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(5));
442 reg_def XMM21g( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(6));
443 reg_def XMM21h( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(7));
444 reg_def XMM21i( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(8));
445 reg_def XMM21j( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(9));
446 reg_def XMM21k( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(10));
447 reg_def XMM21l( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(11));
448 reg_def XMM21m( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(12));
449 reg_def XMM21n( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(13));
450 reg_def XMM21o( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(14));
451 reg_def XMM21p( SOC, SOE, Op_RegF, 21, xmm21->as_VMReg()->next(15));
452
453 reg_def XMM22 ( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg());
454 reg_def XMM22b( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(1));
455 reg_def XMM22c( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(2));
456 reg_def XMM22d( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(3));
457 reg_def XMM22e( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(4));
458 reg_def XMM22f( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(5));
459 reg_def XMM22g( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(6));
460 reg_def XMM22h( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(7));
461 reg_def XMM22i( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(8));
462 reg_def XMM22j( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(9));
463 reg_def XMM22k( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(10));
464 reg_def XMM22l( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(11));
465 reg_def XMM22m( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(12));
466 reg_def XMM22n( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(13));
467 reg_def XMM22o( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(14));
468 reg_def XMM22p( SOC, SOE, Op_RegF, 22, xmm22->as_VMReg()->next(15));
469
470 reg_def XMM23 ( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg());
471 reg_def XMM23b( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(1));
472 reg_def XMM23c( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(2));
473 reg_def XMM23d( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(3));
474 reg_def XMM23e( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(4));
475 reg_def XMM23f( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(5));
476 reg_def XMM23g( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(6));
477 reg_def XMM23h( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(7));
478 reg_def XMM23i( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(8));
479 reg_def XMM23j( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(9));
480 reg_def XMM23k( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(10));
481 reg_def XMM23l( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(11));
482 reg_def XMM23m( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(12));
483 reg_def XMM23n( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(13));
484 reg_def XMM23o( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(14));
485 reg_def XMM23p( SOC, SOE, Op_RegF, 23, xmm23->as_VMReg()->next(15));
486
487 reg_def XMM24 ( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg());
488 reg_def XMM24b( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(1));
489 reg_def XMM24c( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(2));
490 reg_def XMM24d( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(3));
491 reg_def XMM24e( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(4));
492 reg_def XMM24f( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(5));
493 reg_def XMM24g( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(6));
494 reg_def XMM24h( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(7));
495 reg_def XMM24i( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(8));
496 reg_def XMM24j( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(9));
497 reg_def XMM24k( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(10));
498 reg_def XMM24l( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(11));
499 reg_def XMM24m( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(12));
500 reg_def XMM24n( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(13));
501 reg_def XMM24o( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(14));
502 reg_def XMM24p( SOC, SOE, Op_RegF, 24, xmm24->as_VMReg()->next(15));
503
504 reg_def XMM25 ( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg());
505 reg_def XMM25b( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(1));
506 reg_def XMM25c( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(2));
507 reg_def XMM25d( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(3));
508 reg_def XMM25e( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(4));
509 reg_def XMM25f( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(5));
510 reg_def XMM25g( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(6));
511 reg_def XMM25h( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(7));
512 reg_def XMM25i( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(8));
513 reg_def XMM25j( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(9));
514 reg_def XMM25k( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(10));
515 reg_def XMM25l( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(11));
516 reg_def XMM25m( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(12));
517 reg_def XMM25n( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(13));
518 reg_def XMM25o( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(14));
519 reg_def XMM25p( SOC, SOE, Op_RegF, 25, xmm25->as_VMReg()->next(15));
520
521 reg_def XMM26 ( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg());
522 reg_def XMM26b( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(1));
523 reg_def XMM26c( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(2));
524 reg_def XMM26d( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(3));
525 reg_def XMM26e( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(4));
526 reg_def XMM26f( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(5));
527 reg_def XMM26g( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(6));
528 reg_def XMM26h( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(7));
529 reg_def XMM26i( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(8));
530 reg_def XMM26j( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(9));
531 reg_def XMM26k( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(10));
532 reg_def XMM26l( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(11));
533 reg_def XMM26m( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(12));
534 reg_def XMM26n( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(13));
535 reg_def XMM26o( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(14));
536 reg_def XMM26p( SOC, SOE, Op_RegF, 26, xmm26->as_VMReg()->next(15));
537
538 reg_def XMM27g( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(1));
539 reg_def XMM27c( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(2));
540 reg_def XMM27d( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(3));
541 reg_def XMM27e( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(4));
542 reg_def XMM27f( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(5));
543 reg_def XMM27g( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(6));
544 reg_def XMM27h( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(7));
545 reg_def XMM27i( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(8));
546 reg_def XMM27j( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(9));
547 reg_def XMM27k( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(10));
548 reg_def XMM27l( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(11));
549 reg_def XMM27m( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(12));
550 reg_def XMM27n( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(13));
551 reg_def XMM27o( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(14));
552 reg_def XMM27p( SOC, SOE, Op_RegF, 27, xmm27->as_VMReg()->next(15));
553
554 reg_def XMM28 ( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg());
555 reg_def XMM28b( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(1));
556 reg_def XMM28c( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(2));
557 reg_def XMM28d( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(3));
558 reg_def XMM28e( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(4));
559 reg_def XMM28f( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(5));
560 reg_def XMM28g( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(6));
561 reg_def XMM28h( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(7));
562 reg_def XMM28i( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(8));
563 reg_def XMM28j( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(9));
564 reg_def XMM28k( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(10));
565 reg_def XMM28l( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(11));
566 reg_def XMM28m( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(12));
567 reg_def XMM28n( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(13));
568 reg_def XMM28o( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(14));
569 reg_def XMM28p( SOC, SOE, Op_RegF, 28, xmm28->as_VMReg()->next(15));
570
571 reg_def XMM29 ( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg());
572 reg_def XMM29b( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(1));
573 reg_def XMM29c( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(2));
574 reg_def XMM29d( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(3));
575 reg_def XMM29e( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(4));
576 reg_def XMM29f( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(5));
577 reg_def XMM29g( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(6));
578 reg_def XMM29h( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(7));
579 reg_def XMM29i( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(8));
580 reg_def XMM29j( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(9));
581 reg_def XMM29k( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(10));
582 reg_def XMM29l( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(11));
583 reg_def XMM29m( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(12));
584 reg_def XMM29n( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(13));
585 reg_def XMM29o( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(14));
586 reg_def XMM29p( SOC, SOE, Op_RegF, 29, xmm29->as_VMReg()->next(15));
587
588 reg_def XMM30 ( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg());
589 reg_def XMM30b( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(1));
590 reg_def XMM30c( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(2));
591 reg_def XMM30d( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(3));
592 reg_def XMM30e( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(4));
593 reg_def XMM30f( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(5));
594 reg_def XMM30g( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(6));
595 reg_def XMM30h( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(7));
596 reg_def XMM30i( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(8));
597 reg_def XMM30j( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(9));
598 reg_def XMM30k( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(10));
599 reg_def XMM30l( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(11));
600 reg_def XMM30m( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(12));
601 reg_def XMM30n( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(13));
602 reg_def XMM30o( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(14));
603 reg_def XMM30p( SOC, SOE, Op_RegF, 30, xmm30->as_VMReg()->next(15));
604
605 reg_def XMM31 ( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg());
606 reg_def XMM31b( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(1));
607 reg_def XMM31c( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(2));
608 reg_def XMM31d( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(3));
609 reg_def XMM31e( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(4));
610 reg_def XMM31f( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(5));
611 reg_def XMM31g( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(6));
612 reg_def XMM31h( SOC, SOE, Op_RegF, 31, xmm31>-as_VMReg()->next(7));
613 reg_def XMM31i( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(8));
614 reg_def XMM31j( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(9));
615 reg_def XMM31k( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(10));
616 reg_def XMM31l( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(11));
617 reg_def XMM31m( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(12));
618 reg_def XMM31n( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(13));
619 reg_def XMM31o( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(14));
620 reg_def XMM31p( SOC, SOE, Op_RegF, 31, xmm31->as_VMReg()->next(15));
621
622 #else // _WIN64
623
624 reg_def XMM6 ( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg());
625 reg_def XMM6b( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(1));
626 reg_def XMM6c( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(2));
627 reg_def XMM6d( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(3));
628 reg_def XMM6e( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(4));
629 reg_def XMM6f( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(5));
630 reg_def XMM6g( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(6));
631 reg_def XMM6h( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(7));
632 reg_def XMM6i( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(8));
633 reg_def XMM6j( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(9));
634 reg_def XMM6k( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(10));
635 reg_def XMM6l( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(11));
636 reg_def XMM6m( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(12));
637 reg_def XMM6n( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(13));
638 reg_def XMM6o( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(14));
639 reg_def XMM6p( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(15));
640
641 reg_def XMM7 ( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg());
642 reg_def XMM7b( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(1));
643 reg_def XMM7c( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(2));
644 reg_def XMM7d( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(3));
645 reg_def XMM7e( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(4));
646 reg_def XMM7f( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(5));
647 reg_def XMM7g( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(6));
648 reg_def XMM7h( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(7));
649 reg_def XMM7i( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(8));
650 reg_def XMM7j( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(9));
651 reg_def XMM7k( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(10));
652 reg_def XMM7l( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(11));
653 reg_def XMM7m( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(12));
654 reg_def XMM7n( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(13));
655 reg_def XMM7o( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(14));
656 reg_def XMM7p( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(15));
657
658 #ifdef _LP64
659
660 reg_def XMM8 ( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg());
661 reg_def XMM8b( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(1));
662 reg_def XMM8c( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(2));
663 reg_def XMM8d( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(3));
664 reg_def XMM8e( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(4));
665 reg_def XMM8f( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(5));
666 reg_def XMM8g( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(6));
667 reg_def XMM8h( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(7));
668 reg_def XMM8i( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(8));
669 reg_def XMM8j( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(9));
670 reg_def XMM8k( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(10));
671 reg_def XMM8l( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(11));
672 reg_def XMM8m( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(12));
673 reg_def XMM8n( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(13));
674 reg_def XMM8o( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(14));
675 reg_def XMM8p( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(15));
676
677 reg_def XMM9 ( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg());
678 reg_def XMM9b( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(1));
679 reg_def XMM9c( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(2));
680 reg_def XMM9d( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(3));
681 reg_def XMM9e( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(4));
682 reg_def XMM9f( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(5));
683 reg_def XMM9g( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(6));
684 reg_def XMM9h( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(7));
685 reg_def XMM9i( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(8));
686 reg_def XMM9j( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(9));
687 reg_def XMM9k( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(10));
688 reg_def XMM9l( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(11));
689 reg_def XMM9m( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(12));
690 reg_def XMM9n( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(13));
691 reg_def XMM9o( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(14));
692 reg_def XMM9p( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(15));
693
694 reg_def XMM10 ( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg());
695 reg_def XMM10b( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(1));
696 reg_def XMM10c( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(2));
697 reg_def XMM10d( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(3));
698 reg_def XMM10e( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(4));
699 reg_def XMM10f( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(5));
700 reg_def XMM10g( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(6));
701 reg_def XMM10h( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(7));
702 reg_def XMM10i( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(8));
703 reg_def XMM10j( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(9));
704 reg_def XMM10k( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(10));
705 reg_def XMM10l( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(11));
706 reg_def XMM10m( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(12));
707 reg_def XMM10n( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(13));
708 reg_def XMM10o( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(14));
709 reg_def XMM10p( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(15));
710
711 reg_def XMM11 ( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg());
712 reg_def XMM11b( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(1));
713 reg_def XMM11c( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(2));
714 reg_def XMM11d( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(3));
715 reg_def XMM11e( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(4));
716 reg_def XMM11f( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(5));
717 reg_def XMM11g( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(6));
718 reg_def XMM11h( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(7));
719 reg_def XMM11i( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(8));
720 reg_def XMM11j( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(9));
721 reg_def XMM11k( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(10));
722 reg_def XMM11l( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(11));
723 reg_def XMM11m( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(12));
724 reg_def XMM11n( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(13));
725 reg_def XMM11o( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(14));
726 reg_def XMM11p( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(15));
727
728 reg_def XMM12 ( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg());
729 reg_def XMM12b( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(1));
730 reg_def XMM12c( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(2));
731 reg_def XMM12d( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(3));
732 reg_def XMM12e( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(4));
733 reg_def XMM12f( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(5));
734 reg_def XMM12g( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(6));
735 reg_def XMM12h( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(7));
736 reg_def XMM12i( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(8));
737 reg_def XMM12j( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(9));
738 reg_def XMM12k( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(10));
739 reg_def XMM12l( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(11));
740 reg_def XMM12m( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(12));
741 reg_def XMM12n( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(13));
742 reg_def XMM12o( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(14));
743 reg_def XMM12p( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(15));
744
745 reg_def XMM13 ( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg());
746 reg_def XMM13b( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(1));
747 reg_def XMM13c( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(2));
748 reg_def XMM13d( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(3));
749 reg_def XMM13e( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(4));
750 reg_def XMM13f( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(5));
751 reg_def XMM13g( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(6));
752 reg_def XMM13h( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(7));
753 reg_def XMM13i( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(8));
754 reg_def XMM13j( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(9));
755 reg_def XMM13k( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(10));
756 reg_def XMM13l( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(11));
757 reg_def XMM13m( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(12));
758 reg_def XMM13n( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(13));
759 reg_def XMM13o( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(14));
760 reg_def XMM13p( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(15));
761
762 reg_def XMM14 ( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg());
763 reg_def XMM14b( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(1));
764 reg_def XMM14c( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(2));
765 reg_def XMM14d( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(3));
766 reg_def XMM14e( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(4));
767 reg_def XMM14f( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(5));
768 reg_def XMM14g( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(6));
769 reg_def XMM14h( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(7));
770 reg_def XMM14i( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(8));
771 reg_def XMM14j( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(9));
772 reg_def XMM14k( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(10));
773 reg_def XMM14l( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(11));
774 reg_def XMM14m( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(12));
775 reg_def XMM14n( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(13));
776 reg_def XMM14o( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(14));
777 reg_def XMM14p( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(15));
778
779 reg_def XMM15 ( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg());
780 reg_def XMM15b( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(1));
781 reg_def XMM15c( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(2));
782 reg_def XMM15d( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(3));
783 reg_def XMM15e( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(4));
784 reg_def XMM15f( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(5));
785 reg_def XMM15g( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(6));
786 reg_def XMM15h( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(7));
787 reg_def XMM15i( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(8));
788 reg_def XMM15j( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(9));
789 reg_def XMM15k( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(10));
790 reg_def XMM15l( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(11));
791 reg_def XMM15m( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(12));
792 reg_def XMM15n( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(13));
793 reg_def XMM15o( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(14));
794 reg_def XMM15p( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(15));
795
796 reg_def XMM16 ( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg());
797 reg_def XMM16b( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(1));
798 reg_def XMM16c( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(2));
799 reg_def XMM16d( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(3));
800 reg_def XMM16e( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(4));
801 reg_def XMM16f( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(5));
802 reg_def XMM16g( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(6));
803 reg_def XMM16h( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(7));
804 reg_def XMM16i( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(8));
805 reg_def XMM16j( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(9));
806 reg_def XMM16k( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(10));
807 reg_def XMM16l( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(11));
808 reg_def XMM16m( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(12));
809 reg_def XMM16n( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(13));
810 reg_def XMM16o( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(14));
811 reg_def XMM16p( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(15));
812
813 reg_def XMM17 ( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg());
814 reg_def XMM17b( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(1));
815 reg_def XMM17c( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(2));
816 reg_def XMM17d( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(3));
817 reg_def XMM17e( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(4));
818 reg_def XMM17f( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(5));
819 reg_def XMM17g( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(6));
820 reg_def XMM17h( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(7));
821 reg_def XMM17i( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(8));
822 reg_def XMM17j( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(9));
823 reg_def XMM17k( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(10));
824 reg_def XMM17l( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(11));
825 reg_def XMM17m( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(12));
826 reg_def XMM17n( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(13));
827 reg_def XMM17o( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(14));
828 reg_def XMM17p( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(15));
829
830 reg_def XMM18 ( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg());
831 reg_def XMM18b( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(1));
832 reg_def XMM18c( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(2));
833 reg_def XMM18d( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(3));
834 reg_def XMM18e( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(4));
835 reg_def XMM18f( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(5));
836 reg_def XMM18g( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(6));
837 reg_def XMM18h( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(7));
838 reg_def XMM18i( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(8));
839 reg_def XMM18j( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(9));
840 reg_def XMM18k( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(10));
841 reg_def XMM18l( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(11));
842 reg_def XMM18m( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(12));
843 reg_def XMM18n( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(13));
844 reg_def XMM18o( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(14));
845 reg_def XMM18p( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(15));
846
847 reg_def XMM19 ( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg());
848 reg_def XMM19b( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(1));
849 reg_def XMM19c( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(2));
850 reg_def XMM19d( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(3));
851 reg_def XMM19e( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(4));
852 reg_def XMM19f( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(5));
853 reg_def XMM19g( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(6));
854 reg_def XMM19h( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(7));
855 reg_def XMM19i( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(8));
856 reg_def XMM19j( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(9));
857 reg_def XMM19k( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(10));
858 reg_def XMM19l( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(11));
859 reg_def XMM19m( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(12));
860 reg_def XMM19n( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(13));
861 reg_def XMM19o( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(14));
862 reg_def XMM19p( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(15));
863
864 reg_def XMM20 ( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg());
865 reg_def XMM20b( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(1));
866 reg_def XMM20c( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(2));
867 reg_def XMM20d( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(3));
868 reg_def XMM20e( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(4));
869 reg_def XMM20f( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(5));
870 reg_def XMM20g( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(6));
871 reg_def XMM20h( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(7));
872 reg_def XMM20i( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(8));
873 reg_def XMM20j( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(9));
874 reg_def XMM20k( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(10));
875 reg_def XMM20l( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(11));
876 reg_def XMM20m( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(12));
877 reg_def XMM20n( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(13));
878 reg_def XMM20o( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(14));
879 reg_def XMM20p( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(15));
880
881 reg_def XMM21 ( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg());
882 reg_def XMM21b( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(1));
883 reg_def XMM21c( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(2));
884 reg_def XMM21d( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(3));
885 reg_def XMM21e( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(4));
886 reg_def XMM21f( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(5));
887 reg_def XMM21g( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(6));
888 reg_def XMM21h( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(7));
889 reg_def XMM21i( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(8));
890 reg_def XMM21j( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(9));
891 reg_def XMM21k( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(10));
892 reg_def XMM21l( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(11));
893 reg_def XMM21m( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(12));
894 reg_def XMM21n( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(13));
895 reg_def XMM21o( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(14));
896 reg_def XMM21p( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(15));
897
898 reg_def XMM22 ( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg());
899 reg_def XMM22b( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(1));
900 reg_def XMM22c( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(2));
901 reg_def XMM22d( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(3));
902 reg_def XMM22e( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(4));
903 reg_def XMM22f( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(5));
904 reg_def XMM22g( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(6));
905 reg_def XMM22h( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(7));
906 reg_def XMM22i( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(8));
907 reg_def XMM22j( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(9));
908 reg_def XMM22k( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(10));
909 reg_def XMM22l( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(11));
910 reg_def XMM22m( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(12));
911 reg_def XMM22n( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(13));
912 reg_def XMM22o( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(14));
913 reg_def XMM22p( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(15));
914
915 reg_def XMM23 ( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg());
916 reg_def XMM23b( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(1));
917 reg_def XMM23c( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(2));
918 reg_def XMM23d( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(3));
919 reg_def XMM23e( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(4));
920 reg_def XMM23f( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(5));
921 reg_def XMM23g( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(6));
922 reg_def XMM23h( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(7));
923 reg_def XMM23i( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(8));
924 reg_def XMM23j( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(9));
925 reg_def XMM23k( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(10));
926 reg_def XMM23l( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(11));
927 reg_def XMM23m( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(12));
928 reg_def XMM23n( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(13));
929 reg_def XMM23o( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(14));
930 reg_def XMM23p( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(15));
931
932 reg_def XMM24 ( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg());
933 reg_def XMM24b( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(1));
934 reg_def XMM24c( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(2));
935 reg_def XMM24d( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(3));
936 reg_def XMM24e( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(4));
937 reg_def XMM24f( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(5));
938 reg_def XMM24g( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(6));
939 reg_def XMM24h( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(7));
940 reg_def XMM24i( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(8));
941 reg_def XMM24j( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(9));
942 reg_def XMM24k( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(10));
943 reg_def XMM24l( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(11));
944 reg_def XMM24m( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(12));
945 reg_def XMM24n( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(13));
946 reg_def XMM24o( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(14));
947 reg_def XMM24p( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(15));
948
949 reg_def XMM25 ( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg());
950 reg_def XMM25b( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(1));
951 reg_def XMM25c( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(2));
952 reg_def XMM25d( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(3));
953 reg_def XMM25e( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(4));
954 reg_def XMM25f( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(5));
955 reg_def XMM25g( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(6));
956 reg_def XMM25h( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(7));
957 reg_def XMM25i( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(8));
958 reg_def XMM25j( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(9));
959 reg_def XMM25k( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(10));
960 reg_def XMM25l( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(11));
961 reg_def XMM25m( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(12));
962 reg_def XMM25n( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(13));
963 reg_def XMM25o( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(14));
964 reg_def XMM25p( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(15));
965
966 reg_def XMM26 ( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg());
967 reg_def XMM26b( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(1));
968 reg_def XMM26c( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(2));
969 reg_def XMM26d( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(3));
970 reg_def XMM26e( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(4));
971 reg_def XMM26f( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(5));
972 reg_def XMM26g( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(6));
973 reg_def XMM26h( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(7));
974 reg_def XMM26i( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(8));
975 reg_def XMM26j( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(9));
976 reg_def XMM26k( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(10));
977 reg_def XMM26l( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(11));
978 reg_def XMM26m( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(12));
979 reg_def XMM26n( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(13));
980 reg_def XMM26o( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(14));
981 reg_def XMM26p( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(15));
982
983 reg_def XMM27 ( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg());
984 reg_def XMM27b( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(1));
985 reg_def XMM27c( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(2));
986 reg_def XMM27d( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(3));
987 reg_def XMM27e( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(4));
988 reg_def XMM27f( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(5));
989 reg_def XMM27g( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(6));
990 reg_def XMM27h( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(7));
991 reg_def XMM27i( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(8));
992 reg_def XMM27j( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(9));
993 reg_def XMM27k( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(10));
994 reg_def XMM27l( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(11));
995 reg_def XMM27m( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(12));
996 reg_def XMM27n( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(13));
997 reg_def XMM27o( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(14));
998 reg_def XMM27p( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(15));
999
1000 reg_def XMM28 ( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg());
1001 reg_def XMM28b( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(1));
1002 reg_def XMM28c( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(2));
1003 reg_def XMM28d( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(3));
1004 reg_def XMM28e( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(4));
1005 reg_def XMM28f( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(5));
1006 reg_def XMM28g( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(6));
1007 reg_def XMM28h( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(7));
1008 reg_def XMM28i( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(8));
1009 reg_def XMM28j( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(9));
1010 reg_def XMM28k( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(10));
1011 reg_def XMM28l( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(11));
1012 reg_def XMM28m( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(12));
1013 reg_def XMM28n( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(13));
1014 reg_def XMM28o( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(14));
1015 reg_def XMM28p( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(15));
1016
1017 reg_def XMM29 ( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg());
1018 reg_def XMM29b( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(1));
1019 reg_def XMM29c( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(2));
1020 reg_def XMM29d( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(3));
1021 reg_def XMM29e( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(4));
1022 reg_def XMM29f( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(5));
1023 reg_def XMM29g( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(6));
1024 reg_def XMM29h( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(7));
1025 reg_def XMM29i( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(8));
1026 reg_def XMM29j( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(9));
1027 reg_def XMM29k( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(10));
1028 reg_def XMM29l( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(11));
1029 reg_def XMM29m( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(12));
1030 reg_def XMM29n( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(13));
1031 reg_def XMM29o( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(14));
1032 reg_def XMM29p( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(15));
1033
1034 reg_def XMM30 ( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg());
1035 reg_def XMM30b( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(1));
1036 reg_def XMM30c( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(2));
1037 reg_def XMM30d( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(3));
1038 reg_def XMM30e( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(4));
1039 reg_def XMM30f( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(5));
1040 reg_def XMM30g( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(6));
1041 reg_def XMM30h( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(7));
1042 reg_def XMM30i( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(8));
1043 reg_def XMM30j( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(9));
1044 reg_def XMM30k( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(10));
1045 reg_def XMM30l( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(11));
1046 reg_def XMM30m( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(12));
1047 reg_def XMM30n( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(13));
1048 reg_def XMM30o( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(14));
1049 reg_def XMM30p( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(15));
1050
1051 reg_def XMM31 ( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg());
1052 reg_def XMM31b( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(1));
1053 reg_def XMM31c( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(2));
1054 reg_def XMM31d( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(3));
1055 reg_def XMM31e( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(4));
1056 reg_def XMM31f( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(5));
1057 reg_def XMM31g( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(6));
1058 reg_def XMM31h( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(7));
1059 reg_def XMM31i( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(8));
1060 reg_def XMM31j( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(9));
1061 reg_def XMM31k( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(10));
1062 reg_def XMM31l( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(11));
1063 reg_def XMM31m( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(12));
1064 reg_def XMM31n( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(13));
1065 reg_def XMM31o( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(14));
1066 reg_def XMM31p( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(15));
1067
1068 #endif // _LP64
1069
1070 #endif // _WIN64
1071
1072 #ifdef _LP64
1073 reg_def RFLAGS(SOC, SOC, 0, 16, VMRegImpl::Bad());
1074 #else
1075 reg_def RFLAGS(SOC, SOC, 0, 8, VMRegImpl::Bad());
1076 #endif // _LP64
1077
1078 alloc_class chunk1(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
1079 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1080 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1081 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1082 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1083 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1084 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1085 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p
1086 #ifdef _LP64
1087 ,XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1088 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1089 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1090 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1091 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1092 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1093 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1094 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p
1095 ,XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
1096 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
1097 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
1098 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
1099 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
1100 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
1101 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
1102 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
1103 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
1104 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
1105 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
1106 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
1107 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
1108 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
1109 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
1110 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p
1111 #endif
1112 );
1113
1114 // flags allocation class should be last.
1115 alloc_class chunk2(RFLAGS);
1116
1117 // Singleton class for condition codes
1118 reg_class int_flags(RFLAGS);
1119
1120 // Class for pre evex float registers
1121 reg_class float_reg_legacy(XMM0,
1122 XMM1,
1123 XMM2,
1124 XMM3,
1125 XMM4,
1126 XMM5,
1127 XMM6,
1128 XMM7
1129 #ifdef _LP64
1130 ,XMM8,
1131 XMM9,
1132 XMM10,
1133 XMM11,
1134 XMM12,
1135 XMM13,
1136 XMM14,
1137 XMM15
1138 #endif
1139 );
1140
1141 // Class for evex float registers
1142 reg_class float_reg_evex(XMM0,
1143 XMM1,
1144 XMM2,
1145 XMM3,
1146 XMM4,
1147 XMM5,
1148 XMM6,
1149 XMM7
1150 #ifdef _LP64
1151 ,XMM8,
1152 XMM9,
1153 XMM10,
1154 XMM11,
1155 XMM12,
1156 XMM13,
1157 XMM14,
1158 XMM15,
1159 XMM16,
1160 XMM17,
1161 XMM18,
1162 XMM19,
1163 XMM20,
1164 XMM21,
1165 XMM22,
1166 XMM23,
1167 XMM24,
1168 XMM25,
1169 XMM26,
1170 XMM27,
1171 XMM28,
1172 XMM29,
1173 XMM30,
1174 XMM31
1175 #endif
1176 );
1177
1178 reg_class_dynamic float_reg(float_reg_evex, float_reg_legacy, %{ VM_Version::supports_evex() %} );
1179
1180 // Class for pre evex double registers
1181 reg_class double_reg_legacy(XMM0, XMM0b,
1182 XMM1, XMM1b,
1183 XMM2, XMM2b,
1184 XMM3, XMM3b,
1185 XMM4, XMM4b,
1186 XMM5, XMM5b,
1187 XMM6, XMM6b,
1188 XMM7, XMM7b
1189 #ifdef _LP64
1190 ,XMM8, XMM8b,
1191 XMM9, XMM9b,
1192 XMM10, XMM10b,
1193 XMM11, XMM11b,
1194 XMM12, XMM12b,
1195 XMM13, XMM13b,
1196 XMM14, XMM14b,
1197 XMM15, XMM15b
1198 #endif
1199 );
1200
1201 // Class for evex double registers
1202 reg_class double_reg_evex(XMM0, XMM0b,
1203 XMM1, XMM1b,
1204 XMM2, XMM2b,
1205 XMM3, XMM3b,
1206 XMM4, XMM4b,
1207 XMM5, XMM5b,
1208 XMM6, XMM6b,
1209 XMM7, XMM7b
1210 #ifdef _LP64
1211 ,XMM8, XMM8b,
1212 XMM9, XMM9b,
1213 XMM10, XMM10b,
1214 XMM11, XMM11b,
1215 XMM12, XMM12b,
1216 XMM13, XMM13b,
1217 XMM14, XMM14b,
1218 XMM15, XMM15b,
1219 XMM16, XMM16b,
1220 XMM17, XMM17b,
1221 XMM18, XMM18b,
1222 XMM19, XMM19b,
1223 XMM20, XMM20b,
1224 XMM21, XMM21b,
1225 XMM22, XMM22b,
1226 XMM23, XMM23b,
1227 XMM24, XMM24b,
1228 XMM25, XMM25b,
1229 XMM26, XMM26b,
1230 XMM27, XMM27b,
1231 XMM28, XMM28b,
1232 XMM29, XMM29b,
1233 XMM30, XMM30b,
1234 XMM31, XMM31b
1235 #endif
1236 );
1237
1238 reg_class_dynamic double_reg(double_reg_evex, double_reg_legacy, %{ VM_Version::supports_evex() %} );
1239
1240 // Class for pre evex 32bit vector registers
1241 reg_class vectors_reg_legacy(XMM0,
1242 XMM1,
1243 XMM2,
1244 XMM3,
1245 XMM4,
1246 XMM5,
1247 XMM6,
1248 XMM7
1249 #ifdef _LP64
1250 ,XMM8,
1251 XMM9,
1252 XMM10,
1253 XMM11,
1254 XMM12,
1255 XMM13,
1256 XMM14,
1257 XMM15
1258 #endif
1259 );
1260
1261 // Class for evex 32bit vector registers
1262 reg_class vectors_reg_evex(XMM0,
1263 XMM1,
1264 XMM2,
1265 XMM3,
1266 XMM4,
1267 XMM5,
1268 XMM6,
1269 XMM7
1270 #ifdef _LP64
1271 ,XMM8,
1272 XMM9,
1273 XMM10,
1274 XMM11,
1275 XMM12,
1276 XMM13,
1277 XMM14,
1278 XMM15,
1279 XMM16,
1280 XMM17,
1281 XMM18,
1282 XMM19,
1283 XMM20,
1284 XMM21,
1285 XMM22,
1286 XMM23,
1287 XMM24,
1288 XMM25,
1289 XMM26,
1290 XMM27,
1291 XMM28,
1292 XMM29,
1293 XMM30,
1294 XMM31
1295 #endif
1296 );
1297
1298 reg_class_dynamic vectors_reg(vectors_reg_evex, vectors_reg_legacy, %{ VM_Version::supports_evex() %} );
1299
1300 // Class for all 64bit vector registers
1301 reg_class vectord_reg_legacy(XMM0, XMM0b,
1302 XMM1, XMM1b,
1303 XMM2, XMM2b,
1304 XMM3, XMM3b,
1305 XMM4, XMM4b,
1306 XMM5, XMM5b,
1307 XMM6, XMM6b,
1308 XMM7, XMM7b
1309 #ifdef _LP64
1310 ,XMM8, XMM8b,
1311 XMM9, XMM9b,
1312 XMM10, XMM10b,
1313 XMM11, XMM11b,
1314 XMM12, XMM12b,
1315 XMM13, XMM13b,
1316 XMM14, XMM14b,
1317 XMM15, XMM15b
1318 #endif
1319 );
1320
1321 // Class for all 64bit vector registers
1322 reg_class vectord_reg_evex(XMM0, XMM0b,
1323 XMM1, XMM1b,
1324 XMM2, XMM2b,
1325 XMM3, XMM3b,
1326 XMM4, XMM4b,
1327 XMM5, XMM5b,
1328 XMM6, XMM6b,
1329 XMM7, XMM7b
1330 #ifdef _LP64
1331 ,XMM8, XMM8b,
1332 XMM9, XMM9b,
1333 XMM10, XMM10b,
1334 XMM11, XMM11b,
1335 XMM12, XMM12b,
1336 XMM13, XMM13b,
1337 XMM14, XMM14b,
1338 XMM15, XMM15b,
1339 XMM16, XMM16b,
1340 XMM17, XMM17b,
1341 XMM18, XMM18b,
1342 XMM19, XMM19b,
1343 XMM20, XMM20b,
1344 XMM21, XMM21b,
1345 XMM22, XMM22b,
1346 XMM23, XMM23b,
1347 XMM24, XMM24b,
1348 XMM25, XMM25b,
1349 XMM26, XMM26b,
1350 XMM27, XMM27b,
1351 XMM28, XMM28b,
1352 XMM29, XMM29b,
1353 XMM30, XMM30b,
1354 XMM31, XMM31b
1355 #endif
1356 );
1357
1358 reg_class_dynamic vectord_reg(vectord_reg_evex, vectord_reg_legacy, %{ VM_Version::supports_evex() %} );
1359
1360 // Class for all 128bit vector registers
1361 reg_class vectorx_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d,
1362 XMM1, XMM1b, XMM1c, XMM1d,
1363 XMM2, XMM2b, XMM2c, XMM2d,
1364 XMM3, XMM3b, XMM3c, XMM3d,
1365 XMM4, XMM4b, XMM4c, XMM4d,
1366 XMM5, XMM5b, XMM5c, XMM5d,
1367 XMM6, XMM6b, XMM6c, XMM6d,
1368 XMM7, XMM7b, XMM7c, XMM7d
1369 #ifdef _LP64
1370 ,XMM8, XMM8b, XMM8c, XMM8d,
1371 XMM9, XMM9b, XMM9c, XMM9d,
1372 XMM10, XMM10b, XMM10c, XMM10d,
1373 XMM11, XMM11b, XMM11c, XMM11d,
1374 XMM12, XMM12b, XMM12c, XMM12d,
1375 XMM13, XMM13b, XMM13c, XMM13d,
1376 XMM14, XMM14b, XMM14c, XMM14d,
1377 XMM15, XMM15b, XMM15c, XMM15d
1378 #endif
1379 );
1380
1381 // Class for all 128bit vector registers
1382 reg_class vectorx_reg_evex(XMM0, XMM0b, XMM0c, XMM0d,
1383 XMM1, XMM1b, XMM1c, XMM1d,
1384 XMM2, XMM2b, XMM2c, XMM2d,
1385 XMM3, XMM3b, XMM3c, XMM3d,
1386 XMM4, XMM4b, XMM4c, XMM4d,
1387 XMM5, XMM5b, XMM5c, XMM5d,
1388 XMM6, XMM6b, XMM6c, XMM6d,
1389 XMM7, XMM7b, XMM7c, XMM7d
1390 #ifdef _LP64
1391 ,XMM8, XMM8b, XMM8c, XMM8d,
1392 XMM9, XMM9b, XMM9c, XMM9d,
1393 XMM10, XMM10b, XMM10c, XMM10d,
1394 XMM11, XMM11b, XMM11c, XMM11d,
1395 XMM12, XMM12b, XMM12c, XMM12d,
1396 XMM13, XMM13b, XMM13c, XMM13d,
1397 XMM14, XMM14b, XMM14c, XMM14d,
1398 XMM15, XMM15b, XMM15c, XMM15d,
1399 XMM16, XMM16b, XMM16c, XMM16d,
1400 XMM17, XMM17b, XMM17c, XMM17d,
1401 XMM18, XMM18b, XMM18c, XMM18d,
1402 XMM19, XMM19b, XMM19c, XMM19d,
1403 XMM20, XMM20b, XMM20c, XMM20d,
1404 XMM21, XMM21b, XMM21c, XMM21d,
1405 XMM22, XMM22b, XMM22c, XMM22d,
1406 XMM23, XMM23b, XMM23c, XMM23d,
1407 XMM24, XMM24b, XMM24c, XMM24d,
1408 XMM25, XMM25b, XMM25c, XMM25d,
1409 XMM26, XMM26b, XMM26c, XMM26d,
1410 XMM27, XMM27b, XMM27c, XMM27d,
1411 XMM28, XMM28b, XMM28c, XMM28d,
1412 XMM29, XMM29b, XMM29c, XMM29d,
1413 XMM30, XMM30b, XMM30c, XMM30d,
1414 XMM31, XMM31b, XMM31c, XMM31d
1415 #endif
1416 );
1417
1418 reg_class_dynamic vectorx_reg(vectorx_reg_evex, vectorx_reg_legacy, %{ VM_Version::supports_evex() %} );
1419
1420 // Class for all 256bit vector registers
1421 reg_class vectory_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h,
1422 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h,
1423 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h,
1424 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h,
1425 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h,
1426 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h,
1427 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h,
1428 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h
1429 #ifdef _LP64
1430 ,XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h,
1431 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h,
1432 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
1433 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
1434 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
1435 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
1436 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
1437 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h
1438 #endif
1439 );
1440
1441 // Class for all 256bit vector registers
1442 reg_class vectory_reg_evex(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h,
1443 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h,
1444 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h,
1445 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h,
1446 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h,
1447 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h,
1448 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h,
1449 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h
1450 #ifdef _LP64
1451 ,XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h,
1452 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h,
1453 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
1454 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
1455 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
1456 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
1457 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
1458 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h,
1459 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h,
1460 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h,
1461 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h,
1462 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h,
1463 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h,
1464 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h,
1465 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h,
1466 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h,
1467 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h,
1468 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h,
1469 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h,
1470 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h,
1471 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h,
1472 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h,
1473 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h,
1474 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h
1475 #endif
1476 );
1477
1478 reg_class_dynamic vectory_reg(vectory_reg_evex, vectory_reg_legacy, %{ VM_Version::supports_evex() %} );
1479
1480 // Class for all 512bit vector registers
1481 reg_class vectorz_reg(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
1482 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1483 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1484 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1485 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1486 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1487 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1488 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p
1489 #ifdef _LP64
1490 ,XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1491 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1492 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1493 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1494 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1495 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1496 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1497 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p
1498 ,XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
1499 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
1500 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
1501 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
1502 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
1503 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
1504 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
1505 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
1506 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
1507 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
1508 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
1509 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
1510 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
1511 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
1512 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
1513 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p
1514 #endif
1515 );
1516
1517 %}
1518
1519
1520 //----------SOURCE BLOCK-------------------------------------------------------
1521 // This is a block of C++ code which provides values, functions, and
1522 // definitions necessary in the rest of the architecture description
1523
1524 source_hpp %{
1525 // Header information of the source block.
1526 // Method declarations/definitions which are used outside
1527 // the ad-scope can conveniently be defined here.
1528 //
1529 // To keep related declarations/definitions/uses close together,
1530 // we switch between source %{ }% and source_hpp %{ }% freely as needed.
1531
1532 class NativeJump;
1533
1534 class CallStubImpl {
1535
1536 //--------------------------------------------------------------
1537 //---< Used for optimization in Compile::shorten_branches >---
1538 //--------------------------------------------------------------
1539
1540 public:
1541 // Size of call trampoline stub.
1542 static uint size_call_trampoline() {
1543 return 0; // no call trampolines on this platform
1544 }
1545
1546 // number of relocations needed by a call trampoline stub
1547 static uint reloc_call_trampoline() {
1548 return 0; // no call trampolines on this platform
1549 }
1550 };
1551
1552 class HandlerImpl {
1553
1554 public:
1555
1556 static int emit_exception_handler(CodeBuffer &cbuf);
1557 static int emit_deopt_handler(CodeBuffer& cbuf);
1558
1559 static uint size_exception_handler() {
1560 // NativeCall instruction size is the same as NativeJump.
1561 // exception handler starts out as jump and can be patched to
1562 // a call be deoptimization. (4932387)
1563 // Note that this value is also credited (in output.cpp) to
1564 // the size of the code section.
1565 return NativeJump::instruction_size;
1566 }
1567
1568 #ifdef _LP64
1569 static uint size_deopt_handler() {
1570 // three 5 byte instructions
1571 return 15;
1572 }
1573 #else
1574 static uint size_deopt_handler() {
1575 // NativeCall instruction size is the same as NativeJump.
1576 // exception handler starts out as jump and can be patched to
1577 // a call be deoptimization. (4932387)
1578 // Note that this value is also credited (in output.cpp) to
1579 // the size of the code section.
1580 return 5 + NativeJump::instruction_size; // pushl(); jmp;
1581 }
1582 #endif
1583 };
1584
1585 %} // end source_hpp
1586
1587 source %{
1588
1589 // Emit exception handler code.
1590 // Stuff framesize into a register and call a VM stub routine.
1591 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
1592
1593 // Note that the code buffer's insts_mark is always relative to insts.
1594 // That's why we must use the macroassembler to generate a handler.
1595 MacroAssembler _masm(&cbuf);
1596 address base = __ start_a_stub(size_exception_handler());
1597 if (base == NULL) {
1598 ciEnv::current()->record_failure("CodeCache is full");
1599 return 0; // CodeBuffer::expand failed
1600 }
1601 int offset = __ offset();
1602 __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
1603 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1604 __ end_a_stub();
1605 return offset;
1606 }
1607
1608 // Emit deopt handler code.
1609 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
1610
1611 // Note that the code buffer's insts_mark is always relative to insts.
1612 // That's why we must use the macroassembler to generate a handler.
1613 MacroAssembler _masm(&cbuf);
1614 address base = __ start_a_stub(size_deopt_handler());
1615 if (base == NULL) {
1616 ciEnv::current()->record_failure("CodeCache is full");
1617 return 0; // CodeBuffer::expand failed
1618 }
1619 int offset = __ offset();
1620
1621 #ifdef _LP64
1622 address the_pc = (address) __ pc();
1623 Label next;
1624 // push a "the_pc" on the stack without destroying any registers
1625 // as they all may be live.
1626
1627 // push address of "next"
1628 __ call(next, relocInfo::none); // reloc none is fine since it is a disp32
1629 __ bind(next);
1630 // adjust it so it matches "the_pc"
1631 __ subptr(Address(rsp, 0), __ offset() - offset);
1632 #else
1633 InternalAddress here(__ pc());
1634 __ pushptr(here.addr());
1635 #endif
1636
1637 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
1638 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1639 __ end_a_stub();
1640 return offset;
1641 }
1642
1643
1644 //=============================================================================
1645
1646 // Float masks come from different places depending on platform.
1647 #ifdef _LP64
1648 static address float_signmask() { return StubRoutines::x86::float_sign_mask(); }
1649 static address float_signflip() { return StubRoutines::x86::float_sign_flip(); }
1650 static address double_signmask() { return StubRoutines::x86::double_sign_mask(); }
1651 static address double_signflip() { return StubRoutines::x86::double_sign_flip(); }
1652 #else
1653 static address float_signmask() { return (address)float_signmask_pool; }
1654 static address float_signflip() { return (address)float_signflip_pool; }
1655 static address double_signmask() { return (address)double_signmask_pool; }
1656 static address double_signflip() { return (address)double_signflip_pool; }
1657 #endif
1658
1659
1660 const bool Matcher::match_rule_supported(int opcode) {
1661 if (!has_match_rule(opcode))
1662 return false;
1663
1664 bool ret_value = true;
1665 switch (opcode) {
1666 case Op_PopCountI:
1667 case Op_PopCountL:
1668 if (!UsePopCountInstruction)
1669 ret_value = false;
1670 break;
1671 case Op_MulVI:
1672 if ((UseSSE < 4) && (UseAVX < 1)) // only with SSE4_1 or AVX
1673 ret_value = false;
1674 break;
1675 case Op_MulVL:
1676 case Op_MulReductionVL:
1677 if (VM_Version::supports_avx512dq() == false)
1678 ret_value = false;
1679 break;
1680 case Op_AddReductionVL:
1681 if (UseAVX < 3) // only EVEX : vector connectivity becomes an issue here
1682 ret_value = false;
1683 break;
1684 case Op_AddReductionVI:
1685 if (UseSSE < 3) // requires at least SSE3
1686 ret_value = false;
1687 break;
1688 case Op_MulReductionVI:
1689 if (UseSSE < 4) // requires at least SSE4
1690 ret_value = false;
1691 break;
1692 case Op_AddReductionVF:
1693 case Op_AddReductionVD:
1694 case Op_MulReductionVF:
1695 case Op_MulReductionVD:
1696 if (UseSSE < 1) // requires at least SSE
1697 ret_value = false;
1698 break;
1699 case Op_SqrtVD:
1700 if (UseAVX < 1) // enabled for AVX only
1701 ret_value = false;
1702 break;
1703 case Op_CompareAndSwapL:
1704 #ifdef _LP64
1705 case Op_CompareAndSwapP:
1706 #endif
1707 if (!VM_Version::supports_cx8())
1708 ret_value = false;
1709 break;
1710 case Op_CMoveVD:
1711 if (UseAVX > 2)
1712 ret_value = false;
1713 break;
1714 }
1715
1716 return ret_value; // Per default match rules are supported.
1717 }
1718
1719 const int Matcher::float_pressure(int default_pressure_threshold) {
1720 int float_pressure_threshold = default_pressure_threshold;
1721 #ifdef _LP64
1722 if (UseAVX > 2) {
1723 // Increase pressure threshold on machines with AVX3 which have
1724 // 2x more XMM registers.
1725 float_pressure_threshold = default_pressure_threshold * 2;
1726 }
1727 #endif
1728 return float_pressure_threshold;
1729 }
1730
1731 // Max vector size in bytes. 0 if not supported.
1732 const int Matcher::vector_width_in_bytes(BasicType bt) {
1733 assert(is_java_primitive(bt), "only primitive type vectors");
1734 if (UseSSE < 2) return 0;
1735 // SSE2 supports 128bit vectors for all types.
1736 // AVX2 supports 256bit vectors for all types.
1737 // AVX2/EVEX supports 512bit vectors for all types.
1738 int size = (UseAVX > 1) ? (1 << UseAVX) * 8 : 16;
1739 // AVX1 supports 256bit vectors only for FLOAT and DOUBLE.
1740 if (UseAVX > 0 && (bt == T_FLOAT || bt == T_DOUBLE))
1741 size = (UseAVX > 2) ? 64 : 32;
1742 // Use flag to limit vector size.
1743 size = MIN2(size,(int)MaxVectorSize);
1744 // Minimum 2 values in vector (or 4 for bytes).
1745 switch (bt) {
1746 case T_DOUBLE:
1747 case T_LONG:
1748 if (size < 16) return 0;
1749 break;
1750 case T_FLOAT:
1751 case T_INT:
1752 if (size < 8) return 0;
1753 break;
1754 case T_BOOLEAN:
1755 if (size < 4) return 0;
1756 break;
1757 case T_CHAR:
1758 if (size < 4) return 0;
1759 break;
1760 case T_BYTE:
1761 if (size < 4) return 0;
1762 if ((size > 32) && !VM_Version::supports_avx512bw()) return 0;
1763 break;
1764 case T_SHORT:
1765 if (size < 4) return 0;
1766 if ((size > 16) && !VM_Version::supports_avx512bw()) return 0;
1767 break;
1768 default:
1769 ShouldNotReachHere();
1770 }
1771 return size;
1772 }
1773
1774 // Limits on vector size (number of elements) loaded into vector.
1775 const int Matcher::max_vector_size(const BasicType bt) {
1776 return vector_width_in_bytes(bt)/type2aelembytes(bt);
1777 }
1778 const int Matcher::min_vector_size(const BasicType bt) {
1779 int max_size = max_vector_size(bt);
1780 // Min size which can be loaded into vector is 4 bytes.
1781 int size = (type2aelembytes(bt) == 1) ? 4 : 2;
1782 return MIN2(size,max_size);
1783 }
1784
1785 // Vector ideal reg corresponding to specidied size in bytes
1786 const int Matcher::vector_ideal_reg(int size) {
1787 assert(MaxVectorSize >= size, "");
1788 switch(size) {
1789 case 4: return Op_VecS;
1790 case 8: return Op_VecD;
1791 case 16: return Op_VecX;
1792 case 32: return Op_VecY;
1793 case 64: return Op_VecZ;
1794 }
1795 ShouldNotReachHere();
1796 return 0;
1797 }
1798
1799 // Only lowest bits of xmm reg are used for vector shift count.
1800 const int Matcher::vector_shift_count_ideal_reg(int size) {
1801 return Op_VecS;
1802 }
1803
1804 // x86 supports misaligned vectors store/load.
1805 const bool Matcher::misaligned_vectors_ok() {
1806 return !AlignVector; // can be changed by flag
1807 }
1808
1809 // x86 AES instructions are compatible with SunJCE expanded
1810 // keys, hence we do not need to pass the original key to stubs
1811 const bool Matcher::pass_original_key_for_aes() {
1812 return false;
1813 }
1814
1815 // Helper methods for MachSpillCopyNode::implementation().
1816 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1817 int src_hi, int dst_hi, uint ireg, outputStream* st) {
1818 // In 64-bit VM size calculation is very complex. Emitting instructions
1819 // into scratch buffer is used to get size in 64-bit VM.
1820 LP64_ONLY( assert(!do_size, "this method calculates size only for 32-bit VM"); )
1821 assert(ireg == Op_VecS || // 32bit vector
1822 (src_lo & 1) == 0 && (src_lo + 1) == src_hi &&
1823 (dst_lo & 1) == 0 && (dst_lo + 1) == dst_hi,
1824 "no non-adjacent vector moves" );
1825 if (cbuf) {
1826 MacroAssembler _masm(cbuf);
1827 int offset = __ offset();
1828 switch (ireg) {
1829 case Op_VecS: // copy whole register
1830 case Op_VecD:
1831 case Op_VecX:
1832 __ movdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
1833 break;
1834 case Op_VecY:
1835 __ vmovdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
1836 break;
1837 case Op_VecZ:
1838 __ evmovdqul(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 2);
1839 break;
1840 default:
1841 ShouldNotReachHere();
1842 }
1843 int size = __ offset() - offset;
1844 #ifdef ASSERT
1845 // VEX_2bytes prefix is used if UseAVX > 0, so it takes the same 2 bytes as SIMD prefix.
1846 assert(!do_size || size == 4, "incorrect size calculattion");
1847 #endif
1848 return size;
1849 #ifndef PRODUCT
1850 } else if (!do_size) {
1851 switch (ireg) {
1852 case Op_VecS:
1853 case Op_VecD:
1854 case Op_VecX:
1855 st->print("movdqu %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
1856 break;
1857 case Op_VecY:
1858 case Op_VecZ:
1859 st->print("vmovdqu %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
1860 break;
1861 default:
1862 ShouldNotReachHere();
1863 }
1864 #endif
1865 }
1866 // VEX_2bytes prefix is used if UseAVX > 0, and it takes the same 2 bytes as SIMD prefix.
1867 return (UseAVX > 2) ? 6 : 4;
1868 }
1869
1870 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1871 int stack_offset, int reg, uint ireg, outputStream* st) {
1872 // In 64-bit VM size calculation is very complex. Emitting instructions
1873 // into scratch buffer is used to get size in 64-bit VM.
1874 LP64_ONLY( assert(!do_size, "this method calculates size only for 32-bit VM"); )
1875 if (cbuf) {
1876 MacroAssembler _masm(cbuf);
1877 int offset = __ offset();
1878 if (is_load) {
1879 switch (ireg) {
1880 case Op_VecS:
1881 __ movdl(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
1882 break;
1883 case Op_VecD:
1884 __ movq(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
1885 break;
1886 case Op_VecX:
1887 __ movdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
1888 break;
1889 case Op_VecY:
1890 __ vmovdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
1891 break;
1892 case Op_VecZ:
1893 __ evmovdqul(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset), 2);
1894 break;
1895 default:
1896 ShouldNotReachHere();
1897 }
1898 } else { // store
1899 switch (ireg) {
1900 case Op_VecS:
1901 __ movdl(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
1902 break;
1903 case Op_VecD:
1904 __ movq(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
1905 break;
1906 case Op_VecX:
1907 __ movdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
1908 break;
1909 case Op_VecY:
1910 __ vmovdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
1911 break;
1912 case Op_VecZ:
1913 __ evmovdqul(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 2);
1914 break;
1915 default:
1916 ShouldNotReachHere();
1917 }
1918 }
1919 int size = __ offset() - offset;
1920 #ifdef ASSERT
1921 int offset_size = (stack_offset == 0) ? 0 : ((stack_offset < 0x80) ? 1 : (UseAVX > 2) ? 6 : 4);
1922 // VEX_2bytes prefix is used if UseAVX > 0, so it takes the same 2 bytes as SIMD prefix.
1923 assert(!do_size || size == (5+offset_size), "incorrect size calculattion");
1924 #endif
1925 return size;
1926 #ifndef PRODUCT
1927 } else if (!do_size) {
1928 if (is_load) {
1929 switch (ireg) {
1930 case Op_VecS:
1931 st->print("movd %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
1932 break;
1933 case Op_VecD:
1934 st->print("movq %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
1935 break;
1936 case Op_VecX:
1937 st->print("movdqu %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
1938 break;
1939 case Op_VecY:
1940 case Op_VecZ:
1941 st->print("vmovdqu %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
1942 break;
1943 default:
1944 ShouldNotReachHere();
1945 }
1946 } else { // store
1947 switch (ireg) {
1948 case Op_VecS:
1949 st->print("movd [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
1950 break;
1951 case Op_VecD:
1952 st->print("movq [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
1953 break;
1954 case Op_VecX:
1955 st->print("movdqu [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
1956 break;
1957 case Op_VecY:
1958 case Op_VecZ:
1959 st->print("vmovdqu [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
1960 break;
1961 default:
1962 ShouldNotReachHere();
1963 }
1964 }
1965 #endif
1966 }
1967 bool is_single_byte = false;
1968 int vec_len = 0;
1969 if ((UseAVX > 2) && (stack_offset != 0)) {
1970 switch (ireg) {
1971 case Op_VecS:
1972 case Op_VecD:
1973 case Op_VecX:
1974 break;
1975 case Op_VecY:
1976 vec_len = 1;
1977 break;
1978 case Op_VecZ:
1979 vec_len = 2;
1980 break;
1981 }
1982 is_single_byte = Assembler::query_compressed_disp_byte(stack_offset, true, vec_len, Assembler::EVEX_FVM, Assembler::EVEX_32bit, 0);
1983 }
1984 int offset_size = 0;
1985 int size = 5;
1986 if (UseAVX > 2 ) {
1987 if ((VM_Version::supports_avx512vl() == false) && (vec_len == 2)) {
1988 offset_size = (stack_offset == 0) ? 0 : ((is_single_byte) ? 1 : 4);
1989 size += 2; // Need an additional two bytes for EVEX encoding
1990 } else if ((VM_Version::supports_avx512vl() == false) && (vec_len < 2)) {
1991 offset_size = (stack_offset == 0) ? 0 : ((stack_offset <= 127) ? 1 : 4);
1992 } else {
1993 offset_size = (stack_offset == 0) ? 0 : ((is_single_byte) ? 1 : 4);
1994 size += 2; // Need an additional two bytes for EVEX encodding
1995 }
1996 } else {
1997 offset_size = (stack_offset == 0) ? 0 : ((stack_offset <= 127) ? 1 : 4);
1998 }
1999 // VEX_2bytes prefix is used if UseAVX > 0, so it takes the same 2 bytes as SIMD prefix.
2000 return size+offset_size;
2001 }
2002
2003 static inline jfloat replicate4_imm(int con, int width) {
2004 // Load a constant of "width" (in bytes) and replicate it to fill 32bit.
2005 assert(width == 1 || width == 2, "only byte or short types here");
2006 int bit_width = width * 8;
2007 jint val = con;
2008 val &= (1 << bit_width) - 1; // mask off sign bits
2009 while(bit_width < 32) {
2010 val |= (val << bit_width);
2011 bit_width <<= 1;
2012 }
2013 jfloat fval = *((jfloat*) &val); // coerce to float type
2014 return fval;
2015 }
2016
2017 static inline jdouble replicate8_imm(int con, int width) {
2018 // Load a constant of "width" (in bytes) and replicate it to fill 64bit.
2019 assert(width == 1 || width == 2 || width == 4, "only byte, short or int types here");
2020 int bit_width = width * 8;
2021 jlong val = con;
2022 val &= (((jlong) 1) << bit_width) - 1; // mask off sign bits
2023 while(bit_width < 64) {
2024 val |= (val << bit_width);
2025 bit_width <<= 1;
2026 }
2027 jdouble dval = *((jdouble*) &val); // coerce to double type
2028 return dval;
2029 }
2030
2031 #ifndef PRODUCT
2032 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
2033 st->print("nop \t# %d bytes pad for loops and calls", _count);
2034 }
2035 #endif
2036
2037 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
2038 MacroAssembler _masm(&cbuf);
2039 __ nop(_count);
2040 }
2041
2042 uint MachNopNode::size(PhaseRegAlloc*) const {
2043 return _count;
2044 }
2045
2046 #ifndef PRODUCT
2047 void MachBreakpointNode::format(PhaseRegAlloc*, outputStream* st) const {
2048 st->print("# breakpoint");
2049 }
2050 #endif
2051
2052 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc* ra_) const {
2053 MacroAssembler _masm(&cbuf);
2054 __ int3();
2055 }
2056
2057 uint MachBreakpointNode::size(PhaseRegAlloc* ra_) const {
2058 return MachNode::size(ra_);
2059 }
2060
2061 %}
2062
2063 encode %{
2064
2065 enc_class call_epilog %{
2066 if (VerifyStackAtCalls) {
2067 // Check that stack depth is unchanged: find majik cookie on stack
2068 int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
2069 MacroAssembler _masm(&cbuf);
2070 Label L;
2071 __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
2072 __ jccb(Assembler::equal, L);
2073 // Die if stack mismatch
2074 __ int3();
2075 __ bind(L);
2076 }
2077 %}
2078
2079 %}
2080
2081
2082 //----------OPERANDS-----------------------------------------------------------
2083 // Operand definitions must precede instruction definitions for correct parsing
2084 // in the ADLC because operands constitute user defined types which are used in
2085 // instruction definitions.
2086
2087 // This one generically applies only for evex, so only one version
2088 operand vecZ() %{
2089 constraint(ALLOC_IN_RC(vectorz_reg));
2090 match(VecZ);
2091
2092 format %{ %}
2093 interface(REG_INTER);
2094 %}
2095
2096 // Comparison Code for FP conditional move
2097 operand cmpOp_vcmppd() %{
2098 match(Bool);
2099
2100 predicate(n->as_Bool()->_test._test != BoolTest::overflow &&
2101 n->as_Bool()->_test._test != BoolTest::no_overflow);
2102 format %{ "" %}
2103 interface(COND_INTER) %{
2104 equal (0x0, "eq");
2105 less (0x1, "lt");
2106 less_equal (0x2, "le");
2107 not_equal (0xC, "ne");
2108 greater_equal(0xD, "ge");
2109 greater (0xE, "gt");
2110 //TODO cannot compile (adlc breaks) without two next lines with error:
2111 // x86_64.ad(13987) Syntax Error: :In operand cmpOp_vcmppd: Do not support this encode constant: ' %{
2112 // equal' for overflow.
2113 overflow (0x20, "o"); // not really supported by the instruction
2114 no_overflow (0x21, "no"); // not really supported by the instruction
2115 %}
2116 %}
2117
2118
2119 // INSTRUCTIONS -- Platform independent definitions (same for 32- and 64-bit)
2120
2121 // ============================================================================
2122
2123 instruct ShouldNotReachHere() %{
2124 match(Halt);
2125 format %{ "int3\t# ShouldNotReachHere" %}
2126 ins_encode %{
2127 __ int3();
2128 %}
2129 ins_pipe(pipe_slow);
2130 %}
2131
2132 // ============================================================================
2133
2134 instruct addF_reg(regF dst, regF src) %{
2135 predicate((UseSSE>=1) && (UseAVX == 0));
2136 match(Set dst (AddF dst src));
2137
2138 format %{ "addss $dst, $src" %}
2139 ins_cost(150);
2140 ins_encode %{
2141 __ addss($dst$$XMMRegister, $src$$XMMRegister);
2142 %}
2143 ins_pipe(pipe_slow);
2144 %}
2145
2146 instruct addF_mem(regF dst, memory src) %{
2147 predicate((UseSSE>=1) && (UseAVX == 0));
2148 match(Set dst (AddF dst (LoadF src)));
2149
2150 format %{ "addss $dst, $src" %}
2151 ins_cost(150);
2152 ins_encode %{
2153 __ addss($dst$$XMMRegister, $src$$Address);
2154 %}
2155 ins_pipe(pipe_slow);
2156 %}
2157
2158 instruct addF_imm(regF dst, immF con) %{
2159 predicate((UseSSE>=1) && (UseAVX == 0));
2160 match(Set dst (AddF dst con));
2161 format %{ "addss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
2162 ins_cost(150);
2163 ins_encode %{
2164 __ addss($dst$$XMMRegister, $constantaddress($con));
2165 %}
2166 ins_pipe(pipe_slow);
2167 %}
2168
2169 instruct addF_reg_reg(regF dst, regF src1, regF src2) %{
2170 predicate(UseAVX > 0);
2171 match(Set dst (AddF src1 src2));
2172
2173 format %{ "vaddss $dst, $src1, $src2" %}
2174 ins_cost(150);
2175 ins_encode %{
2176 __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2177 %}
2178 ins_pipe(pipe_slow);
2179 %}
2180
2181 instruct addF_reg_mem(regF dst, regF src1, memory src2) %{
2182 predicate(UseAVX > 0);
2183 match(Set dst (AddF src1 (LoadF src2)));
2184
2185 format %{ "vaddss $dst, $src1, $src2" %}
2186 ins_cost(150);
2187 ins_encode %{
2188 __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2189 %}
2190 ins_pipe(pipe_slow);
2191 %}
2192
2193 instruct addF_reg_imm(regF dst, regF src, immF con) %{
2194 predicate(UseAVX > 0);
2195 match(Set dst (AddF src con));
2196
2197 format %{ "vaddss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
2198 ins_cost(150);
2199 ins_encode %{
2200 __ vaddss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2201 %}
2202 ins_pipe(pipe_slow);
2203 %}
2204
2205 instruct addD_reg(regD dst, regD src) %{
2206 predicate((UseSSE>=2) && (UseAVX == 0));
2207 match(Set dst (AddD dst src));
2208
2209 format %{ "addsd $dst, $src" %}
2210 ins_cost(150);
2211 ins_encode %{
2212 __ addsd($dst$$XMMRegister, $src$$XMMRegister);
2213 %}
2214 ins_pipe(pipe_slow);
2215 %}
2216
2217 instruct addD_mem(regD dst, memory src) %{
2218 predicate((UseSSE>=2) && (UseAVX == 0));
2219 match(Set dst (AddD dst (LoadD src)));
2220
2221 format %{ "addsd $dst, $src" %}
2222 ins_cost(150);
2223 ins_encode %{
2224 __ addsd($dst$$XMMRegister, $src$$Address);
2225 %}
2226 ins_pipe(pipe_slow);
2227 %}
2228
2229 instruct addD_imm(regD dst, immD con) %{
2230 predicate((UseSSE>=2) && (UseAVX == 0));
2231 match(Set dst (AddD dst con));
2232 format %{ "addsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
2233 ins_cost(150);
2234 ins_encode %{
2235 __ addsd($dst$$XMMRegister, $constantaddress($con));
2236 %}
2237 ins_pipe(pipe_slow);
2238 %}
2239
2240 instruct addD_reg_reg(regD dst, regD src1, regD src2) %{
2241 predicate(UseAVX > 0);
2242 match(Set dst (AddD src1 src2));
2243
2244 format %{ "vaddsd $dst, $src1, $src2" %}
2245 ins_cost(150);
2246 ins_encode %{
2247 __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2248 %}
2249 ins_pipe(pipe_slow);
2250 %}
2251
2252 instruct addD_reg_mem(regD dst, regD src1, memory src2) %{
2253 predicate(UseAVX > 0);
2254 match(Set dst (AddD src1 (LoadD src2)));
2255
2256 format %{ "vaddsd $dst, $src1, $src2" %}
2257 ins_cost(150);
2258 ins_encode %{
2259 __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2260 %}
2261 ins_pipe(pipe_slow);
2262 %}
2263
2264 instruct addD_reg_imm(regD dst, regD src, immD con) %{
2265 predicate(UseAVX > 0);
2266 match(Set dst (AddD src con));
2267
2268 format %{ "vaddsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
2269 ins_cost(150);
2270 ins_encode %{
2271 __ vaddsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2272 %}
2273 ins_pipe(pipe_slow);
2274 %}
2275
2276 instruct subF_reg(regF dst, regF src) %{
2277 predicate((UseSSE>=1) && (UseAVX == 0));
2278 match(Set dst (SubF dst src));
2279
2280 format %{ "subss $dst, $src" %}
2281 ins_cost(150);
2282 ins_encode %{
2283 __ subss($dst$$XMMRegister, $src$$XMMRegister);
2284 %}
2285 ins_pipe(pipe_slow);
2286 %}
2287
2288 instruct subF_mem(regF dst, memory src) %{
2289 predicate((UseSSE>=1) && (UseAVX == 0));
2290 match(Set dst (SubF dst (LoadF src)));
2291
2292 format %{ "subss $dst, $src" %}
2293 ins_cost(150);
2294 ins_encode %{
2295 __ subss($dst$$XMMRegister, $src$$Address);
2296 %}
2297 ins_pipe(pipe_slow);
2298 %}
2299
2300 instruct subF_imm(regF dst, immF con) %{
2301 predicate((UseSSE>=1) && (UseAVX == 0));
2302 match(Set dst (SubF dst con));
2303 format %{ "subss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
2304 ins_cost(150);
2305 ins_encode %{
2306 __ subss($dst$$XMMRegister, $constantaddress($con));
2307 %}
2308 ins_pipe(pipe_slow);
2309 %}
2310
2311 instruct subF_reg_reg(regF dst, regF src1, regF src2) %{
2312 predicate(UseAVX > 0);
2313 match(Set dst (SubF src1 src2));
2314
2315 format %{ "vsubss $dst, $src1, $src2" %}
2316 ins_cost(150);
2317 ins_encode %{
2318 __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2319 %}
2320 ins_pipe(pipe_slow);
2321 %}
2322
2323 instruct subF_reg_mem(regF dst, regF src1, memory src2) %{
2324 predicate(UseAVX > 0);
2325 match(Set dst (SubF src1 (LoadF src2)));
2326
2327 format %{ "vsubss $dst, $src1, $src2" %}
2328 ins_cost(150);
2329 ins_encode %{
2330 __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2331 %}
2332 ins_pipe(pipe_slow);
2333 %}
2334
2335 instruct subF_reg_imm(regF dst, regF src, immF con) %{
2336 predicate(UseAVX > 0);
2337 match(Set dst (SubF src con));
2338
2339 format %{ "vsubss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
2340 ins_cost(150);
2341 ins_encode %{
2342 __ vsubss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2343 %}
2344 ins_pipe(pipe_slow);
2345 %}
2346
2347 instruct subD_reg(regD dst, regD src) %{
2348 predicate((UseSSE>=2) && (UseAVX == 0));
2349 match(Set dst (SubD dst src));
2350
2351 format %{ "subsd $dst, $src" %}
2352 ins_cost(150);
2353 ins_encode %{
2354 __ subsd($dst$$XMMRegister, $src$$XMMRegister);
2355 %}
2356 ins_pipe(pipe_slow);
2357 %}
2358
2359 instruct subD_mem(regD dst, memory src) %{
2360 predicate((UseSSE>=2) && (UseAVX == 0));
2361 match(Set dst (SubD dst (LoadD src)));
2362
2363 format %{ "subsd $dst, $src" %}
2364 ins_cost(150);
2365 ins_encode %{
2366 __ subsd($dst$$XMMRegister, $src$$Address);
2367 %}
2368 ins_pipe(pipe_slow);
2369 %}
2370
2371 instruct subD_imm(regD dst, immD con) %{
2372 predicate((UseSSE>=2) && (UseAVX == 0));
2373 match(Set dst (SubD dst con));
2374 format %{ "subsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
2375 ins_cost(150);
2376 ins_encode %{
2377 __ subsd($dst$$XMMRegister, $constantaddress($con));
2378 %}
2379 ins_pipe(pipe_slow);
2380 %}
2381
2382 instruct subD_reg_reg(regD dst, regD src1, regD src2) %{
2383 predicate(UseAVX > 0);
2384 match(Set dst (SubD src1 src2));
2385
2386 format %{ "vsubsd $dst, $src1, $src2" %}
2387 ins_cost(150);
2388 ins_encode %{
2389 __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2390 %}
2391 ins_pipe(pipe_slow);
2392 %}
2393
2394 instruct subD_reg_mem(regD dst, regD src1, memory src2) %{
2395 predicate(UseAVX > 0);
2396 match(Set dst (SubD src1 (LoadD src2)));
2397
2398 format %{ "vsubsd $dst, $src1, $src2" %}
2399 ins_cost(150);
2400 ins_encode %{
2401 __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2402 %}
2403 ins_pipe(pipe_slow);
2404 %}
2405
2406 instruct subD_reg_imm(regD dst, regD src, immD con) %{
2407 predicate(UseAVX > 0);
2408 match(Set dst (SubD src con));
2409
2410 format %{ "vsubsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
2411 ins_cost(150);
2412 ins_encode %{
2413 __ vsubsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2414 %}
2415 ins_pipe(pipe_slow);
2416 %}
2417
2418 instruct mulF_reg(regF dst, regF src) %{
2419 predicate((UseSSE>=1) && (UseAVX == 0));
2420 match(Set dst (MulF dst src));
2421
2422 format %{ "mulss $dst, $src" %}
2423 ins_cost(150);
2424 ins_encode %{
2425 __ mulss($dst$$XMMRegister, $src$$XMMRegister);
2426 %}
2427 ins_pipe(pipe_slow);
2428 %}
2429
2430 instruct mulF_mem(regF dst, memory src) %{
2431 predicate((UseSSE>=1) && (UseAVX == 0));
2432 match(Set dst (MulF dst (LoadF src)));
2433
2434 format %{ "mulss $dst, $src" %}
2435 ins_cost(150);
2436 ins_encode %{
2437 __ mulss($dst$$XMMRegister, $src$$Address);
2438 %}
2439 ins_pipe(pipe_slow);
2440 %}
2441
2442 instruct mulF_imm(regF dst, immF con) %{
2443 predicate((UseSSE>=1) && (UseAVX == 0));
2444 match(Set dst (MulF dst con));
2445 format %{ "mulss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
2446 ins_cost(150);
2447 ins_encode %{
2448 __ mulss($dst$$XMMRegister, $constantaddress($con));
2449 %}
2450 ins_pipe(pipe_slow);
2451 %}
2452
2453 instruct mulF_reg_reg(regF dst, regF src1, regF src2) %{
2454 predicate(UseAVX > 0);
2455 match(Set dst (MulF src1 src2));
2456
2457 format %{ "vmulss $dst, $src1, $src2" %}
2458 ins_cost(150);
2459 ins_encode %{
2460 __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2461 %}
2462 ins_pipe(pipe_slow);
2463 %}
2464
2465 instruct mulF_reg_mem(regF dst, regF src1, memory src2) %{
2466 predicate(UseAVX > 0);
2467 match(Set dst (MulF src1 (LoadF src2)));
2468
2469 format %{ "vmulss $dst, $src1, $src2" %}
2470 ins_cost(150);
2471 ins_encode %{
2472 __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2473 %}
2474 ins_pipe(pipe_slow);
2475 %}
2476
2477 instruct mulF_reg_imm(regF dst, regF src, immF con) %{
2478 predicate(UseAVX > 0);
2479 match(Set dst (MulF src con));
2480
2481 format %{ "vmulss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
2482 ins_cost(150);
2483 ins_encode %{
2484 __ vmulss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2485 %}
2486 ins_pipe(pipe_slow);
2487 %}
2488
2489 instruct mulD_reg(regD dst, regD src) %{
2490 predicate((UseSSE>=2) && (UseAVX == 0));
2491 match(Set dst (MulD dst src));
2492
2493 format %{ "mulsd $dst, $src" %}
2494 ins_cost(150);
2495 ins_encode %{
2496 __ mulsd($dst$$XMMRegister, $src$$XMMRegister);
2497 %}
2498 ins_pipe(pipe_slow);
2499 %}
2500
2501 instruct mulD_mem(regD dst, memory src) %{
2502 predicate((UseSSE>=2) && (UseAVX == 0));
2503 match(Set dst (MulD dst (LoadD src)));
2504
2505 format %{ "mulsd $dst, $src" %}
2506 ins_cost(150);
2507 ins_encode %{
2508 __ mulsd($dst$$XMMRegister, $src$$Address);
2509 %}
2510 ins_pipe(pipe_slow);
2511 %}
2512
2513 instruct mulD_imm(regD dst, immD con) %{
2514 predicate((UseSSE>=2) && (UseAVX == 0));
2515 match(Set dst (MulD dst con));
2516 format %{ "mulsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
2517 ins_cost(150);
2518 ins_encode %{
2519 __ mulsd($dst$$XMMRegister, $constantaddress($con));
2520 %}
2521 ins_pipe(pipe_slow);
2522 %}
2523
2524 instruct mulD_reg_reg(regD dst, regD src1, regD src2) %{
2525 predicate(UseAVX > 0);
2526 match(Set dst (MulD src1 src2));
2527
2528 format %{ "vmulsd $dst, $src1, $src2" %}
2529 ins_cost(150);
2530 ins_encode %{
2531 __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2532 %}
2533 ins_pipe(pipe_slow);
2534 %}
2535
2536 instruct mulD_reg_mem(regD dst, regD src1, memory src2) %{
2537 predicate(UseAVX > 0);
2538 match(Set dst (MulD src1 (LoadD src2)));
2539
2540 format %{ "vmulsd $dst, $src1, $src2" %}
2541 ins_cost(150);
2542 ins_encode %{
2543 __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2544 %}
2545 ins_pipe(pipe_slow);
2546 %}
2547
2548 instruct mulD_reg_imm(regD dst, regD src, immD con) %{
2549 predicate(UseAVX > 0);
2550 match(Set dst (MulD src con));
2551
2552 format %{ "vmulsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
2553 ins_cost(150);
2554 ins_encode %{
2555 __ vmulsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2556 %}
2557 ins_pipe(pipe_slow);
2558 %}
2559
2560 instruct divF_reg(regF dst, regF src) %{
2561 predicate((UseSSE>=1) && (UseAVX == 0));
2562 match(Set dst (DivF dst src));
2563
2564 format %{ "divss $dst, $src" %}
2565 ins_cost(150);
2566 ins_encode %{
2567 __ divss($dst$$XMMRegister, $src$$XMMRegister);
2568 %}
2569 ins_pipe(pipe_slow);
2570 %}
2571
2572 instruct divF_mem(regF dst, memory src) %{
2573 predicate((UseSSE>=1) && (UseAVX == 0));
2574 match(Set dst (DivF dst (LoadF src)));
2575
2576 format %{ "divss $dst, $src" %}
2577 ins_cost(150);
2578 ins_encode %{
2579 __ divss($dst$$XMMRegister, $src$$Address);
2580 %}
2581 ins_pipe(pipe_slow);
2582 %}
2583
2584 instruct divF_imm(regF dst, immF con) %{
2585 predicate((UseSSE>=1) && (UseAVX == 0));
2586 match(Set dst (DivF dst con));
2587 format %{ "divss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
2588 ins_cost(150);
2589 ins_encode %{
2590 __ divss($dst$$XMMRegister, $constantaddress($con));
2591 %}
2592 ins_pipe(pipe_slow);
2593 %}
2594
2595 instruct divF_reg_reg(regF dst, regF src1, regF src2) %{
2596 predicate(UseAVX > 0);
2597 match(Set dst (DivF src1 src2));
2598
2599 format %{ "vdivss $dst, $src1, $src2" %}
2600 ins_cost(150);
2601 ins_encode %{
2602 __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2603 %}
2604 ins_pipe(pipe_slow);
2605 %}
2606
2607 instruct divF_reg_mem(regF dst, regF src1, memory src2) %{
2608 predicate(UseAVX > 0);
2609 match(Set dst (DivF src1 (LoadF src2)));
2610
2611 format %{ "vdivss $dst, $src1, $src2" %}
2612 ins_cost(150);
2613 ins_encode %{
2614 __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2615 %}
2616 ins_pipe(pipe_slow);
2617 %}
2618
2619 instruct divF_reg_imm(regF dst, regF src, immF con) %{
2620 predicate(UseAVX > 0);
2621 match(Set dst (DivF src con));
2622
2623 format %{ "vdivss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
2624 ins_cost(150);
2625 ins_encode %{
2626 __ vdivss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2627 %}
2628 ins_pipe(pipe_slow);
2629 %}
2630
2631 instruct divD_reg(regD dst, regD src) %{
2632 predicate((UseSSE>=2) && (UseAVX == 0));
2633 match(Set dst (DivD dst src));
2634
2635 format %{ "divsd $dst, $src" %}
2636 ins_cost(150);
2637 ins_encode %{
2638 __ divsd($dst$$XMMRegister, $src$$XMMRegister);
2639 %}
2640 ins_pipe(pipe_slow);
2641 %}
2642
2643 instruct divD_mem(regD dst, memory src) %{
2644 predicate((UseSSE>=2) && (UseAVX == 0));
2645 match(Set dst (DivD dst (LoadD src)));
2646
2647 format %{ "divsd $dst, $src" %}
2648 ins_cost(150);
2649 ins_encode %{
2650 __ divsd($dst$$XMMRegister, $src$$Address);
2651 %}
2652 ins_pipe(pipe_slow);
2653 %}
2654
2655 instruct divD_imm(regD dst, immD con) %{
2656 predicate((UseSSE>=2) && (UseAVX == 0));
2657 match(Set dst (DivD dst con));
2658 format %{ "divsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
2659 ins_cost(150);
2660 ins_encode %{
2661 __ divsd($dst$$XMMRegister, $constantaddress($con));
2662 %}
2663 ins_pipe(pipe_slow);
2664 %}
2665
2666 instruct divD_reg_reg(regD dst, regD src1, regD src2) %{
2667 predicate(UseAVX > 0);
2668 match(Set dst (DivD src1 src2));
2669
2670 format %{ "vdivsd $dst, $src1, $src2" %}
2671 ins_cost(150);
2672 ins_encode %{
2673 __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2674 %}
2675 ins_pipe(pipe_slow);
2676 %}
2677
2678 instruct divD_reg_mem(regD dst, regD src1, memory src2) %{
2679 predicate(UseAVX > 0);
2680 match(Set dst (DivD src1 (LoadD src2)));
2681
2682 format %{ "vdivsd $dst, $src1, $src2" %}
2683 ins_cost(150);
2684 ins_encode %{
2685 __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2686 %}
2687 ins_pipe(pipe_slow);
2688 %}
2689
2690 instruct divD_reg_imm(regD dst, regD src, immD con) %{
2691 predicate(UseAVX > 0);
2692 match(Set dst (DivD src con));
2693
2694 format %{ "vdivsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
2695 ins_cost(150);
2696 ins_encode %{
2697 __ vdivsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2698 %}
2699 ins_pipe(pipe_slow);
2700 %}
2701
2702 instruct absF_reg(regF dst) %{
2703 predicate((UseSSE>=1) && (UseAVX == 0));
2704 match(Set dst (AbsF dst));
2705 ins_cost(150);
2706 format %{ "andps $dst, [0x7fffffff]\t# abs float by sign masking" %}
2707 ins_encode %{
2708 __ andps($dst$$XMMRegister, ExternalAddress(float_signmask()));
2709 %}
2710 ins_pipe(pipe_slow);
2711 %}
2712
2713 instruct absF_reg_reg(regF dst, regF src) %{
2714 predicate(UseAVX > 0);
2715 match(Set dst (AbsF src));
2716 ins_cost(150);
2717 format %{ "vandps $dst, $src, [0x7fffffff]\t# abs float by sign masking" %}
2718 ins_encode %{
2719 int vector_len = 0;
2720 __ vandps($dst$$XMMRegister, $src$$XMMRegister,
2721 ExternalAddress(float_signmask()), vector_len);
2722 %}
2723 ins_pipe(pipe_slow);
2724 %}
2725
2726 instruct absD_reg(regD dst) %{
2727 predicate((UseSSE>=2) && (UseAVX == 0));
2728 match(Set dst (AbsD dst));
2729 ins_cost(150);
2730 format %{ "andpd $dst, [0x7fffffffffffffff]\t"
2731 "# abs double by sign masking" %}
2732 ins_encode %{
2733 __ andpd($dst$$XMMRegister, ExternalAddress(double_signmask()));
2734 %}
2735 ins_pipe(pipe_slow);
2736 %}
2737
2738 instruct absD_reg_reg(regD dst, regD src) %{
2739 predicate(UseAVX > 0);
2740 match(Set dst (AbsD src));
2741 ins_cost(150);
2742 format %{ "vandpd $dst, $src, [0x7fffffffffffffff]\t"
2743 "# abs double by sign masking" %}
2744 ins_encode %{
2745 int vector_len = 0;
2746 __ vandpd($dst$$XMMRegister, $src$$XMMRegister,
2747 ExternalAddress(double_signmask()), vector_len);
2748 %}
2749 ins_pipe(pipe_slow);
2750 %}
2751
2752 instruct negF_reg(regF dst) %{
2753 predicate((UseSSE>=1) && (UseAVX == 0));
2754 match(Set dst (NegF dst));
2755 ins_cost(150);
2756 format %{ "xorps $dst, [0x80000000]\t# neg float by sign flipping" %}
2757 ins_encode %{
2758 __ xorps($dst$$XMMRegister, ExternalAddress(float_signflip()));
2759 %}
2760 ins_pipe(pipe_slow);
2761 %}
2762
2763 instruct negF_reg_reg(regF dst, regF src) %{
2764 predicate(UseAVX > 0);
2765 match(Set dst (NegF src));
2766 ins_cost(150);
2767 format %{ "vnegatess $dst, $src, [0x80000000]\t# neg float by sign flipping" %}
2768 ins_encode %{
2769 __ vnegatess($dst$$XMMRegister, $src$$XMMRegister,
2770 ExternalAddress(float_signflip()));
2771 %}
2772 ins_pipe(pipe_slow);
2773 %}
2774
2775 instruct negD_reg(regD dst) %{
2776 predicate((UseSSE>=2) && (UseAVX == 0));
2777 match(Set dst (NegD dst));
2778 ins_cost(150);
2779 format %{ "xorpd $dst, [0x8000000000000000]\t"
2780 "# neg double by sign flipping" %}
2781 ins_encode %{
2782 __ xorpd($dst$$XMMRegister, ExternalAddress(double_signflip()));
2783 %}
2784 ins_pipe(pipe_slow);
2785 %}
2786
2787 instruct negD_reg_reg(regD dst, regD src) %{
2788 predicate(UseAVX > 0);
2789 match(Set dst (NegD src));
2790 ins_cost(150);
2791 format %{ "vnegatess $dst, $src, [0x8000000000000000]\t"
2792 "# neg double by sign flipping" %}
2793 ins_encode %{
2794 __ vnegatesd($dst$$XMMRegister, $src$$XMMRegister,
2795 ExternalAddress(double_signflip()));
2796 %}
2797 ins_pipe(pipe_slow);
2798 %}
2799
2800 instruct sqrtF_reg(regF dst, regF src) %{
2801 predicate(UseSSE>=1);
2802 match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
2803
2804 format %{ "sqrtss $dst, $src" %}
2805 ins_cost(150);
2806 ins_encode %{
2807 __ sqrtss($dst$$XMMRegister, $src$$XMMRegister);
2808 %}
2809 ins_pipe(pipe_slow);
2810 %}
2811
2812 instruct sqrtF_mem(regF dst, memory src) %{
2813 predicate(UseSSE>=1);
2814 match(Set dst (ConvD2F (SqrtD (ConvF2D (LoadF src)))));
2815
2816 format %{ "sqrtss $dst, $src" %}
2817 ins_cost(150);
2818 ins_encode %{
2819 __ sqrtss($dst$$XMMRegister, $src$$Address);
2820 %}
2821 ins_pipe(pipe_slow);
2822 %}
2823
2824 instruct sqrtF_imm(regF dst, immF con) %{
2825 predicate(UseSSE>=1);
2826 match(Set dst (ConvD2F (SqrtD (ConvF2D con))));
2827 format %{ "sqrtss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
2828 ins_cost(150);
2829 ins_encode %{
2830 __ sqrtss($dst$$XMMRegister, $constantaddress($con));
2831 %}
2832 ins_pipe(pipe_slow);
2833 %}
2834
2835 instruct sqrtD_reg(regD dst, regD src) %{
2836 predicate(UseSSE>=2);
2837 match(Set dst (SqrtD src));
2838
2839 format %{ "sqrtsd $dst, $src" %}
2840 ins_cost(150);
2841 ins_encode %{
2842 __ sqrtsd($dst$$XMMRegister, $src$$XMMRegister);
2843 %}
2844 ins_pipe(pipe_slow);
2845 %}
2846
2847 instruct sqrtD_mem(regD dst, memory src) %{
2848 predicate(UseSSE>=2);
2849 match(Set dst (SqrtD (LoadD src)));
2850
2851 format %{ "sqrtsd $dst, $src" %}
2852 ins_cost(150);
2853 ins_encode %{
2854 __ sqrtsd($dst$$XMMRegister, $src$$Address);
2855 %}
2856 ins_pipe(pipe_slow);
2857 %}
2858
2859 instruct sqrtD_imm(regD dst, immD con) %{
2860 predicate(UseSSE>=2);
2861 match(Set dst (SqrtD con));
2862 format %{ "sqrtsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
2863 ins_cost(150);
2864 ins_encode %{
2865 __ sqrtsd($dst$$XMMRegister, $constantaddress($con));
2866 %}
2867 ins_pipe(pipe_slow);
2868 %}
2869
2870 // ====================VECTOR INSTRUCTIONS=====================================
2871
2872 // Load vectors (4 bytes long)
2873 instruct loadV4(vecS dst, memory mem) %{
2874 predicate(n->as_LoadVector()->memory_size() == 4);
2875 match(Set dst (LoadVector mem));
2876 ins_cost(125);
2877 format %{ "movd $dst,$mem\t! load vector (4 bytes)" %}
2878 ins_encode %{
2879 __ movdl($dst$$XMMRegister, $mem$$Address);
2880 %}
2881 ins_pipe( pipe_slow );
2882 %}
2883
2884 // Load vectors (8 bytes long)
2885 instruct loadV8(vecD dst, memory mem) %{
2886 predicate(n->as_LoadVector()->memory_size() == 8);
2887 match(Set dst (LoadVector mem));
2888 ins_cost(125);
2889 format %{ "movq $dst,$mem\t! load vector (8 bytes)" %}
2890 ins_encode %{
2891 __ movq($dst$$XMMRegister, $mem$$Address);
2892 %}
2893 ins_pipe( pipe_slow );
2894 %}
2895
2896 // Load vectors (16 bytes long)
2897 instruct loadV16(vecX dst, memory mem) %{
2898 predicate(n->as_LoadVector()->memory_size() == 16);
2899 match(Set dst (LoadVector mem));
2900 ins_cost(125);
2901 format %{ "movdqu $dst,$mem\t! load vector (16 bytes)" %}
2902 ins_encode %{
2903 __ movdqu($dst$$XMMRegister, $mem$$Address);
2904 %}
2905 ins_pipe( pipe_slow );
2906 %}
2907
2908 // Load vectors (32 bytes long)
2909 instruct loadV32(vecY dst, memory mem) %{
2910 predicate(n->as_LoadVector()->memory_size() == 32);
2911 match(Set dst (LoadVector mem));
2912 ins_cost(125);
2913 format %{ "vmovdqu $dst,$mem\t! load vector (32 bytes)" %}
2914 ins_encode %{
2915 __ vmovdqu($dst$$XMMRegister, $mem$$Address);
2916 %}
2917 ins_pipe( pipe_slow );
2918 %}
2919
2920 // Load vectors (64 bytes long)
2921 instruct loadV64(vecZ dst, memory mem) %{
2922 predicate(n->as_LoadVector()->memory_size() == 64);
2923 match(Set dst (LoadVector mem));
2924 ins_cost(125);
2925 format %{ "vmovdqu $dst k0,$mem\t! load vector (64 bytes)" %}
2926 ins_encode %{
2927 int vector_len = 2;
2928 __ evmovdqul($dst$$XMMRegister, $mem$$Address, vector_len);
2929 %}
2930 ins_pipe( pipe_slow );
2931 %}
2932
2933 // Store vectors
2934 instruct storeV4(memory mem, vecS src) %{
2935 predicate(n->as_StoreVector()->memory_size() == 4);
2936 match(Set mem (StoreVector mem src));
2937 ins_cost(145);
2938 format %{ "movd $mem,$src\t! store vector (4 bytes)" %}
2939 ins_encode %{
2940 __ movdl($mem$$Address, $src$$XMMRegister);
2941 %}
2942 ins_pipe( pipe_slow );
2943 %}
2944
2945 instruct storeV8(memory mem, vecD src) %{
2946 predicate(n->as_StoreVector()->memory_size() == 8);
2947 match(Set mem (StoreVector mem src));
2948 ins_cost(145);
2949 format %{ "movq $mem,$src\t! store vector (8 bytes)" %}
2950 ins_encode %{
2951 __ movq($mem$$Address, $src$$XMMRegister);
2952 %}
2953 ins_pipe( pipe_slow );
2954 %}
2955
2956 instruct storeV16(memory mem, vecX src) %{
2957 predicate(n->as_StoreVector()->memory_size() == 16);
2958 match(Set mem (StoreVector mem src));
2959 ins_cost(145);
2960 format %{ "movdqu $mem,$src\t! store vector (16 bytes)" %}
2961 ins_encode %{
2962 __ movdqu($mem$$Address, $src$$XMMRegister);
2963 %}
2964 ins_pipe( pipe_slow );
2965 %}
2966
2967 instruct storeV32(memory mem, vecY src) %{
2968 predicate(n->as_StoreVector()->memory_size() == 32);
2969 match(Set mem (StoreVector mem src));
2970 ins_cost(145);
2971 format %{ "vmovdqu $mem,$src\t! store vector (32 bytes)" %}
2972 ins_encode %{
2973 __ vmovdqu($mem$$Address, $src$$XMMRegister);
2974 %}
2975 ins_pipe( pipe_slow );
2976 %}
2977
2978 instruct storeV64(memory mem, vecZ src) %{
2979 predicate(n->as_StoreVector()->memory_size() == 64);
2980 match(Set mem (StoreVector mem src));
2981 ins_cost(145);
2982 format %{ "vmovdqu $mem k0,$src\t! store vector (64 bytes)" %}
2983 ins_encode %{
2984 int vector_len = 2;
2985 __ evmovdqul($mem$$Address, $src$$XMMRegister, vector_len);
2986 %}
2987 ins_pipe( pipe_slow );
2988 %}
2989
2990 // ====================LEGACY REPLICATE=======================================
2991
2992 instruct Repl4B_mem(vecS dst, memory mem) %{
2993 predicate(n->as_Vector()->length() == 4 && UseAVX > 0 && !VM_Version::supports_avx512vlbw());
2994 match(Set dst (ReplicateB (LoadB mem)));
2995 format %{ "punpcklbw $dst,$mem\n\t"
2996 "pshuflw $dst,$dst,0x00\t! replicate4B" %}
2997 ins_encode %{
2998 __ punpcklbw($dst$$XMMRegister, $mem$$Address);
2999 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3000 %}
3001 ins_pipe( pipe_slow );
3002 %}
3003
3004 instruct Repl8B_mem(vecD dst, memory mem) %{
3005 predicate(n->as_Vector()->length() == 8 && UseAVX > 0 && !VM_Version::supports_avx512vlbw());
3006 match(Set dst (ReplicateB (LoadB mem)));
3007 format %{ "punpcklbw $dst,$mem\n\t"
3008 "pshuflw $dst,$dst,0x00\t! replicate8B" %}
3009 ins_encode %{
3010 __ punpcklbw($dst$$XMMRegister, $mem$$Address);
3011 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3012 %}
3013 ins_pipe( pipe_slow );
3014 %}
3015
3016 instruct Repl16B(vecX dst, rRegI src) %{
3017 predicate(n->as_Vector()->length() == 16 && !VM_Version::supports_avx512vlbw());
3018 match(Set dst (ReplicateB src));
3019 format %{ "movd $dst,$src\n\t"
3020 "punpcklbw $dst,$dst\n\t"
3021 "pshuflw $dst,$dst,0x00\n\t"
3022 "punpcklqdq $dst,$dst\t! replicate16B" %}
3023 ins_encode %{
3024 __ movdl($dst$$XMMRegister, $src$$Register);
3025 __ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
3026 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3027 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3028 %}
3029 ins_pipe( pipe_slow );
3030 %}
3031
3032 instruct Repl16B_mem(vecX dst, memory mem) %{
3033 predicate(n->as_Vector()->length() == 16 && UseAVX > 0 && !VM_Version::supports_avx512vlbw());
3034 match(Set dst (ReplicateB (LoadB mem)));
3035 format %{ "punpcklbw $dst,$mem\n\t"
3036 "pshuflw $dst,$dst,0x00\n\t"
3037 "punpcklqdq $dst,$dst\t! replicate16B" %}
3038 ins_encode %{
3039 __ punpcklbw($dst$$XMMRegister, $mem$$Address);
3040 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3041 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3042 %}
3043 ins_pipe( pipe_slow );
3044 %}
3045
3046 instruct Repl32B(vecY dst, rRegI src) %{
3047 predicate(n->as_Vector()->length() == 32 && !VM_Version::supports_avx512vlbw());
3048 match(Set dst (ReplicateB src));
3049 format %{ "movd $dst,$src\n\t"
3050 "punpcklbw $dst,$dst\n\t"
3051 "pshuflw $dst,$dst,0x00\n\t"
3052 "punpcklqdq $dst,$dst\n\t"
3053 "vinserti128h $dst,$dst,$dst\t! replicate32B" %}
3054 ins_encode %{
3055 __ movdl($dst$$XMMRegister, $src$$Register);
3056 __ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
3057 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3058 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3059 __ vinserti128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3060 %}
3061 ins_pipe( pipe_slow );
3062 %}
3063
3064 instruct Repl32B_mem(vecY dst, memory mem) %{
3065 predicate(n->as_Vector()->length() == 32 && !VM_Version::supports_avx512vlbw());
3066 match(Set dst (ReplicateB (LoadB mem)));
3067 format %{ "punpcklbw $dst,$mem\n\t"
3068 "pshuflw $dst,$dst,0x00\n\t"
3069 "punpcklqdq $dst,$dst\n\t"
3070 "vinserti128h $dst,$dst,$dst\t! replicate32B" %}
3071 ins_encode %{
3072 __ punpcklbw($dst$$XMMRegister, $mem$$Address);
3073 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3074 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3075 __ vinserti128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3076 %}
3077 ins_pipe( pipe_slow );
3078 %}
3079
3080 instruct Repl16B_imm(vecX dst, immI con) %{
3081 predicate(n->as_Vector()->length() == 16 && !VM_Version::supports_avx512vlbw());
3082 match(Set dst (ReplicateB con));
3083 format %{ "movq $dst,[$constantaddress]\n\t"
3084 "punpcklqdq $dst,$dst\t! replicate16B($con)" %}
3085 ins_encode %{
3086 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 1)));
3087 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3088 %}
3089 ins_pipe( pipe_slow );
3090 %}
3091
3092 instruct Repl32B_imm(vecY dst, immI con) %{
3093 predicate(n->as_Vector()->length() == 32 && !VM_Version::supports_avx512vlbw());
3094 match(Set dst (ReplicateB con));
3095 format %{ "movq $dst,[$constantaddress]\n\t"
3096 "punpcklqdq $dst,$dst\n\t"
3097 "vinserti128h $dst,$dst,$dst\t! lreplicate32B($con)" %}
3098 ins_encode %{
3099 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 1)));
3100 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3101 __ vinserti128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3102 %}
3103 ins_pipe( pipe_slow );
3104 %}
3105
3106 instruct Repl4S(vecD dst, rRegI src) %{
3107 predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vlbw());
3108 match(Set dst (ReplicateS src));
3109 format %{ "movd $dst,$src\n\t"
3110 "pshuflw $dst,$dst,0x00\t! replicate4S" %}
3111 ins_encode %{
3112 __ movdl($dst$$XMMRegister, $src$$Register);
3113 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3114 %}
3115 ins_pipe( pipe_slow );
3116 %}
3117
3118 instruct Repl4S_mem(vecD dst, memory mem) %{
3119 predicate(n->as_Vector()->length() == 4 && UseAVX > 0 && !VM_Version::supports_avx512vlbw());
3120 match(Set dst (ReplicateS (LoadS mem)));
3121 format %{ "pshuflw $dst,$mem,0x00\t! replicate4S" %}
3122 ins_encode %{
3123 __ pshuflw($dst$$XMMRegister, $mem$$Address, 0x00);
3124 %}
3125 ins_pipe( pipe_slow );
3126 %}
3127
3128 instruct Repl8S(vecX dst, rRegI src) %{
3129 predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vlbw());
3130 match(Set dst (ReplicateS src));
3131 format %{ "movd $dst,$src\n\t"
3132 "pshuflw $dst,$dst,0x00\n\t"
3133 "punpcklqdq $dst,$dst\t! replicate8S" %}
3134 ins_encode %{
3135 __ movdl($dst$$XMMRegister, $src$$Register);
3136 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3137 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3138 %}
3139 ins_pipe( pipe_slow );
3140 %}
3141
3142 instruct Repl8S_mem(vecX dst, memory mem) %{
3143 predicate(n->as_Vector()->length() == 8 && UseAVX > 0 && !VM_Version::supports_avx512vlbw());
3144 match(Set dst (ReplicateS (LoadS mem)));
3145 format %{ "pshuflw $dst,$mem,0x00\n\t"
3146 "punpcklqdq $dst,$dst\t! replicate8S" %}
3147 ins_encode %{
3148 __ pshuflw($dst$$XMMRegister, $mem$$Address, 0x00);
3149 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3150 %}
3151 ins_pipe( pipe_slow );
3152 %}
3153
3154 instruct Repl8S_imm(vecX dst, immI con) %{
3155 predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vlbw());
3156 match(Set dst (ReplicateS con));
3157 format %{ "movq $dst,[$constantaddress]\n\t"
3158 "punpcklqdq $dst,$dst\t! replicate8S($con)" %}
3159 ins_encode %{
3160 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 2)));
3161 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3162 %}
3163 ins_pipe( pipe_slow );
3164 %}
3165
3166 instruct Repl16S(vecY dst, rRegI src) %{
3167 predicate(n->as_Vector()->length() == 16 && !VM_Version::supports_avx512vlbw());
3168 match(Set dst (ReplicateS src));
3169 format %{ "movd $dst,$src\n\t"
3170 "pshuflw $dst,$dst,0x00\n\t"
3171 "punpcklqdq $dst,$dst\n\t"
3172 "vinserti128h $dst,$dst,$dst\t! replicate16S" %}
3173 ins_encode %{
3174 __ movdl($dst$$XMMRegister, $src$$Register);
3175 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3176 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3177 __ vinserti128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3178 %}
3179 ins_pipe( pipe_slow );
3180 %}
3181
3182 instruct Repl16S_mem(vecY dst, memory mem) %{
3183 predicate(n->as_Vector()->length() == 16 && !VM_Version::supports_avx512vlbw());
3184 match(Set dst (ReplicateS (LoadS mem)));
3185 format %{ "pshuflw $dst,$mem,0x00\n\t"
3186 "punpcklqdq $dst,$dst\n\t"
3187 "vinserti128h $dst,$dst,$dst\t! replicate16S" %}
3188 ins_encode %{
3189 __ pshuflw($dst$$XMMRegister, $mem$$Address, 0x00);
3190 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3191 __ vinserti128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3192 %}
3193 ins_pipe( pipe_slow );
3194 %}
3195
3196 instruct Repl16S_imm(vecY dst, immI con) %{
3197 predicate(n->as_Vector()->length() == 16 && !VM_Version::supports_avx512vlbw());
3198 match(Set dst (ReplicateS con));
3199 format %{ "movq $dst,[$constantaddress]\n\t"
3200 "punpcklqdq $dst,$dst\n\t"
3201 "vinserti128h $dst,$dst,$dst\t! replicate16S($con)" %}
3202 ins_encode %{
3203 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 2)));
3204 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3205 __ vinserti128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3206 %}
3207 ins_pipe( pipe_slow );
3208 %}
3209
3210 instruct Repl4I(vecX dst, rRegI src) %{
3211 predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3212 match(Set dst (ReplicateI src));
3213 format %{ "movd $dst,$src\n\t"
3214 "pshufd $dst,$dst,0x00\t! replicate4I" %}
3215 ins_encode %{
3216 __ movdl($dst$$XMMRegister, $src$$Register);
3217 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3218 %}
3219 ins_pipe( pipe_slow );
3220 %}
3221
3222 instruct Repl4I_mem(vecX dst, memory mem) %{
3223 predicate(n->as_Vector()->length() == 4 && UseAVX > 0 && !VM_Version::supports_avx512vl());
3224 match(Set dst (ReplicateI (LoadI mem)));
3225 format %{ "pshufd $dst,$mem,0x00\t! replicate4I" %}
3226 ins_encode %{
3227 __ pshufd($dst$$XMMRegister, $mem$$Address, 0x00);
3228 %}
3229 ins_pipe( pipe_slow );
3230 %}
3231
3232 instruct Repl8I(vecY dst, rRegI src) %{
3233 predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vl());
3234 match(Set dst (ReplicateI src));
3235 format %{ "movd $dst,$src\n\t"
3236 "pshufd $dst,$dst,0x00\n\t"
3237 "vinserti128h $dst,$dst,$dst\t! replicate8I" %}
3238 ins_encode %{
3239 __ movdl($dst$$XMMRegister, $src$$Register);
3240 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3241 __ vinserti128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3242 %}
3243 ins_pipe( pipe_slow );
3244 %}
3245
3246 instruct Repl8I_mem(vecY dst, memory mem) %{
3247 predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vl());
3248 match(Set dst (ReplicateI (LoadI mem)));
3249 format %{ "pshufd $dst,$mem,0x00\n\t"
3250 "vinserti128h $dst,$dst,$dst\t! replicate8I" %}
3251 ins_encode %{
3252 __ pshufd($dst$$XMMRegister, $mem$$Address, 0x00);
3253 __ vinserti128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3254 %}
3255 ins_pipe( pipe_slow );
3256 %}
3257
3258 instruct Repl4I_imm(vecX dst, immI con) %{
3259 predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3260 match(Set dst (ReplicateI con));
3261 format %{ "movq $dst,[$constantaddress]\t! replicate4I($con)\n\t"
3262 "punpcklqdq $dst,$dst" %}
3263 ins_encode %{
3264 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 4)));
3265 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3266 %}
3267 ins_pipe( pipe_slow );
3268 %}
3269
3270 instruct Repl8I_imm(vecY dst, immI con) %{
3271 predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vl());
3272 match(Set dst (ReplicateI con));
3273 format %{ "movq $dst,[$constantaddress]\t! replicate8I($con)\n\t"
3274 "punpcklqdq $dst,$dst\n\t"
3275 "vinserti128h $dst,$dst,$dst" %}
3276 ins_encode %{
3277 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 4)));
3278 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3279 __ vinserti128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3280 %}
3281 ins_pipe( pipe_slow );
3282 %}
3283
3284 // Long could be loaded into xmm register directly from memory.
3285 instruct Repl2L_mem(vecX dst, memory mem) %{
3286 predicate(n->as_Vector()->length() == 2 && !VM_Version::supports_avx512vlbw());
3287 match(Set dst (ReplicateL (LoadL mem)));
3288 format %{ "movq $dst,$mem\n\t"
3289 "punpcklqdq $dst,$dst\t! replicate2L" %}
3290 ins_encode %{
3291 __ movq($dst$$XMMRegister, $mem$$Address);
3292 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3293 %}
3294 ins_pipe( pipe_slow );
3295 %}
3296
3297 // Replicate long (8 byte) scalar to be vector
3298 #ifdef _LP64
3299 instruct Repl4L(vecY dst, rRegL src) %{
3300 predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3301 match(Set dst (ReplicateL src));
3302 format %{ "movdq $dst,$src\n\t"
3303 "punpcklqdq $dst,$dst\n\t"
3304 "vinserti128h $dst,$dst,$dst\t! replicate4L" %}
3305 ins_encode %{
3306 __ movdq($dst$$XMMRegister, $src$$Register);
3307 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3308 __ vinserti128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3309 %}
3310 ins_pipe( pipe_slow );
3311 %}
3312 #else // _LP64
3313 instruct Repl4L(vecY dst, eRegL src, regD tmp) %{
3314 predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3315 match(Set dst (ReplicateL src));
3316 effect(TEMP dst, USE src, TEMP tmp);
3317 format %{ "movdl $dst,$src.lo\n\t"
3318 "movdl $tmp,$src.hi\n\t"
3319 "punpckldq $dst,$tmp\n\t"
3320 "punpcklqdq $dst,$dst\n\t"
3321 "vinserti128h $dst,$dst,$dst\t! replicate4L" %}
3322 ins_encode %{
3323 __ movdl($dst$$XMMRegister, $src$$Register);
3324 __ movdl($tmp$$XMMRegister, HIGH_FROM_LOW($src$$Register));
3325 __ punpckldq($dst$$XMMRegister, $tmp$$XMMRegister);
3326 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3327 __ vinserti128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3328 %}
3329 ins_pipe( pipe_slow );
3330 %}
3331 #endif // _LP64
3332
3333 instruct Repl4L_imm(vecY dst, immL con) %{
3334 predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3335 match(Set dst (ReplicateL con));
3336 format %{ "movq $dst,[$constantaddress]\n\t"
3337 "punpcklqdq $dst,$dst\n\t"
3338 "vinserti128h $dst,$dst,$dst\t! replicate4L($con)" %}
3339 ins_encode %{
3340 __ movq($dst$$XMMRegister, $constantaddress($con));
3341 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3342 __ vinserti128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3343 %}
3344 ins_pipe( pipe_slow );
3345 %}
3346
3347 instruct Repl4L_mem(vecY dst, memory mem) %{
3348 predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3349 match(Set dst (ReplicateL (LoadL mem)));
3350 format %{ "movq $dst,$mem\n\t"
3351 "punpcklqdq $dst,$dst\n\t"
3352 "vinserti128h $dst,$dst,$dst\t! replicate4L" %}
3353 ins_encode %{
3354 __ movq($dst$$XMMRegister, $mem$$Address);
3355 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3356 __ vinserti128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3357 %}
3358 ins_pipe( pipe_slow );
3359 %}
3360
3361 instruct Repl2F_mem(vecD dst, memory mem) %{
3362 predicate(n->as_Vector()->length() == 2 && UseAVX > 0 && !VM_Version::supports_avx512vl());
3363 match(Set dst (ReplicateF (LoadF mem)));
3364 format %{ "pshufd $dst,$mem,0x00\t! replicate2F" %}
3365 ins_encode %{
3366 __ pshufd($dst$$XMMRegister, $mem$$Address, 0x00);
3367 %}
3368 ins_pipe( pipe_slow );
3369 %}
3370
3371 instruct Repl4F_mem(vecX dst, memory mem) %{
3372 predicate(n->as_Vector()->length() == 4 && UseAVX > 0 && !VM_Version::supports_avx512vl());
3373 match(Set dst (ReplicateF (LoadF mem)));
3374 format %{ "pshufd $dst,$mem,0x00\t! replicate4F" %}
3375 ins_encode %{
3376 __ pshufd($dst$$XMMRegister, $mem$$Address, 0x00);
3377 %}
3378 ins_pipe( pipe_slow );
3379 %}
3380
3381 instruct Repl8F(vecY dst, regF src) %{
3382 predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vl());
3383 match(Set dst (ReplicateF src));
3384 format %{ "pshufd $dst,$src,0x00\n\t"
3385 "vinsertf128h $dst,$dst,$dst\t! replicate8F" %}
3386 ins_encode %{
3387 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
3388 __ vinsertf128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3389 %}
3390 ins_pipe( pipe_slow );
3391 %}
3392
3393 instruct Repl8F_mem(vecY dst, memory mem) %{
3394 predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vl());
3395 match(Set dst (ReplicateF (LoadF mem)));
3396 format %{ "pshufd $dst,$mem,0x00\n\t"
3397 "vinsertf128h $dst,$dst,$dst\t! replicate8F" %}
3398 ins_encode %{
3399 __ pshufd($dst$$XMMRegister, $mem$$Address, 0x00);
3400 __ vinsertf128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3401 %}
3402 ins_pipe( pipe_slow );
3403 %}
3404
3405 instruct Repl2F_zero(vecD dst, immF0 zero) %{
3406 predicate(n->as_Vector()->length() == 2 && UseAVX < 3);
3407 match(Set dst (ReplicateF zero));
3408 format %{ "xorps $dst,$dst\t! replicate2F zero" %}
3409 ins_encode %{
3410 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
3411 %}
3412 ins_pipe( fpu_reg_reg );
3413 %}
3414
3415 instruct Repl4F_zero(vecX dst, immF0 zero) %{
3416 predicate(n->as_Vector()->length() == 4 && UseAVX < 3);
3417 match(Set dst (ReplicateF zero));
3418 format %{ "xorps $dst,$dst\t! replicate4F zero" %}
3419 ins_encode %{
3420 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
3421 %}
3422 ins_pipe( fpu_reg_reg );
3423 %}
3424
3425 instruct Repl8F_zero(vecY dst, immF0 zero) %{
3426 predicate(n->as_Vector()->length() == 8 && UseAVX < 3);
3427 match(Set dst (ReplicateF zero));
3428 format %{ "vxorps $dst,$dst,$dst\t! replicate8F zero" %}
3429 ins_encode %{
3430 int vector_len = 1;
3431 __ vxorps($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
3432 %}
3433 ins_pipe( fpu_reg_reg );
3434 %}
3435
3436 instruct Repl2D_mem(vecX dst, memory mem) %{
3437 predicate(n->as_Vector()->length() == 2 && UseAVX > 0 && !VM_Version::supports_avx512vl());
3438 match(Set dst (ReplicateD (LoadD mem)));
3439 format %{ "pshufd $dst,$mem,0x44\t! replicate2D" %}
3440 ins_encode %{
3441 __ pshufd($dst$$XMMRegister, $mem$$Address, 0x44);
3442 %}
3443 ins_pipe( pipe_slow );
3444 %}
3445
3446 instruct Repl4D(vecY dst, regD src) %{
3447 predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3448 match(Set dst (ReplicateD src));
3449 format %{ "pshufd $dst,$src,0x44\n\t"
3450 "vinsertf128h $dst,$dst,$dst\t! replicate4D" %}
3451 ins_encode %{
3452 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x44);
3453 __ vinsertf128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3454 %}
3455 ins_pipe( pipe_slow );
3456 %}
3457
3458 instruct Repl4D_mem(vecY dst, memory mem) %{
3459 predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3460 match(Set dst (ReplicateD (LoadD mem)));
3461 format %{ "pshufd $dst,$mem,0x44\n\t"
3462 "vinsertf128h $dst,$dst,$dst\t! replicate4D" %}
3463 ins_encode %{
3464 __ pshufd($dst$$XMMRegister, $mem$$Address, 0x44);
3465 __ vinsertf128h($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister);
3466 %}
3467 ins_pipe( pipe_slow );
3468 %}
3469
3470 // Replicate double (8 byte) scalar zero to be vector
3471 instruct Repl2D_zero(vecX dst, immD0 zero) %{
3472 predicate(n->as_Vector()->length() == 2 && UseAVX < 3);
3473 match(Set dst (ReplicateD zero));
3474 format %{ "xorpd $dst,$dst\t! replicate2D zero" %}
3475 ins_encode %{
3476 __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
3477 %}
3478 ins_pipe( fpu_reg_reg );
3479 %}
3480
3481 instruct Repl4D_zero(vecY dst, immD0 zero) %{
3482 predicate(n->as_Vector()->length() == 4 && UseAVX < 3);
3483 match(Set dst (ReplicateD zero));
3484 format %{ "vxorpd $dst,$dst,$dst,vect256\t! replicate4D zero" %}
3485 ins_encode %{
3486 int vector_len = 1;
3487 __ vxorpd($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
3488 %}
3489 ins_pipe( fpu_reg_reg );
3490 %}
3491
3492 // ====================GENERIC REPLICATE==========================================
3493
3494 // Replicate byte scalar to be vector
3495 instruct Repl4B(vecS dst, rRegI src) %{
3496 predicate(n->as_Vector()->length() == 4);
3497 match(Set dst (ReplicateB src));
3498 format %{ "movd $dst,$src\n\t"
3499 "punpcklbw $dst,$dst\n\t"
3500 "pshuflw $dst,$dst,0x00\t! replicate4B" %}
3501 ins_encode %{
3502 __ movdl($dst$$XMMRegister, $src$$Register);
3503 __ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
3504 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3505 %}
3506 ins_pipe( pipe_slow );
3507 %}
3508
3509 instruct Repl8B(vecD dst, rRegI src) %{
3510 predicate(n->as_Vector()->length() == 8);
3511 match(Set dst (ReplicateB src));
3512 format %{ "movd $dst,$src\n\t"
3513 "punpcklbw $dst,$dst\n\t"
3514 "pshuflw $dst,$dst,0x00\t! replicate8B" %}
3515 ins_encode %{
3516 __ movdl($dst$$XMMRegister, $src$$Register);
3517 __ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
3518 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3519 %}
3520 ins_pipe( pipe_slow );
3521 %}
3522
3523 // Replicate byte scalar immediate to be vector by loading from const table.
3524 instruct Repl4B_imm(vecS dst, immI con) %{
3525 predicate(n->as_Vector()->length() == 4);
3526 match(Set dst (ReplicateB con));
3527 format %{ "movdl $dst,[$constantaddress]\t! replicate4B($con)" %}
3528 ins_encode %{
3529 __ movdl($dst$$XMMRegister, $constantaddress(replicate4_imm($con$$constant, 1)));
3530 %}
3531 ins_pipe( pipe_slow );
3532 %}
3533
3534 instruct Repl8B_imm(vecD dst, immI con) %{
3535 predicate(n->as_Vector()->length() == 8);
3536 match(Set dst (ReplicateB con));
3537 format %{ "movq $dst,[$constantaddress]\t! replicate8B($con)" %}
3538 ins_encode %{
3539 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 1)));
3540 %}
3541 ins_pipe( pipe_slow );
3542 %}
3543
3544 // Replicate byte scalar zero to be vector
3545 instruct Repl4B_zero(vecS dst, immI0 zero) %{
3546 predicate(n->as_Vector()->length() == 4);
3547 match(Set dst (ReplicateB zero));
3548 format %{ "pxor $dst,$dst\t! replicate4B zero" %}
3549 ins_encode %{
3550 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
3551 %}
3552 ins_pipe( fpu_reg_reg );
3553 %}
3554
3555 instruct Repl8B_zero(vecD dst, immI0 zero) %{
3556 predicate(n->as_Vector()->length() == 8);
3557 match(Set dst (ReplicateB zero));
3558 format %{ "pxor $dst,$dst\t! replicate8B zero" %}
3559 ins_encode %{
3560 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
3561 %}
3562 ins_pipe( fpu_reg_reg );
3563 %}
3564
3565 instruct Repl16B_zero(vecX dst, immI0 zero) %{
3566 predicate(n->as_Vector()->length() == 16);
3567 match(Set dst (ReplicateB zero));
3568 format %{ "pxor $dst,$dst\t! replicate16B zero" %}
3569 ins_encode %{
3570 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
3571 %}
3572 ins_pipe( fpu_reg_reg );
3573 %}
3574
3575 instruct Repl32B_zero(vecY dst, immI0 zero) %{
3576 predicate(n->as_Vector()->length() == 32);
3577 match(Set dst (ReplicateB zero));
3578 format %{ "vpxor $dst,$dst,$dst\t! replicate32B zero" %}
3579 ins_encode %{
3580 // Use vxorpd since AVX does not have vpxor for 256-bit (AVX2 will have it).
3581 int vector_len = 1;
3582 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
3583 %}
3584 ins_pipe( fpu_reg_reg );
3585 %}
3586
3587 // Replicate char/short (2 byte) scalar to be vector
3588 instruct Repl2S(vecS dst, rRegI src) %{
3589 predicate(n->as_Vector()->length() == 2);
3590 match(Set dst (ReplicateS src));
3591 format %{ "movd $dst,$src\n\t"
3592 "pshuflw $dst,$dst,0x00\t! replicate2S" %}
3593 ins_encode %{
3594 __ movdl($dst$$XMMRegister, $src$$Register);
3595 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3596 %}
3597 ins_pipe( fpu_reg_reg );
3598 %}
3599
3600 // Replicate char/short (2 byte) scalar immediate to be vector by loading from const table.
3601 instruct Repl2S_imm(vecS dst, immI con) %{
3602 predicate(n->as_Vector()->length() == 2);
3603 match(Set dst (ReplicateS con));
3604 format %{ "movdl $dst,[$constantaddress]\t! replicate2S($con)" %}
3605 ins_encode %{
3606 __ movdl($dst$$XMMRegister, $constantaddress(replicate4_imm($con$$constant, 2)));
3607 %}
3608 ins_pipe( fpu_reg_reg );
3609 %}
3610
3611 instruct Repl4S_imm(vecD dst, immI con) %{
3612 predicate(n->as_Vector()->length() == 4);
3613 match(Set dst (ReplicateS con));
3614 format %{ "movq $dst,[$constantaddress]\t! replicate4S($con)" %}
3615 ins_encode %{
3616 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 2)));
3617 %}
3618 ins_pipe( fpu_reg_reg );
3619 %}
3620
3621 // Replicate char/short (2 byte) scalar zero to be vector
3622 instruct Repl2S_zero(vecS dst, immI0 zero) %{
3623 predicate(n->as_Vector()->length() == 2);
3624 match(Set dst (ReplicateS zero));
3625 format %{ "pxor $dst,$dst\t! replicate2S zero" %}
3626 ins_encode %{
3627 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
3628 %}
3629 ins_pipe( fpu_reg_reg );
3630 %}
3631
3632 instruct Repl4S_zero(vecD dst, immI0 zero) %{
3633 predicate(n->as_Vector()->length() == 4);
3634 match(Set dst (ReplicateS zero));
3635 format %{ "pxor $dst,$dst\t! replicate4S zero" %}
3636 ins_encode %{
3637 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
3638 %}
3639 ins_pipe( fpu_reg_reg );
3640 %}
3641
3642 instruct Repl8S_zero(vecX dst, immI0 zero) %{
3643 predicate(n->as_Vector()->length() == 8);
3644 match(Set dst (ReplicateS zero));
3645 format %{ "pxor $dst,$dst\t! replicate8S zero" %}
3646 ins_encode %{
3647 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
3648 %}
3649 ins_pipe( fpu_reg_reg );
3650 %}
3651
3652 instruct Repl16S_zero(vecY dst, immI0 zero) %{
3653 predicate(n->as_Vector()->length() == 16);
3654 match(Set dst (ReplicateS zero));
3655 format %{ "vpxor $dst,$dst,$dst\t! replicate16S zero" %}
3656 ins_encode %{
3657 // Use vxorpd since AVX does not have vpxor for 256-bit (AVX2 will have it).
3658 int vector_len = 1;
3659 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
3660 %}
3661 ins_pipe( fpu_reg_reg );
3662 %}
3663
3664 // Replicate integer (4 byte) scalar to be vector
3665 instruct Repl2I(vecD dst, rRegI src) %{
3666 predicate(n->as_Vector()->length() == 2);
3667 match(Set dst (ReplicateI src));
3668 format %{ "movd $dst,$src\n\t"
3669 "pshufd $dst,$dst,0x00\t! replicate2I" %}
3670 ins_encode %{
3671 __ movdl($dst$$XMMRegister, $src$$Register);
3672 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3673 %}
3674 ins_pipe( fpu_reg_reg );
3675 %}
3676
3677 // Integer could be loaded into xmm register directly from memory.
3678 instruct Repl2I_mem(vecD dst, memory mem) %{
3679 predicate(n->as_Vector()->length() == 2);
3680 match(Set dst (ReplicateI (LoadI mem)));
3681 format %{ "movd $dst,$mem\n\t"
3682 "pshufd $dst,$dst,0x00\t! replicate2I" %}
3683 ins_encode %{
3684 __ movdl($dst$$XMMRegister, $mem$$Address);
3685 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3686 %}
3687 ins_pipe( fpu_reg_reg );
3688 %}
3689
3690 // Replicate integer (4 byte) scalar immediate to be vector by loading from const table.
3691 instruct Repl2I_imm(vecD dst, immI con) %{
3692 predicate(n->as_Vector()->length() == 2);
3693 match(Set dst (ReplicateI con));
3694 format %{ "movq $dst,[$constantaddress]\t! replicate2I($con)" %}
3695 ins_encode %{
3696 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 4)));
3697 %}
3698 ins_pipe( fpu_reg_reg );
3699 %}
3700
3701 // Replicate integer (4 byte) scalar zero to be vector
3702 instruct Repl2I_zero(vecD dst, immI0 zero) %{
3703 predicate(n->as_Vector()->length() == 2);
3704 match(Set dst (ReplicateI zero));
3705 format %{ "pxor $dst,$dst\t! replicate2I" %}
3706 ins_encode %{
3707 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
3708 %}
3709 ins_pipe( fpu_reg_reg );
3710 %}
3711
3712 instruct Repl4I_zero(vecX dst, immI0 zero) %{
3713 predicate(n->as_Vector()->length() == 4);
3714 match(Set dst (ReplicateI zero));
3715 format %{ "pxor $dst,$dst\t! replicate4I zero)" %}
3716 ins_encode %{
3717 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
3718 %}
3719 ins_pipe( fpu_reg_reg );
3720 %}
3721
3722 instruct Repl8I_zero(vecY dst, immI0 zero) %{
3723 predicate(n->as_Vector()->length() == 8);
3724 match(Set dst (ReplicateI zero));
3725 format %{ "vpxor $dst,$dst,$dst\t! replicate8I zero" %}
3726 ins_encode %{
3727 // Use vxorpd since AVX does not have vpxor for 256-bit (AVX2 will have it).
3728 int vector_len = 1;
3729 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
3730 %}
3731 ins_pipe( fpu_reg_reg );
3732 %}
3733
3734 // Replicate long (8 byte) scalar to be vector
3735 #ifdef _LP64
3736 instruct Repl2L(vecX dst, rRegL src) %{
3737 predicate(n->as_Vector()->length() == 2);
3738 match(Set dst (ReplicateL src));
3739 format %{ "movdq $dst,$src\n\t"
3740 "punpcklqdq $dst,$dst\t! replicate2L" %}
3741 ins_encode %{
3742 __ movdq($dst$$XMMRegister, $src$$Register);
3743 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3744 %}
3745 ins_pipe( pipe_slow );
3746 %}
3747 #else // _LP64
3748 instruct Repl2L(vecX dst, eRegL src, regD tmp) %{
3749 predicate(n->as_Vector()->length() == 2);
3750 match(Set dst (ReplicateL src));
3751 effect(TEMP dst, USE src, TEMP tmp);
3752 format %{ "movdl $dst,$src.lo\n\t"
3753 "movdl $tmp,$src.hi\n\t"
3754 "punpckldq $dst,$tmp\n\t"
3755 "punpcklqdq $dst,$dst\t! replicate2L"%}
3756 ins_encode %{
3757 __ movdl($dst$$XMMRegister, $src$$Register);
3758 __ movdl($tmp$$XMMRegister, HIGH_FROM_LOW($src$$Register));
3759 __ punpckldq($dst$$XMMRegister, $tmp$$XMMRegister);
3760 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3761 %}
3762 ins_pipe( pipe_slow );
3763 %}
3764 #endif // _LP64
3765
3766 // Replicate long (8 byte) scalar immediate to be vector by loading from const table.
3767 instruct Repl2L_imm(vecX dst, immL con) %{
3768 predicate(n->as_Vector()->length() == 2);
3769 match(Set dst (ReplicateL con));
3770 format %{ "movq $dst,[$constantaddress]\n\t"
3771 "punpcklqdq $dst,$dst\t! replicate2L($con)" %}
3772 ins_encode %{
3773 __ movq($dst$$XMMRegister, $constantaddress($con));
3774 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3775 %}
3776 ins_pipe( pipe_slow );
3777 %}
3778
3779 // Replicate long (8 byte) scalar zero to be vector
3780 instruct Repl2L_zero(vecX dst, immL0 zero) %{
3781 predicate(n->as_Vector()->length() == 2);
3782 match(Set dst (ReplicateL zero));
3783 format %{ "pxor $dst,$dst\t! replicate2L zero" %}
3784 ins_encode %{
3785 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
3786 %}
3787 ins_pipe( fpu_reg_reg );
3788 %}
3789
3790 instruct Repl4L_zero(vecY dst, immL0 zero) %{
3791 predicate(n->as_Vector()->length() == 4);
3792 match(Set dst (ReplicateL zero));
3793 format %{ "vpxor $dst,$dst,$dst\t! replicate4L zero" %}
3794 ins_encode %{
3795 // Use vxorpd since AVX does not have vpxor for 256-bit (AVX2 will have it).
3796 int vector_len = 1;
3797 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
3798 %}
3799 ins_pipe( fpu_reg_reg );
3800 %}
3801
3802 // Replicate float (4 byte) scalar to be vector
3803 instruct Repl2F(vecD dst, regF src) %{
3804 predicate(n->as_Vector()->length() == 2);
3805 match(Set dst (ReplicateF src));
3806 format %{ "pshufd $dst,$dst,0x00\t! replicate2F" %}
3807 ins_encode %{
3808 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
3809 %}
3810 ins_pipe( fpu_reg_reg );
3811 %}
3812
3813 instruct Repl4F(vecX dst, regF src) %{
3814 predicate(n->as_Vector()->length() == 4);
3815 match(Set dst (ReplicateF src));
3816 format %{ "pshufd $dst,$dst,0x00\t! replicate4F" %}
3817 ins_encode %{
3818 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
3819 %}
3820 ins_pipe( pipe_slow );
3821 %}
3822
3823 // Replicate double (8 bytes) scalar to be vector
3824 instruct Repl2D(vecX dst, regD src) %{
3825 predicate(n->as_Vector()->length() == 2);
3826 match(Set dst (ReplicateD src));
3827 format %{ "pshufd $dst,$src,0x44\t! replicate2D" %}
3828 ins_encode %{
3829 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x44);
3830 %}
3831 ins_pipe( pipe_slow );
3832 %}
3833
3834 // ====================EVEX REPLICATE=============================================
3835
3836 instruct Repl4B_mem_evex(vecS dst, memory mem) %{
3837 predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vlbw());
3838 match(Set dst (ReplicateB (LoadB mem)));
3839 format %{ "vpbroadcastb $dst,$mem\t! replicate4B" %}
3840 ins_encode %{
3841 int vector_len = 0;
3842 __ evpbroadcastb($dst$$XMMRegister, $mem$$Address, vector_len);
3843 %}
3844 ins_pipe( pipe_slow );
3845 %}
3846
3847 instruct Repl8B_mem_evex(vecD dst, memory mem) %{
3848 predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vlbw());
3849 match(Set dst (ReplicateB (LoadB mem)));
3850 format %{ "vpbroadcastb $dst,$mem\t! replicate8B" %}
3851 ins_encode %{
3852 int vector_len = 0;
3853 __ evpbroadcastb($dst$$XMMRegister, $mem$$Address, vector_len);
3854 %}
3855 ins_pipe( pipe_slow );
3856 %}
3857
3858 instruct Repl16B_evex(vecX dst, rRegI src) %{
3859 predicate(n->as_Vector()->length() == 16 && VM_Version::supports_avx512vlbw());
3860 match(Set dst (ReplicateB src));
3861 format %{ "vpbroadcastb $dst,$src\t! replicate16B" %}
3862 ins_encode %{
3863 int vector_len = 0;
3864 __ evpbroadcastb($dst$$XMMRegister, $src$$Register, vector_len);
3865 %}
3866 ins_pipe( pipe_slow );
3867 %}
3868
3869 instruct Repl16B_mem_evex(vecX dst, memory mem) %{
3870 predicate(n->as_Vector()->length() == 16 && VM_Version::supports_avx512vlbw());
3871 match(Set dst (ReplicateB (LoadB mem)));
3872 format %{ "vpbroadcastb $dst,$mem\t! replicate16B" %}
3873 ins_encode %{
3874 int vector_len = 0;
3875 __ evpbroadcastb($dst$$XMMRegister, $mem$$Address, vector_len);
3876 %}
3877 ins_pipe( pipe_slow );
3878 %}
3879
3880 instruct Repl32B_evex(vecY dst, rRegI src) %{
3881 predicate(n->as_Vector()->length() == 32 && VM_Version::supports_avx512vlbw());
3882 match(Set dst (ReplicateB src));
3883 format %{ "vpbroadcastb $dst,$src\t! replicate32B" %}
3884 ins_encode %{
3885 int vector_len = 1;
3886 __ evpbroadcastb($dst$$XMMRegister, $src$$Register, vector_len);
3887 %}
3888 ins_pipe( pipe_slow );
3889 %}
3890
3891 instruct Repl32B_mem_evex(vecY dst, memory mem) %{
3892 predicate(n->as_Vector()->length() == 32 && VM_Version::supports_avx512vlbw());
3893 match(Set dst (ReplicateB (LoadB mem)));
3894 format %{ "vpbroadcastb $dst,$mem\t! replicate32B" %}
3895 ins_encode %{
3896 int vector_len = 1;
3897 __ evpbroadcastb($dst$$XMMRegister, $mem$$Address, vector_len);
3898 %}
3899 ins_pipe( pipe_slow );
3900 %}
3901
3902 instruct Repl64B_evex(vecZ dst, rRegI src) %{
3903 predicate(n->as_Vector()->length() == 64 && VM_Version::supports_avx512bw());
3904 match(Set dst (ReplicateB src));
3905 format %{ "vpbroadcastb $dst,$src\t! upper replicate64B" %}
3906 ins_encode %{
3907 int vector_len = 2;
3908 __ evpbroadcastb($dst$$XMMRegister, $src$$Register, vector_len);
3909 %}
3910 ins_pipe( pipe_slow );
3911 %}
3912
3913 instruct Repl64B_mem_evex(vecZ dst, memory mem) %{
3914 predicate(n->as_Vector()->length() == 64 && VM_Version::supports_avx512bw());
3915 match(Set dst (ReplicateB (LoadB mem)));
3916 format %{ "vpbroadcastb $dst,$mem\t! replicate64B" %}
3917 ins_encode %{
3918 int vector_len = 2;
3919 __ evpbroadcastb($dst$$XMMRegister, $mem$$Address, vector_len);
3920 %}
3921 ins_pipe( pipe_slow );
3922 %}
3923
3924 instruct Repl16B_imm_evex(vecX dst, immI con) %{
3925 predicate(n->as_Vector()->length() == 16 && VM_Version::supports_avx512vlbw());
3926 match(Set dst (ReplicateB con));
3927 format %{ "movq $dst,[$constantaddress]\n\t"
3928 "vpbroadcastb $dst,$dst\t! replicate16B" %}
3929 ins_encode %{
3930 int vector_len = 0;
3931 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 1)));
3932 __ evpbroadcastb($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
3933 %}
3934 ins_pipe( pipe_slow );
3935 %}
3936
3937 instruct Repl32B_imm_evex(vecY dst, immI con) %{
3938 predicate(n->as_Vector()->length() == 32 && VM_Version::supports_avx512vlbw());
3939 match(Set dst (ReplicateB con));
3940 format %{ "movq $dst,[$constantaddress]\n\t"
3941 "vpbroadcastb $dst,$dst\t! replicate32B" %}
3942 ins_encode %{
3943 int vector_len = 1;
3944 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 1)));
3945 __ evpbroadcastb($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
3946 %}
3947 ins_pipe( pipe_slow );
3948 %}
3949
3950 instruct Repl64B_imm_evex(vecZ dst, immI con) %{
3951 predicate(n->as_Vector()->length() == 64 && VM_Version::supports_avx512bw());
3952 match(Set dst (ReplicateB con));
3953 format %{ "movq $dst,[$constantaddress]\n\t"
3954 "vpbroadcastb $dst,$dst\t! upper replicate64B" %}
3955 ins_encode %{
3956 int vector_len = 2;
3957 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 1)));
3958 __ evpbroadcastb($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
3959 %}
3960 ins_pipe( pipe_slow );
3961 %}
3962
3963 instruct Repl64B_zero_evex(vecZ dst, immI0 zero) %{
3964 predicate(n->as_Vector()->length() == 64 && UseAVX > 2);
3965 match(Set dst (ReplicateB zero));
3966 format %{ "vpxor $dst k0,$dst,$dst\t! replicate64B zero" %}
3967 ins_encode %{
3968 // Use vxorpd since AVX does not have vpxor for 512-bit (EVEX will have it).
3969 int vector_len = 2;
3970 __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
3971 %}
3972 ins_pipe( fpu_reg_reg );
3973 %}
3974
3975 instruct Repl4S_evex(vecD dst, rRegI src) %{
3976 predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vlbw());
3977 match(Set dst (ReplicateS src));
3978 format %{ "vpbroadcastw $dst,$src\t! replicate4S" %}
3979 ins_encode %{
3980 int vector_len = 0;
3981 __ evpbroadcastw($dst$$XMMRegister, $src$$Register, vector_len);
3982 %}
3983 ins_pipe( pipe_slow );
3984 %}
3985
3986 instruct Repl4S_mem_evex(vecD dst, memory mem) %{
3987 predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vlbw());
3988 match(Set dst (ReplicateS (LoadS mem)));
3989 format %{ "vpbroadcastw $dst,$mem\t! replicate4S" %}
3990 ins_encode %{
3991 int vector_len = 0;
3992 __ evpbroadcastw($dst$$XMMRegister, $mem$$Address, vector_len);
3993 %}
3994 ins_pipe( pipe_slow );
3995 %}
3996
3997 instruct Repl8S_evex(vecX dst, rRegI src) %{
3998 predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vlbw());
3999 match(Set dst (ReplicateS src));
4000 format %{ "vpbroadcastw $dst,$src\t! replicate8S" %}
4001 ins_encode %{
4002 int vector_len = 0;
4003 __ evpbroadcastw($dst$$XMMRegister, $src$$Register, vector_len);
4004 %}
4005 ins_pipe( pipe_slow );
4006 %}
4007
4008 instruct Repl8S_mem_evex(vecX dst, memory mem) %{
4009 predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vlbw());
4010 match(Set dst (ReplicateS (LoadS mem)));
4011 format %{ "vpbroadcastw $dst,$mem\t! replicate8S" %}
4012 ins_encode %{
4013 int vector_len = 0;
4014 __ evpbroadcastw($dst$$XMMRegister, $mem$$Address, vector_len);
4015 %}
4016 ins_pipe( pipe_slow );
4017 %}
4018
4019 instruct Repl16S_evex(vecY dst, rRegI src) %{
4020 predicate(n->as_Vector()->length() == 16 && VM_Version::supports_avx512vlbw());
4021 match(Set dst (ReplicateS src));
4022 format %{ "vpbroadcastw $dst,$src\t! replicate16S" %}
4023 ins_encode %{
4024 int vector_len = 1;
4025 __ evpbroadcastw($dst$$XMMRegister, $src$$Register, vector_len);
4026 %}
4027 ins_pipe( pipe_slow );
4028 %}
4029
4030 instruct Repl16S_mem_evex(vecY dst, memory mem) %{
4031 predicate(n->as_Vector()->length() == 16 && VM_Version::supports_avx512vlbw());
4032 match(Set dst (ReplicateS (LoadS mem)));
4033 format %{ "vpbroadcastw $dst,$mem\t! replicate16S" %}
4034 ins_encode %{
4035 int vector_len = 1;
4036 __ evpbroadcastw($dst$$XMMRegister, $mem$$Address, vector_len);
4037 %}
4038 ins_pipe( pipe_slow );
4039 %}
4040
4041 instruct Repl32S_evex(vecZ dst, rRegI src) %{
4042 predicate(n->as_Vector()->length() == 32 && VM_Version::supports_avx512bw());
4043 match(Set dst (ReplicateS src));
4044 format %{ "vpbroadcastw $dst,$src\t! replicate32S" %}
4045 ins_encode %{
4046 int vector_len = 2;
4047 __ evpbroadcastw($dst$$XMMRegister, $src$$Register, vector_len);
4048 %}
4049 ins_pipe( pipe_slow );
4050 %}
4051
4052 instruct Repl32S_mem_evex(vecZ dst, memory mem) %{
4053 predicate(n->as_Vector()->length() == 32 && VM_Version::supports_avx512bw());
4054 match(Set dst (ReplicateS (LoadS mem)));
4055 format %{ "vpbroadcastw $dst,$mem\t! replicate32S" %}
4056 ins_encode %{
4057 int vector_len = 2;
4058 __ evpbroadcastw($dst$$XMMRegister, $mem$$Address, vector_len);
4059 %}
4060 ins_pipe( pipe_slow );
4061 %}
4062
4063 instruct Repl8S_imm_evex(vecX dst, immI con) %{
4064 predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vlbw());
4065 match(Set dst (ReplicateS con));
4066 format %{ "movq $dst,[$constantaddress]\n\t"
4067 "vpbroadcastw $dst,$dst\t! replicate8S" %}
4068 ins_encode %{
4069 int vector_len = 0;
4070 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 2)));
4071 __ evpbroadcastw($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4072 %}
4073 ins_pipe( pipe_slow );
4074 %}
4075
4076 instruct Repl16S_imm_evex(vecY dst, immI con) %{
4077 predicate(n->as_Vector()->length() == 16 && VM_Version::supports_avx512vlbw());
4078 match(Set dst (ReplicateS con));
4079 format %{ "movq $dst,[$constantaddress]\n\t"
4080 "vpbroadcastw $dst,$dst\t! replicate16S" %}
4081 ins_encode %{
4082 int vector_len = 1;
4083 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 2)));
4084 __ evpbroadcastw($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4085 %}
4086 ins_pipe( pipe_slow );
4087 %}
4088
4089 instruct Repl32S_imm_evex(vecZ dst, immI con) %{
4090 predicate(n->as_Vector()->length() == 32 && VM_Version::supports_avx512bw());
4091 match(Set dst (ReplicateS con));
4092 format %{ "movq $dst,[$constantaddress]\n\t"
4093 "vpbroadcastw $dst,$dst\t! replicate32S" %}
4094 ins_encode %{
4095 int vector_len = 2;
4096 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 2)));
4097 __ evpbroadcastw($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4098 %}
4099 ins_pipe( pipe_slow );
4100 %}
4101
4102 instruct Repl32S_zero_evex(vecZ dst, immI0 zero) %{
4103 predicate(n->as_Vector()->length() == 32 && UseAVX > 2);
4104 match(Set dst (ReplicateS zero));
4105 format %{ "vpxor $dst k0,$dst,$dst\t! replicate32S zero" %}
4106 ins_encode %{
4107 // Use vxorpd since AVX does not have vpxor for 512-bit (EVEX will have it).
4108 int vector_len = 2;
4109 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4110 %}
4111 ins_pipe( fpu_reg_reg );
4112 %}
4113
4114 instruct Repl4I_evex(vecX dst, rRegI src) %{
4115 predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4116 match(Set dst (ReplicateI src));
4117 format %{ "vpbroadcastd $dst,$src\t! replicate4I" %}
4118 ins_encode %{
4119 int vector_len = 0;
4120 __ evpbroadcastd($dst$$XMMRegister, $src$$Register, vector_len);
4121 %}
4122 ins_pipe( pipe_slow );
4123 %}
4124
4125 instruct Repl4I_mem_evex(vecX dst, memory mem) %{
4126 predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4127 match(Set dst (ReplicateI (LoadI mem)));
4128 format %{ "vpbroadcastd $dst,$mem\t! replicate4I" %}
4129 ins_encode %{
4130 int vector_len = 0;
4131 __ evpbroadcastd($dst$$XMMRegister, $mem$$Address, vector_len);
4132 %}
4133 ins_pipe( pipe_slow );
4134 %}
4135
4136 instruct Repl8I_evex(vecY dst, rRegI src) %{
4137 predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vl());
4138 match(Set dst (ReplicateI src));
4139 format %{ "vpbroadcastd $dst,$src\t! replicate8I" %}
4140 ins_encode %{
4141 int vector_len = 1;
4142 __ evpbroadcastd($dst$$XMMRegister, $src$$Register, vector_len);
4143 %}
4144 ins_pipe( pipe_slow );
4145 %}
4146
4147 instruct Repl8I_mem_evex(vecY dst, memory mem) %{
4148 predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vl());
4149 match(Set dst (ReplicateI (LoadI mem)));
4150 format %{ "vpbroadcastd $dst,$mem\t! replicate8I" %}
4151 ins_encode %{
4152 int vector_len = 1;
4153 __ evpbroadcastd($dst$$XMMRegister, $mem$$Address, vector_len);
4154 %}
4155 ins_pipe( pipe_slow );
4156 %}
4157
4158 instruct Repl16I_evex(vecZ dst, rRegI src) %{
4159 predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
4160 match(Set dst (ReplicateI src));
4161 format %{ "vpbroadcastd $dst,$src\t! replicate16I" %}
4162 ins_encode %{
4163 int vector_len = 2;
4164 __ evpbroadcastd($dst$$XMMRegister, $src$$Register, vector_len);
4165 %}
4166 ins_pipe( pipe_slow );
4167 %}
4168
4169 instruct Repl16I_mem_evex(vecZ dst, memory mem) %{
4170 predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
4171 match(Set dst (ReplicateI (LoadI mem)));
4172 format %{ "vpbroadcastd $dst,$mem\t! replicate16I" %}
4173 ins_encode %{
4174 int vector_len = 2;
4175 __ evpbroadcastd($dst$$XMMRegister, $mem$$Address, vector_len);
4176 %}
4177 ins_pipe( pipe_slow );
4178 %}
4179
4180 instruct Repl4I_imm_evex(vecX dst, immI con) %{
4181 predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4182 match(Set dst (ReplicateI con));
4183 format %{ "movq $dst,[$constantaddress]\t! replicate8I($con)\n\t"
4184 "vpbroadcastd $dst,$dst\t! replicate4I" %}
4185 ins_encode %{
4186 int vector_len = 0;
4187 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 4)));
4188 __ evpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4189 %}
4190 ins_pipe( pipe_slow );
4191 %}
4192
4193 instruct Repl8I_imm_evex(vecY dst, immI con) %{
4194 predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vl());
4195 match(Set dst (ReplicateI con));
4196 format %{ "movq $dst,[$constantaddress]\t! replicate8I($con)\n\t"
4197 "vpbroadcastd $dst,$dst\t! replicate8I" %}
4198 ins_encode %{
4199 int vector_len = 1;
4200 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 4)));
4201 __ evpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4202 %}
4203 ins_pipe( pipe_slow );
4204 %}
4205
4206 instruct Repl16I_imm_evex(vecZ dst, immI con) %{
4207 predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
4208 match(Set dst (ReplicateI con));
4209 format %{ "movq $dst,[$constantaddress]\t! replicate16I($con)\n\t"
4210 "vpbroadcastd $dst,$dst\t! replicate16I" %}
4211 ins_encode %{
4212 int vector_len = 2;
4213 __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 4)));
4214 __ evpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4215 %}
4216 ins_pipe( pipe_slow );
4217 %}
4218
4219 instruct Repl16I_zero_evex(vecZ dst, immI0 zero) %{
4220 predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
4221 match(Set dst (ReplicateI zero));
4222 format %{ "vpxor $dst k0,$dst,$dst\t! replicate16I zero" %}
4223 ins_encode %{
4224 // Use vxorpd since AVX does not have vpxor for 512-bit (AVX2 will have it).
4225 int vector_len = 2;
4226 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4227 %}
4228 ins_pipe( fpu_reg_reg );
4229 %}
4230
4231 // Replicate long (8 byte) scalar to be vector
4232 #ifdef _LP64
4233 instruct Repl4L_evex(vecY dst, rRegL src) %{
4234 predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4235 match(Set dst (ReplicateL src));
4236 format %{ "vpbroadcastq $dst,$src\t! replicate4L" %}
4237 ins_encode %{
4238 int vector_len = 1;
4239 __ evpbroadcastq($dst$$XMMRegister, $src$$Register, vector_len);
4240 %}
4241 ins_pipe( pipe_slow );
4242 %}
4243
4244 instruct Repl8L_evex(vecZ dst, rRegL src) %{
4245 predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4246 match(Set dst (ReplicateL src));
4247 format %{ "vpbroadcastq $dst,$src\t! replicate8L" %}
4248 ins_encode %{
4249 int vector_len = 2;
4250 __ evpbroadcastq($dst$$XMMRegister, $src$$Register, vector_len);
4251 %}
4252 ins_pipe( pipe_slow );
4253 %}
4254 #else // _LP64
4255 instruct Repl4L_evex(vecY dst, eRegL src, regD tmp) %{
4256 predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4257 match(Set dst (ReplicateL src));
4258 effect(TEMP dst, USE src, TEMP tmp);
4259 format %{ "movdl $dst,$src.lo\n\t"
4260 "movdl $tmp,$src.hi\n\t"
4261 "punpckldq $dst,$tmp\n\t"
4262 "vpbroadcastq $dst,$dst\t! replicate4L" %}
4263 ins_encode %{
4264 int vector_len = 1;
4265 __ movdl($dst$$XMMRegister, $src$$Register);
4266 __ movdl($tmp$$XMMRegister, HIGH_FROM_LOW($src$$Register));
4267 __ punpckldq($dst$$XMMRegister, $tmp$$XMMRegister);
4268 __ evpbroadcastq($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4269 %}
4270 ins_pipe( pipe_slow );
4271 %}
4272
4273 instruct Repl8L_evex(vecZ dst, eRegL src, regD tmp) %{
4274 predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4275 match(Set dst (ReplicateL src));
4276 effect(TEMP dst, USE src, TEMP tmp);
4277 format %{ "movdl $dst,$src.lo\n\t"
4278 "movdl $tmp,$src.hi\n\t"
4279 "punpckldq $dst,$tmp\n\t"
4280 "vpbroadcastq $dst,$dst\t! replicate8L" %}
4281 ins_encode %{
4282 int vector_len = 2;
4283 __ movdl($dst$$XMMRegister, $src$$Register);
4284 __ movdl($tmp$$XMMRegister, HIGH_FROM_LOW($src$$Register));
4285 __ punpckldq($dst$$XMMRegister, $tmp$$XMMRegister);
4286 __ evpbroadcastq($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4287 %}
4288 ins_pipe( pipe_slow );
4289 %}
4290 #endif // _LP64
4291
4292 instruct Repl4L_imm_evex(vecY dst, immL con) %{
4293 predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4294 match(Set dst (ReplicateL con));
4295 format %{ "movq $dst,[$constantaddress]\n\t"
4296 "vpbroadcastq $dst,$dst\t! replicate4L" %}
4297 ins_encode %{
4298 int vector_len = 1;
4299 __ movq($dst$$XMMRegister, $constantaddress($con));
4300 __ evpbroadcastq($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4301 %}
4302 ins_pipe( pipe_slow );
4303 %}
4304
4305 instruct Repl8L_imm_evex(vecZ dst, immL con) %{
4306 predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4307 match(Set dst (ReplicateL con));
4308 format %{ "movq $dst,[$constantaddress]\n\t"
4309 "vpbroadcastq $dst,$dst\t! replicate8L" %}
4310 ins_encode %{
4311 int vector_len = 2;
4312 __ movq($dst$$XMMRegister, $constantaddress($con));
4313 __ evpbroadcastq($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4314 %}
4315 ins_pipe( pipe_slow );
4316 %}
4317
4318 instruct Repl2L_mem_evex(vecX dst, memory mem) %{
4319 predicate(n->as_Vector()->length() == 2 && VM_Version::supports_avx512vl());
4320 match(Set dst (ReplicateL (LoadL mem)));
4321 format %{ "vpbroadcastd $dst,$mem\t! replicate2L" %}
4322 ins_encode %{
4323 int vector_len = 0;
4324 __ evpbroadcastq($dst$$XMMRegister, $mem$$Address, vector_len);
4325 %}
4326 ins_pipe( pipe_slow );
4327 %}
4328
4329 instruct Repl4L_mem_evex(vecY dst, memory mem) %{
4330 predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4331 match(Set dst (ReplicateL (LoadL mem)));
4332 format %{ "vpbroadcastd $dst,$mem\t! replicate4L" %}
4333 ins_encode %{
4334 int vector_len = 1;
4335 __ evpbroadcastq($dst$$XMMRegister, $mem$$Address, vector_len);
4336 %}
4337 ins_pipe( pipe_slow );
4338 %}
4339
4340 instruct Repl8L_mem_evex(vecZ dst, memory mem) %{
4341 predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4342 match(Set dst (ReplicateL (LoadL mem)));
4343 format %{ "vpbroadcastd $dst,$mem\t! replicate8L" %}
4344 ins_encode %{
4345 int vector_len = 2;
4346 __ evpbroadcastq($dst$$XMMRegister, $mem$$Address, vector_len);
4347 %}
4348 ins_pipe( pipe_slow );
4349 %}
4350
4351 instruct Repl8L_zero_evex(vecZ dst, immL0 zero) %{
4352 predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4353 match(Set dst (ReplicateL zero));
4354 format %{ "vpxor $dst k0,$dst,$dst\t! replicate8L zero" %}
4355 ins_encode %{
4356 // Use vxorpd since AVX does not have vpxor for 512-bit (EVEX will have it).
4357 int vector_len = 2;
4358 __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4359 %}
4360 ins_pipe( fpu_reg_reg );
4361 %}
4362
4363 instruct Repl8F_evex(vecY dst, regF src) %{
4364 predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vl());
4365 match(Set dst (ReplicateF src));
4366 format %{ "vbroadcastss $dst,$src\t! replicate8F" %}
4367 ins_encode %{
4368 int vector_len = 1;
4369 __ evpbroadcastss($dst$$XMMRegister, $src$$XMMRegister, vector_len);
4370 %}
4371 ins_pipe( pipe_slow );
4372 %}
4373
4374 instruct Repl8F_mem_evex(vecY dst, memory mem) %{
4375 predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vl());
4376 match(Set dst (ReplicateF (LoadF mem)));
4377 format %{ "vbroadcastss $dst,$mem\t! replicate8F" %}
4378 ins_encode %{
4379 int vector_len = 1;
4380 __ evpbroadcastss($dst$$XMMRegister, $mem$$Address, vector_len);
4381 %}
4382 ins_pipe( pipe_slow );
4383 %}
4384
4385 instruct Repl16F_evex(vecZ dst, regF src) %{
4386 predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
4387 match(Set dst (ReplicateF src));
4388 format %{ "vbroadcastss $dst,$src\t! replicate16F" %}
4389 ins_encode %{
4390 int vector_len = 2;
4391 __ evpbroadcastss($dst$$XMMRegister, $src$$XMMRegister, vector_len);
4392 %}
4393 ins_pipe( pipe_slow );
4394 %}
4395
4396 instruct Repl16F_mem_evex(vecZ dst, memory mem) %{
4397 predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
4398 match(Set dst (ReplicateF (LoadF mem)));
4399 format %{ "vbroadcastss $dst,$mem\t! replicate16F" %}
4400 ins_encode %{
4401 int vector_len = 2;
4402 __ evpbroadcastss($dst$$XMMRegister, $mem$$Address, vector_len);
4403 %}
4404 ins_pipe( pipe_slow );
4405 %}
4406
4407 instruct Repl2F_zero_evex(vecD dst, immF0 zero) %{
4408 predicate(n->as_Vector()->length() == 2 && UseAVX > 2);
4409 match(Set dst (ReplicateF zero));
4410 format %{ "vpxor $dst k0,$dst,$dst\t! replicate2F zero" %}
4411 ins_encode %{
4412 // Use vpxor in place of vxorps since EVEX has a constriant on dq for vxorps: this is a 512-bit operation
4413 int vector_len = 2;
4414 __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4415 %}
4416 ins_pipe( fpu_reg_reg );
4417 %}
4418
4419 instruct Repl4F_zero_evex(vecX dst, immF0 zero) %{
4420 predicate(n->as_Vector()->length() == 4 && UseAVX > 2);
4421 match(Set dst (ReplicateF zero));
4422 format %{ "vpxor $dst k0,$dst,$dst\t! replicate4F zero" %}
4423 ins_encode %{
4424 // Use vpxor in place of vxorps since EVEX has a constriant on dq for vxorps: this is a 512-bit operation
4425 int vector_len = 2;
4426 __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4427 %}
4428 ins_pipe( fpu_reg_reg );
4429 %}
4430
4431 instruct Repl8F_zero_evex(vecY dst, immF0 zero) %{
4432 predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4433 match(Set dst (ReplicateF zero));
4434 format %{ "vpxor $dst k0,$dst,$dst\t! replicate8F zero" %}
4435 ins_encode %{
4436 // Use vpxor in place of vxorps since EVEX has a constriant on dq for vxorps: this is a 512-bit operation
4437 int vector_len = 2;
4438 __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4439 %}
4440 ins_pipe( fpu_reg_reg );
4441 %}
4442
4443 instruct Repl16F_zero_evex(vecZ dst, immF0 zero) %{
4444 predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
4445 match(Set dst (ReplicateF zero));
4446 format %{ "vpxor $dst k0,$dst,$dst\t! replicate16F zero" %}
4447 ins_encode %{
4448 // Use vpxor in place of vxorps since EVEX has a constriant on dq for vxorps: this is a 512-bit operation
4449 int vector_len = 2;
4450 __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4451 %}
4452 ins_pipe( fpu_reg_reg );
4453 %}
4454
4455 instruct Repl4D_evex(vecY dst, regD src) %{
4456 predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4457 match(Set dst (ReplicateD src));
4458 format %{ "vbroadcastsd $dst,$src\t! replicate4D" %}
4459 ins_encode %{
4460 int vector_len = 1;
4461 __ evpbroadcastsd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
4462 %}
4463 ins_pipe( pipe_slow );
4464 %}
4465
4466 instruct Repl4D_mem_evex(vecY dst, memory mem) %{
4467 predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4468 match(Set dst (ReplicateD (LoadD mem)));
4469 format %{ "vbroadcastsd $dst,$mem\t! replicate4D" %}
4470 ins_encode %{
4471 int vector_len = 1;
4472 __ evpbroadcastsd($dst$$XMMRegister, $mem$$Address, vector_len);
4473 %}
4474 ins_pipe( pipe_slow );
4475 %}
4476
4477 instruct Repl8D_evex(vecZ dst, regD src) %{
4478 predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4479 match(Set dst (ReplicateD src));
4480 format %{ "vbroadcastsd $dst,$src\t! replicate8D" %}
4481 ins_encode %{
4482 int vector_len = 2;
4483 __ evpbroadcastsd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
4484 %}
4485 ins_pipe( pipe_slow );
4486 %}
4487
4488 instruct Repl8D_mem_evex(vecZ dst, memory mem) %{
4489 predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4490 match(Set dst (ReplicateD (LoadD mem)));
4491 format %{ "vbroadcastsd $dst,$mem\t! replicate8D" %}
4492 ins_encode %{
4493 int vector_len = 2;
4494 __ evpbroadcastsd($dst$$XMMRegister, $mem$$Address, vector_len);
4495 %}
4496 ins_pipe( pipe_slow );
4497 %}
4498
4499 instruct Repl2D_zero_evex(vecX dst, immD0 zero) %{
4500 predicate(n->as_Vector()->length() == 2 && UseAVX > 2);
4501 match(Set dst (ReplicateD zero));
4502 format %{ "vpxor $dst k0,$dst,$dst\t! replicate2D zero" %}
4503 ins_encode %{
4504 // Use vpxor in place of vxorpd since EVEX has a constriant on dq for vxorpd: this is a 512-bit operation
4505 int vector_len = 2;
4506 __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4507 %}
4508 ins_pipe( fpu_reg_reg );
4509 %}
4510
4511 instruct Repl4D_zero_evex(vecY dst, immD0 zero) %{
4512 predicate(n->as_Vector()->length() == 4 && UseAVX > 2);
4513 match(Set dst (ReplicateD zero));
4514 format %{ "vpxor $dst k0,$dst,$dst\t! replicate4D zero" %}
4515 ins_encode %{
4516 // Use vpxor in place of vxorpd since EVEX has a constriant on dq for vxorpd: this is a 512-bit operation
4517 int vector_len = 2;
4518 __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4519 %}
4520 ins_pipe( fpu_reg_reg );
4521 %}
4522
4523 instruct Repl8D_zero_evex(vecZ dst, immD0 zero) %{
4524 predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4525 match(Set dst (ReplicateD zero));
4526 format %{ "vpxor $dst k0,$dst,$dst,vect512\t! replicate8D zero" %}
4527 ins_encode %{
4528 // Use vpxor in place of vxorpd since EVEX has a constriant on dq for vxorpd: this is a 512-bit operation
4529 int vector_len = 2;
4530 __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4531 %}
4532 ins_pipe( fpu_reg_reg );
4533 %}
4534
4535 // ====================REDUCTION ARITHMETIC=======================================
4536
4537 instruct rsadd2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
4538 predicate(UseSSE > 2 && UseAVX == 0);
4539 match(Set dst (AddReductionVI src1 src2));
4540 effect(TEMP tmp2, TEMP tmp);
4541 format %{ "movdqu $tmp2,$src2\n\t"
4542 "phaddd $tmp2,$tmp2\n\t"
4543 "movd $tmp,$src1\n\t"
4544 "paddd $tmp,$tmp2\n\t"
4545 "movd $dst,$tmp\t! add reduction2I" %}
4546 ins_encode %{
4547 __ movdqu($tmp2$$XMMRegister, $src2$$XMMRegister);
4548 __ phaddd($tmp2$$XMMRegister, $tmp2$$XMMRegister);
4549 __ movdl($tmp$$XMMRegister, $src1$$Register);
4550 __ paddd($tmp$$XMMRegister, $tmp2$$XMMRegister);
4551 __ movdl($dst$$Register, $tmp$$XMMRegister);
4552 %}
4553 ins_pipe( pipe_slow );
4554 %}
4555
4556 instruct rvadd2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
4557 predicate(UseAVX > 0 && UseAVX < 3);
4558 match(Set dst (AddReductionVI src1 src2));
4559 effect(TEMP tmp, TEMP tmp2);
4560 format %{ "vphaddd $tmp,$src2,$src2\n\t"
4561 "movd $tmp2,$src1\n\t"
4562 "vpaddd $tmp2,$tmp2,$tmp\n\t"
4563 "movd $dst,$tmp2\t! add reduction2I" %}
4564 ins_encode %{
4565 int vector_len = 0;
4566 __ vphaddd($tmp$$XMMRegister, $src2$$XMMRegister, $src2$$XMMRegister, vector_len);
4567 __ movdl($tmp2$$XMMRegister, $src1$$Register);
4568 __ vpaddd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
4569 __ movdl($dst$$Register, $tmp2$$XMMRegister);
4570 %}
4571 ins_pipe( pipe_slow );
4572 %}
4573
4574 instruct rvadd2I_reduction_reg_evex(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
4575 predicate(UseAVX > 2);
4576 match(Set dst (AddReductionVI src1 src2));
4577 effect(TEMP tmp, TEMP tmp2);
4578 format %{ "pshufd $tmp2,$src2,0x1\n\t"
4579 "vpaddd $tmp,$src2,$tmp2\n\t"
4580 "movd $tmp2,$src1\n\t"
4581 "vpaddd $tmp2,$tmp,$tmp2\n\t"
4582 "movd $dst,$tmp2\t! add reduction2I" %}
4583 ins_encode %{
4584 int vector_len = 0;
4585 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
4586 __ vpaddd($tmp$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, vector_len);
4587 __ movdl($tmp2$$XMMRegister, $src1$$Register);
4588 __ vpaddd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
4589 __ movdl($dst$$Register, $tmp2$$XMMRegister);
4590 %}
4591 ins_pipe( pipe_slow );
4592 %}
4593
4594 instruct rsadd4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
4595 predicate(UseSSE > 2 && UseAVX == 0);
4596 match(Set dst (AddReductionVI src1 src2));
4597 effect(TEMP tmp2, TEMP tmp);
4598 format %{ "movdqu $tmp2,$src2\n\t"
4599 "phaddd $tmp2,$tmp2\n\t"
4600 "phaddd $tmp2,$tmp2\n\t"
4601 "movd $tmp,$src1\n\t"
4602 "paddd $tmp,$tmp2\n\t"
4603 "movd $dst,$tmp\t! add reduction4I" %}
4604 ins_encode %{
4605 __ movdqu($tmp2$$XMMRegister, $src2$$XMMRegister);
4606 __ phaddd($tmp2$$XMMRegister, $tmp2$$XMMRegister);
4607 __ phaddd($tmp2$$XMMRegister, $tmp2$$XMMRegister);
4608 __ movdl($tmp$$XMMRegister, $src1$$Register);
4609 __ paddd($tmp$$XMMRegister, $tmp2$$XMMRegister);
4610 __ movdl($dst$$Register, $tmp$$XMMRegister);
4611 %}
4612 ins_pipe( pipe_slow );
4613 %}
4614
4615 instruct rvadd4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
4616 predicate(UseAVX > 0 && UseAVX < 3);
4617 match(Set dst (AddReductionVI src1 src2));
4618 effect(TEMP tmp, TEMP tmp2);
4619 format %{ "vphaddd $tmp,$src2,$src2\n\t"
4620 "vphaddd $tmp,$tmp,$tmp2\n\t"
4621 "movd $tmp2,$src1\n\t"
4622 "vpaddd $tmp2,$tmp2,$tmp\n\t"
4623 "movd $dst,$tmp2\t! add reduction4I" %}
4624 ins_encode %{
4625 int vector_len = 0;
4626 __ vphaddd($tmp$$XMMRegister, $src2$$XMMRegister, $src2$$XMMRegister, vector_len);
4627 __ vphaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
4628 __ movdl($tmp2$$XMMRegister, $src1$$Register);
4629 __ vpaddd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
4630 __ movdl($dst$$Register, $tmp2$$XMMRegister);
4631 %}
4632 ins_pipe( pipe_slow );
4633 %}
4634
4635 instruct rvadd4I_reduction_reg_evex(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
4636 predicate(UseAVX > 2);
4637 match(Set dst (AddReductionVI src1 src2));
4638 effect(TEMP tmp, TEMP tmp2);
4639 format %{ "pshufd $tmp2,$src2,0xE\n\t"
4640 "vpaddd $tmp,$src2,$tmp2\n\t"
4641 "pshufd $tmp2,$tmp,0x1\n\t"
4642 "vpaddd $tmp,$tmp,$tmp2\n\t"
4643 "movd $tmp2,$src1\n\t"
4644 "vpaddd $tmp2,$tmp,$tmp2\n\t"
4645 "movd $dst,$tmp2\t! add reduction4I" %}
4646 ins_encode %{
4647 int vector_len = 0;
4648 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
4649 __ vpaddd($tmp$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, vector_len);
4650 __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
4651 __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
4652 __ movdl($tmp2$$XMMRegister, $src1$$Register);
4653 __ vpaddd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
4654 __ movdl($dst$$Register, $tmp2$$XMMRegister);
4655 %}
4656 ins_pipe( pipe_slow );
4657 %}
4658
4659 instruct rvadd8I_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2) %{
4660 predicate(UseAVX > 0 && UseAVX < 3);
4661 match(Set dst (AddReductionVI src1 src2));
4662 effect(TEMP tmp, TEMP tmp2);
4663 format %{ "vphaddd $tmp,$src2,$src2\n\t"
4664 "vphaddd $tmp,$tmp,$tmp2\n\t"
4665 "vextracti128 $tmp2,$tmp\n\t"
4666 "vpaddd $tmp,$tmp,$tmp2\n\t"
4667 "movd $tmp2,$src1\n\t"
4668 "vpaddd $tmp2,$tmp2,$tmp\n\t"
4669 "movd $dst,$tmp2\t! add reduction8I" %}
4670 ins_encode %{
4671 int vector_len = 1;
4672 __ vphaddd($tmp$$XMMRegister, $src2$$XMMRegister, $src2$$XMMRegister, vector_len);
4673 __ vphaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
4674 __ vextracti128h($tmp2$$XMMRegister, $tmp$$XMMRegister);
4675 __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
4676 __ movdl($tmp2$$XMMRegister, $src1$$Register);
4677 __ vpaddd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
4678 __ movdl($dst$$Register, $tmp2$$XMMRegister);
4679 %}
4680 ins_pipe( pipe_slow );
4681 %}
4682
4683 instruct rvadd8I_reduction_reg_evex(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2) %{
4684 predicate(UseAVX > 2);
4685 match(Set dst (AddReductionVI src1 src2));
4686 effect(TEMP tmp, TEMP tmp2);
4687 format %{ "vextracti128 $tmp,$src2\n\t"
4688 "vpaddd $tmp,$tmp,$src2\n\t"
4689 "pshufd $tmp2,$tmp,0xE\n\t"
4690 "vpaddd $tmp,$tmp,$tmp2\n\t"
4691 "pshufd $tmp2,$tmp,0x1\n\t"
4692 "vpaddd $tmp,$tmp,$tmp2\n\t"
4693 "movd $tmp2,$src1\n\t"
4694 "vpaddd $tmp2,$tmp,$tmp2\n\t"
4695 "movd $dst,$tmp2\t! add reduction8I" %}
4696 ins_encode %{
4697 int vector_len = 0;
4698 __ vextracti128h($tmp$$XMMRegister, $src2$$XMMRegister);
4699 __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
4700 __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
4701 __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
4702 __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
4703 __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
4704 __ movdl($tmp2$$XMMRegister, $src1$$Register);
4705 __ vpaddd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
4706 __ movdl($dst$$Register, $tmp2$$XMMRegister);
4707 %}
4708 ins_pipe( pipe_slow );
4709 %}
4710
4711 instruct rvadd16I_reduction_reg_evex(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, regF tmp3) %{
4712 predicate(UseAVX > 2);
4713 match(Set dst (AddReductionVI src1 src2));
4714 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
4715 format %{ "vextracti64x4 $tmp3,$src2\n\t"
4716 "vpaddd $tmp3,$tmp3,$src2\n\t"
4717 "vextracti128 $tmp,$tmp3\n\t"
4718 "vpaddd $tmp,$tmp,$tmp3\n\t"
4719 "pshufd $tmp2,$tmp,0xE\n\t"
4720 "vpaddd $tmp,$tmp,$tmp2\n\t"
4721 "pshufd $tmp2,$tmp,0x1\n\t"
4722 "vpaddd $tmp,$tmp,$tmp2\n\t"
4723 "movd $tmp2,$src1\n\t"
4724 "vpaddd $tmp2,$tmp,$tmp2\n\t"
4725 "movd $dst,$tmp2\t! mul reduction16I" %}
4726 ins_encode %{
4727 __ vextracti64x4h($tmp3$$XMMRegister, $src2$$XMMRegister);
4728 __ vpaddd($tmp3$$XMMRegister, $tmp3$$XMMRegister, $src2$$XMMRegister, 1);
4729 __ vextracti128h($tmp$$XMMRegister, $tmp3$$XMMRegister);
4730 __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp3$$XMMRegister, 0);
4731 __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
4732 __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
4733 __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
4734 __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
4735 __ movdl($tmp2$$XMMRegister, $src1$$Register);
4736 __ vpaddd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
4737 __ movdl($dst$$Register, $tmp2$$XMMRegister);
4738 %}
4739 ins_pipe( pipe_slow );
4740 %}
4741
4742 #ifdef _LP64
4743 instruct rvadd2L_reduction_reg(rRegL dst, rRegL src1, vecX src2, regF tmp, regF tmp2) %{
4744 predicate(UseAVX > 2);
4745 match(Set dst (AddReductionVL src1 src2));
4746 effect(TEMP tmp, TEMP tmp2);
4747 format %{ "pshufd $tmp2,$src2,0xE\n\t"
4748 "vpaddq $tmp,$src2,$tmp2\n\t"
4749 "movdq $tmp2,$src1\n\t"
4750 "vpaddq $tmp2,$tmp,$tmp2\n\t"
4751 "movdq $dst,$tmp2\t! add reduction2L" %}
4752 ins_encode %{
4753 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
4754 __ vpaddq($tmp$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, 0);
4755 __ movdq($tmp2$$XMMRegister, $src1$$Register);
4756 __ vpaddq($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
4757 __ movdq($dst$$Register, $tmp2$$XMMRegister);
4758 %}
4759 ins_pipe( pipe_slow );
4760 %}
4761
4762 instruct rvadd4L_reduction_reg(rRegL dst, rRegL src1, vecY src2, regF tmp, regF tmp2) %{
4763 predicate(UseAVX > 2);
4764 match(Set dst (AddReductionVL src1 src2));
4765 effect(TEMP tmp, TEMP tmp2);
4766 format %{ "vextracti64x2 $tmp,$src2, 0x1\n\t"
4767 "vpaddq $tmp2,$tmp,$src2\n\t"
4768 "pshufd $tmp,$tmp2,0xE\n\t"
4769 "vpaddq $tmp2,$tmp2,$tmp\n\t"
4770 "movdq $tmp,$src1\n\t"
4771 "vpaddq $tmp2,$tmp2,$tmp\n\t"
4772 "movdq $dst,$tmp2\t! add reduction4L" %}
4773 ins_encode %{
4774 __ vextracti64x2h($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
4775 __ vpaddq($tmp2$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, 0);
4776 __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
4777 __ vpaddq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
4778 __ movdq($tmp$$XMMRegister, $src1$$Register);
4779 __ vpaddq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
4780 __ movdq($dst$$Register, $tmp2$$XMMRegister);
4781 %}
4782 ins_pipe( pipe_slow );
4783 %}
4784
4785 instruct rvadd8L_reduction_reg(rRegL dst, rRegL src1, vecZ src2, regF tmp, regF tmp2) %{
4786 predicate(UseAVX > 2);
4787 match(Set dst (AddReductionVL src1 src2));
4788 effect(TEMP tmp, TEMP tmp2);
4789 format %{ "vextracti64x4 $tmp2,$src2\n\t"
4790 "vpaddq $tmp2,$tmp2,$src2\n\t"
4791 "vextracti128 $tmp,$tmp2\n\t"
4792 "vpaddq $tmp2,$tmp2,$tmp\n\t"
4793 "pshufd $tmp,$tmp2,0xE\n\t"
4794 "vpaddq $tmp2,$tmp2,$tmp\n\t"
4795 "movdq $tmp,$src1\n\t"
4796 "vpaddq $tmp2,$tmp2,$tmp\n\t"
4797 "movdq $dst,$tmp2\t! add reduction8L" %}
4798 ins_encode %{
4799 __ vextracti64x4h($tmp2$$XMMRegister, $src2$$XMMRegister);
4800 __ vpaddq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
4801 __ vextracti128h($tmp$$XMMRegister, $tmp2$$XMMRegister);
4802 __ vpaddq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
4803 __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
4804 __ vpaddq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
4805 __ movdq($tmp$$XMMRegister, $src1$$Register);
4806 __ vpaddq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
4807 __ movdq($dst$$Register, $tmp2$$XMMRegister);
4808 %}
4809 ins_pipe( pipe_slow );
4810 %}
4811 #endif
4812
4813 instruct rsadd2F_reduction_reg(regF dst, regF src1, vecD src2, regF tmp, regF tmp2) %{
4814 predicate(UseSSE >= 1 && UseAVX == 0);
4815 match(Set dst (AddReductionVF src1 src2));
4816 effect(TEMP tmp, TEMP tmp2);
4817 format %{ "movdqu $tmp,$src1\n\t"
4818 "addss $tmp,$src2\n\t"
4819 "pshufd $tmp2,$src2,0x01\n\t"
4820 "addss $tmp,$tmp2\n\t"
4821 "movdqu $dst,$tmp\t! add reduction2F" %}
4822 ins_encode %{
4823 __ movdqu($tmp$$XMMRegister, $src1$$XMMRegister);
4824 __ addss($tmp$$XMMRegister, $src2$$XMMRegister);
4825 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x01);
4826 __ addss($tmp$$XMMRegister, $tmp2$$XMMRegister);
4827 __ movdqu($dst$$XMMRegister, $tmp$$XMMRegister);
4828 %}
4829 ins_pipe( pipe_slow );
4830 %}
4831
4832 instruct rvadd2F_reduction_reg(regF dst, regF src1, vecD src2, regF tmp, regF tmp2) %{
4833 predicate(UseAVX > 0);
4834 match(Set dst (AddReductionVF src1 src2));
4835 effect(TEMP tmp2, TEMP tmp);
4836 format %{ "vaddss $tmp2,$src1,$src2\n\t"
4837 "pshufd $tmp,$src2,0x01\n\t"
4838 "vaddss $dst,$tmp2,$tmp\t! add reduction2F" %}
4839 ins_encode %{
4840 __ vaddss($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
4841 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
4842 __ vaddss($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4843 %}
4844 ins_pipe( pipe_slow );
4845 %}
4846
4847 instruct rsadd4F_reduction_reg(regF dst, regF src1, vecX src2, regF tmp, regF tmp2) %{
4848 predicate(UseSSE >= 1 && UseAVX == 0);
4849 match(Set dst (AddReductionVF src1 src2));
4850 effect(TEMP tmp, TEMP tmp2);
4851 format %{ "movdqu $tmp,$src1\n\t"
4852 "addss $tmp,$src2\n\t"
4853 "pshufd $tmp2,$src2,0x01\n\t"
4854 "addss $tmp,$tmp2\n\t"
4855 "pshufd $tmp2,$src2,0x02\n\t"
4856 "addss $tmp,$tmp2\n\t"
4857 "pshufd $tmp2,$src2,0x03\n\t"
4858 "addss $tmp,$tmp2\n\t"
4859 "movdqu $dst,$tmp\t! add reduction4F" %}
4860 ins_encode %{
4861 __ movdqu($tmp$$XMMRegister, $src1$$XMMRegister);
4862 __ addss($tmp$$XMMRegister, $src2$$XMMRegister);
4863 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x01);
4864 __ addss($tmp$$XMMRegister, $tmp2$$XMMRegister);
4865 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x02);
4866 __ addss($tmp$$XMMRegister, $tmp2$$XMMRegister);
4867 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x03);
4868 __ addss($tmp$$XMMRegister, $tmp2$$XMMRegister);
4869 __ movdqu($dst$$XMMRegister, $tmp$$XMMRegister);
4870 %}
4871 ins_pipe( pipe_slow );
4872 %}
4873
4874 instruct rvadd4F_reduction_reg(regF dst, regF src1, vecX src2, regF tmp, regF tmp2) %{
4875 predicate(UseAVX > 0);
4876 match(Set dst (AddReductionVF src1 src2));
4877 effect(TEMP tmp, TEMP tmp2);
4878 format %{ "vaddss $tmp2,$src1,$src2\n\t"
4879 "pshufd $tmp,$src2,0x01\n\t"
4880 "vaddss $tmp2,$tmp2,$tmp\n\t"
4881 "pshufd $tmp,$src2,0x02\n\t"
4882 "vaddss $tmp2,$tmp2,$tmp\n\t"
4883 "pshufd $tmp,$src2,0x03\n\t"
4884 "vaddss $dst,$tmp2,$tmp\t! add reduction4F" %}
4885 ins_encode %{
4886 __ vaddss($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
4887 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
4888 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4889 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
4890 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4891 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
4892 __ vaddss($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4893 %}
4894 ins_pipe( pipe_slow );
4895 %}
4896
4897 instruct radd8F_reduction_reg(regF dst, regF src1, vecY src2, regF tmp, regF tmp2, regF tmp3) %{
4898 predicate(UseAVX > 0);
4899 match(Set dst (AddReductionVF src1 src2));
4900 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
4901 format %{ "vaddss $tmp2,$src1,$src2\n\t"
4902 "pshufd $tmp,$src2,0x01\n\t"
4903 "vaddss $tmp2,$tmp2,$tmp\n\t"
4904 "pshufd $tmp,$src2,0x02\n\t"
4905 "vaddss $tmp2,$tmp2,$tmp\n\t"
4906 "pshufd $tmp,$src2,0x03\n\t"
4907 "vaddss $tmp2,$tmp2,$tmp\n\t"
4908 "vextractf128 $tmp3,$src2\n\t"
4909 "vaddss $tmp2,$tmp2,$tmp3\n\t"
4910 "pshufd $tmp,$tmp3,0x01\n\t"
4911 "vaddss $tmp2,$tmp2,$tmp\n\t"
4912 "pshufd $tmp,$tmp3,0x02\n\t"
4913 "vaddss $tmp2,$tmp2,$tmp\n\t"
4914 "pshufd $tmp,$tmp3,0x03\n\t"
4915 "vaddss $dst,$tmp2,$tmp\t! add reduction8F" %}
4916 ins_encode %{
4917 __ vaddss($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
4918 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
4919 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4920 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
4921 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4922 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
4923 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4924 __ vextractf128h($tmp3$$XMMRegister, $src2$$XMMRegister);
4925 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
4926 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x01);
4927 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4928 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x02);
4929 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4930 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x03);
4931 __ vaddss($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4932 %}
4933 ins_pipe( pipe_slow );
4934 %}
4935
4936 instruct radd16F_reduction_reg(regF dst, regF src1, vecZ src2, regF tmp, regF tmp2, regF tmp3) %{
4937 predicate(UseAVX > 2);
4938 match(Set dst (AddReductionVF src1 src2));
4939 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
4940 format %{ "vaddss $tmp2,$src1,$src2\n\t"
4941 "pshufd $tmp,$src2,0x01\n\t"
4942 "vaddss $tmp2,$tmp2,$tmp\n\t"
4943 "pshufd $tmp,$src2,0x02\n\t"
4944 "vaddss $tmp2,$tmp2,$tmp\n\t"
4945 "pshufd $tmp,$src2,0x03\n\t"
4946 "vaddss $tmp2,$tmp2,$tmp\n\t"
4947 "vextractf64x2 $tmp3,$src2, 0x1\n\t"
4948 "vaddss $tmp2,$tmp2,$tmp3\n\t"
4949 "pshufd $tmp,$tmp3,0x01\n\t"
4950 "vaddss $tmp2,$tmp2,$tmp\n\t"
4951 "pshufd $tmp,$tmp3,0x02\n\t"
4952 "vaddss $tmp2,$tmp2,$tmp\n\t"
4953 "pshufd $tmp,$tmp3,0x03\n\t"
4954 "vaddss $tmp2,$tmp2,$tmp\n\t"
4955 "vextractf64x2 $tmp3,$src2, 0x2\n\t"
4956 "vaddss $tmp2,$tmp2,$tmp3\n\t"
4957 "pshufd $tmp,$tmp3,0x01\n\t"
4958 "vaddss $tmp2,$tmp2,$tmp\n\t"
4959 "pshufd $tmp,$tmp3,0x02\n\t"
4960 "vaddss $tmp2,$tmp2,$tmp\n\t"
4961 "pshufd $tmp,$tmp3,0x03\n\t"
4962 "vaddss $tmp2,$tmp2,$tmp\n\t"
4963 "vextractf64x2 $tmp3,$src2, 0x3\n\t"
4964 "vaddss $tmp2,$tmp2,$tmp3\n\t"
4965 "pshufd $tmp,$tmp3,0x01\n\t"
4966 "vaddss $tmp2,$tmp2,$tmp\n\t"
4967 "pshufd $tmp,$tmp3,0x02\n\t"
4968 "vaddss $tmp2,$tmp2,$tmp\n\t"
4969 "pshufd $tmp,$tmp3,0x03\n\t"
4970 "vaddss $dst,$tmp2,$tmp\t! add reduction16F" %}
4971 ins_encode %{
4972 __ vaddss($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
4973 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
4974 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4975 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
4976 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4977 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
4978 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4979 __ vextractf32x4h($tmp3$$XMMRegister, $src2$$XMMRegister, 0x1);
4980 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
4981 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x01);
4982 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4983 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x02);
4984 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4985 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x03);
4986 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4987 __ vextractf32x4h($tmp3$$XMMRegister, $src2$$XMMRegister, 0x2);
4988 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
4989 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x01);
4990 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4991 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x02);
4992 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4993 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x03);
4994 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4995 __ vextractf32x4h($tmp3$$XMMRegister, $src2$$XMMRegister, 0x3);
4996 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
4997 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x01);
4998 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
4999 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x02);
5000 __ vaddss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5001 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x03);
5002 __ vaddss($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5003 %}
5004 ins_pipe( pipe_slow );
5005 %}
5006
5007 instruct rsadd2D_reduction_reg(regD dst, regD src1, vecX src2, regD tmp) %{
5008 predicate(UseSSE >= 1 && UseAVX == 0);
5009 match(Set dst (AddReductionVD src1 src2));
5010 effect(TEMP tmp, TEMP dst);
5011 format %{ "movdqu $tmp,$src1\n\t"
5012 "addsd $tmp,$src2\n\t"
5013 "pshufd $dst,$src2,0xE\n\t"
5014 "addsd $dst,$tmp\t! add reduction2D" %}
5015 ins_encode %{
5016 __ movdqu($tmp$$XMMRegister, $src1$$XMMRegister);
5017 __ addsd($tmp$$XMMRegister, $src2$$XMMRegister);
5018 __ pshufd($dst$$XMMRegister, $src2$$XMMRegister, 0xE);
5019 __ addsd($dst$$XMMRegister, $tmp$$XMMRegister);
5020 %}
5021 ins_pipe( pipe_slow );
5022 %}
5023
5024 instruct rvadd2D_reduction_reg(regD dst, regD src1, vecX src2, regD tmp, regD tmp2) %{
5025 predicate(UseAVX > 0);
5026 match(Set dst (AddReductionVD src1 src2));
5027 effect(TEMP tmp, TEMP tmp2);
5028 format %{ "vaddsd $tmp2,$src1,$src2\n\t"
5029 "pshufd $tmp,$src2,0xE\n\t"
5030 "vaddsd $dst,$tmp2,$tmp\t! add reduction2D" %}
5031 ins_encode %{
5032 __ vaddsd($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
5033 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
5034 __ vaddsd($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5035 %}
5036 ins_pipe( pipe_slow );
5037 %}
5038
5039 instruct rvadd4D_reduction_reg(regD dst, regD src1, vecY src2, regD tmp, regD tmp2, regD tmp3) %{
5040 predicate(UseAVX > 0);
5041 match(Set dst (AddReductionVD src1 src2));
5042 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
5043 format %{ "vaddsd $tmp2,$src1,$src2\n\t"
5044 "pshufd $tmp,$src2,0xE\n\t"
5045 "vaddsd $tmp2,$tmp2,$tmp\n\t"
5046 "vextractf128 $tmp3,$src2\n\t"
5047 "vaddsd $tmp2,$tmp2,$tmp3\n\t"
5048 "pshufd $tmp,$tmp3,0xE\n\t"
5049 "vaddsd $dst,$tmp2,$tmp\t! add reduction4D" %}
5050 ins_encode %{
5051 __ vaddsd($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
5052 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
5053 __ vaddsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5054 __ vextractf128h($tmp3$$XMMRegister, $src2$$XMMRegister);
5055 __ vaddsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
5056 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0xE);
5057 __ vaddsd($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5058 %}
5059 ins_pipe( pipe_slow );
5060 %}
5061
5062 instruct rvadd8D_reduction_reg(regD dst, regD src1, vecZ src2, regD tmp, regD tmp2, regD tmp3) %{
5063 predicate(UseAVX > 2);
5064 match(Set dst (AddReductionVD src1 src2));
5065 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
5066 format %{ "vaddsd $tmp2,$src1,$src2\n\t"
5067 "pshufd $tmp,$src2,0xE\n\t"
5068 "vaddsd $tmp2,$tmp2,$tmp\n\t"
5069 "vextractf64x2 $tmp3,$src2, 0x1\n\t"
5070 "vaddsd $tmp2,$tmp2,$tmp3\n\t"
5071 "pshufd $tmp,$tmp3,0xE\n\t"
5072 "vaddsd $tmp2,$tmp2,$tmp\n\t"
5073 "vextractf64x2 $tmp3,$src2, 0x2\n\t"
5074 "vaddsd $tmp2,$tmp2,$tmp3\n\t"
5075 "pshufd $tmp,$tmp3,0xE\n\t"
5076 "vaddsd $tmp2,$tmp2,$tmp\n\t"
5077 "vextractf64x2 $tmp3,$src2, 0x3\n\t"
5078 "vaddsd $tmp2,$tmp2,$tmp3\n\t"
5079 "pshufd $tmp,$tmp3,0xE\n\t"
5080 "vaddsd $dst,$tmp2,$tmp\t! add reduction8D" %}
5081 ins_encode %{
5082 __ vaddsd($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
5083 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
5084 __ vaddsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5085 __ vextractf64x2h($tmp3$$XMMRegister, $src2$$XMMRegister, 0x1);
5086 __ vaddsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
5087 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0xE);
5088 __ vaddsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5089 __ vextractf64x2h($tmp3$$XMMRegister, $src2$$XMMRegister, 0x2);
5090 __ vaddsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
5091 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0xE);
5092 __ vaddsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5093 __ vextractf64x2h($tmp3$$XMMRegister, $src2$$XMMRegister, 0x3);
5094 __ vaddsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
5095 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0xE);
5096 __ vaddsd($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5097 %}
5098 ins_pipe( pipe_slow );
5099 %}
5100
5101 instruct rsmul2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
5102 predicate(UseSSE > 3 && UseAVX == 0);
5103 match(Set dst (MulReductionVI src1 src2));
5104 effect(TEMP tmp, TEMP tmp2);
5105 format %{ "pshufd $tmp2,$src2,0x1\n\t"
5106 "pmulld $tmp2,$src2\n\t"
5107 "movd $tmp,$src1\n\t"
5108 "pmulld $tmp2,$tmp\n\t"
5109 "movd $dst,$tmp2\t! mul reduction2I" %}
5110 ins_encode %{
5111 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
5112 __ pmulld($tmp2$$XMMRegister, $src2$$XMMRegister);
5113 __ movdl($tmp$$XMMRegister, $src1$$Register);
5114 __ pmulld($tmp2$$XMMRegister, $tmp$$XMMRegister);
5115 __ movdl($dst$$Register, $tmp2$$XMMRegister);
5116 %}
5117 ins_pipe( pipe_slow );
5118 %}
5119
5120 instruct rvmul2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
5121 predicate(UseAVX > 0);
5122 match(Set dst (MulReductionVI src1 src2));
5123 effect(TEMP tmp, TEMP tmp2);
5124 format %{ "pshufd $tmp2,$src2,0x1\n\t"
5125 "vpmulld $tmp,$src2,$tmp2\n\t"
5126 "movd $tmp2,$src1\n\t"
5127 "vpmulld $tmp2,$tmp,$tmp2\n\t"
5128 "movd $dst,$tmp2\t! mul reduction2I" %}
5129 ins_encode %{
5130 int vector_len = 0;
5131 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
5132 __ vpmulld($tmp$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, vector_len);
5133 __ movdl($tmp2$$XMMRegister, $src1$$Register);
5134 __ vpmulld($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
5135 __ movdl($dst$$Register, $tmp2$$XMMRegister);
5136 %}
5137 ins_pipe( pipe_slow );
5138 %}
5139
5140 instruct rsmul4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
5141 predicate(UseSSE > 3 && UseAVX == 0);
5142 match(Set dst (MulReductionVI src1 src2));
5143 effect(TEMP tmp, TEMP tmp2);
5144 format %{ "pshufd $tmp2,$src2,0xE\n\t"
5145 "pmulld $tmp2,$src2\n\t"
5146 "pshufd $tmp,$tmp2,0x1\n\t"
5147 "pmulld $tmp2,$tmp\n\t"
5148 "movd $tmp,$src1\n\t"
5149 "pmulld $tmp2,$tmp\n\t"
5150 "movd $dst,$tmp2\t! mul reduction4I" %}
5151 ins_encode %{
5152 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
5153 __ pmulld($tmp2$$XMMRegister, $src2$$XMMRegister);
5154 __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x1);
5155 __ pmulld($tmp2$$XMMRegister, $tmp$$XMMRegister);
5156 __ movdl($tmp$$XMMRegister, $src1$$Register);
5157 __ pmulld($tmp2$$XMMRegister, $tmp$$XMMRegister);
5158 __ movdl($dst$$Register, $tmp2$$XMMRegister);
5159 %}
5160 ins_pipe( pipe_slow );
5161 %}
5162
5163 instruct rvmul4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
5164 predicate(UseAVX > 0);
5165 match(Set dst (MulReductionVI src1 src2));
5166 effect(TEMP tmp, TEMP tmp2);
5167 format %{ "pshufd $tmp2,$src2,0xE\n\t"
5168 "vpmulld $tmp,$src2,$tmp2\n\t"
5169 "pshufd $tmp2,$tmp,0x1\n\t"
5170 "vpmulld $tmp,$tmp,$tmp2\n\t"
5171 "movd $tmp2,$src1\n\t"
5172 "vpmulld $tmp2,$tmp,$tmp2\n\t"
5173 "movd $dst,$tmp2\t! mul reduction4I" %}
5174 ins_encode %{
5175 int vector_len = 0;
5176 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
5177 __ vpmulld($tmp$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, vector_len);
5178 __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
5179 __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
5180 __ movdl($tmp2$$XMMRegister, $src1$$Register);
5181 __ vpmulld($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
5182 __ movdl($dst$$Register, $tmp2$$XMMRegister);
5183 %}
5184 ins_pipe( pipe_slow );
5185 %}
5186
5187 instruct rvmul8I_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2) %{
5188 predicate(UseAVX > 0);
5189 match(Set dst (MulReductionVI src1 src2));
5190 effect(TEMP tmp, TEMP tmp2);
5191 format %{ "vextracti128 $tmp,$src2\n\t"
5192 "vpmulld $tmp,$tmp,$src2\n\t"
5193 "pshufd $tmp2,$tmp,0xE\n\t"
5194 "vpmulld $tmp,$tmp,$tmp2\n\t"
5195 "pshufd $tmp2,$tmp,0x1\n\t"
5196 "vpmulld $tmp,$tmp,$tmp2\n\t"
5197 "movd $tmp2,$src1\n\t"
5198 "vpmulld $tmp2,$tmp,$tmp2\n\t"
5199 "movd $dst,$tmp2\t! mul reduction8I" %}
5200 ins_encode %{
5201 int vector_len = 0;
5202 __ vextracti128h($tmp$$XMMRegister, $src2$$XMMRegister);
5203 __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
5204 __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
5205 __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
5206 __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
5207 __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
5208 __ movdl($tmp2$$XMMRegister, $src1$$Register);
5209 __ vpmulld($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
5210 __ movdl($dst$$Register, $tmp2$$XMMRegister);
5211 %}
5212 ins_pipe( pipe_slow );
5213 %}
5214
5215 instruct rvmul16I_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, regF tmp3) %{
5216 predicate(UseAVX > 2);
5217 match(Set dst (MulReductionVI src1 src2));
5218 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
5219 format %{ "vextracti64x4 $tmp3,$src2\n\t"
5220 "vpmulld $tmp3,$tmp3,$src2\n\t"
5221 "vextracti128 $tmp,$tmp3\n\t"
5222 "vpmulld $tmp,$tmp,$src2\n\t"
5223 "pshufd $tmp2,$tmp,0xE\n\t"
5224 "vpmulld $tmp,$tmp,$tmp2\n\t"
5225 "pshufd $tmp2,$tmp,0x1\n\t"
5226 "vpmulld $tmp,$tmp,$tmp2\n\t"
5227 "movd $tmp2,$src1\n\t"
5228 "vpmulld $tmp2,$tmp,$tmp2\n\t"
5229 "movd $dst,$tmp2\t! mul reduction16I" %}
5230 ins_encode %{
5231 __ vextracti64x4h($tmp3$$XMMRegister, $src2$$XMMRegister);
5232 __ vpmulld($tmp3$$XMMRegister, $tmp3$$XMMRegister, $src2$$XMMRegister, 1);
5233 __ vextracti128h($tmp$$XMMRegister, $tmp3$$XMMRegister);
5234 __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp3$$XMMRegister, 0);
5235 __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
5236 __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
5237 __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
5238 __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
5239 __ movdl($tmp2$$XMMRegister, $src1$$Register);
5240 __ vpmulld($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
5241 __ movdl($dst$$Register, $tmp2$$XMMRegister);
5242 %}
5243 ins_pipe( pipe_slow );
5244 %}
5245
5246 #ifdef _LP64
5247 instruct rvmul2L_reduction_reg(rRegL dst, rRegL src1, vecX src2, regF tmp, regF tmp2) %{
5248 predicate(UseAVX > 2 && VM_Version::supports_avx512dq());
5249 match(Set dst (MulReductionVL src1 src2));
5250 effect(TEMP tmp, TEMP tmp2);
5251 format %{ "pshufd $tmp2,$src2,0xE\n\t"
5252 "vpmullq $tmp,$src2,$tmp2\n\t"
5253 "movdq $tmp2,$src1\n\t"
5254 "vpmullq $tmp2,$tmp,$tmp2\n\t"
5255 "movdq $dst,$tmp2\t! mul reduction2L" %}
5256 ins_encode %{
5257 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
5258 __ vpmullq($tmp$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, 0);
5259 __ movdq($tmp2$$XMMRegister, $src1$$Register);
5260 __ vpmullq($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
5261 __ movdq($dst$$Register, $tmp2$$XMMRegister);
5262 %}
5263 ins_pipe( pipe_slow );
5264 %}
5265
5266 instruct rvmul4L_reduction_reg(rRegL dst, rRegL src1, vecY src2, regF tmp, regF tmp2) %{
5267 predicate(UseAVX > 2 && VM_Version::supports_avx512dq());
5268 match(Set dst (MulReductionVL src1 src2));
5269 effect(TEMP tmp, TEMP tmp2);
5270 format %{ "vextracti64x2 $tmp,$src2, 0x1\n\t"
5271 "vpmullq $tmp2,$tmp,$src2\n\t"
5272 "pshufd $tmp,$tmp2,0xE\n\t"
5273 "vpmullq $tmp2,$tmp2,$tmp\n\t"
5274 "movdq $tmp,$src1\n\t"
5275 "vpmullq $tmp2,$tmp2,$tmp\n\t"
5276 "movdq $dst,$tmp2\t! mul reduction4L" %}
5277 ins_encode %{
5278 __ vextracti64x2h($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
5279 __ vpmullq($tmp2$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, 0);
5280 __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
5281 __ vpmullq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
5282 __ movdq($tmp$$XMMRegister, $src1$$Register);
5283 __ vpmullq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
5284 __ movdq($dst$$Register, $tmp2$$XMMRegister);
5285 %}
5286 ins_pipe( pipe_slow );
5287 %}
5288
5289 instruct rvmul8L_reduction_reg(rRegL dst, rRegL src1, vecZ src2, regF tmp, regF tmp2) %{
5290 predicate(UseAVX > 2 && VM_Version::supports_avx512dq());
5291 match(Set dst (MulReductionVL src1 src2));
5292 effect(TEMP tmp, TEMP tmp2);
5293 format %{ "vextracti64x4 $tmp2,$src2\n\t"
5294 "vpmullq $tmp2,$tmp2,$src2\n\t"
5295 "vextracti128 $tmp,$tmp2\n\t"
5296 "vpmullq $tmp2,$tmp2,$tmp\n\t"
5297 "pshufd $tmp,$tmp2,0xE\n\t"
5298 "vpmullq $tmp2,$tmp2,$tmp\n\t"
5299 "movdq $tmp,$src1\n\t"
5300 "vpmullq $tmp2,$tmp2,$tmp\n\t"
5301 "movdq $dst,$tmp2\t! mul reduction8L" %}
5302 ins_encode %{
5303 __ vextracti64x4h($tmp2$$XMMRegister, $src2$$XMMRegister);
5304 __ vpmullq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
5305 __ vextracti128h($tmp$$XMMRegister, $tmp2$$XMMRegister);
5306 __ vpmullq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
5307 __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
5308 __ vpmullq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
5309 __ movdq($tmp$$XMMRegister, $src1$$Register);
5310 __ vpmullq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
5311 __ movdq($dst$$Register, $tmp2$$XMMRegister);
5312 %}
5313 ins_pipe( pipe_slow );
5314 %}
5315 #endif
5316
5317 instruct rsmul2F_reduction(regF dst, regF src1, vecD src2, regF tmp, regF tmp2) %{
5318 predicate(UseSSE >= 1 && UseAVX == 0);
5319 match(Set dst (MulReductionVF src1 src2));
5320 effect(TEMP tmp, TEMP tmp2);
5321 format %{ "movdqu $tmp,$src1\n\t"
5322 "mulss $tmp,$src2\n\t"
5323 "pshufd $tmp2,$src2,0x01\n\t"
5324 "mulss $tmp,$tmp2\n\t"
5325 "movdqu $dst,$tmp\t! mul reduction2F" %}
5326 ins_encode %{
5327 __ movdqu($tmp$$XMMRegister, $src1$$XMMRegister);
5328 __ mulss($tmp$$XMMRegister, $src2$$XMMRegister);
5329 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x01);
5330 __ mulss($tmp$$XMMRegister, $tmp2$$XMMRegister);
5331 __ movdqu($dst$$XMMRegister, $tmp$$XMMRegister);
5332 %}
5333 ins_pipe( pipe_slow );
5334 %}
5335
5336 instruct rvmul2F_reduction_reg(regF dst, regF src1, vecD src2, regF tmp, regF tmp2) %{
5337 predicate(UseAVX > 0);
5338 match(Set dst (MulReductionVF src1 src2));
5339 effect(TEMP tmp, TEMP tmp2);
5340 format %{ "vmulss $tmp2,$src1,$src2\n\t"
5341 "pshufd $tmp,$src2,0x01\n\t"
5342 "vmulss $dst,$tmp2,$tmp\t! mul reduction2F" %}
5343 ins_encode %{
5344 __ vmulss($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
5345 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
5346 __ vmulss($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5347 %}
5348 ins_pipe( pipe_slow );
5349 %}
5350
5351 instruct rsmul4F_reduction_reg(regF dst, regF src1, vecX src2, regF tmp, regF tmp2) %{
5352 predicate(UseSSE >= 1 && UseAVX == 0);
5353 match(Set dst (MulReductionVF src1 src2));
5354 effect(TEMP tmp, TEMP tmp2);
5355 format %{ "movdqu $tmp,$src1\n\t"
5356 "mulss $tmp,$src2\n\t"
5357 "pshufd $tmp2,$src2,0x01\n\t"
5358 "mulss $tmp,$tmp2\n\t"
5359 "pshufd $tmp2,$src2,0x02\n\t"
5360 "mulss $tmp,$tmp2\n\t"
5361 "pshufd $tmp2,$src2,0x03\n\t"
5362 "mulss $tmp,$tmp2\n\t"
5363 "movdqu $dst,$tmp\t! mul reduction4F" %}
5364 ins_encode %{
5365 __ movdqu($tmp$$XMMRegister, $src1$$XMMRegister);
5366 __ mulss($tmp$$XMMRegister, $src2$$XMMRegister);
5367 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x01);
5368 __ mulss($tmp$$XMMRegister, $tmp2$$XMMRegister);
5369 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x02);
5370 __ mulss($tmp$$XMMRegister, $tmp2$$XMMRegister);
5371 __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x03);
5372 __ mulss($tmp$$XMMRegister, $tmp2$$XMMRegister);
5373 __ movdqu($dst$$XMMRegister, $tmp$$XMMRegister);
5374 %}
5375 ins_pipe( pipe_slow );
5376 %}
5377
5378 instruct rvmul4F_reduction_reg(regF dst, regF src1, vecX src2, regF tmp, regF tmp2) %{
5379 predicate(UseAVX > 0);
5380 match(Set dst (MulReductionVF src1 src2));
5381 effect(TEMP tmp, TEMP tmp2);
5382 format %{ "vmulss $tmp2,$src1,$src2\n\t"
5383 "pshufd $tmp,$src2,0x01\n\t"
5384 "vmulss $tmp2,$tmp2,$tmp\n\t"
5385 "pshufd $tmp,$src2,0x02\n\t"
5386 "vmulss $tmp2,$tmp2,$tmp\n\t"
5387 "pshufd $tmp,$src2,0x03\n\t"
5388 "vmulss $dst,$tmp2,$tmp\t! mul reduction4F" %}
5389 ins_encode %{
5390 __ vmulss($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
5391 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
5392 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5393 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
5394 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5395 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
5396 __ vmulss($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5397 %}
5398 ins_pipe( pipe_slow );
5399 %}
5400
5401 instruct rvmul8F_reduction_reg(regF dst, regF src1, vecY src2, regF tmp, regF tmp2, regF tmp3) %{
5402 predicate(UseAVX > 0);
5403 match(Set dst (MulReductionVF src1 src2));
5404 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
5405 format %{ "vmulss $tmp2,$src1,$src2\n\t"
5406 "pshufd $tmp,$src2,0x01\n\t"
5407 "vmulss $tmp2,$tmp2,$tmp\n\t"
5408 "pshufd $tmp,$src2,0x02\n\t"
5409 "vmulss $tmp2,$tmp2,$tmp\n\t"
5410 "pshufd $tmp,$src2,0x03\n\t"
5411 "vmulss $tmp2,$tmp2,$tmp\n\t"
5412 "vextractf128 $tmp3,$src2\n\t"
5413 "vmulss $tmp2,$tmp2,$tmp3\n\t"
5414 "pshufd $tmp,$tmp3,0x01\n\t"
5415 "vmulss $tmp2,$tmp2,$tmp\n\t"
5416 "pshufd $tmp,$tmp3,0x02\n\t"
5417 "vmulss $tmp2,$tmp2,$tmp\n\t"
5418 "pshufd $tmp,$tmp3,0x03\n\t"
5419 "vmulss $dst,$tmp2,$tmp\t! mul reduction8F" %}
5420 ins_encode %{
5421 __ vmulss($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
5422 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
5423 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5424 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
5425 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5426 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
5427 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5428 __ vextractf128h($tmp3$$XMMRegister, $src2$$XMMRegister);
5429 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
5430 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x01);
5431 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5432 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x02);
5433 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5434 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x03);
5435 __ vmulss($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5436 %}
5437 ins_pipe( pipe_slow );
5438 %}
5439
5440 instruct rvmul16F_reduction_reg(regF dst, regF src1, vecZ src2, regF tmp, regF tmp2, regF tmp3) %{
5441 predicate(UseAVX > 2);
5442 match(Set dst (MulReductionVF src1 src2));
5443 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
5444 format %{ "vmulss $tmp2,$src1,$src2\n\t"
5445 "pshufd $tmp,$src2,0x01\n\t"
5446 "vmulss $tmp2,$tmp2,$tmp\n\t"
5447 "pshufd $tmp,$src2,0x02\n\t"
5448 "vmulss $tmp2,$tmp2,$tmp\n\t"
5449 "pshufd $tmp,$src2,0x03\n\t"
5450 "vmulss $tmp2,$tmp2,$tmp\n\t"
5451 "vextractf32x4 $tmp3,$src2, 0x1\n\t"
5452 "vmulss $tmp2,$tmp2,$tmp3\n\t"
5453 "pshufd $tmp,$tmp3,0x01\n\t"
5454 "vmulss $tmp2,$tmp2,$tmp\n\t"
5455 "pshufd $tmp,$tmp3,0x02\n\t"
5456 "vmulss $tmp2,$tmp2,$tmp\n\t"
5457 "pshufd $tmp,$tmp3,0x03\n\t"
5458 "vmulss $tmp2,$tmp2,$tmp\n\t"
5459 "vextractf32x4 $tmp3,$src2, 0x2\n\t"
5460 "vmulss $tmp2,$tmp2,$tmp3\n\t"
5461 "pshufd $tmp,$tmp3,0x01\n\t"
5462 "vmulss $tmp2,$tmp2,$tmp\n\t"
5463 "pshufd $tmp,$tmp3,0x02\n\t"
5464 "vmulss $tmp2,$tmp2,$tmp\n\t"
5465 "pshufd $tmp,$tmp3,0x03\n\t"
5466 "vmulss $tmp2,$tmp2,$tmp\n\t"
5467 "vextractf32x4 $tmp3,$src2, 0x3\n\t"
5468 "vmulss $tmp2,$tmp2,$tmp3\n\t"
5469 "pshufd $tmp,$tmp3,0x01\n\t"
5470 "vmulss $tmp2,$tmp2,$tmp\n\t"
5471 "pshufd $tmp,$tmp3,0x02\n\t"
5472 "vmulss $tmp2,$tmp2,$tmp\n\t"
5473 "pshufd $tmp,$tmp3,0x03\n\t"
5474 "vmulss $dst,$tmp2,$tmp\t! mul reduction16F" %}
5475 ins_encode %{
5476 __ vmulss($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
5477 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
5478 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5479 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
5480 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5481 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
5482 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5483 __ vextractf32x4h($tmp3$$XMMRegister, $src2$$XMMRegister, 0x1);
5484 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
5485 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x01);
5486 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5487 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x02);
5488 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5489 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x03);
5490 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5491 __ vextractf32x4h($tmp3$$XMMRegister, $src2$$XMMRegister, 0x2);
5492 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
5493 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x01);
5494 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5495 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x02);
5496 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5497 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x03);
5498 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5499 __ vextractf32x4h($tmp3$$XMMRegister, $src2$$XMMRegister, 0x3);
5500 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
5501 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x01);
5502 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5503 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x02);
5504 __ vmulss($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5505 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0x03);
5506 __ vmulss($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5507 %}
5508 ins_pipe( pipe_slow );
5509 %}
5510
5511 instruct rsmul2D_reduction_reg(regD dst, regD src1, vecX src2, regD tmp) %{
5512 predicate(UseSSE >= 1 && UseAVX == 0);
5513 match(Set dst (MulReductionVD src1 src2));
5514 effect(TEMP tmp, TEMP dst);
5515 format %{ "movdqu $tmp,$src1\n\t"
5516 "mulsd $tmp,$src2\n\t"
5517 "pshufd $dst,$src2,0xE\n\t"
5518 "mulsd $dst,$tmp\t! mul reduction2D" %}
5519 ins_encode %{
5520 __ movdqu($tmp$$XMMRegister, $src1$$XMMRegister);
5521 __ mulsd($tmp$$XMMRegister, $src2$$XMMRegister);
5522 __ pshufd($dst$$XMMRegister, $src2$$XMMRegister, 0xE);
5523 __ mulsd($dst$$XMMRegister, $tmp$$XMMRegister);
5524 %}
5525 ins_pipe( pipe_slow );
5526 %}
5527
5528 instruct rvmul2D_reduction_reg(regD dst, regD src1, vecX src2, regD tmp, regD tmp2) %{
5529 predicate(UseAVX > 0);
5530 match(Set dst (MulReductionVD src1 src2));
5531 effect(TEMP tmp, TEMP tmp2);
5532 format %{ "vmulsd $tmp2,$src1,$src2\n\t"
5533 "pshufd $tmp,$src2,0xE\n\t"
5534 "vmulsd $dst,$tmp2,$tmp\t! mul reduction2D" %}
5535 ins_encode %{
5536 __ vmulsd($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
5537 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
5538 __ vmulsd($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5539 %}
5540 ins_pipe( pipe_slow );
5541 %}
5542
5543 instruct rvmul4D_reduction_reg(regD dst, regD src1, vecY src2, regD tmp, regD tmp2, regD tmp3) %{
5544 predicate(UseAVX > 0);
5545 match(Set dst (MulReductionVD src1 src2));
5546 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
5547 format %{ "vmulsd $tmp2,$src1,$src2\n\t"
5548 "pshufd $tmp,$src2,0xE\n\t"
5549 "vmulsd $tmp2,$tmp2,$tmp\n\t"
5550 "vextractf128 $tmp3,$src2\n\t"
5551 "vmulsd $tmp2,$tmp2,$tmp3\n\t"
5552 "pshufd $tmp,$tmp3,0xE\n\t"
5553 "vmulsd $dst,$tmp2,$tmp\t! mul reduction4D" %}
5554 ins_encode %{
5555 __ vmulsd($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
5556 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
5557 __ vmulsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5558 __ vextractf128h($tmp3$$XMMRegister, $src2$$XMMRegister);
5559 __ vmulsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
5560 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0xE);
5561 __ vmulsd($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5562 %}
5563 ins_pipe( pipe_slow );
5564 %}
5565
5566 instruct rvmul8D_reduction_reg(regD dst, regD src1, vecZ src2, regD tmp, regD tmp2, regD tmp3) %{
5567 predicate(UseAVX > 2);
5568 match(Set dst (MulReductionVD src1 src2));
5569 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
5570 format %{ "vmulsd $tmp2,$src1,$src2\n\t"
5571 "pshufd $tmp,$src2,0xE\n\t"
5572 "vmulsd $tmp2,$tmp2,$tmp\n\t"
5573 "vextractf64x2 $tmp3,$src2, 0x1\n\t"
5574 "vmulsd $tmp2,$tmp2,$tmp3\n\t"
5575 "pshufd $tmp,$src2,0xE\n\t"
5576 "vmulsd $tmp2,$tmp2,$tmp\n\t"
5577 "vextractf64x2 $tmp3,$src2, 0x2\n\t"
5578 "vmulsd $tmp2,$tmp2,$tmp3\n\t"
5579 "pshufd $tmp,$tmp3,0xE\n\t"
5580 "vmulsd $tmp2,$tmp2,$tmp\n\t"
5581 "vextractf64x2 $tmp3,$src2, 0x3\n\t"
5582 "vmulsd $tmp2,$tmp2,$tmp3\n\t"
5583 "pshufd $tmp,$tmp3,0xE\n\t"
5584 "vmulsd $dst,$tmp2,$tmp\t! mul reduction8D" %}
5585 ins_encode %{
5586 __ vmulsd($tmp2$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
5587 __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
5588 __ vmulsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5589 __ vextractf64x2h($tmp3$$XMMRegister, $src2$$XMMRegister, 0x1);
5590 __ vmulsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
5591 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0xE);
5592 __ vmulsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5593 __ vextractf64x2h($tmp3$$XMMRegister, $src2$$XMMRegister, 0x2);
5594 __ vmulsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
5595 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0xE);
5596 __ vmulsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5597 __ vextractf64x2h($tmp3$$XMMRegister, $src2$$XMMRegister, 0x3);
5598 __ vmulsd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister);
5599 __ pshufd($tmp$$XMMRegister, $tmp3$$XMMRegister, 0xE);
5600 __ vmulsd($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister);
5601 %}
5602 ins_pipe( pipe_slow );
5603 %}
5604
5605 // ====================VECTOR ARITHMETIC=======================================
5606
5607 // --------------------------------- ADD --------------------------------------
5608
5609 // Bytes vector add
5610 instruct vadd4B(vecS dst, vecS src) %{
5611 predicate(n->as_Vector()->length() == 4);
5612 match(Set dst (AddVB dst src));
5613 format %{ "paddb $dst,$src\t! add packed4B" %}
5614 ins_encode %{
5615 __ paddb($dst$$XMMRegister, $src$$XMMRegister);
5616 %}
5617 ins_pipe( pipe_slow );
5618 %}
5619
5620 instruct vadd4B_reg(vecS dst, vecS src1, vecS src2) %{
5621 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
5622 match(Set dst (AddVB src1 src2));
5623 format %{ "vpaddb $dst,$src1,$src2\t! add packed4B" %}
5624 ins_encode %{
5625 int vector_len = 0;
5626 __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5627 %}
5628 ins_pipe( pipe_slow );
5629 %}
5630
5631 instruct vadd4B_mem(vecS dst, vecS src, memory mem) %{
5632 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
5633 match(Set dst (AddVB src (LoadVector mem)));
5634 format %{ "vpaddb $dst,$src,$mem\t! add packed4B" %}
5635 ins_encode %{
5636 int vector_len = 0;
5637 __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5638 %}
5639 ins_pipe( pipe_slow );
5640 %}
5641
5642 instruct vadd8B(vecD dst, vecD src) %{
5643 predicate(n->as_Vector()->length() == 8);
5644 match(Set dst (AddVB dst src));
5645 format %{ "paddb $dst,$src\t! add packed8B" %}
5646 ins_encode %{
5647 __ paddb($dst$$XMMRegister, $src$$XMMRegister);
5648 %}
5649 ins_pipe( pipe_slow );
5650 %}
5651
5652 instruct vadd8B_reg(vecD dst, vecD src1, vecD src2) %{
5653 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
5654 match(Set dst (AddVB src1 src2));
5655 format %{ "vpaddb $dst,$src1,$src2\t! add packed8B" %}
5656 ins_encode %{
5657 int vector_len = 0;
5658 __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5659 %}
5660 ins_pipe( pipe_slow );
5661 %}
5662
5663 instruct vadd8B_mem(vecD dst, vecD src, memory mem) %{
5664 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
5665 match(Set dst (AddVB src (LoadVector mem)));
5666 format %{ "vpaddb $dst,$src,$mem\t! add packed8B" %}
5667 ins_encode %{
5668 int vector_len = 0;
5669 __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5670 %}
5671 ins_pipe( pipe_slow );
5672 %}
5673
5674 instruct vadd16B(vecX dst, vecX src) %{
5675 predicate(n->as_Vector()->length() == 16);
5676 match(Set dst (AddVB dst src));
5677 format %{ "paddb $dst,$src\t! add packed16B" %}
5678 ins_encode %{
5679 __ paddb($dst$$XMMRegister, $src$$XMMRegister);
5680 %}
5681 ins_pipe( pipe_slow );
5682 %}
5683
5684 instruct vadd16B_reg(vecX dst, vecX src1, vecX src2) %{
5685 predicate(UseAVX > 0 && n->as_Vector()->length() == 16);
5686 match(Set dst (AddVB src1 src2));
5687 format %{ "vpaddb $dst,$src1,$src2\t! add packed16B" %}
5688 ins_encode %{
5689 int vector_len = 0;
5690 __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5691 %}
5692 ins_pipe( pipe_slow );
5693 %}
5694
5695 instruct vadd16B_mem(vecX dst, vecX src, memory mem) %{
5696 predicate(UseAVX > 0 && n->as_Vector()->length() == 16);
5697 match(Set dst (AddVB src (LoadVector mem)));
5698 format %{ "vpaddb $dst,$src,$mem\t! add packed16B" %}
5699 ins_encode %{
5700 int vector_len = 0;
5701 __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5702 %}
5703 ins_pipe( pipe_slow );
5704 %}
5705
5706 instruct vadd32B_reg(vecY dst, vecY src1, vecY src2) %{
5707 predicate(UseAVX > 1 && n->as_Vector()->length() == 32);
5708 match(Set dst (AddVB src1 src2));
5709 format %{ "vpaddb $dst,$src1,$src2\t! add packed32B" %}
5710 ins_encode %{
5711 int vector_len = 1;
5712 __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5713 %}
5714 ins_pipe( pipe_slow );
5715 %}
5716
5717 instruct vadd32B_mem(vecY dst, vecY src, memory mem) %{
5718 predicate(UseAVX > 1 && n->as_Vector()->length() == 32);
5719 match(Set dst (AddVB src (LoadVector mem)));
5720 format %{ "vpaddb $dst,$src,$mem\t! add packed32B" %}
5721 ins_encode %{
5722 int vector_len = 1;
5723 __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5724 %}
5725 ins_pipe( pipe_slow );
5726 %}
5727
5728 instruct vadd64B_reg(vecZ dst, vecZ src1, vecZ src2) %{
5729 predicate(UseAVX > 2 && n->as_Vector()->length() == 64);
5730 match(Set dst (AddVB src1 src2));
5731 format %{ "vpaddb $dst,$src1,$src2\t! add packed64B" %}
5732 ins_encode %{
5733 int vector_len = 2;
5734 __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5735 %}
5736 ins_pipe( pipe_slow );
5737 %}
5738
5739 instruct vadd64B_mem(vecZ dst, vecZ src, memory mem) %{
5740 predicate(UseAVX > 2 && n->as_Vector()->length() == 64);
5741 match(Set dst (AddVB src (LoadVector mem)));
5742 format %{ "vpaddb $dst,$src,$mem\t! add packed64B" %}
5743 ins_encode %{
5744 int vector_len = 2;
5745 __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5746 %}
5747 ins_pipe( pipe_slow );
5748 %}
5749
5750 // Shorts/Chars vector add
5751 instruct vadd2S(vecS dst, vecS src) %{
5752 predicate(n->as_Vector()->length() == 2);
5753 match(Set dst (AddVS dst src));
5754 format %{ "paddw $dst,$src\t! add packed2S" %}
5755 ins_encode %{
5756 __ paddw($dst$$XMMRegister, $src$$XMMRegister);
5757 %}
5758 ins_pipe( pipe_slow );
5759 %}
5760
5761 instruct vadd2S_reg(vecS dst, vecS src1, vecS src2) %{
5762 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
5763 match(Set dst (AddVS src1 src2));
5764 format %{ "vpaddw $dst,$src1,$src2\t! add packed2S" %}
5765 ins_encode %{
5766 int vector_len = 0;
5767 __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5768 %}
5769 ins_pipe( pipe_slow );
5770 %}
5771
5772 instruct vadd2S_mem(vecS dst, vecS src, memory mem) %{
5773 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
5774 match(Set dst (AddVS src (LoadVector mem)));
5775 format %{ "vpaddw $dst,$src,$mem\t! add packed2S" %}
5776 ins_encode %{
5777 int vector_len = 0;
5778 __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5779 %}
5780 ins_pipe( pipe_slow );
5781 %}
5782
5783 instruct vadd4S(vecD dst, vecD src) %{
5784 predicate(n->as_Vector()->length() == 4);
5785 match(Set dst (AddVS dst src));
5786 format %{ "paddw $dst,$src\t! add packed4S" %}
5787 ins_encode %{
5788 __ paddw($dst$$XMMRegister, $src$$XMMRegister);
5789 %}
5790 ins_pipe( pipe_slow );
5791 %}
5792
5793 instruct vadd4S_reg(vecD dst, vecD src1, vecD src2) %{
5794 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
5795 match(Set dst (AddVS src1 src2));
5796 format %{ "vpaddw $dst,$src1,$src2\t! add packed4S" %}
5797 ins_encode %{
5798 int vector_len = 0;
5799 __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5800 %}
5801 ins_pipe( pipe_slow );
5802 %}
5803
5804 instruct vadd4S_mem(vecD dst, vecD src, memory mem) %{
5805 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
5806 match(Set dst (AddVS src (LoadVector mem)));
5807 format %{ "vpaddw $dst,$src,$mem\t! add packed4S" %}
5808 ins_encode %{
5809 int vector_len = 0;
5810 __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5811 %}
5812 ins_pipe( pipe_slow );
5813 %}
5814
5815 instruct vadd8S(vecX dst, vecX src) %{
5816 predicate(n->as_Vector()->length() == 8);
5817 match(Set dst (AddVS dst src));
5818 format %{ "paddw $dst,$src\t! add packed8S" %}
5819 ins_encode %{
5820 __ paddw($dst$$XMMRegister, $src$$XMMRegister);
5821 %}
5822 ins_pipe( pipe_slow );
5823 %}
5824
5825 instruct vadd8S_reg(vecX dst, vecX src1, vecX src2) %{
5826 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
5827 match(Set dst (AddVS src1 src2));
5828 format %{ "vpaddw $dst,$src1,$src2\t! add packed8S" %}
5829 ins_encode %{
5830 int vector_len = 0;
5831 __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5832 %}
5833 ins_pipe( pipe_slow );
5834 %}
5835
5836 instruct vadd8S_mem(vecX dst, vecX src, memory mem) %{
5837 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
5838 match(Set dst (AddVS src (LoadVector mem)));
5839 format %{ "vpaddw $dst,$src,$mem\t! add packed8S" %}
5840 ins_encode %{
5841 int vector_len = 0;
5842 __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5843 %}
5844 ins_pipe( pipe_slow );
5845 %}
5846
5847 instruct vadd16S_reg(vecY dst, vecY src1, vecY src2) %{
5848 predicate(UseAVX > 1 && n->as_Vector()->length() == 16);
5849 match(Set dst (AddVS src1 src2));
5850 format %{ "vpaddw $dst,$src1,$src2\t! add packed16S" %}
5851 ins_encode %{
5852 int vector_len = 1;
5853 __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5854 %}
5855 ins_pipe( pipe_slow );
5856 %}
5857
5858 instruct vadd16S_mem(vecY dst, vecY src, memory mem) %{
5859 predicate(UseAVX > 1 && n->as_Vector()->length() == 16);
5860 match(Set dst (AddVS src (LoadVector mem)));
5861 format %{ "vpaddw $dst,$src,$mem\t! add packed16S" %}
5862 ins_encode %{
5863 int vector_len = 1;
5864 __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5865 %}
5866 ins_pipe( pipe_slow );
5867 %}
5868
5869 instruct vadd32S_reg(vecZ dst, vecZ src1, vecZ src2) %{
5870 predicate(UseAVX > 2 && n->as_Vector()->length() == 32);
5871 match(Set dst (AddVS src1 src2));
5872 format %{ "vpaddw $dst,$src1,$src2\t! add packed32S" %}
5873 ins_encode %{
5874 int vector_len = 2;
5875 __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5876 %}
5877 ins_pipe( pipe_slow );
5878 %}
5879
5880 instruct vadd32S_mem(vecZ dst, vecZ src, memory mem) %{
5881 predicate(UseAVX > 2 && n->as_Vector()->length() == 32);
5882 match(Set dst (AddVS src (LoadVector mem)));
5883 format %{ "vpaddw $dst,$src,$mem\t! add packed32S" %}
5884 ins_encode %{
5885 int vector_len = 2;
5886 __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5887 %}
5888 ins_pipe( pipe_slow );
5889 %}
5890
5891 // Integers vector add
5892 instruct vadd2I(vecD dst, vecD src) %{
5893 predicate(n->as_Vector()->length() == 2);
5894 match(Set dst (AddVI dst src));
5895 format %{ "paddd $dst,$src\t! add packed2I" %}
5896 ins_encode %{
5897 __ paddd($dst$$XMMRegister, $src$$XMMRegister);
5898 %}
5899 ins_pipe( pipe_slow );
5900 %}
5901
5902 instruct vadd2I_reg(vecD dst, vecD src1, vecD src2) %{
5903 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
5904 match(Set dst (AddVI src1 src2));
5905 format %{ "vpaddd $dst,$src1,$src2\t! add packed2I" %}
5906 ins_encode %{
5907 int vector_len = 0;
5908 __ vpaddd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5909 %}
5910 ins_pipe( pipe_slow );
5911 %}
5912
5913 instruct vadd2I_mem(vecD dst, vecD src, memory mem) %{
5914 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
5915 match(Set dst (AddVI src (LoadVector mem)));
5916 format %{ "vpaddd $dst,$src,$mem\t! add packed2I" %}
5917 ins_encode %{
5918 int vector_len = 0;
5919 __ vpaddd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5920 %}
5921 ins_pipe( pipe_slow );
5922 %}
5923
5924 instruct vadd4I(vecX dst, vecX src) %{
5925 predicate(n->as_Vector()->length() == 4);
5926 match(Set dst (AddVI dst src));
5927 format %{ "paddd $dst,$src\t! add packed4I" %}
5928 ins_encode %{
5929 __ paddd($dst$$XMMRegister, $src$$XMMRegister);
5930 %}
5931 ins_pipe( pipe_slow );
5932 %}
5933
5934 instruct vadd4I_reg(vecX dst, vecX src1, vecX src2) %{
5935 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
5936 match(Set dst (AddVI src1 src2));
5937 format %{ "vpaddd $dst,$src1,$src2\t! add packed4I" %}
5938 ins_encode %{
5939 int vector_len = 0;
5940 __ vpaddd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5941 %}
5942 ins_pipe( pipe_slow );
5943 %}
5944
5945 instruct vadd4I_mem(vecX dst, vecX src, memory mem) %{
5946 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
5947 match(Set dst (AddVI src (LoadVector mem)));
5948 format %{ "vpaddd $dst,$src,$mem\t! add packed4I" %}
5949 ins_encode %{
5950 int vector_len = 0;
5951 __ vpaddd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5952 %}
5953 ins_pipe( pipe_slow );
5954 %}
5955
5956 instruct vadd8I_reg(vecY dst, vecY src1, vecY src2) %{
5957 predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
5958 match(Set dst (AddVI src1 src2));
5959 format %{ "vpaddd $dst,$src1,$src2\t! add packed8I" %}
5960 ins_encode %{
5961 int vector_len = 1;
5962 __ vpaddd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5963 %}
5964 ins_pipe( pipe_slow );
5965 %}
5966
5967 instruct vadd8I_mem(vecY dst, vecY src, memory mem) %{
5968 predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
5969 match(Set dst (AddVI src (LoadVector mem)));
5970 format %{ "vpaddd $dst,$src,$mem\t! add packed8I" %}
5971 ins_encode %{
5972 int vector_len = 1;
5973 __ vpaddd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5974 %}
5975 ins_pipe( pipe_slow );
5976 %}
5977
5978 instruct vadd16I_reg(vecZ dst, vecZ src1, vecZ src2) %{
5979 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
5980 match(Set dst (AddVI src1 src2));
5981 format %{ "vpaddd $dst,$src1,$src2\t! add packed16I" %}
5982 ins_encode %{
5983 int vector_len = 2;
5984 __ vpaddd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
5985 %}
5986 ins_pipe( pipe_slow );
5987 %}
5988
5989 instruct vadd16I_mem(vecZ dst, vecZ src, memory mem) %{
5990 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
5991 match(Set dst (AddVI src (LoadVector mem)));
5992 format %{ "vpaddd $dst,$src,$mem\t! add packed16I" %}
5993 ins_encode %{
5994 int vector_len = 2;
5995 __ vpaddd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
5996 %}
5997 ins_pipe( pipe_slow );
5998 %}
5999
6000 // Longs vector add
6001 instruct vadd2L(vecX dst, vecX src) %{
6002 predicate(n->as_Vector()->length() == 2);
6003 match(Set dst (AddVL dst src));
6004 format %{ "paddq $dst,$src\t! add packed2L" %}
6005 ins_encode %{
6006 __ paddq($dst$$XMMRegister, $src$$XMMRegister);
6007 %}
6008 ins_pipe( pipe_slow );
6009 %}
6010
6011 instruct vadd2L_reg(vecX dst, vecX src1, vecX src2) %{
6012 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6013 match(Set dst (AddVL src1 src2));
6014 format %{ "vpaddq $dst,$src1,$src2\t! add packed2L" %}
6015 ins_encode %{
6016 int vector_len = 0;
6017 __ vpaddq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6018 %}
6019 ins_pipe( pipe_slow );
6020 %}
6021
6022 instruct vadd2L_mem(vecX dst, vecX src, memory mem) %{
6023 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6024 match(Set dst (AddVL src (LoadVector mem)));
6025 format %{ "vpaddq $dst,$src,$mem\t! add packed2L" %}
6026 ins_encode %{
6027 int vector_len = 0;
6028 __ vpaddq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6029 %}
6030 ins_pipe( pipe_slow );
6031 %}
6032
6033 instruct vadd4L_reg(vecY dst, vecY src1, vecY src2) %{
6034 predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
6035 match(Set dst (AddVL src1 src2));
6036 format %{ "vpaddq $dst,$src1,$src2\t! add packed4L" %}
6037 ins_encode %{
6038 int vector_len = 1;
6039 __ vpaddq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6040 %}
6041 ins_pipe( pipe_slow );
6042 %}
6043
6044 instruct vadd4L_mem(vecY dst, vecY src, memory mem) %{
6045 predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
6046 match(Set dst (AddVL src (LoadVector mem)));
6047 format %{ "vpaddq $dst,$src,$mem\t! add packed4L" %}
6048 ins_encode %{
6049 int vector_len = 1;
6050 __ vpaddq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6051 %}
6052 ins_pipe( pipe_slow );
6053 %}
6054
6055 instruct vadd8L_reg(vecZ dst, vecZ src1, vecZ src2) %{
6056 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
6057 match(Set dst (AddVL src1 src2));
6058 format %{ "vpaddq $dst,$src1,$src2\t! add packed8L" %}
6059 ins_encode %{
6060 int vector_len = 2;
6061 __ vpaddq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6062 %}
6063 ins_pipe( pipe_slow );
6064 %}
6065
6066 instruct vadd8L_mem(vecZ dst, vecZ src, memory mem) %{
6067 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
6068 match(Set dst (AddVL src (LoadVector mem)));
6069 format %{ "vpaddq $dst,$src,$mem\t! add packed8L" %}
6070 ins_encode %{
6071 int vector_len = 2;
6072 __ vpaddq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6073 %}
6074 ins_pipe( pipe_slow );
6075 %}
6076
6077 // Floats vector add
6078 instruct vadd2F(vecD dst, vecD src) %{
6079 predicate(n->as_Vector()->length() == 2);
6080 match(Set dst (AddVF dst src));
6081 format %{ "addps $dst,$src\t! add packed2F" %}
6082 ins_encode %{
6083 __ addps($dst$$XMMRegister, $src$$XMMRegister);
6084 %}
6085 ins_pipe( pipe_slow );
6086 %}
6087
6088 instruct vadd2F_reg(vecD dst, vecD src1, vecD src2) %{
6089 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6090 match(Set dst (AddVF src1 src2));
6091 format %{ "vaddps $dst,$src1,$src2\t! add packed2F" %}
6092 ins_encode %{
6093 int vector_len = 0;
6094 __ vaddps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6095 %}
6096 ins_pipe( pipe_slow );
6097 %}
6098
6099 instruct vadd2F_mem(vecD dst, vecD src, memory mem) %{
6100 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6101 match(Set dst (AddVF src (LoadVector mem)));
6102 format %{ "vaddps $dst,$src,$mem\t! add packed2F" %}
6103 ins_encode %{
6104 int vector_len = 0;
6105 __ vaddps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6106 %}
6107 ins_pipe( pipe_slow );
6108 %}
6109
6110 instruct vadd4F(vecX dst, vecX src) %{
6111 predicate(n->as_Vector()->length() == 4);
6112 match(Set dst (AddVF dst src));
6113 format %{ "addps $dst,$src\t! add packed4F" %}
6114 ins_encode %{
6115 __ addps($dst$$XMMRegister, $src$$XMMRegister);
6116 %}
6117 ins_pipe( pipe_slow );
6118 %}
6119
6120 instruct vadd4F_reg(vecX dst, vecX src1, vecX src2) %{
6121 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6122 match(Set dst (AddVF src1 src2));
6123 format %{ "vaddps $dst,$src1,$src2\t! add packed4F" %}
6124 ins_encode %{
6125 int vector_len = 0;
6126 __ vaddps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6127 %}
6128 ins_pipe( pipe_slow );
6129 %}
6130
6131 instruct vadd4F_mem(vecX dst, vecX src, memory mem) %{
6132 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6133 match(Set dst (AddVF src (LoadVector mem)));
6134 format %{ "vaddps $dst,$src,$mem\t! add packed4F" %}
6135 ins_encode %{
6136 int vector_len = 0;
6137 __ vaddps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6138 %}
6139 ins_pipe( pipe_slow );
6140 %}
6141
6142 instruct vadd8F_reg(vecY dst, vecY src1, vecY src2) %{
6143 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
6144 match(Set dst (AddVF src1 src2));
6145 format %{ "vaddps $dst,$src1,$src2\t! add packed8F" %}
6146 ins_encode %{
6147 int vector_len = 1;
6148 __ vaddps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6149 %}
6150 ins_pipe( pipe_slow );
6151 %}
6152
6153 instruct vadd8F_mem(vecY dst, vecY src, memory mem) %{
6154 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
6155 match(Set dst (AddVF src (LoadVector mem)));
6156 format %{ "vaddps $dst,$src,$mem\t! add packed8F" %}
6157 ins_encode %{
6158 int vector_len = 1;
6159 __ vaddps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6160 %}
6161 ins_pipe( pipe_slow );
6162 %}
6163
6164 instruct vadd16F_reg(vecZ dst, vecZ src1, vecZ src2) %{
6165 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
6166 match(Set dst (AddVF src1 src2));
6167 format %{ "vaddps $dst,$src1,$src2\t! add packed16F" %}
6168 ins_encode %{
6169 int vector_len = 2;
6170 __ vaddps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6171 %}
6172 ins_pipe( pipe_slow );
6173 %}
6174
6175 instruct vadd16F_mem(vecZ dst, vecZ src, memory mem) %{
6176 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
6177 match(Set dst (AddVF src (LoadVector mem)));
6178 format %{ "vaddps $dst,$src,$mem\t! add packed16F" %}
6179 ins_encode %{
6180 int vector_len = 2;
6181 __ vaddps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6182 %}
6183 ins_pipe( pipe_slow );
6184 %}
6185
6186 // Doubles vector add
6187 instruct vadd2D(vecX dst, vecX src) %{
6188 predicate(n->as_Vector()->length() == 2);
6189 match(Set dst (AddVD dst src));
6190 format %{ "addpd $dst,$src\t! add packed2D" %}
6191 ins_encode %{
6192 __ addpd($dst$$XMMRegister, $src$$XMMRegister);
6193 %}
6194 ins_pipe( pipe_slow );
6195 %}
6196
6197 instruct vadd2D_reg(vecX dst, vecX src1, vecX src2) %{
6198 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6199 match(Set dst (AddVD src1 src2));
6200 format %{ "vaddpd $dst,$src1,$src2\t! add packed2D" %}
6201 ins_encode %{
6202 int vector_len = 0;
6203 __ vaddpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6204 %}
6205 ins_pipe( pipe_slow );
6206 %}
6207
6208 instruct vadd2D_mem(vecX dst, vecX src, memory mem) %{
6209 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6210 match(Set dst (AddVD src (LoadVector mem)));
6211 format %{ "vaddpd $dst,$src,$mem\t! add packed2D" %}
6212 ins_encode %{
6213 int vector_len = 0;
6214 __ vaddpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6215 %}
6216 ins_pipe( pipe_slow );
6217 %}
6218
6219 instruct vadd4D_reg(vecY dst, vecY src1, vecY src2) %{
6220 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6221 match(Set dst (AddVD src1 src2));
6222 format %{ "vaddpd $dst,$src1,$src2\t! add packed4D" %}
6223 ins_encode %{
6224 int vector_len = 1;
6225 __ vaddpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6226 %}
6227 ins_pipe( pipe_slow );
6228 %}
6229
6230 instruct vadd4D_mem(vecY dst, vecY src, memory mem) %{
6231 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6232 match(Set dst (AddVD src (LoadVector mem)));
6233 format %{ "vaddpd $dst,$src,$mem\t! add packed4D" %}
6234 ins_encode %{
6235 int vector_len = 1;
6236 __ vaddpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6237 %}
6238 ins_pipe( pipe_slow );
6239 %}
6240
6241 instruct vadd8D_reg(vecZ dst, vecZ src1, vecZ src2) %{
6242 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
6243 match(Set dst (AddVD src1 src2));
6244 format %{ "vaddpd $dst,$src1,$src2\t! add packed8D" %}
6245 ins_encode %{
6246 int vector_len = 2;
6247 __ vaddpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6248 %}
6249 ins_pipe( pipe_slow );
6250 %}
6251
6252 instruct vadd8D_mem(vecZ dst, vecZ src, memory mem) %{
6253 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
6254 match(Set dst (AddVD src (LoadVector mem)));
6255 format %{ "vaddpd $dst,$src,$mem\t! add packed8D" %}
6256 ins_encode %{
6257 int vector_len = 2;
6258 __ vaddpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6259 %}
6260 ins_pipe( pipe_slow );
6261 %}
6262
6263 // --------------------------------- SUB --------------------------------------
6264
6265 // Bytes vector sub
6266 instruct vsub4B(vecS dst, vecS src) %{
6267 predicate(n->as_Vector()->length() == 4);
6268 match(Set dst (SubVB dst src));
6269 format %{ "psubb $dst,$src\t! sub packed4B" %}
6270 ins_encode %{
6271 __ psubb($dst$$XMMRegister, $src$$XMMRegister);
6272 %}
6273 ins_pipe( pipe_slow );
6274 %}
6275
6276 instruct vsub4B_reg(vecS dst, vecS src1, vecS src2) %{
6277 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6278 match(Set dst (SubVB src1 src2));
6279 format %{ "vpsubb $dst,$src1,$src2\t! sub packed4B" %}
6280 ins_encode %{
6281 int vector_len = 0;
6282 __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6283 %}
6284 ins_pipe( pipe_slow );
6285 %}
6286
6287 instruct vsub4B_mem(vecS dst, vecS src, memory mem) %{
6288 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6289 match(Set dst (SubVB src (LoadVector mem)));
6290 format %{ "vpsubb $dst,$src,$mem\t! sub packed4B" %}
6291 ins_encode %{
6292 int vector_len = 0;
6293 __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6294 %}
6295 ins_pipe( pipe_slow );
6296 %}
6297
6298 instruct vsub8B(vecD dst, vecD src) %{
6299 predicate(n->as_Vector()->length() == 8);
6300 match(Set dst (SubVB dst src));
6301 format %{ "psubb $dst,$src\t! sub packed8B" %}
6302 ins_encode %{
6303 __ psubb($dst$$XMMRegister, $src$$XMMRegister);
6304 %}
6305 ins_pipe( pipe_slow );
6306 %}
6307
6308 instruct vsub8B_reg(vecD dst, vecD src1, vecD src2) %{
6309 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
6310 match(Set dst (SubVB src1 src2));
6311 format %{ "vpsubb $dst,$src1,$src2\t! sub packed8B" %}
6312 ins_encode %{
6313 int vector_len = 0;
6314 __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6315 %}
6316 ins_pipe( pipe_slow );
6317 %}
6318
6319 instruct vsub8B_mem(vecD dst, vecD src, memory mem) %{
6320 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
6321 match(Set dst (SubVB src (LoadVector mem)));
6322 format %{ "vpsubb $dst,$src,$mem\t! sub packed8B" %}
6323 ins_encode %{
6324 int vector_len = 0;
6325 __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6326 %}
6327 ins_pipe( pipe_slow );
6328 %}
6329
6330 instruct vsub16B(vecX dst, vecX src) %{
6331 predicate(n->as_Vector()->length() == 16);
6332 match(Set dst (SubVB dst src));
6333 format %{ "psubb $dst,$src\t! sub packed16B" %}
6334 ins_encode %{
6335 __ psubb($dst$$XMMRegister, $src$$XMMRegister);
6336 %}
6337 ins_pipe( pipe_slow );
6338 %}
6339
6340 instruct vsub16B_reg(vecX dst, vecX src1, vecX src2) %{
6341 predicate(UseAVX > 0 && n->as_Vector()->length() == 16);
6342 match(Set dst (SubVB src1 src2));
6343 format %{ "vpsubb $dst,$src1,$src2\t! sub packed16B" %}
6344 ins_encode %{
6345 int vector_len = 0;
6346 __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6347 %}
6348 ins_pipe( pipe_slow );
6349 %}
6350
6351 instruct vsub16B_mem(vecX dst, vecX src, memory mem) %{
6352 predicate(UseAVX > 0 && n->as_Vector()->length() == 16);
6353 match(Set dst (SubVB src (LoadVector mem)));
6354 format %{ "vpsubb $dst,$src,$mem\t! sub packed16B" %}
6355 ins_encode %{
6356 int vector_len = 0;
6357 __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6358 %}
6359 ins_pipe( pipe_slow );
6360 %}
6361
6362 instruct vsub32B_reg(vecY dst, vecY src1, vecY src2) %{
6363 predicate(UseAVX > 1 && n->as_Vector()->length() == 32);
6364 match(Set dst (SubVB src1 src2));
6365 format %{ "vpsubb $dst,$src1,$src2\t! sub packed32B" %}
6366 ins_encode %{
6367 int vector_len = 1;
6368 __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6369 %}
6370 ins_pipe( pipe_slow );
6371 %}
6372
6373 instruct vsub32B_mem(vecY dst, vecY src, memory mem) %{
6374 predicate(UseAVX > 1 && n->as_Vector()->length() == 32);
6375 match(Set dst (SubVB src (LoadVector mem)));
6376 format %{ "vpsubb $dst,$src,$mem\t! sub packed32B" %}
6377 ins_encode %{
6378 int vector_len = 1;
6379 __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6380 %}
6381 ins_pipe( pipe_slow );
6382 %}
6383
6384 instruct vsub64B_reg(vecZ dst, vecZ src1, vecZ src2) %{
6385 predicate(UseAVX > 2 && n->as_Vector()->length() == 64);
6386 match(Set dst (SubVB src1 src2));
6387 format %{ "vpsubb $dst,$src1,$src2\t! sub packed64B" %}
6388 ins_encode %{
6389 int vector_len = 2;
6390 __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6391 %}
6392 ins_pipe( pipe_slow );
6393 %}
6394
6395 instruct vsub64B_mem(vecZ dst, vecZ src, memory mem) %{
6396 predicate(UseAVX > 2 && n->as_Vector()->length() == 64);
6397 match(Set dst (SubVB src (LoadVector mem)));
6398 format %{ "vpsubb $dst,$src,$mem\t! sub packed64B" %}
6399 ins_encode %{
6400 int vector_len = 2;
6401 __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6402 %}
6403 ins_pipe( pipe_slow );
6404 %}
6405
6406 // Shorts/Chars vector sub
6407 instruct vsub2S(vecS dst, vecS src) %{
6408 predicate(n->as_Vector()->length() == 2);
6409 match(Set dst (SubVS dst src));
6410 format %{ "psubw $dst,$src\t! sub packed2S" %}
6411 ins_encode %{
6412 __ psubw($dst$$XMMRegister, $src$$XMMRegister);
6413 %}
6414 ins_pipe( pipe_slow );
6415 %}
6416
6417 instruct vsub2S_reg(vecS dst, vecS src1, vecS src2) %{
6418 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6419 match(Set dst (SubVS src1 src2));
6420 format %{ "vpsubw $dst,$src1,$src2\t! sub packed2S" %}
6421 ins_encode %{
6422 int vector_len = 0;
6423 __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6424 %}
6425 ins_pipe( pipe_slow );
6426 %}
6427
6428 instruct vsub2S_mem(vecS dst, vecS src, memory mem) %{
6429 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6430 match(Set dst (SubVS src (LoadVector mem)));
6431 format %{ "vpsubw $dst,$src,$mem\t! sub packed2S" %}
6432 ins_encode %{
6433 int vector_len = 0;
6434 __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6435 %}
6436 ins_pipe( pipe_slow );
6437 %}
6438
6439 instruct vsub4S(vecD dst, vecD src) %{
6440 predicate(n->as_Vector()->length() == 4);
6441 match(Set dst (SubVS dst src));
6442 format %{ "psubw $dst,$src\t! sub packed4S" %}
6443 ins_encode %{
6444 __ psubw($dst$$XMMRegister, $src$$XMMRegister);
6445 %}
6446 ins_pipe( pipe_slow );
6447 %}
6448
6449 instruct vsub4S_reg(vecD dst, vecD src1, vecD src2) %{
6450 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6451 match(Set dst (SubVS src1 src2));
6452 format %{ "vpsubw $dst,$src1,$src2\t! sub packed4S" %}
6453 ins_encode %{
6454 int vector_len = 0;
6455 __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6456 %}
6457 ins_pipe( pipe_slow );
6458 %}
6459
6460 instruct vsub4S_mem(vecD dst, vecD src, memory mem) %{
6461 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6462 match(Set dst (SubVS src (LoadVector mem)));
6463 format %{ "vpsubw $dst,$src,$mem\t! sub packed4S" %}
6464 ins_encode %{
6465 int vector_len = 0;
6466 __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6467 %}
6468 ins_pipe( pipe_slow );
6469 %}
6470
6471 instruct vsub8S(vecX dst, vecX src) %{
6472 predicate(n->as_Vector()->length() == 8);
6473 match(Set dst (SubVS dst src));
6474 format %{ "psubw $dst,$src\t! sub packed8S" %}
6475 ins_encode %{
6476 __ psubw($dst$$XMMRegister, $src$$XMMRegister);
6477 %}
6478 ins_pipe( pipe_slow );
6479 %}
6480
6481 instruct vsub8S_reg(vecX dst, vecX src1, vecX src2) %{
6482 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
6483 match(Set dst (SubVS src1 src2));
6484 format %{ "vpsubw $dst,$src1,$src2\t! sub packed8S" %}
6485 ins_encode %{
6486 int vector_len = 0;
6487 __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6488 %}
6489 ins_pipe( pipe_slow );
6490 %}
6491
6492 instruct vsub8S_mem(vecX dst, vecX src, memory mem) %{
6493 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
6494 match(Set dst (SubVS src (LoadVector mem)));
6495 format %{ "vpsubw $dst,$src,$mem\t! sub packed8S" %}
6496 ins_encode %{
6497 int vector_len = 0;
6498 __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6499 %}
6500 ins_pipe( pipe_slow );
6501 %}
6502
6503 instruct vsub16S_reg(vecY dst, vecY src1, vecY src2) %{
6504 predicate(UseAVX > 1 && n->as_Vector()->length() == 16);
6505 match(Set dst (SubVS src1 src2));
6506 format %{ "vpsubw $dst,$src1,$src2\t! sub packed16S" %}
6507 ins_encode %{
6508 int vector_len = 1;
6509 __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6510 %}
6511 ins_pipe( pipe_slow );
6512 %}
6513
6514 instruct vsub16S_mem(vecY dst, vecY src, memory mem) %{
6515 predicate(UseAVX > 1 && n->as_Vector()->length() == 16);
6516 match(Set dst (SubVS src (LoadVector mem)));
6517 format %{ "vpsubw $dst,$src,$mem\t! sub packed16S" %}
6518 ins_encode %{
6519 int vector_len = 1;
6520 __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6521 %}
6522 ins_pipe( pipe_slow );
6523 %}
6524
6525 instruct vsub32S_reg(vecZ dst, vecZ src1, vecZ src2) %{
6526 predicate(UseAVX > 2 && n->as_Vector()->length() == 32);
6527 match(Set dst (SubVS src1 src2));
6528 format %{ "vpsubw $dst,$src1,$src2\t! sub packed32S" %}
6529 ins_encode %{
6530 int vector_len = 2;
6531 __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6532 %}
6533 ins_pipe( pipe_slow );
6534 %}
6535
6536 instruct vsub32S_mem(vecZ dst, vecZ src, memory mem) %{
6537 predicate(UseAVX > 2 && n->as_Vector()->length() == 32);
6538 match(Set dst (SubVS src (LoadVector mem)));
6539 format %{ "vpsubw $dst,$src,$mem\t! sub packed32S" %}
6540 ins_encode %{
6541 int vector_len = 2;
6542 __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6543 %}
6544 ins_pipe( pipe_slow );
6545 %}
6546
6547 // Integers vector sub
6548 instruct vsub2I(vecD dst, vecD src) %{
6549 predicate(n->as_Vector()->length() == 2);
6550 match(Set dst (SubVI dst src));
6551 format %{ "psubd $dst,$src\t! sub packed2I" %}
6552 ins_encode %{
6553 __ psubd($dst$$XMMRegister, $src$$XMMRegister);
6554 %}
6555 ins_pipe( pipe_slow );
6556 %}
6557
6558 instruct vsub2I_reg(vecD dst, vecD src1, vecD src2) %{
6559 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6560 match(Set dst (SubVI src1 src2));
6561 format %{ "vpsubd $dst,$src1,$src2\t! sub packed2I" %}
6562 ins_encode %{
6563 int vector_len = 0;
6564 __ vpsubd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6565 %}
6566 ins_pipe( pipe_slow );
6567 %}
6568
6569 instruct vsub2I_mem(vecD dst, vecD src, memory mem) %{
6570 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6571 match(Set dst (SubVI src (LoadVector mem)));
6572 format %{ "vpsubd $dst,$src,$mem\t! sub packed2I" %}
6573 ins_encode %{
6574 int vector_len = 0;
6575 __ vpsubd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6576 %}
6577 ins_pipe( pipe_slow );
6578 %}
6579
6580 instruct vsub4I(vecX dst, vecX src) %{
6581 predicate(n->as_Vector()->length() == 4);
6582 match(Set dst (SubVI dst src));
6583 format %{ "psubd $dst,$src\t! sub packed4I" %}
6584 ins_encode %{
6585 __ psubd($dst$$XMMRegister, $src$$XMMRegister);
6586 %}
6587 ins_pipe( pipe_slow );
6588 %}
6589
6590 instruct vsub4I_reg(vecX dst, vecX src1, vecX src2) %{
6591 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6592 match(Set dst (SubVI src1 src2));
6593 format %{ "vpsubd $dst,$src1,$src2\t! sub packed4I" %}
6594 ins_encode %{
6595 int vector_len = 0;
6596 __ vpsubd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6597 %}
6598 ins_pipe( pipe_slow );
6599 %}
6600
6601 instruct vsub4I_mem(vecX dst, vecX src, memory mem) %{
6602 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6603 match(Set dst (SubVI src (LoadVector mem)));
6604 format %{ "vpsubd $dst,$src,$mem\t! sub packed4I" %}
6605 ins_encode %{
6606 int vector_len = 0;
6607 __ vpsubd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6608 %}
6609 ins_pipe( pipe_slow );
6610 %}
6611
6612 instruct vsub8I_reg(vecY dst, vecY src1, vecY src2) %{
6613 predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
6614 match(Set dst (SubVI src1 src2));
6615 format %{ "vpsubd $dst,$src1,$src2\t! sub packed8I" %}
6616 ins_encode %{
6617 int vector_len = 1;
6618 __ vpsubd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6619 %}
6620 ins_pipe( pipe_slow );
6621 %}
6622
6623 instruct vsub8I_mem(vecY dst, vecY src, memory mem) %{
6624 predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
6625 match(Set dst (SubVI src (LoadVector mem)));
6626 format %{ "vpsubd $dst,$src,$mem\t! sub packed8I" %}
6627 ins_encode %{
6628 int vector_len = 1;
6629 __ vpsubd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6630 %}
6631 ins_pipe( pipe_slow );
6632 %}
6633
6634 instruct vsub16I_reg(vecZ dst, vecZ src1, vecZ src2) %{
6635 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
6636 match(Set dst (SubVI src1 src2));
6637 format %{ "vpsubd $dst,$src1,$src2\t! sub packed16I" %}
6638 ins_encode %{
6639 int vector_len = 2;
6640 __ vpsubd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6641 %}
6642 ins_pipe( pipe_slow );
6643 %}
6644
6645 instruct vsub16I_mem(vecZ dst, vecZ src, memory mem) %{
6646 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
6647 match(Set dst (SubVI src (LoadVector mem)));
6648 format %{ "vpsubd $dst,$src,$mem\t! sub packed16I" %}
6649 ins_encode %{
6650 int vector_len = 2;
6651 __ vpsubd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6652 %}
6653 ins_pipe( pipe_slow );
6654 %}
6655
6656 // Longs vector sub
6657 instruct vsub2L(vecX dst, vecX src) %{
6658 predicate(n->as_Vector()->length() == 2);
6659 match(Set dst (SubVL dst src));
6660 format %{ "psubq $dst,$src\t! sub packed2L" %}
6661 ins_encode %{
6662 __ psubq($dst$$XMMRegister, $src$$XMMRegister);
6663 %}
6664 ins_pipe( pipe_slow );
6665 %}
6666
6667 instruct vsub2L_reg(vecX dst, vecX src1, vecX src2) %{
6668 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6669 match(Set dst (SubVL src1 src2));
6670 format %{ "vpsubq $dst,$src1,$src2\t! sub packed2L" %}
6671 ins_encode %{
6672 int vector_len = 0;
6673 __ vpsubq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6674 %}
6675 ins_pipe( pipe_slow );
6676 %}
6677
6678 instruct vsub2L_mem(vecX dst, vecX src, memory mem) %{
6679 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6680 match(Set dst (SubVL src (LoadVector mem)));
6681 format %{ "vpsubq $dst,$src,$mem\t! sub packed2L" %}
6682 ins_encode %{
6683 int vector_len = 0;
6684 __ vpsubq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6685 %}
6686 ins_pipe( pipe_slow );
6687 %}
6688
6689 instruct vsub4L_reg(vecY dst, vecY src1, vecY src2) %{
6690 predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
6691 match(Set dst (SubVL src1 src2));
6692 format %{ "vpsubq $dst,$src1,$src2\t! sub packed4L" %}
6693 ins_encode %{
6694 int vector_len = 1;
6695 __ vpsubq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6696 %}
6697 ins_pipe( pipe_slow );
6698 %}
6699
6700 instruct vsub4L_mem(vecY dst, vecY src, memory mem) %{
6701 predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
6702 match(Set dst (SubVL src (LoadVector mem)));
6703 format %{ "vpsubq $dst,$src,$mem\t! sub packed4L" %}
6704 ins_encode %{
6705 int vector_len = 1;
6706 __ vpsubq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6707 %}
6708 ins_pipe( pipe_slow );
6709 %}
6710
6711 instruct vsub8L_reg(vecZ dst, vecZ src1, vecZ src2) %{
6712 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
6713 match(Set dst (SubVL src1 src2));
6714 format %{ "vpsubq $dst,$src1,$src2\t! sub packed8L" %}
6715 ins_encode %{
6716 int vector_len = 2;
6717 __ vpsubq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6718 %}
6719 ins_pipe( pipe_slow );
6720 %}
6721
6722 instruct vsub8L_mem(vecZ dst, vecZ src, memory mem) %{
6723 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
6724 match(Set dst (SubVL src (LoadVector mem)));
6725 format %{ "vpsubq $dst,$src,$mem\t! sub packed8L" %}
6726 ins_encode %{
6727 int vector_len = 2;
6728 __ vpsubq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6729 %}
6730 ins_pipe( pipe_slow );
6731 %}
6732
6733 // Floats vector sub
6734 instruct vsub2F(vecD dst, vecD src) %{
6735 predicate(n->as_Vector()->length() == 2);
6736 match(Set dst (SubVF dst src));
6737 format %{ "subps $dst,$src\t! sub packed2F" %}
6738 ins_encode %{
6739 __ subps($dst$$XMMRegister, $src$$XMMRegister);
6740 %}
6741 ins_pipe( pipe_slow );
6742 %}
6743
6744 instruct vsub2F_reg(vecD dst, vecD src1, vecD src2) %{
6745 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6746 match(Set dst (SubVF src1 src2));
6747 format %{ "vsubps $dst,$src1,$src2\t! sub packed2F" %}
6748 ins_encode %{
6749 int vector_len = 0;
6750 __ vsubps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6751 %}
6752 ins_pipe( pipe_slow );
6753 %}
6754
6755 instruct vsub2F_mem(vecD dst, vecD src, memory mem) %{
6756 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6757 match(Set dst (SubVF src (LoadVector mem)));
6758 format %{ "vsubps $dst,$src,$mem\t! sub packed2F" %}
6759 ins_encode %{
6760 int vector_len = 0;
6761 __ vsubps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6762 %}
6763 ins_pipe( pipe_slow );
6764 %}
6765
6766 instruct vsub4F(vecX dst, vecX src) %{
6767 predicate(n->as_Vector()->length() == 4);
6768 match(Set dst (SubVF dst src));
6769 format %{ "subps $dst,$src\t! sub packed4F" %}
6770 ins_encode %{
6771 __ subps($dst$$XMMRegister, $src$$XMMRegister);
6772 %}
6773 ins_pipe( pipe_slow );
6774 %}
6775
6776 instruct vsub4F_reg(vecX dst, vecX src1, vecX src2) %{
6777 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6778 match(Set dst (SubVF src1 src2));
6779 format %{ "vsubps $dst,$src1,$src2\t! sub packed4F" %}
6780 ins_encode %{
6781 int vector_len = 0;
6782 __ vsubps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6783 %}
6784 ins_pipe( pipe_slow );
6785 %}
6786
6787 instruct vsub4F_mem(vecX dst, vecX src, memory mem) %{
6788 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6789 match(Set dst (SubVF src (LoadVector mem)));
6790 format %{ "vsubps $dst,$src,$mem\t! sub packed4F" %}
6791 ins_encode %{
6792 int vector_len = 0;
6793 __ vsubps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6794 %}
6795 ins_pipe( pipe_slow );
6796 %}
6797
6798 instruct vsub8F_reg(vecY dst, vecY src1, vecY src2) %{
6799 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
6800 match(Set dst (SubVF src1 src2));
6801 format %{ "vsubps $dst,$src1,$src2\t! sub packed8F" %}
6802 ins_encode %{
6803 int vector_len = 1;
6804 __ vsubps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6805 %}
6806 ins_pipe( pipe_slow );
6807 %}
6808
6809 instruct vsub8F_mem(vecY dst, vecY src, memory mem) %{
6810 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
6811 match(Set dst (SubVF src (LoadVector mem)));
6812 format %{ "vsubps $dst,$src,$mem\t! sub packed8F" %}
6813 ins_encode %{
6814 int vector_len = 1;
6815 __ vsubps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6816 %}
6817 ins_pipe( pipe_slow );
6818 %}
6819
6820 instruct vsub16F_reg(vecZ dst, vecZ src1, vecZ src2) %{
6821 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
6822 match(Set dst (SubVF src1 src2));
6823 format %{ "vsubps $dst,$src1,$src2\t! sub packed16F" %}
6824 ins_encode %{
6825 int vector_len = 2;
6826 __ vsubps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6827 %}
6828 ins_pipe( pipe_slow );
6829 %}
6830
6831 instruct vsub16F_mem(vecZ dst, vecZ src, memory mem) %{
6832 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
6833 match(Set dst (SubVF src (LoadVector mem)));
6834 format %{ "vsubps $dst,$src,$mem\t! sub packed16F" %}
6835 ins_encode %{
6836 int vector_len = 2;
6837 __ vsubps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6838 %}
6839 ins_pipe( pipe_slow );
6840 %}
6841
6842 // Doubles vector sub
6843 instruct vsub2D(vecX dst, vecX src) %{
6844 predicate(n->as_Vector()->length() == 2);
6845 match(Set dst (SubVD dst src));
6846 format %{ "subpd $dst,$src\t! sub packed2D" %}
6847 ins_encode %{
6848 __ subpd($dst$$XMMRegister, $src$$XMMRegister);
6849 %}
6850 ins_pipe( pipe_slow );
6851 %}
6852
6853 instruct vsub2D_reg(vecX dst, vecX src1, vecX src2) %{
6854 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6855 match(Set dst (SubVD src1 src2));
6856 format %{ "vsubpd $dst,$src1,$src2\t! sub packed2D" %}
6857 ins_encode %{
6858 int vector_len = 0;
6859 __ vsubpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6860 %}
6861 ins_pipe( pipe_slow );
6862 %}
6863
6864 instruct vsub2D_mem(vecX dst, vecX src, memory mem) %{
6865 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6866 match(Set dst (SubVD src (LoadVector mem)));
6867 format %{ "vsubpd $dst,$src,$mem\t! sub packed2D" %}
6868 ins_encode %{
6869 int vector_len = 0;
6870 __ vsubpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6871 %}
6872 ins_pipe( pipe_slow );
6873 %}
6874
6875 instruct vsub4D_reg(vecY dst, vecY src1, vecY src2) %{
6876 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6877 match(Set dst (SubVD src1 src2));
6878 format %{ "vsubpd $dst,$src1,$src2\t! sub packed4D" %}
6879 ins_encode %{
6880 int vector_len = 1;
6881 __ vsubpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6882 %}
6883 ins_pipe( pipe_slow );
6884 %}
6885
6886 instruct vsub4D_mem(vecY dst, vecY src, memory mem) %{
6887 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6888 match(Set dst (SubVD src (LoadVector mem)));
6889 format %{ "vsubpd $dst,$src,$mem\t! sub packed4D" %}
6890 ins_encode %{
6891 int vector_len = 1;
6892 __ vsubpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6893 %}
6894 ins_pipe( pipe_slow );
6895 %}
6896
6897 instruct vsub8D_reg(vecZ dst, vecZ src1, vecZ src2) %{
6898 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
6899 match(Set dst (SubVD src1 src2));
6900 format %{ "vsubpd $dst,$src1,$src2\t! sub packed8D" %}
6901 ins_encode %{
6902 int vector_len = 2;
6903 __ vsubpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6904 %}
6905 ins_pipe( pipe_slow );
6906 %}
6907
6908 instruct vsub8D_mem(vecZ dst, vecZ src, memory mem) %{
6909 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
6910 match(Set dst (SubVD src (LoadVector mem)));
6911 format %{ "vsubpd $dst,$src,$mem\t! sub packed8D" %}
6912 ins_encode %{
6913 int vector_len = 2;
6914 __ vsubpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6915 %}
6916 ins_pipe( pipe_slow );
6917 %}
6918
6919 // --------------------------------- MUL --------------------------------------
6920
6921 // Shorts/Chars vector mul
6922 instruct vmul2S(vecS dst, vecS src) %{
6923 predicate(n->as_Vector()->length() == 2);
6924 match(Set dst (MulVS dst src));
6925 format %{ "pmullw $dst,$src\t! mul packed2S" %}
6926 ins_encode %{
6927 __ pmullw($dst$$XMMRegister, $src$$XMMRegister);
6928 %}
6929 ins_pipe( pipe_slow );
6930 %}
6931
6932 instruct vmul2S_reg(vecS dst, vecS src1, vecS src2) %{
6933 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6934 match(Set dst (MulVS src1 src2));
6935 format %{ "vpmullw $dst,$src1,$src2\t! mul packed2S" %}
6936 ins_encode %{
6937 int vector_len = 0;
6938 __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6939 %}
6940 ins_pipe( pipe_slow );
6941 %}
6942
6943 instruct vmul2S_mem(vecS dst, vecS src, memory mem) %{
6944 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
6945 match(Set dst (MulVS src (LoadVector mem)));
6946 format %{ "vpmullw $dst,$src,$mem\t! mul packed2S" %}
6947 ins_encode %{
6948 int vector_len = 0;
6949 __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6950 %}
6951 ins_pipe( pipe_slow );
6952 %}
6953
6954 instruct vmul4S(vecD dst, vecD src) %{
6955 predicate(n->as_Vector()->length() == 4);
6956 match(Set dst (MulVS dst src));
6957 format %{ "pmullw $dst,$src\t! mul packed4S" %}
6958 ins_encode %{
6959 __ pmullw($dst$$XMMRegister, $src$$XMMRegister);
6960 %}
6961 ins_pipe( pipe_slow );
6962 %}
6963
6964 instruct vmul4S_reg(vecD dst, vecD src1, vecD src2) %{
6965 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6966 match(Set dst (MulVS src1 src2));
6967 format %{ "vpmullw $dst,$src1,$src2\t! mul packed4S" %}
6968 ins_encode %{
6969 int vector_len = 0;
6970 __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
6971 %}
6972 ins_pipe( pipe_slow );
6973 %}
6974
6975 instruct vmul4S_mem(vecD dst, vecD src, memory mem) %{
6976 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
6977 match(Set dst (MulVS src (LoadVector mem)));
6978 format %{ "vpmullw $dst,$src,$mem\t! mul packed4S" %}
6979 ins_encode %{
6980 int vector_len = 0;
6981 __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
6982 %}
6983 ins_pipe( pipe_slow );
6984 %}
6985
6986 instruct vmul8S(vecX dst, vecX src) %{
6987 predicate(n->as_Vector()->length() == 8);
6988 match(Set dst (MulVS dst src));
6989 format %{ "pmullw $dst,$src\t! mul packed8S" %}
6990 ins_encode %{
6991 __ pmullw($dst$$XMMRegister, $src$$XMMRegister);
6992 %}
6993 ins_pipe( pipe_slow );
6994 %}
6995
6996 instruct vmul8S_reg(vecX dst, vecX src1, vecX src2) %{
6997 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
6998 match(Set dst (MulVS src1 src2));
6999 format %{ "vpmullw $dst,$src1,$src2\t! mul packed8S" %}
7000 ins_encode %{
7001 int vector_len = 0;
7002 __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7003 %}
7004 ins_pipe( pipe_slow );
7005 %}
7006
7007 instruct vmul8S_mem(vecX dst, vecX src, memory mem) %{
7008 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
7009 match(Set dst (MulVS src (LoadVector mem)));
7010 format %{ "vpmullw $dst,$src,$mem\t! mul packed8S" %}
7011 ins_encode %{
7012 int vector_len = 0;
7013 __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7014 %}
7015 ins_pipe( pipe_slow );
7016 %}
7017
7018 instruct vmul16S_reg(vecY dst, vecY src1, vecY src2) %{
7019 predicate(UseAVX > 1 && n->as_Vector()->length() == 16);
7020 match(Set dst (MulVS src1 src2));
7021 format %{ "vpmullw $dst,$src1,$src2\t! mul packed16S" %}
7022 ins_encode %{
7023 int vector_len = 1;
7024 __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7025 %}
7026 ins_pipe( pipe_slow );
7027 %}
7028
7029 instruct vmul16S_mem(vecY dst, vecY src, memory mem) %{
7030 predicate(UseAVX > 1 && n->as_Vector()->length() == 16);
7031 match(Set dst (MulVS src (LoadVector mem)));
7032 format %{ "vpmullw $dst,$src,$mem\t! mul packed16S" %}
7033 ins_encode %{
7034 int vector_len = 1;
7035 __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7036 %}
7037 ins_pipe( pipe_slow );
7038 %}
7039
7040 instruct vmul32S_reg(vecZ dst, vecZ src1, vecZ src2) %{
7041 predicate(UseAVX > 2 && n->as_Vector()->length() == 32);
7042 match(Set dst (MulVS src1 src2));
7043 format %{ "vpmullw $dst,$src1,$src2\t! mul packed32S" %}
7044 ins_encode %{
7045 int vector_len = 2;
7046 __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7047 %}
7048 ins_pipe( pipe_slow );
7049 %}
7050
7051 instruct vmul32S_mem(vecZ dst, vecZ src, memory mem) %{
7052 predicate(UseAVX > 2 && n->as_Vector()->length() == 32);
7053 match(Set dst (MulVS src (LoadVector mem)));
7054 format %{ "vpmullw $dst,$src,$mem\t! mul packed32S" %}
7055 ins_encode %{
7056 int vector_len = 2;
7057 __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7058 %}
7059 ins_pipe( pipe_slow );
7060 %}
7061
7062 // Integers vector mul (sse4_1)
7063 instruct vmul2I(vecD dst, vecD src) %{
7064 predicate(UseSSE > 3 && n->as_Vector()->length() == 2);
7065 match(Set dst (MulVI dst src));
7066 format %{ "pmulld $dst,$src\t! mul packed2I" %}
7067 ins_encode %{
7068 __ pmulld($dst$$XMMRegister, $src$$XMMRegister);
7069 %}
7070 ins_pipe( pipe_slow );
7071 %}
7072
7073 instruct vmul2I_reg(vecD dst, vecD src1, vecD src2) %{
7074 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7075 match(Set dst (MulVI src1 src2));
7076 format %{ "vpmulld $dst,$src1,$src2\t! mul packed2I" %}
7077 ins_encode %{
7078 int vector_len = 0;
7079 __ vpmulld($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7080 %}
7081 ins_pipe( pipe_slow );
7082 %}
7083
7084 instruct vmul2I_mem(vecD dst, vecD src, memory mem) %{
7085 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7086 match(Set dst (MulVI src (LoadVector mem)));
7087 format %{ "vpmulld $dst,$src,$mem\t! mul packed2I" %}
7088 ins_encode %{
7089 int vector_len = 0;
7090 __ vpmulld($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7091 %}
7092 ins_pipe( pipe_slow );
7093 %}
7094
7095 instruct vmul4I(vecX dst, vecX src) %{
7096 predicate(UseSSE > 3 && n->as_Vector()->length() == 4);
7097 match(Set dst (MulVI dst src));
7098 format %{ "pmulld $dst,$src\t! mul packed4I" %}
7099 ins_encode %{
7100 __ pmulld($dst$$XMMRegister, $src$$XMMRegister);
7101 %}
7102 ins_pipe( pipe_slow );
7103 %}
7104
7105 instruct vmul4I_reg(vecX dst, vecX src1, vecX src2) %{
7106 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7107 match(Set dst (MulVI src1 src2));
7108 format %{ "vpmulld $dst,$src1,$src2\t! mul packed4I" %}
7109 ins_encode %{
7110 int vector_len = 0;
7111 __ vpmulld($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7112 %}
7113 ins_pipe( pipe_slow );
7114 %}
7115
7116 instruct vmul4I_mem(vecX dst, vecX src, memory mem) %{
7117 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7118 match(Set dst (MulVI src (LoadVector mem)));
7119 format %{ "vpmulld $dst,$src,$mem\t! mul packed4I" %}
7120 ins_encode %{
7121 int vector_len = 0;
7122 __ vpmulld($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7123 %}
7124 ins_pipe( pipe_slow );
7125 %}
7126
7127 instruct vmul2L_reg(vecX dst, vecX src1, vecX src2) %{
7128 predicate(UseAVX > 2 && n->as_Vector()->length() == 2 && VM_Version::supports_avx512dq());
7129 match(Set dst (MulVL src1 src2));
7130 format %{ "vpmullq $dst,$src1,$src2\t! mul packed2L" %}
7131 ins_encode %{
7132 int vector_len = 0;
7133 __ vpmullq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7134 %}
7135 ins_pipe( pipe_slow );
7136 %}
7137
7138 instruct vmul2L_mem(vecX dst, vecX src, memory mem) %{
7139 predicate(UseAVX > 2 && n->as_Vector()->length() == 2 && VM_Version::supports_avx512dq());
7140 match(Set dst (MulVL src (LoadVector mem)));
7141 format %{ "vpmullq $dst,$src,$mem\t! mul packed2L" %}
7142 ins_encode %{
7143 int vector_len = 0;
7144 __ vpmullq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7145 %}
7146 ins_pipe( pipe_slow );
7147 %}
7148
7149 instruct vmul4L_reg(vecY dst, vecY src1, vecY src2) %{
7150 predicate(UseAVX > 2 && n->as_Vector()->length() == 4 && VM_Version::supports_avx512dq());
7151 match(Set dst (MulVL src1 src2));
7152 format %{ "vpmullq $dst,$src1,$src2\t! mul packed4L" %}
7153 ins_encode %{
7154 int vector_len = 1;
7155 __ vpmullq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7156 %}
7157 ins_pipe( pipe_slow );
7158 %}
7159
7160 instruct vmul4L_mem(vecY dst, vecY src, memory mem) %{
7161 predicate(UseAVX > 2 && n->as_Vector()->length() == 4 && VM_Version::supports_avx512dq());
7162 match(Set dst (MulVL src (LoadVector mem)));
7163 format %{ "vpmullq $dst,$src,$mem\t! mul packed4L" %}
7164 ins_encode %{
7165 int vector_len = 1;
7166 __ vpmullq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7167 %}
7168 ins_pipe( pipe_slow );
7169 %}
7170
7171 instruct vmul8L_reg(vecZ dst, vecZ src1, vecZ src2) %{
7172 predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && VM_Version::supports_avx512dq());
7173 match(Set dst (MulVL src1 src2));
7174 format %{ "vpmullq $dst,$src1,$src2\t! mul packed8L" %}
7175 ins_encode %{
7176 int vector_len = 2;
7177 __ vpmullq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7178 %}
7179 ins_pipe( pipe_slow );
7180 %}
7181
7182 instruct vmul8L_mem(vecZ dst, vecZ src, memory mem) %{
7183 predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && VM_Version::supports_avx512dq());
7184 match(Set dst (MulVL src (LoadVector mem)));
7185 format %{ "vpmullq $dst,$src,$mem\t! mul packed8L" %}
7186 ins_encode %{
7187 int vector_len = 2;
7188 __ vpmullq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7189 %}
7190 ins_pipe( pipe_slow );
7191 %}
7192
7193 instruct vmul8I_reg(vecY dst, vecY src1, vecY src2) %{
7194 predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
7195 match(Set dst (MulVI src1 src2));
7196 format %{ "vpmulld $dst,$src1,$src2\t! mul packed8I" %}
7197 ins_encode %{
7198 int vector_len = 1;
7199 __ vpmulld($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7200 %}
7201 ins_pipe( pipe_slow );
7202 %}
7203
7204 instruct vmul8I_mem(vecY dst, vecY src, memory mem) %{
7205 predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
7206 match(Set dst (MulVI src (LoadVector mem)));
7207 format %{ "vpmulld $dst,$src,$mem\t! mul packed8I" %}
7208 ins_encode %{
7209 int vector_len = 1;
7210 __ vpmulld($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7211 %}
7212 ins_pipe( pipe_slow );
7213 %}
7214
7215 instruct vmul16I_reg(vecZ dst, vecZ src1, vecZ src2) %{
7216 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
7217 match(Set dst (MulVI src1 src2));
7218 format %{ "vpmulld $dst,$src1,$src2\t! mul packed16I" %}
7219 ins_encode %{
7220 int vector_len = 2;
7221 __ vpmulld($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7222 %}
7223 ins_pipe( pipe_slow );
7224 %}
7225
7226 instruct vmul16I_mem(vecZ dst, vecZ src, memory mem) %{
7227 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
7228 match(Set dst (MulVI src (LoadVector mem)));
7229 format %{ "vpmulld $dst,$src,$mem\t! mul packed16I" %}
7230 ins_encode %{
7231 int vector_len = 2;
7232 __ vpmulld($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7233 %}
7234 ins_pipe( pipe_slow );
7235 %}
7236
7237 // Floats vector mul
7238 instruct vmul2F(vecD dst, vecD src) %{
7239 predicate(n->as_Vector()->length() == 2);
7240 match(Set dst (MulVF dst src));
7241 format %{ "mulps $dst,$src\t! mul packed2F" %}
7242 ins_encode %{
7243 __ mulps($dst$$XMMRegister, $src$$XMMRegister);
7244 %}
7245 ins_pipe( pipe_slow );
7246 %}
7247
7248 instruct vmul2F_reg(vecD dst, vecD src1, vecD src2) %{
7249 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7250 match(Set dst (MulVF src1 src2));
7251 format %{ "vmulps $dst,$src1,$src2\t! mul packed2F" %}
7252 ins_encode %{
7253 int vector_len = 0;
7254 __ vmulps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7255 %}
7256 ins_pipe( pipe_slow );
7257 %}
7258
7259 instruct vmul2F_mem(vecD dst, vecD src, memory mem) %{
7260 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7261 match(Set dst (MulVF src (LoadVector mem)));
7262 format %{ "vmulps $dst,$src,$mem\t! mul packed2F" %}
7263 ins_encode %{
7264 int vector_len = 0;
7265 __ vmulps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7266 %}
7267 ins_pipe( pipe_slow );
7268 %}
7269
7270 instruct vmul4F(vecX dst, vecX src) %{
7271 predicate(n->as_Vector()->length() == 4);
7272 match(Set dst (MulVF dst src));
7273 format %{ "mulps $dst,$src\t! mul packed4F" %}
7274 ins_encode %{
7275 __ mulps($dst$$XMMRegister, $src$$XMMRegister);
7276 %}
7277 ins_pipe( pipe_slow );
7278 %}
7279
7280 instruct vmul4F_reg(vecX dst, vecX src1, vecX src2) %{
7281 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7282 match(Set dst (MulVF src1 src2));
7283 format %{ "vmulps $dst,$src1,$src2\t! mul packed4F" %}
7284 ins_encode %{
7285 int vector_len = 0;
7286 __ vmulps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7287 %}
7288 ins_pipe( pipe_slow );
7289 %}
7290
7291 instruct vmul4F_mem(vecX dst, vecX src, memory mem) %{
7292 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7293 match(Set dst (MulVF src (LoadVector mem)));
7294 format %{ "vmulps $dst,$src,$mem\t! mul packed4F" %}
7295 ins_encode %{
7296 int vector_len = 0;
7297 __ vmulps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7298 %}
7299 ins_pipe( pipe_slow );
7300 %}
7301
7302 instruct vmul8F_reg(vecY dst, vecY src1, vecY src2) %{
7303 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
7304 match(Set dst (MulVF src1 src2));
7305 format %{ "vmulps $dst,$src1,$src2\t! mul packed8F" %}
7306 ins_encode %{
7307 int vector_len = 1;
7308 __ vmulps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7309 %}
7310 ins_pipe( pipe_slow );
7311 %}
7312
7313 instruct vmul8F_mem(vecY dst, vecY src, memory mem) %{
7314 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
7315 match(Set dst (MulVF src (LoadVector mem)));
7316 format %{ "vmulps $dst,$src,$mem\t! mul packed8F" %}
7317 ins_encode %{
7318 int vector_len = 1;
7319 __ vmulps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7320 %}
7321 ins_pipe( pipe_slow );
7322 %}
7323
7324 instruct vmul16F_reg(vecZ dst, vecZ src1, vecZ src2) %{
7325 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
7326 match(Set dst (MulVF src1 src2));
7327 format %{ "vmulps $dst,$src1,$src2\t! mul packed16F" %}
7328 ins_encode %{
7329 int vector_len = 2;
7330 __ vmulps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7331 %}
7332 ins_pipe( pipe_slow );
7333 %}
7334
7335 instruct vmul16F_mem(vecZ dst, vecZ src, memory mem) %{
7336 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
7337 match(Set dst (MulVF src (LoadVector mem)));
7338 format %{ "vmulps $dst,$src,$mem\t! mul packed16F" %}
7339 ins_encode %{
7340 int vector_len = 2;
7341 __ vmulps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7342 %}
7343 ins_pipe( pipe_slow );
7344 %}
7345
7346 // Doubles vector mul
7347 instruct vmul2D(vecX dst, vecX src) %{
7348 predicate(n->as_Vector()->length() == 2);
7349 match(Set dst (MulVD dst src));
7350 format %{ "mulpd $dst,$src\t! mul packed2D" %}
7351 ins_encode %{
7352 __ mulpd($dst$$XMMRegister, $src$$XMMRegister);
7353 %}
7354 ins_pipe( pipe_slow );
7355 %}
7356
7357 instruct vmul2D_reg(vecX dst, vecX src1, vecX src2) %{
7358 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7359 match(Set dst (MulVD src1 src2));
7360 format %{ "vmulpd $dst,$src1,$src2\t! mul packed2D" %}
7361 ins_encode %{
7362 int vector_len = 0;
7363 __ vmulpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7364 %}
7365 ins_pipe( pipe_slow );
7366 %}
7367
7368 instruct vmul2D_mem(vecX dst, vecX src, memory mem) %{
7369 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7370 match(Set dst (MulVD src (LoadVector mem)));
7371 format %{ "vmulpd $dst,$src,$mem\t! mul packed2D" %}
7372 ins_encode %{
7373 int vector_len = 0;
7374 __ vmulpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7375 %}
7376 ins_pipe( pipe_slow );
7377 %}
7378
7379 instruct vmul4D_reg(vecY dst, vecY src1, vecY src2) %{
7380 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7381 match(Set dst (MulVD src1 src2));
7382 format %{ "vmulpd $dst,$src1,$src2\t! mul packed4D" %}
7383 ins_encode %{
7384 int vector_len = 1;
7385 __ vmulpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7386 %}
7387 ins_pipe( pipe_slow );
7388 %}
7389
7390 instruct vmul4D_mem(vecY dst, vecY src, memory mem) %{
7391 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7392 match(Set dst (MulVD src (LoadVector mem)));
7393 format %{ "vmulpd $dst,$src,$mem\t! mul packed4D" %}
7394 ins_encode %{
7395 int vector_len = 1;
7396 __ vmulpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7397 %}
7398 ins_pipe( pipe_slow );
7399 %}
7400
7401 instruct vmul8D_reg(vecZ dst, vecZ src1, vecZ src2) %{
7402 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
7403 match(Set dst (MulVD src1 src2));
7404 format %{ "vmulpd $dst k0,$src1,$src2\t! mul packed8D" %}
7405 ins_encode %{
7406 int vector_len = 2;
7407 __ vmulpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7408 %}
7409 ins_pipe( pipe_slow );
7410 %}
7411
7412 instruct vmul8D_mem(vecZ dst, vecZ src, memory mem) %{
7413 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
7414 match(Set dst (MulVD src (LoadVector mem)));
7415 format %{ "vmulpd $dst k0,$src,$mem\t! mul packed8D" %}
7416 ins_encode %{
7417 int vector_len = 2;
7418 __ vmulpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7419 %}
7420 ins_pipe( pipe_slow );
7421 %}
7422
7423 instruct vcmov4D_reg(vecY dst, vecY src1, vecY src2, immI8 cop, cmpOp_vcmppd copnd) %{
7424 predicate(UseAVX > 0 && UseAVX < 3 && n->as_Vector()->length() == 4);
7425 match(Set dst (CMoveVD (Binary copnd cop) (Binary src1 src2)));
7426 effect(TEMP dst, USE src1, USE src2);
7427 format %{ "cmppd.$copnd $dst, $src1, $src2 ! vcmovevd, cond=$cop\n\t"
7428 "vpblendd $dst,$src1,$src2,$dst ! vcmovevd\n\t"
7429 %}
7430 ins_encode %{
7431 int vector_len = 1;
7432 int cond = (Assembler::Condition)($copnd$$cmpcode);
7433 __ cmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cond, vector_len);
7434 __ vpblendd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $dst$$XMMRegister, vector_len);
7435 %}
7436 ins_pipe( pipe_slow );
7437 %}
7438
7439 // --------------------------------- DIV --------------------------------------
7440
7441 // Floats vector div
7442 instruct vdiv2F(vecD dst, vecD src) %{
7443 predicate(n->as_Vector()->length() == 2);
7444 match(Set dst (DivVF dst src));
7445 format %{ "divps $dst,$src\t! div packed2F" %}
7446 ins_encode %{
7447 __ divps($dst$$XMMRegister, $src$$XMMRegister);
7448 %}
7449 ins_pipe( pipe_slow );
7450 %}
7451
7452 instruct vdiv2F_reg(vecD dst, vecD src1, vecD src2) %{
7453 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7454 match(Set dst (DivVF src1 src2));
7455 format %{ "vdivps $dst,$src1,$src2\t! div packed2F" %}
7456 ins_encode %{
7457 int vector_len = 0;
7458 __ vdivps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7459 %}
7460 ins_pipe( pipe_slow );
7461 %}
7462
7463 instruct vdiv2F_mem(vecD dst, vecD src, memory mem) %{
7464 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7465 match(Set dst (DivVF src (LoadVector mem)));
7466 format %{ "vdivps $dst,$src,$mem\t! div packed2F" %}
7467 ins_encode %{
7468 int vector_len = 0;
7469 __ vdivps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7470 %}
7471 ins_pipe( pipe_slow );
7472 %}
7473
7474 instruct vdiv4F(vecX dst, vecX src) %{
7475 predicate(n->as_Vector()->length() == 4);
7476 match(Set dst (DivVF dst src));
7477 format %{ "divps $dst,$src\t! div packed4F" %}
7478 ins_encode %{
7479 __ divps($dst$$XMMRegister, $src$$XMMRegister);
7480 %}
7481 ins_pipe( pipe_slow );
7482 %}
7483
7484 instruct vdiv4F_reg(vecX dst, vecX src1, vecX src2) %{
7485 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7486 match(Set dst (DivVF src1 src2));
7487 format %{ "vdivps $dst,$src1,$src2\t! div packed4F" %}
7488 ins_encode %{
7489 int vector_len = 0;
7490 __ vdivps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7491 %}
7492 ins_pipe( pipe_slow );
7493 %}
7494
7495 instruct vdiv4F_mem(vecX dst, vecX src, memory mem) %{
7496 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7497 match(Set dst (DivVF src (LoadVector mem)));
7498 format %{ "vdivps $dst,$src,$mem\t! div packed4F" %}
7499 ins_encode %{
7500 int vector_len = 0;
7501 __ vdivps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7502 %}
7503 ins_pipe( pipe_slow );
7504 %}
7505
7506 instruct vdiv8F_reg(vecY dst, vecY src1, vecY src2) %{
7507 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
7508 match(Set dst (DivVF src1 src2));
7509 format %{ "vdivps $dst,$src1,$src2\t! div packed8F" %}
7510 ins_encode %{
7511 int vector_len = 1;
7512 __ vdivps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7513 %}
7514 ins_pipe( pipe_slow );
7515 %}
7516
7517 instruct vdiv8F_mem(vecY dst, vecY src, memory mem) %{
7518 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
7519 match(Set dst (DivVF src (LoadVector mem)));
7520 format %{ "vdivps $dst,$src,$mem\t! div packed8F" %}
7521 ins_encode %{
7522 int vector_len = 1;
7523 __ vdivps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7524 %}
7525 ins_pipe( pipe_slow );
7526 %}
7527
7528 instruct vdiv16F_reg(vecZ dst, vecZ src1, vecZ src2) %{
7529 predicate(UseAVX > 0 && n->as_Vector()->length() == 16);
7530 match(Set dst (DivVF src1 src2));
7531 format %{ "vdivps $dst,$src1,$src2\t! div packed16F" %}
7532 ins_encode %{
7533 int vector_len = 2;
7534 __ vdivps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7535 %}
7536 ins_pipe( pipe_slow );
7537 %}
7538
7539 instruct vdiv16F_mem(vecZ dst, vecZ src, memory mem) %{
7540 predicate(UseAVX > 0 && n->as_Vector()->length() == 16);
7541 match(Set dst (DivVF src (LoadVector mem)));
7542 format %{ "vdivps $dst,$src,$mem\t! div packed16F" %}
7543 ins_encode %{
7544 int vector_len = 2;
7545 __ vdivps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7546 %}
7547 ins_pipe( pipe_slow );
7548 %}
7549
7550 // Doubles vector div
7551 instruct vdiv2D(vecX dst, vecX src) %{
7552 predicate(n->as_Vector()->length() == 2);
7553 match(Set dst (DivVD dst src));
7554 format %{ "divpd $dst,$src\t! div packed2D" %}
7555 ins_encode %{
7556 __ divpd($dst$$XMMRegister, $src$$XMMRegister);
7557 %}
7558 ins_pipe( pipe_slow );
7559 %}
7560
7561 instruct vdiv2D_reg(vecX dst, vecX src1, vecX src2) %{
7562 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7563 match(Set dst (DivVD src1 src2));
7564 format %{ "vdivpd $dst,$src1,$src2\t! div packed2D" %}
7565 ins_encode %{
7566 int vector_len = 0;
7567 __ vdivpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7568 %}
7569 ins_pipe( pipe_slow );
7570 %}
7571
7572 instruct vdiv2D_mem(vecX dst, vecX src, memory mem) %{
7573 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7574 match(Set dst (DivVD src (LoadVector mem)));
7575 format %{ "vdivpd $dst,$src,$mem\t! div packed2D" %}
7576 ins_encode %{
7577 int vector_len = 0;
7578 __ vdivpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7579 %}
7580 ins_pipe( pipe_slow );
7581 %}
7582
7583 instruct vdiv4D_reg(vecY dst, vecY src1, vecY src2) %{
7584 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7585 match(Set dst (DivVD src1 src2));
7586 format %{ "vdivpd $dst,$src1,$src2\t! div packed4D" %}
7587 ins_encode %{
7588 int vector_len = 1;
7589 __ vdivpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7590 %}
7591 ins_pipe( pipe_slow );
7592 %}
7593
7594 instruct vdiv4D_mem(vecY dst, vecY src, memory mem) %{
7595 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7596 match(Set dst (DivVD src (LoadVector mem)));
7597 format %{ "vdivpd $dst,$src,$mem\t! div packed4D" %}
7598 ins_encode %{
7599 int vector_len = 1;
7600 __ vdivpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7601 %}
7602 ins_pipe( pipe_slow );
7603 %}
7604
7605 instruct vdiv8D_reg(vecZ dst, vecZ src1, vecZ src2) %{
7606 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
7607 match(Set dst (DivVD src1 src2));
7608 format %{ "vdivpd $dst,$src1,$src2\t! div packed8D" %}
7609 ins_encode %{
7610 int vector_len = 2;
7611 __ vdivpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
7612 %}
7613 ins_pipe( pipe_slow );
7614 %}
7615
7616 instruct vdiv8D_mem(vecZ dst, vecZ src, memory mem) %{
7617 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
7618 match(Set dst (DivVD src (LoadVector mem)));
7619 format %{ "vdivpd $dst,$src,$mem\t! div packed8D" %}
7620 ins_encode %{
7621 int vector_len = 2;
7622 __ vdivpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
7623 %}
7624 ins_pipe( pipe_slow );
7625 %}
7626
7627 // ------------------------------ Shift ---------------------------------------
7628
7629 // Left and right shift count vectors are the same on x86
7630 // (only lowest bits of xmm reg are used for count).
7631 instruct vshiftcnt(vecS dst, rRegI cnt) %{
7632 match(Set dst (LShiftCntV cnt));
7633 match(Set dst (RShiftCntV cnt));
7634 format %{ "movd $dst,$cnt\t! load shift count" %}
7635 ins_encode %{
7636 __ movdl($dst$$XMMRegister, $cnt$$Register);
7637 %}
7638 ins_pipe( pipe_slow );
7639 %}
7640
7641 // --------------------------------- Sqrt --------------------------------------
7642
7643 // Floating point vector sqrt - double precision only
7644 instruct vsqrt2D_reg(vecX dst, vecX src) %{
7645 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7646 match(Set dst (SqrtVD src));
7647 format %{ "vsqrtpd $dst,$src\t! sqrt packed2D" %}
7648 ins_encode %{
7649 int vector_len = 0;
7650 __ vsqrtpd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
7651 %}
7652 ins_pipe( pipe_slow );
7653 %}
7654
7655 instruct vsqrt2D_mem(vecX dst, memory mem) %{
7656 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7657 match(Set dst (SqrtVD (LoadVector mem)));
7658 format %{ "vsqrtpd $dst,$mem\t! sqrt packed2D" %}
7659 ins_encode %{
7660 int vector_len = 0;
7661 __ vsqrtpd($dst$$XMMRegister, $mem$$Address, vector_len);
7662 %}
7663 ins_pipe( pipe_slow );
7664 %}
7665
7666 instruct vsqrt4D_reg(vecY dst, vecY src) %{
7667 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7668 match(Set dst (SqrtVD src));
7669 format %{ "vsqrtpd $dst,$src\t! sqrt packed4D" %}
7670 ins_encode %{
7671 int vector_len = 1;
7672 __ vsqrtpd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
7673 %}
7674 ins_pipe( pipe_slow );
7675 %}
7676
7677 instruct vsqrt4D_mem(vecY dst, memory mem) %{
7678 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7679 match(Set dst (SqrtVD (LoadVector mem)));
7680 format %{ "vsqrtpd $dst,$mem\t! sqrt packed4D" %}
7681 ins_encode %{
7682 int vector_len = 1;
7683 __ vsqrtpd($dst$$XMMRegister, $mem$$Address, vector_len);
7684 %}
7685 ins_pipe( pipe_slow );
7686 %}
7687
7688 instruct vsqrt8D_reg(vecZ dst, vecZ src) %{
7689 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
7690 match(Set dst (SqrtVD src));
7691 format %{ "vsqrtpd $dst,$src\t! sqrt packed8D" %}
7692 ins_encode %{
7693 int vector_len = 2;
7694 __ vsqrtpd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
7695 %}
7696 ins_pipe( pipe_slow );
7697 %}
7698
7699 instruct vsqrt8D_mem(vecZ dst, memory mem) %{
7700 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
7701 match(Set dst (SqrtVD (LoadVector mem)));
7702 format %{ "vsqrtpd $dst,$mem\t! sqrt packed8D" %}
7703 ins_encode %{
7704 int vector_len = 2;
7705 __ vsqrtpd($dst$$XMMRegister, $mem$$Address, vector_len);
7706 %}
7707 ins_pipe( pipe_slow );
7708 %}
7709
7710 // ------------------------------ LeftShift -----------------------------------
7711
7712 // Shorts/Chars vector left shift
7713 instruct vsll2S(vecS dst, vecS shift) %{
7714 predicate(n->as_Vector()->length() == 2);
7715 match(Set dst (LShiftVS dst shift));
7716 format %{ "psllw $dst,$shift\t! left shift packed2S" %}
7717 ins_encode %{
7718 __ psllw($dst$$XMMRegister, $shift$$XMMRegister);
7719 %}
7720 ins_pipe( pipe_slow );
7721 %}
7722
7723 instruct vsll2S_imm(vecS dst, immI8 shift) %{
7724 predicate(n->as_Vector()->length() == 2);
7725 match(Set dst (LShiftVS dst shift));
7726 format %{ "psllw $dst,$shift\t! left shift packed2S" %}
7727 ins_encode %{
7728 __ psllw($dst$$XMMRegister, (int)$shift$$constant);
7729 %}
7730 ins_pipe( pipe_slow );
7731 %}
7732
7733 instruct vsll2S_reg(vecS dst, vecS src, vecS shift) %{
7734 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7735 match(Set dst (LShiftVS src shift));
7736 format %{ "vpsllw $dst,$src,$shift\t! left shift packed2S" %}
7737 ins_encode %{
7738 int vector_len = 0;
7739 __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
7740 %}
7741 ins_pipe( pipe_slow );
7742 %}
7743
7744 instruct vsll2S_reg_imm(vecS dst, vecS src, immI8 shift) %{
7745 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7746 match(Set dst (LShiftVS src shift));
7747 format %{ "vpsllw $dst,$src,$shift\t! left shift packed2S" %}
7748 ins_encode %{
7749 int vector_len = 0;
7750 __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
7751 %}
7752 ins_pipe( pipe_slow );
7753 %}
7754
7755 instruct vsll4S(vecD dst, vecS shift) %{
7756 predicate(n->as_Vector()->length() == 4);
7757 match(Set dst (LShiftVS dst shift));
7758 format %{ "psllw $dst,$shift\t! left shift packed4S" %}
7759 ins_encode %{
7760 __ psllw($dst$$XMMRegister, $shift$$XMMRegister);
7761 %}
7762 ins_pipe( pipe_slow );
7763 %}
7764
7765 instruct vsll4S_imm(vecD dst, immI8 shift) %{
7766 predicate(n->as_Vector()->length() == 4);
7767 match(Set dst (LShiftVS dst shift));
7768 format %{ "psllw $dst,$shift\t! left shift packed4S" %}
7769 ins_encode %{
7770 __ psllw($dst$$XMMRegister, (int)$shift$$constant);
7771 %}
7772 ins_pipe( pipe_slow );
7773 %}
7774
7775 instruct vsll4S_reg(vecD dst, vecD src, vecS shift) %{
7776 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7777 match(Set dst (LShiftVS src shift));
7778 format %{ "vpsllw $dst,$src,$shift\t! left shift packed4S" %}
7779 ins_encode %{
7780 int vector_len = 0;
7781 __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
7782 %}
7783 ins_pipe( pipe_slow );
7784 %}
7785
7786 instruct vsll4S_reg_imm(vecD dst, vecD src, immI8 shift) %{
7787 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7788 match(Set dst (LShiftVS src shift));
7789 format %{ "vpsllw $dst,$src,$shift\t! left shift packed4S" %}
7790 ins_encode %{
7791 int vector_len = 0;
7792 __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
7793 %}
7794 ins_pipe( pipe_slow );
7795 %}
7796
7797 instruct vsll8S(vecX dst, vecS shift) %{
7798 predicate(n->as_Vector()->length() == 8);
7799 match(Set dst (LShiftVS dst shift));
7800 format %{ "psllw $dst,$shift\t! left shift packed8S" %}
7801 ins_encode %{
7802 __ psllw($dst$$XMMRegister, $shift$$XMMRegister);
7803 %}
7804 ins_pipe( pipe_slow );
7805 %}
7806
7807 instruct vsll8S_imm(vecX dst, immI8 shift) %{
7808 predicate(n->as_Vector()->length() == 8);
7809 match(Set dst (LShiftVS dst shift));
7810 format %{ "psllw $dst,$shift\t! left shift packed8S" %}
7811 ins_encode %{
7812 __ psllw($dst$$XMMRegister, (int)$shift$$constant);
7813 %}
7814 ins_pipe( pipe_slow );
7815 %}
7816
7817 instruct vsll8S_reg(vecX dst, vecX src, vecS shift) %{
7818 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
7819 match(Set dst (LShiftVS src shift));
7820 format %{ "vpsllw $dst,$src,$shift\t! left shift packed8S" %}
7821 ins_encode %{
7822 int vector_len = 0;
7823 __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
7824 %}
7825 ins_pipe( pipe_slow );
7826 %}
7827
7828 instruct vsll8S_reg_imm(vecX dst, vecX src, immI8 shift) %{
7829 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
7830 match(Set dst (LShiftVS src shift));
7831 format %{ "vpsllw $dst,$src,$shift\t! left shift packed8S" %}
7832 ins_encode %{
7833 int vector_len = 0;
7834 __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
7835 %}
7836 ins_pipe( pipe_slow );
7837 %}
7838
7839 instruct vsll16S_reg(vecY dst, vecY src, vecS shift) %{
7840 predicate(UseAVX > 1 && n->as_Vector()->length() == 16);
7841 match(Set dst (LShiftVS src shift));
7842 format %{ "vpsllw $dst,$src,$shift\t! left shift packed16S" %}
7843 ins_encode %{
7844 int vector_len = 1;
7845 __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
7846 %}
7847 ins_pipe( pipe_slow );
7848 %}
7849
7850 instruct vsll16S_reg_imm(vecY dst, vecY src, immI8 shift) %{
7851 predicate(UseAVX > 1 && n->as_Vector()->length() == 16);
7852 match(Set dst (LShiftVS src shift));
7853 format %{ "vpsllw $dst,$src,$shift\t! left shift packed16S" %}
7854 ins_encode %{
7855 int vector_len = 1;
7856 __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
7857 %}
7858 ins_pipe( pipe_slow );
7859 %}
7860
7861 instruct vsll32S_reg(vecZ dst, vecZ src, vecS shift) %{
7862 predicate(UseAVX > 2 && n->as_Vector()->length() == 32);
7863 match(Set dst (LShiftVS src shift));
7864 format %{ "vpsllw $dst,$src,$shift\t! left shift packed32S" %}
7865 ins_encode %{
7866 int vector_len = 2;
7867 __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
7868 %}
7869 ins_pipe( pipe_slow );
7870 %}
7871
7872 instruct vsll32S_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
7873 predicate(UseAVX > 2 && n->as_Vector()->length() == 32);
7874 match(Set dst (LShiftVS src shift));
7875 format %{ "vpsllw $dst,$src,$shift\t! left shift packed32S" %}
7876 ins_encode %{
7877 int vector_len = 2;
7878 __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
7879 %}
7880 ins_pipe( pipe_slow );
7881 %}
7882
7883 // Integers vector left shift
7884 instruct vsll2I(vecD dst, vecS shift) %{
7885 predicate(n->as_Vector()->length() == 2);
7886 match(Set dst (LShiftVI dst shift));
7887 format %{ "pslld $dst,$shift\t! left shift packed2I" %}
7888 ins_encode %{
7889 __ pslld($dst$$XMMRegister, $shift$$XMMRegister);
7890 %}
7891 ins_pipe( pipe_slow );
7892 %}
7893
7894 instruct vsll2I_imm(vecD dst, immI8 shift) %{
7895 predicate(n->as_Vector()->length() == 2);
7896 match(Set dst (LShiftVI dst shift));
7897 format %{ "pslld $dst,$shift\t! left shift packed2I" %}
7898 ins_encode %{
7899 __ pslld($dst$$XMMRegister, (int)$shift$$constant);
7900 %}
7901 ins_pipe( pipe_slow );
7902 %}
7903
7904 instruct vsll2I_reg(vecD dst, vecD src, vecS shift) %{
7905 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7906 match(Set dst (LShiftVI src shift));
7907 format %{ "vpslld $dst,$src,$shift\t! left shift packed2I" %}
7908 ins_encode %{
7909 int vector_len = 0;
7910 __ vpslld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
7911 %}
7912 ins_pipe( pipe_slow );
7913 %}
7914
7915 instruct vsll2I_reg_imm(vecD dst, vecD src, immI8 shift) %{
7916 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
7917 match(Set dst (LShiftVI src shift));
7918 format %{ "vpslld $dst,$src,$shift\t! left shift packed2I" %}
7919 ins_encode %{
7920 int vector_len = 0;
7921 __ vpslld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
7922 %}
7923 ins_pipe( pipe_slow );
7924 %}
7925
7926 instruct vsll4I(vecX dst, vecS shift) %{
7927 predicate(n->as_Vector()->length() == 4);
7928 match(Set dst (LShiftVI dst shift));
7929 format %{ "pslld $dst,$shift\t! left shift packed4I" %}
7930 ins_encode %{
7931 __ pslld($dst$$XMMRegister, $shift$$XMMRegister);
7932 %}
7933 ins_pipe( pipe_slow );
7934 %}
7935
7936 instruct vsll4I_imm(vecX dst, immI8 shift) %{
7937 predicate(n->as_Vector()->length() == 4);
7938 match(Set dst (LShiftVI dst shift));
7939 format %{ "pslld $dst,$shift\t! left shift packed4I" %}
7940 ins_encode %{
7941 __ pslld($dst$$XMMRegister, (int)$shift$$constant);
7942 %}
7943 ins_pipe( pipe_slow );
7944 %}
7945
7946 instruct vsll4I_reg(vecX dst, vecX src, vecS shift) %{
7947 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7948 match(Set dst (LShiftVI src shift));
7949 format %{ "vpslld $dst,$src,$shift\t! left shift packed4I" %}
7950 ins_encode %{
7951 int vector_len = 0;
7952 __ vpslld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
7953 %}
7954 ins_pipe( pipe_slow );
7955 %}
7956
7957 instruct vsll4I_reg_imm(vecX dst, vecX src, immI8 shift) %{
7958 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
7959 match(Set dst (LShiftVI src shift));
7960 format %{ "vpslld $dst,$src,$shift\t! left shift packed4I" %}
7961 ins_encode %{
7962 int vector_len = 0;
7963 __ vpslld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
7964 %}
7965 ins_pipe( pipe_slow );
7966 %}
7967
7968 instruct vsll8I_reg(vecY dst, vecY src, vecS shift) %{
7969 predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
7970 match(Set dst (LShiftVI src shift));
7971 format %{ "vpslld $dst,$src,$shift\t! left shift packed8I" %}
7972 ins_encode %{
7973 int vector_len = 1;
7974 __ vpslld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
7975 %}
7976 ins_pipe( pipe_slow );
7977 %}
7978
7979 instruct vsll8I_reg_imm(vecY dst, vecY src, immI8 shift) %{
7980 predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
7981 match(Set dst (LShiftVI src shift));
7982 format %{ "vpslld $dst,$src,$shift\t! left shift packed8I" %}
7983 ins_encode %{
7984 int vector_len = 1;
7985 __ vpslld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
7986 %}
7987 ins_pipe( pipe_slow );
7988 %}
7989
7990 instruct vsll16I_reg(vecZ dst, vecZ src, vecS shift) %{
7991 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
7992 match(Set dst (LShiftVI src shift));
7993 format %{ "vpslld $dst,$src,$shift\t! left shift packed16I" %}
7994 ins_encode %{
7995 int vector_len = 2;
7996 __ vpslld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
7997 %}
7998 ins_pipe( pipe_slow );
7999 %}
8000
8001 instruct vsll16I_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
8002 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
8003 match(Set dst (LShiftVI src shift));
8004 format %{ "vpslld $dst,$src,$shift\t! left shift packed16I" %}
8005 ins_encode %{
8006 int vector_len = 2;
8007 __ vpslld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8008 %}
8009 ins_pipe( pipe_slow );
8010 %}
8011
8012 // Longs vector left shift
8013 instruct vsll2L(vecX dst, vecS shift) %{
8014 predicate(n->as_Vector()->length() == 2);
8015 match(Set dst (LShiftVL dst shift));
8016 format %{ "psllq $dst,$shift\t! left shift packed2L" %}
8017 ins_encode %{
8018 __ psllq($dst$$XMMRegister, $shift$$XMMRegister);
8019 %}
8020 ins_pipe( pipe_slow );
8021 %}
8022
8023 instruct vsll2L_imm(vecX dst, immI8 shift) %{
8024 predicate(n->as_Vector()->length() == 2);
8025 match(Set dst (LShiftVL dst shift));
8026 format %{ "psllq $dst,$shift\t! left shift packed2L" %}
8027 ins_encode %{
8028 __ psllq($dst$$XMMRegister, (int)$shift$$constant);
8029 %}
8030 ins_pipe( pipe_slow );
8031 %}
8032
8033 instruct vsll2L_reg(vecX dst, vecX src, vecS shift) %{
8034 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
8035 match(Set dst (LShiftVL src shift));
8036 format %{ "vpsllq $dst,$src,$shift\t! left shift packed2L" %}
8037 ins_encode %{
8038 int vector_len = 0;
8039 __ vpsllq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8040 %}
8041 ins_pipe( pipe_slow );
8042 %}
8043
8044 instruct vsll2L_reg_imm(vecX dst, vecX src, immI8 shift) %{
8045 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
8046 match(Set dst (LShiftVL src shift));
8047 format %{ "vpsllq $dst,$src,$shift\t! left shift packed2L" %}
8048 ins_encode %{
8049 int vector_len = 0;
8050 __ vpsllq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8051 %}
8052 ins_pipe( pipe_slow );
8053 %}
8054
8055 instruct vsll4L_reg(vecY dst, vecY src, vecS shift) %{
8056 predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
8057 match(Set dst (LShiftVL src shift));
8058 format %{ "vpsllq $dst,$src,$shift\t! left shift packed4L" %}
8059 ins_encode %{
8060 int vector_len = 1;
8061 __ vpsllq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8062 %}
8063 ins_pipe( pipe_slow );
8064 %}
8065
8066 instruct vsll4L_reg_imm(vecY dst, vecY src, immI8 shift) %{
8067 predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
8068 match(Set dst (LShiftVL src shift));
8069 format %{ "vpsllq $dst,$src,$shift\t! left shift packed4L" %}
8070 ins_encode %{
8071 int vector_len = 1;
8072 __ vpsllq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8073 %}
8074 ins_pipe( pipe_slow );
8075 %}
8076
8077 instruct vsll8L_reg(vecZ dst, vecZ src, vecS shift) %{
8078 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
8079 match(Set dst (LShiftVL src shift));
8080 format %{ "vpsllq $dst,$src,$shift\t! left shift packed8L" %}
8081 ins_encode %{
8082 int vector_len = 2;
8083 __ vpsllq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8084 %}
8085 ins_pipe( pipe_slow );
8086 %}
8087
8088 instruct vsll8L_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
8089 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
8090 match(Set dst (LShiftVL src shift));
8091 format %{ "vpsllq $dst,$src,$shift\t! left shift packed8L" %}
8092 ins_encode %{
8093 int vector_len = 2;
8094 __ vpsllq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8095 %}
8096 ins_pipe( pipe_slow );
8097 %}
8098
8099 // ----------------------- LogicalRightShift -----------------------------------
8100
8101 // Shorts vector logical right shift produces incorrect Java result
8102 // for negative data because java code convert short value into int with
8103 // sign extension before a shift. But char vectors are fine since chars are
8104 // unsigned values.
8105
8106 instruct vsrl2S(vecS dst, vecS shift) %{
8107 predicate(n->as_Vector()->length() == 2);
8108 match(Set dst (URShiftVS dst shift));
8109 format %{ "psrlw $dst,$shift\t! logical right shift packed2S" %}
8110 ins_encode %{
8111 __ psrlw($dst$$XMMRegister, $shift$$XMMRegister);
8112 %}
8113 ins_pipe( pipe_slow );
8114 %}
8115
8116 instruct vsrl2S_imm(vecS dst, immI8 shift) %{
8117 predicate(n->as_Vector()->length() == 2);
8118 match(Set dst (URShiftVS dst shift));
8119 format %{ "psrlw $dst,$shift\t! logical right shift packed2S" %}
8120 ins_encode %{
8121 __ psrlw($dst$$XMMRegister, (int)$shift$$constant);
8122 %}
8123 ins_pipe( pipe_slow );
8124 %}
8125
8126 instruct vsrl2S_reg(vecS dst, vecS src, vecS shift) %{
8127 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
8128 match(Set dst (URShiftVS src shift));
8129 format %{ "vpsrlw $dst,$src,$shift\t! logical right shift packed2S" %}
8130 ins_encode %{
8131 int vector_len = 0;
8132 __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8133 %}
8134 ins_pipe( pipe_slow );
8135 %}
8136
8137 instruct vsrl2S_reg_imm(vecS dst, vecS src, immI8 shift) %{
8138 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
8139 match(Set dst (URShiftVS src shift));
8140 format %{ "vpsrlw $dst,$src,$shift\t! logical right shift packed2S" %}
8141 ins_encode %{
8142 int vector_len = 0;
8143 __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8144 %}
8145 ins_pipe( pipe_slow );
8146 %}
8147
8148 instruct vsrl4S(vecD dst, vecS shift) %{
8149 predicate(n->as_Vector()->length() == 4);
8150 match(Set dst (URShiftVS dst shift));
8151 format %{ "psrlw $dst,$shift\t! logical right shift packed4S" %}
8152 ins_encode %{
8153 __ psrlw($dst$$XMMRegister, $shift$$XMMRegister);
8154 %}
8155 ins_pipe( pipe_slow );
8156 %}
8157
8158 instruct vsrl4S_imm(vecD dst, immI8 shift) %{
8159 predicate(n->as_Vector()->length() == 4);
8160 match(Set dst (URShiftVS dst shift));
8161 format %{ "psrlw $dst,$shift\t! logical right shift packed4S" %}
8162 ins_encode %{
8163 __ psrlw($dst$$XMMRegister, (int)$shift$$constant);
8164 %}
8165 ins_pipe( pipe_slow );
8166 %}
8167
8168 instruct vsrl4S_reg(vecD dst, vecD src, vecS shift) %{
8169 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
8170 match(Set dst (URShiftVS src shift));
8171 format %{ "vpsrlw $dst,$src,$shift\t! logical right shift packed4S" %}
8172 ins_encode %{
8173 int vector_len = 0;
8174 __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8175 %}
8176 ins_pipe( pipe_slow );
8177 %}
8178
8179 instruct vsrl4S_reg_imm(vecD dst, vecD src, immI8 shift) %{
8180 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
8181 match(Set dst (URShiftVS src shift));
8182 format %{ "vpsrlw $dst,$src,$shift\t! logical right shift packed4S" %}
8183 ins_encode %{
8184 int vector_len = 0;
8185 __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8186 %}
8187 ins_pipe( pipe_slow );
8188 %}
8189
8190 instruct vsrl8S(vecX dst, vecS shift) %{
8191 predicate(n->as_Vector()->length() == 8);
8192 match(Set dst (URShiftVS dst shift));
8193 format %{ "psrlw $dst,$shift\t! logical right shift packed8S" %}
8194 ins_encode %{
8195 __ psrlw($dst$$XMMRegister, $shift$$XMMRegister);
8196 %}
8197 ins_pipe( pipe_slow );
8198 %}
8199
8200 instruct vsrl8S_imm(vecX dst, immI8 shift) %{
8201 predicate(n->as_Vector()->length() == 8);
8202 match(Set dst (URShiftVS dst shift));
8203 format %{ "psrlw $dst,$shift\t! logical right shift packed8S" %}
8204 ins_encode %{
8205 __ psrlw($dst$$XMMRegister, (int)$shift$$constant);
8206 %}
8207 ins_pipe( pipe_slow );
8208 %}
8209
8210 instruct vsrl8S_reg(vecX dst, vecX src, vecS shift) %{
8211 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
8212 match(Set dst (URShiftVS src shift));
8213 format %{ "vpsrlw $dst,$src,$shift\t! logical right shift packed8S" %}
8214 ins_encode %{
8215 int vector_len = 0;
8216 __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8217 %}
8218 ins_pipe( pipe_slow );
8219 %}
8220
8221 instruct vsrl8S_reg_imm(vecX dst, vecX src, immI8 shift) %{
8222 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
8223 match(Set dst (URShiftVS src shift));
8224 format %{ "vpsrlw $dst,$src,$shift\t! logical right shift packed8S" %}
8225 ins_encode %{
8226 int vector_len = 0;
8227 __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8228 %}
8229 ins_pipe( pipe_slow );
8230 %}
8231
8232 instruct vsrl16S_reg(vecY dst, vecY src, vecS shift) %{
8233 predicate(UseAVX > 1 && n->as_Vector()->length() == 16);
8234 match(Set dst (URShiftVS src shift));
8235 format %{ "vpsrlw $dst,$src,$shift\t! logical right shift packed16S" %}
8236 ins_encode %{
8237 int vector_len = 1;
8238 __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8239 %}
8240 ins_pipe( pipe_slow );
8241 %}
8242
8243 instruct vsrl16S_reg_imm(vecY dst, vecY src, immI8 shift) %{
8244 predicate(UseAVX > 1 && n->as_Vector()->length() == 16);
8245 match(Set dst (URShiftVS src shift));
8246 format %{ "vpsrlw $dst,$src,$shift\t! logical right shift packed16S" %}
8247 ins_encode %{
8248 int vector_len = 1;
8249 __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8250 %}
8251 ins_pipe( pipe_slow );
8252 %}
8253
8254 instruct vsrl32S_reg(vecZ dst, vecZ src, vecS shift) %{
8255 predicate(UseAVX > 2 && n->as_Vector()->length() == 32);
8256 match(Set dst (URShiftVS src shift));
8257 format %{ "vpsrlw $dst,$src,$shift\t! logical right shift packed32S" %}
8258 ins_encode %{
8259 int vector_len = 2;
8260 __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8261 %}
8262 ins_pipe( pipe_slow );
8263 %}
8264
8265 instruct vsrl32S_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
8266 predicate(UseAVX > 2 && n->as_Vector()->length() == 32);
8267 match(Set dst (URShiftVS src shift));
8268 format %{ "vpsrlw $dst,$src,$shift\t! logical right shift packed32S" %}
8269 ins_encode %{
8270 int vector_len = 2;
8271 __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8272 %}
8273 ins_pipe( pipe_slow );
8274 %}
8275
8276 // Integers vector logical right shift
8277 instruct vsrl2I(vecD dst, vecS shift) %{
8278 predicate(n->as_Vector()->length() == 2);
8279 match(Set dst (URShiftVI dst shift));
8280 format %{ "psrld $dst,$shift\t! logical right shift packed2I" %}
8281 ins_encode %{
8282 __ psrld($dst$$XMMRegister, $shift$$XMMRegister);
8283 %}
8284 ins_pipe( pipe_slow );
8285 %}
8286
8287 instruct vsrl2I_imm(vecD dst, immI8 shift) %{
8288 predicate(n->as_Vector()->length() == 2);
8289 match(Set dst (URShiftVI dst shift));
8290 format %{ "psrld $dst,$shift\t! logical right shift packed2I" %}
8291 ins_encode %{
8292 __ psrld($dst$$XMMRegister, (int)$shift$$constant);
8293 %}
8294 ins_pipe( pipe_slow );
8295 %}
8296
8297 instruct vsrl2I_reg(vecD dst, vecD src, vecS shift) %{
8298 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
8299 match(Set dst (URShiftVI src shift));
8300 format %{ "vpsrld $dst,$src,$shift\t! logical right shift packed2I" %}
8301 ins_encode %{
8302 int vector_len = 0;
8303 __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8304 %}
8305 ins_pipe( pipe_slow );
8306 %}
8307
8308 instruct vsrl2I_reg_imm(vecD dst, vecD src, immI8 shift) %{
8309 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
8310 match(Set dst (URShiftVI src shift));
8311 format %{ "vpsrld $dst,$src,$shift\t! logical right shift packed2I" %}
8312 ins_encode %{
8313 int vector_len = 0;
8314 __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8315 %}
8316 ins_pipe( pipe_slow );
8317 %}
8318
8319 instruct vsrl4I(vecX dst, vecS shift) %{
8320 predicate(n->as_Vector()->length() == 4);
8321 match(Set dst (URShiftVI dst shift));
8322 format %{ "psrld $dst,$shift\t! logical right shift packed4I" %}
8323 ins_encode %{
8324 __ psrld($dst$$XMMRegister, $shift$$XMMRegister);
8325 %}
8326 ins_pipe( pipe_slow );
8327 %}
8328
8329 instruct vsrl4I_imm(vecX dst, immI8 shift) %{
8330 predicate(n->as_Vector()->length() == 4);
8331 match(Set dst (URShiftVI dst shift));
8332 format %{ "psrld $dst,$shift\t! logical right shift packed4I" %}
8333 ins_encode %{
8334 __ psrld($dst$$XMMRegister, (int)$shift$$constant);
8335 %}
8336 ins_pipe( pipe_slow );
8337 %}
8338
8339 instruct vsrl4I_reg(vecX dst, vecX src, vecS shift) %{
8340 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
8341 match(Set dst (URShiftVI src shift));
8342 format %{ "vpsrld $dst,$src,$shift\t! logical right shift packed4I" %}
8343 ins_encode %{
8344 int vector_len = 0;
8345 __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8346 %}
8347 ins_pipe( pipe_slow );
8348 %}
8349
8350 instruct vsrl4I_reg_imm(vecX dst, vecX src, immI8 shift) %{
8351 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
8352 match(Set dst (URShiftVI src shift));
8353 format %{ "vpsrld $dst,$src,$shift\t! logical right shift packed4I" %}
8354 ins_encode %{
8355 int vector_len = 0;
8356 __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8357 %}
8358 ins_pipe( pipe_slow );
8359 %}
8360
8361 instruct vsrl8I_reg(vecY dst, vecY src, vecS shift) %{
8362 predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
8363 match(Set dst (URShiftVI src shift));
8364 format %{ "vpsrld $dst,$src,$shift\t! logical right shift packed8I" %}
8365 ins_encode %{
8366 int vector_len = 1;
8367 __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8368 %}
8369 ins_pipe( pipe_slow );
8370 %}
8371
8372 instruct vsrl8I_reg_imm(vecY dst, vecY src, immI8 shift) %{
8373 predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
8374 match(Set dst (URShiftVI src shift));
8375 format %{ "vpsrld $dst,$src,$shift\t! logical right shift packed8I" %}
8376 ins_encode %{
8377 int vector_len = 1;
8378 __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8379 %}
8380 ins_pipe( pipe_slow );
8381 %}
8382
8383 instruct vsrl16I_reg(vecZ dst, vecZ src, vecS shift) %{
8384 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
8385 match(Set dst (URShiftVI src shift));
8386 format %{ "vpsrld $dst,$src,$shift\t! logical right shift packed16I" %}
8387 ins_encode %{
8388 int vector_len = 2;
8389 __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8390 %}
8391 ins_pipe( pipe_slow );
8392 %}
8393
8394 instruct vsrl16I_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
8395 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
8396 match(Set dst (URShiftVI src shift));
8397 format %{ "vpsrld $dst,$src,$shift\t! logical right shift packed16I" %}
8398 ins_encode %{
8399 int vector_len = 2;
8400 __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8401 %}
8402 ins_pipe( pipe_slow );
8403 %}
8404
8405 // Longs vector logical right shift
8406 instruct vsrl2L(vecX dst, vecS shift) %{
8407 predicate(n->as_Vector()->length() == 2);
8408 match(Set dst (URShiftVL dst shift));
8409 format %{ "psrlq $dst,$shift\t! logical right shift packed2L" %}
8410 ins_encode %{
8411 __ psrlq($dst$$XMMRegister, $shift$$XMMRegister);
8412 %}
8413 ins_pipe( pipe_slow );
8414 %}
8415
8416 instruct vsrl2L_imm(vecX dst, immI8 shift) %{
8417 predicate(n->as_Vector()->length() == 2);
8418 match(Set dst (URShiftVL dst shift));
8419 format %{ "psrlq $dst,$shift\t! logical right shift packed2L" %}
8420 ins_encode %{
8421 __ psrlq($dst$$XMMRegister, (int)$shift$$constant);
8422 %}
8423 ins_pipe( pipe_slow );
8424 %}
8425
8426 instruct vsrl2L_reg(vecX dst, vecX src, vecS shift) %{
8427 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
8428 match(Set dst (URShiftVL src shift));
8429 format %{ "vpsrlq $dst,$src,$shift\t! logical right shift packed2L" %}
8430 ins_encode %{
8431 int vector_len = 0;
8432 __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8433 %}
8434 ins_pipe( pipe_slow );
8435 %}
8436
8437 instruct vsrl2L_reg_imm(vecX dst, vecX src, immI8 shift) %{
8438 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
8439 match(Set dst (URShiftVL src shift));
8440 format %{ "vpsrlq $dst,$src,$shift\t! logical right shift packed2L" %}
8441 ins_encode %{
8442 int vector_len = 0;
8443 __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8444 %}
8445 ins_pipe( pipe_slow );
8446 %}
8447
8448 instruct vsrl4L_reg(vecY dst, vecY src, vecS shift) %{
8449 predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
8450 match(Set dst (URShiftVL src shift));
8451 format %{ "vpsrlq $dst,$src,$shift\t! logical right shift packed4L" %}
8452 ins_encode %{
8453 int vector_len = 1;
8454 __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8455 %}
8456 ins_pipe( pipe_slow );
8457 %}
8458
8459 instruct vsrl4L_reg_imm(vecY dst, vecY src, immI8 shift) %{
8460 predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
8461 match(Set dst (URShiftVL src shift));
8462 format %{ "vpsrlq $dst,$src,$shift\t! logical right shift packed4L" %}
8463 ins_encode %{
8464 int vector_len = 1;
8465 __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8466 %}
8467 ins_pipe( pipe_slow );
8468 %}
8469
8470 instruct vsrl8L_reg(vecZ dst, vecZ src, vecS shift) %{
8471 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
8472 match(Set dst (URShiftVL src shift));
8473 format %{ "vpsrlq $dst,$src,$shift\t! logical right shift packed8L" %}
8474 ins_encode %{
8475 int vector_len = 2;
8476 __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8477 %}
8478 ins_pipe( pipe_slow );
8479 %}
8480
8481 instruct vsrl8L_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
8482 predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
8483 match(Set dst (URShiftVL src shift));
8484 format %{ "vpsrlq $dst,$src,$shift\t! logical right shift packed8L" %}
8485 ins_encode %{
8486 int vector_len = 2;
8487 __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8488 %}
8489 ins_pipe( pipe_slow );
8490 %}
8491
8492 // ------------------- ArithmeticRightShift -----------------------------------
8493
8494 // Shorts/Chars vector arithmetic right shift
8495 instruct vsra2S(vecS dst, vecS shift) %{
8496 predicate(n->as_Vector()->length() == 2);
8497 match(Set dst (RShiftVS dst shift));
8498 format %{ "psraw $dst,$shift\t! arithmetic right shift packed2S" %}
8499 ins_encode %{
8500 __ psraw($dst$$XMMRegister, $shift$$XMMRegister);
8501 %}
8502 ins_pipe( pipe_slow );
8503 %}
8504
8505 instruct vsra2S_imm(vecS dst, immI8 shift) %{
8506 predicate(n->as_Vector()->length() == 2);
8507 match(Set dst (RShiftVS dst shift));
8508 format %{ "psraw $dst,$shift\t! arithmetic right shift packed2S" %}
8509 ins_encode %{
8510 __ psraw($dst$$XMMRegister, (int)$shift$$constant);
8511 %}
8512 ins_pipe( pipe_slow );
8513 %}
8514
8515 instruct vsra2S_reg(vecS dst, vecS src, vecS shift) %{
8516 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
8517 match(Set dst (RShiftVS src shift));
8518 format %{ "vpsraw $dst,$src,$shift\t! arithmetic right shift packed2S" %}
8519 ins_encode %{
8520 int vector_len = 0;
8521 __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8522 %}
8523 ins_pipe( pipe_slow );
8524 %}
8525
8526 instruct vsra2S_reg_imm(vecS dst, vecS src, immI8 shift) %{
8527 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
8528 match(Set dst (RShiftVS src shift));
8529 format %{ "vpsraw $dst,$src,$shift\t! arithmetic right shift packed2S" %}
8530 ins_encode %{
8531 int vector_len = 0;
8532 __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8533 %}
8534 ins_pipe( pipe_slow );
8535 %}
8536
8537 instruct vsra4S(vecD dst, vecS shift) %{
8538 predicate(n->as_Vector()->length() == 4);
8539 match(Set dst (RShiftVS dst shift));
8540 format %{ "psraw $dst,$shift\t! arithmetic right shift packed4S" %}
8541 ins_encode %{
8542 __ psraw($dst$$XMMRegister, $shift$$XMMRegister);
8543 %}
8544 ins_pipe( pipe_slow );
8545 %}
8546
8547 instruct vsra4S_imm(vecD dst, immI8 shift) %{
8548 predicate(n->as_Vector()->length() == 4);
8549 match(Set dst (RShiftVS dst shift));
8550 format %{ "psraw $dst,$shift\t! arithmetic right shift packed4S" %}
8551 ins_encode %{
8552 __ psraw($dst$$XMMRegister, (int)$shift$$constant);
8553 %}
8554 ins_pipe( pipe_slow );
8555 %}
8556
8557 instruct vsra4S_reg(vecD dst, vecD src, vecS shift) %{
8558 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
8559 match(Set dst (RShiftVS src shift));
8560 format %{ "vpsraw $dst,$src,$shift\t! arithmetic right shift packed4S" %}
8561 ins_encode %{
8562 int vector_len = 0;
8563 __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8564 %}
8565 ins_pipe( pipe_slow );
8566 %}
8567
8568 instruct vsra4S_reg_imm(vecD dst, vecD src, immI8 shift) %{
8569 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
8570 match(Set dst (RShiftVS src shift));
8571 format %{ "vpsraw $dst,$src,$shift\t! arithmetic right shift packed4S" %}
8572 ins_encode %{
8573 int vector_len = 0;
8574 __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8575 %}
8576 ins_pipe( pipe_slow );
8577 %}
8578
8579 instruct vsra8S(vecX dst, vecS shift) %{
8580 predicate(n->as_Vector()->length() == 8);
8581 match(Set dst (RShiftVS dst shift));
8582 format %{ "psraw $dst,$shift\t! arithmetic right shift packed8S" %}
8583 ins_encode %{
8584 __ psraw($dst$$XMMRegister, $shift$$XMMRegister);
8585 %}
8586 ins_pipe( pipe_slow );
8587 %}
8588
8589 instruct vsra8S_imm(vecX dst, immI8 shift) %{
8590 predicate(n->as_Vector()->length() == 8);
8591 match(Set dst (RShiftVS dst shift));
8592 format %{ "psraw $dst,$shift\t! arithmetic right shift packed8S" %}
8593 ins_encode %{
8594 __ psraw($dst$$XMMRegister, (int)$shift$$constant);
8595 %}
8596 ins_pipe( pipe_slow );
8597 %}
8598
8599 instruct vsra8S_reg(vecX dst, vecX src, vecS shift) %{
8600 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
8601 match(Set dst (RShiftVS src shift));
8602 format %{ "vpsraw $dst,$src,$shift\t! arithmetic right shift packed8S" %}
8603 ins_encode %{
8604 int vector_len = 0;
8605 __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8606 %}
8607 ins_pipe( pipe_slow );
8608 %}
8609
8610 instruct vsra8S_reg_imm(vecX dst, vecX src, immI8 shift) %{
8611 predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
8612 match(Set dst (RShiftVS src shift));
8613 format %{ "vpsraw $dst,$src,$shift\t! arithmetic right shift packed8S" %}
8614 ins_encode %{
8615 int vector_len = 0;
8616 __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8617 %}
8618 ins_pipe( pipe_slow );
8619 %}
8620
8621 instruct vsra16S_reg(vecY dst, vecY src, vecS shift) %{
8622 predicate(UseAVX > 1 && n->as_Vector()->length() == 16);
8623 match(Set dst (RShiftVS src shift));
8624 format %{ "vpsraw $dst,$src,$shift\t! arithmetic right shift packed16S" %}
8625 ins_encode %{
8626 int vector_len = 1;
8627 __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8628 %}
8629 ins_pipe( pipe_slow );
8630 %}
8631
8632 instruct vsra16S_reg_imm(vecY dst, vecY src, immI8 shift) %{
8633 predicate(UseAVX > 1 && n->as_Vector()->length() == 16);
8634 match(Set dst (RShiftVS src shift));
8635 format %{ "vpsraw $dst,$src,$shift\t! arithmetic right shift packed16S" %}
8636 ins_encode %{
8637 int vector_len = 1;
8638 __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8639 %}
8640 ins_pipe( pipe_slow );
8641 %}
8642
8643 instruct vsra32S_reg(vecZ dst, vecZ src, vecS shift) %{
8644 predicate(UseAVX > 2 && n->as_Vector()->length() == 32);
8645 match(Set dst (RShiftVS src shift));
8646 format %{ "vpsraw $dst,$src,$shift\t! arithmetic right shift packed32S" %}
8647 ins_encode %{
8648 int vector_len = 2;
8649 __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8650 %}
8651 ins_pipe( pipe_slow );
8652 %}
8653
8654 instruct vsra32S_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
8655 predicate(UseAVX > 2 && n->as_Vector()->length() == 32);
8656 match(Set dst (RShiftVS src shift));
8657 format %{ "vpsraw $dst,$src,$shift\t! arithmetic right shift packed32S" %}
8658 ins_encode %{
8659 int vector_len = 2;
8660 __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8661 %}
8662 ins_pipe( pipe_slow );
8663 %}
8664
8665 // Integers vector arithmetic right shift
8666 instruct vsra2I(vecD dst, vecS shift) %{
8667 predicate(n->as_Vector()->length() == 2);
8668 match(Set dst (RShiftVI dst shift));
8669 format %{ "psrad $dst,$shift\t! arithmetic right shift packed2I" %}
8670 ins_encode %{
8671 __ psrad($dst$$XMMRegister, $shift$$XMMRegister);
8672 %}
8673 ins_pipe( pipe_slow );
8674 %}
8675
8676 instruct vsra2I_imm(vecD dst, immI8 shift) %{
8677 predicate(n->as_Vector()->length() == 2);
8678 match(Set dst (RShiftVI dst shift));
8679 format %{ "psrad $dst,$shift\t! arithmetic right shift packed2I" %}
8680 ins_encode %{
8681 __ psrad($dst$$XMMRegister, (int)$shift$$constant);
8682 %}
8683 ins_pipe( pipe_slow );
8684 %}
8685
8686 instruct vsra2I_reg(vecD dst, vecD src, vecS shift) %{
8687 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
8688 match(Set dst (RShiftVI src shift));
8689 format %{ "vpsrad $dst,$src,$shift\t! arithmetic right shift packed2I" %}
8690 ins_encode %{
8691 int vector_len = 0;
8692 __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8693 %}
8694 ins_pipe( pipe_slow );
8695 %}
8696
8697 instruct vsra2I_reg_imm(vecD dst, vecD src, immI8 shift) %{
8698 predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
8699 match(Set dst (RShiftVI src shift));
8700 format %{ "vpsrad $dst,$src,$shift\t! arithmetic right shift packed2I" %}
8701 ins_encode %{
8702 int vector_len = 0;
8703 __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8704 %}
8705 ins_pipe( pipe_slow );
8706 %}
8707
8708 instruct vsra4I(vecX dst, vecS shift) %{
8709 predicate(n->as_Vector()->length() == 4);
8710 match(Set dst (RShiftVI dst shift));
8711 format %{ "psrad $dst,$shift\t! arithmetic right shift packed4I" %}
8712 ins_encode %{
8713 __ psrad($dst$$XMMRegister, $shift$$XMMRegister);
8714 %}
8715 ins_pipe( pipe_slow );
8716 %}
8717
8718 instruct vsra4I_imm(vecX dst, immI8 shift) %{
8719 predicate(n->as_Vector()->length() == 4);
8720 match(Set dst (RShiftVI dst shift));
8721 format %{ "psrad $dst,$shift\t! arithmetic right shift packed4I" %}
8722 ins_encode %{
8723 __ psrad($dst$$XMMRegister, (int)$shift$$constant);
8724 %}
8725 ins_pipe( pipe_slow );
8726 %}
8727
8728 instruct vsra4I_reg(vecX dst, vecX src, vecS shift) %{
8729 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
8730 match(Set dst (RShiftVI src shift));
8731 format %{ "vpsrad $dst,$src,$shift\t! arithmetic right shift packed4I" %}
8732 ins_encode %{
8733 int vector_len = 0;
8734 __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8735 %}
8736 ins_pipe( pipe_slow );
8737 %}
8738
8739 instruct vsra4I_reg_imm(vecX dst, vecX src, immI8 shift) %{
8740 predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
8741 match(Set dst (RShiftVI src shift));
8742 format %{ "vpsrad $dst,$src,$shift\t! arithmetic right shift packed4I" %}
8743 ins_encode %{
8744 int vector_len = 0;
8745 __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8746 %}
8747 ins_pipe( pipe_slow );
8748 %}
8749
8750 instruct vsra8I_reg(vecY dst, vecY src, vecS shift) %{
8751 predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
8752 match(Set dst (RShiftVI src shift));
8753 format %{ "vpsrad $dst,$src,$shift\t! arithmetic right shift packed8I" %}
8754 ins_encode %{
8755 int vector_len = 1;
8756 __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8757 %}
8758 ins_pipe( pipe_slow );
8759 %}
8760
8761 instruct vsra8I_reg_imm(vecY dst, vecY src, immI8 shift) %{
8762 predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
8763 match(Set dst (RShiftVI src shift));
8764 format %{ "vpsrad $dst,$src,$shift\t! arithmetic right shift packed8I" %}
8765 ins_encode %{
8766 int vector_len = 1;
8767 __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8768 %}
8769 ins_pipe( pipe_slow );
8770 %}
8771
8772 instruct vsra16I_reg(vecZ dst, vecZ src, vecS shift) %{
8773 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
8774 match(Set dst (RShiftVI src shift));
8775 format %{ "vpsrad $dst,$src,$shift\t! arithmetic right shift packed16I" %}
8776 ins_encode %{
8777 int vector_len = 2;
8778 __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
8779 %}
8780 ins_pipe( pipe_slow );
8781 %}
8782
8783 instruct vsra16I_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
8784 predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
8785 match(Set dst (RShiftVI src shift));
8786 format %{ "vpsrad $dst,$src,$shift\t! arithmetic right shift packed16I" %}
8787 ins_encode %{
8788 int vector_len = 2;
8789 __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
8790 %}
8791 ins_pipe( pipe_slow );
8792 %}
8793
8794 // There are no longs vector arithmetic right shift instructions.
8795
8796
8797 // --------------------------------- AND --------------------------------------
8798
8799 instruct vand4B(vecS dst, vecS src) %{
8800 predicate(n->as_Vector()->length_in_bytes() == 4);
8801 match(Set dst (AndV dst src));
8802 format %{ "pand $dst,$src\t! and vectors (4 bytes)" %}
8803 ins_encode %{
8804 __ pand($dst$$XMMRegister, $src$$XMMRegister);
8805 %}
8806 ins_pipe( pipe_slow );
8807 %}
8808
8809 instruct vand4B_reg(vecS dst, vecS src1, vecS src2) %{
8810 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
8811 match(Set dst (AndV src1 src2));
8812 format %{ "vpand $dst,$src1,$src2\t! and vectors (4 bytes)" %}
8813 ins_encode %{
8814 int vector_len = 0;
8815 __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
8816 %}
8817 ins_pipe( pipe_slow );
8818 %}
8819
8820 instruct vand4B_mem(vecS dst, vecS src, memory mem) %{
8821 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
8822 match(Set dst (AndV src (LoadVector mem)));
8823 format %{ "vpand $dst,$src,$mem\t! and vectors (4 bytes)" %}
8824 ins_encode %{
8825 int vector_len = 0;
8826 __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
8827 %}
8828 ins_pipe( pipe_slow );
8829 %}
8830
8831 instruct vand8B(vecD dst, vecD src) %{
8832 predicate(n->as_Vector()->length_in_bytes() == 8);
8833 match(Set dst (AndV dst src));
8834 format %{ "pand $dst,$src\t! and vectors (8 bytes)" %}
8835 ins_encode %{
8836 __ pand($dst$$XMMRegister, $src$$XMMRegister);
8837 %}
8838 ins_pipe( pipe_slow );
8839 %}
8840
8841 instruct vand8B_reg(vecD dst, vecD src1, vecD src2) %{
8842 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 8);
8843 match(Set dst (AndV src1 src2));
8844 format %{ "vpand $dst,$src1,$src2\t! and vectors (8 bytes)" %}
8845 ins_encode %{
8846 int vector_len = 0;
8847 __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
8848 %}
8849 ins_pipe( pipe_slow );
8850 %}
8851
8852 instruct vand8B_mem(vecD dst, vecD src, memory mem) %{
8853 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 8);
8854 match(Set dst (AndV src (LoadVector mem)));
8855 format %{ "vpand $dst,$src,$mem\t! and vectors (8 bytes)" %}
8856 ins_encode %{
8857 int vector_len = 0;
8858 __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
8859 %}
8860 ins_pipe( pipe_slow );
8861 %}
8862
8863 instruct vand16B(vecX dst, vecX src) %{
8864 predicate(n->as_Vector()->length_in_bytes() == 16);
8865 match(Set dst (AndV dst src));
8866 format %{ "pand $dst,$src\t! and vectors (16 bytes)" %}
8867 ins_encode %{
8868 __ pand($dst$$XMMRegister, $src$$XMMRegister);
8869 %}
8870 ins_pipe( pipe_slow );
8871 %}
8872
8873 instruct vand16B_reg(vecX dst, vecX src1, vecX src2) %{
8874 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 16);
8875 match(Set dst (AndV src1 src2));
8876 format %{ "vpand $dst,$src1,$src2\t! and vectors (16 bytes)" %}
8877 ins_encode %{
8878 int vector_len = 0;
8879 __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
8880 %}
8881 ins_pipe( pipe_slow );
8882 %}
8883
8884 instruct vand16B_mem(vecX dst, vecX src, memory mem) %{
8885 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 16);
8886 match(Set dst (AndV src (LoadVector mem)));
8887 format %{ "vpand $dst,$src,$mem\t! and vectors (16 bytes)" %}
8888 ins_encode %{
8889 int vector_len = 0;
8890 __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
8891 %}
8892 ins_pipe( pipe_slow );
8893 %}
8894
8895 instruct vand32B_reg(vecY dst, vecY src1, vecY src2) %{
8896 predicate(UseAVX > 1 && n->as_Vector()->length_in_bytes() == 32);
8897 match(Set dst (AndV src1 src2));
8898 format %{ "vpand $dst,$src1,$src2\t! and vectors (32 bytes)" %}
8899 ins_encode %{
8900 int vector_len = 1;
8901 __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
8902 %}
8903 ins_pipe( pipe_slow );
8904 %}
8905
8906 instruct vand32B_mem(vecY dst, vecY src, memory mem) %{
8907 predicate(UseAVX > 1 && n->as_Vector()->length_in_bytes() == 32);
8908 match(Set dst (AndV src (LoadVector mem)));
8909 format %{ "vpand $dst,$src,$mem\t! and vectors (32 bytes)" %}
8910 ins_encode %{
8911 int vector_len = 1;
8912 __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
8913 %}
8914 ins_pipe( pipe_slow );
8915 %}
8916
8917 instruct vand64B_reg(vecZ dst, vecZ src1, vecZ src2) %{
8918 predicate(UseAVX > 2 && n->as_Vector()->length_in_bytes() == 64);
8919 match(Set dst (AndV src1 src2));
8920 format %{ "vpand $dst,$src1,$src2\t! and vectors (64 bytes)" %}
8921 ins_encode %{
8922 int vector_len = 2;
8923 __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
8924 %}
8925 ins_pipe( pipe_slow );
8926 %}
8927
8928 instruct vand64B_mem(vecZ dst, vecZ src, memory mem) %{
8929 predicate(UseAVX > 2 && n->as_Vector()->length_in_bytes() == 64);
8930 match(Set dst (AndV src (LoadVector mem)));
8931 format %{ "vpand $dst,$src,$mem\t! and vectors (64 bytes)" %}
8932 ins_encode %{
8933 int vector_len = 2;
8934 __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
8935 %}
8936 ins_pipe( pipe_slow );
8937 %}
8938
8939 // --------------------------------- OR ---------------------------------------
8940
8941 instruct vor4B(vecS dst, vecS src) %{
8942 predicate(n->as_Vector()->length_in_bytes() == 4);
8943 match(Set dst (OrV dst src));
8944 format %{ "por $dst,$src\t! or vectors (4 bytes)" %}
8945 ins_encode %{
8946 __ por($dst$$XMMRegister, $src$$XMMRegister);
8947 %}
8948 ins_pipe( pipe_slow );
8949 %}
8950
8951 instruct vor4B_reg(vecS dst, vecS src1, vecS src2) %{
8952 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
8953 match(Set dst (OrV src1 src2));
8954 format %{ "vpor $dst,$src1,$src2\t! or vectors (4 bytes)" %}
8955 ins_encode %{
8956 int vector_len = 0;
8957 __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
8958 %}
8959 ins_pipe( pipe_slow );
8960 %}
8961
8962 instruct vor4B_mem(vecS dst, vecS src, memory mem) %{
8963 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
8964 match(Set dst (OrV src (LoadVector mem)));
8965 format %{ "vpor $dst,$src,$mem\t! or vectors (4 bytes)" %}
8966 ins_encode %{
8967 int vector_len = 0;
8968 __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
8969 %}
8970 ins_pipe( pipe_slow );
8971 %}
8972
8973 instruct vor8B(vecD dst, vecD src) %{
8974 predicate(n->as_Vector()->length_in_bytes() == 8);
8975 match(Set dst (OrV dst src));
8976 format %{ "por $dst,$src\t! or vectors (8 bytes)" %}
8977 ins_encode %{
8978 __ por($dst$$XMMRegister, $src$$XMMRegister);
8979 %}
8980 ins_pipe( pipe_slow );
8981 %}
8982
8983 instruct vor8B_reg(vecD dst, vecD src1, vecD src2) %{
8984 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 8);
8985 match(Set dst (OrV src1 src2));
8986 format %{ "vpor $dst,$src1,$src2\t! or vectors (8 bytes)" %}
8987 ins_encode %{
8988 int vector_len = 0;
8989 __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
8990 %}
8991 ins_pipe( pipe_slow );
8992 %}
8993
8994 instruct vor8B_mem(vecD dst, vecD src, memory mem) %{
8995 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
8996 match(Set dst (OrV src (LoadVector mem)));
8997 format %{ "vpor $dst,$src,$mem\t! or vectors (8 bytes)" %}
8998 ins_encode %{
8999 int vector_len = 0;
9000 __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
9001 %}
9002 ins_pipe( pipe_slow );
9003 %}
9004
9005 instruct vor16B(vecX dst, vecX src) %{
9006 predicate(n->as_Vector()->length_in_bytes() == 16);
9007 match(Set dst (OrV dst src));
9008 format %{ "por $dst,$src\t! or vectors (16 bytes)" %}
9009 ins_encode %{
9010 __ por($dst$$XMMRegister, $src$$XMMRegister);
9011 %}
9012 ins_pipe( pipe_slow );
9013 %}
9014
9015 instruct vor16B_reg(vecX dst, vecX src1, vecX src2) %{
9016 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 16);
9017 match(Set dst (OrV src1 src2));
9018 format %{ "vpor $dst,$src1,$src2\t! or vectors (16 bytes)" %}
9019 ins_encode %{
9020 int vector_len = 0;
9021 __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
9022 %}
9023 ins_pipe( pipe_slow );
9024 %}
9025
9026 instruct vor16B_mem(vecX dst, vecX src, memory mem) %{
9027 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 16);
9028 match(Set dst (OrV src (LoadVector mem)));
9029 format %{ "vpor $dst,$src,$mem\t! or vectors (16 bytes)" %}
9030 ins_encode %{
9031 int vector_len = 0;
9032 __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
9033 %}
9034 ins_pipe( pipe_slow );
9035 %}
9036
9037 instruct vor32B_reg(vecY dst, vecY src1, vecY src2) %{
9038 predicate(UseAVX > 1 && n->as_Vector()->length_in_bytes() == 32);
9039 match(Set dst (OrV src1 src2));
9040 format %{ "vpor $dst,$src1,$src2\t! or vectors (32 bytes)" %}
9041 ins_encode %{
9042 int vector_len = 1;
9043 __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
9044 %}
9045 ins_pipe( pipe_slow );
9046 %}
9047
9048 instruct vor32B_mem(vecY dst, vecY src, memory mem) %{
9049 predicate(UseAVX > 1 && n->as_Vector()->length_in_bytes() == 32);
9050 match(Set dst (OrV src (LoadVector mem)));
9051 format %{ "vpor $dst,$src,$mem\t! or vectors (32 bytes)" %}
9052 ins_encode %{
9053 int vector_len = 1;
9054 __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
9055 %}
9056 ins_pipe( pipe_slow );
9057 %}
9058
9059 instruct vor64B_reg(vecZ dst, vecZ src1, vecZ src2) %{
9060 predicate(UseAVX > 2 && n->as_Vector()->length_in_bytes() == 64);
9061 match(Set dst (OrV src1 src2));
9062 format %{ "vpor $dst,$src1,$src2\t! or vectors (64 bytes)" %}
9063 ins_encode %{
9064 int vector_len = 2;
9065 __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
9066 %}
9067 ins_pipe( pipe_slow );
9068 %}
9069
9070 instruct vor64B_mem(vecZ dst, vecZ src, memory mem) %{
9071 predicate(UseAVX > 2 && n->as_Vector()->length_in_bytes() == 64);
9072 match(Set dst (OrV src (LoadVector mem)));
9073 format %{ "vpor $dst,$src,$mem\t! or vectors (64 bytes)" %}
9074 ins_encode %{
9075 int vector_len = 2;
9076 __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
9077 %}
9078 ins_pipe( pipe_slow );
9079 %}
9080
9081 // --------------------------------- XOR --------------------------------------
9082
9083 instruct vxor4B(vecS dst, vecS src) %{
9084 predicate(n->as_Vector()->length_in_bytes() == 4);
9085 match(Set dst (XorV dst src));
9086 format %{ "pxor $dst,$src\t! xor vectors (4 bytes)" %}
9087 ins_encode %{
9088 __ pxor($dst$$XMMRegister, $src$$XMMRegister);
9089 %}
9090 ins_pipe( pipe_slow );
9091 %}
9092
9093 instruct vxor4B_reg(vecS dst, vecS src1, vecS src2) %{
9094 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
9095 match(Set dst (XorV src1 src2));
9096 format %{ "vpxor $dst,$src1,$src2\t! xor vectors (4 bytes)" %}
9097 ins_encode %{
9098 int vector_len = 0;
9099 __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
9100 %}
9101 ins_pipe( pipe_slow );
9102 %}
9103
9104 instruct vxor4B_mem(vecS dst, vecS src, memory mem) %{
9105 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
9106 match(Set dst (XorV src (LoadVector mem)));
9107 format %{ "vpxor $dst,$src,$mem\t! xor vectors (4 bytes)" %}
9108 ins_encode %{
9109 int vector_len = 0;
9110 __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
9111 %}
9112 ins_pipe( pipe_slow );
9113 %}
9114
9115 instruct vxor8B(vecD dst, vecD src) %{
9116 predicate(n->as_Vector()->length_in_bytes() == 8);
9117 match(Set dst (XorV dst src));
9118 format %{ "pxor $dst,$src\t! xor vectors (8 bytes)" %}
9119 ins_encode %{
9120 __ pxor($dst$$XMMRegister, $src$$XMMRegister);
9121 %}
9122 ins_pipe( pipe_slow );
9123 %}
9124
9125 instruct vxor8B_reg(vecD dst, vecD src1, vecD src2) %{
9126 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 8);
9127 match(Set dst (XorV src1 src2));
9128 format %{ "vpxor $dst,$src1,$src2\t! xor vectors (8 bytes)" %}
9129 ins_encode %{
9130 int vector_len = 0;
9131 __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
9132 %}
9133 ins_pipe( pipe_slow );
9134 %}
9135
9136 instruct vxor8B_mem(vecD dst, vecD src, memory mem) %{
9137 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 8);
9138 match(Set dst (XorV src (LoadVector mem)));
9139 format %{ "vpxor $dst,$src,$mem\t! xor vectors (8 bytes)" %}
9140 ins_encode %{
9141 int vector_len = 0;
9142 __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
9143 %}
9144 ins_pipe( pipe_slow );
9145 %}
9146
9147 instruct vxor16B(vecX dst, vecX src) %{
9148 predicate(n->as_Vector()->length_in_bytes() == 16);
9149 match(Set dst (XorV dst src));
9150 format %{ "pxor $dst,$src\t! xor vectors (16 bytes)" %}
9151 ins_encode %{
9152 __ pxor($dst$$XMMRegister, $src$$XMMRegister);
9153 %}
9154 ins_pipe( pipe_slow );
9155 %}
9156
9157 instruct vxor16B_reg(vecX dst, vecX src1, vecX src2) %{
9158 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 16);
9159 match(Set dst (XorV src1 src2));
9160 format %{ "vpxor $dst,$src1,$src2\t! xor vectors (16 bytes)" %}
9161 ins_encode %{
9162 int vector_len = 0;
9163 __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
9164 %}
9165 ins_pipe( pipe_slow );
9166 %}
9167
9168 instruct vxor16B_mem(vecX dst, vecX src, memory mem) %{
9169 predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 16);
9170 match(Set dst (XorV src (LoadVector mem)));
9171 format %{ "vpxor $dst,$src,$mem\t! xor vectors (16 bytes)" %}
9172 ins_encode %{
9173 int vector_len = 0;
9174 __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
9175 %}
9176 ins_pipe( pipe_slow );
9177 %}
9178
9179 instruct vxor32B_reg(vecY dst, vecY src1, vecY src2) %{
9180 predicate(UseAVX > 1 && n->as_Vector()->length_in_bytes() == 32);
9181 match(Set dst (XorV src1 src2));
9182 format %{ "vpxor $dst,$src1,$src2\t! xor vectors (32 bytes)" %}
9183 ins_encode %{
9184 int vector_len = 1;
9185 __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
9186 %}
9187 ins_pipe( pipe_slow );
9188 %}
9189
9190 instruct vxor32B_mem(vecY dst, vecY src, memory mem) %{
9191 predicate(UseAVX > 1 && n->as_Vector()->length_in_bytes() == 32);
9192 match(Set dst (XorV src (LoadVector mem)));
9193 format %{ "vpxor $dst,$src,$mem\t! xor vectors (32 bytes)" %}
9194 ins_encode %{
9195 int vector_len = 1;
9196 __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
9197 %}
9198 ins_pipe( pipe_slow );
9199 %}
9200
9201 instruct vxor64B_reg(vecZ dst, vecZ src1, vecZ src2) %{
9202 predicate(UseAVX > 2 && n->as_Vector()->length_in_bytes() == 64);
9203 match(Set dst (XorV src1 src2));
9204 format %{ "vpxor $dst,$src1,$src2\t! xor vectors (64 bytes)" %}
9205 ins_encode %{
9206 int vector_len = 2;
9207 __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
9208 %}
9209 ins_pipe( pipe_slow );
9210 %}
9211
9212 instruct vxor64B_mem(vecZ dst, vecZ src, memory mem) %{
9213 predicate(UseAVX > 2 && n->as_Vector()->length_in_bytes() == 64);
9214 match(Set dst (XorV src (LoadVector mem)));
9215 format %{ "vpxor $dst,$src,$mem\t! xor vectors (64 bytes)" %}
9216 ins_encode %{
9217 int vector_len = 2;
9218 __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
9219 %}
9220 ins_pipe( pipe_slow );
9221 %}
9222