1 /*
2 * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 package jdk.internal.vm.vector;
25
26 import jdk.internal.HotSpotIntrinsicCandidate;
27 import jdk.internal.misc.Unsafe;
28 import jdk.internal.vm.annotation.ForceInline;
29
30 import java.nio.Buffer;
31 import java.nio.ByteBuffer;
32 import java.util.Objects;
33 import java.util.function.*;
34
35 public class VectorSupport {
36 static {
37 registerNatives();
38 }
39
40 private static final Unsafe U = Unsafe.getUnsafe();
41
42 // Unary
43 public static final int VECTOR_OP_ABS = 0;
44 public static final int VECTOR_OP_NEG = 1;
45 public static final int VECTOR_OP_SQRT = 2;
46
47 // Binary
48 public static final int VECTOR_OP_ADD = 4;
49 public static final int VECTOR_OP_SUB = 5;
50 public static final int VECTOR_OP_MUL = 6;
51 public static final int VECTOR_OP_DIV = 7;
52 public static final int VECTOR_OP_MIN = 8;
53 public static final int VECTOR_OP_MAX = 9;
54
55 public static final int VECTOR_OP_AND = 10;
56 public static final int VECTOR_OP_OR = 11;
57 public static final int VECTOR_OP_XOR = 12;
58
59 // Ternary
60 public static final int VECTOR_OP_FMA = 13;
61
62 // Broadcast int
63 public static final int VECTOR_OP_LSHIFT = 14;
64 public static final int VECTOR_OP_RSHIFT = 15;
65 public static final int VECTOR_OP_URSHIFT = 16;
66
67 public static final int VECTOR_OP_CAST = 17;
68 public static final int VECTOR_OP_REINTERPRET = 18;
69
70 // enum BoolTest
71 public static final int BT_eq = 0;
72 public static final int BT_ne = 4;
73 public static final int BT_le = 5;
74 public static final int BT_ge = 7;
75 public static final int BT_lt = 3;
76 public static final int BT_gt = 1;
77 public static final int BT_overflow = 2;
78 public static final int BT_no_overflow = 6;
79
80 // BasicType codes, for primitives only:
81 public static final int
82 T_FLOAT = 6,
83 T_DOUBLE = 7,
84 T_BYTE = 8,
85 T_SHORT = 9,
86 T_INT = 10,
87 T_LONG = 11;
88
89 /* ============================================================================ */
90
91 public static class VectorSpecies<E> {}
92
93 public static class VectorPayload {
94 private final Object payload; // array of primitives
95
96 public VectorPayload(Object payload) {
97 this.payload = payload;
98 }
99
100 protected final Object getPayload() {
101 return VectorSupport.maybeRebox(this).payload;
102 }
103 }
104
105 public static class Vector<E> extends VectorPayload {
106 public Vector(Object payload) {
107 super(payload);
108 }
109 }
110
111 public static class VectorShuffle<E> extends VectorPayload {
112 public VectorShuffle(Object payload) {
113 super(payload);
114 }
115 }
116 public static class VectorMask<E> extends VectorPayload {
117 public VectorMask(Object payload) {
118 super(payload);
119 }
120 }
121
122 /* ============================================================================ */
123 public interface BroadcastOperation<VM, E, S extends VectorSpecies<E>> {
124 VM broadcast(long l, S s);
125 }
126
127 @HotSpotIntrinsicCandidate
128 public static
129 <VM, E, S extends VectorSpecies<E>>
130 VM broadcastCoerced(Class<? extends VM> vmClass, Class<E> E, int length,
131 long bits, S s,
132 BroadcastOperation<VM, E, S> defaultImpl) {
133 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
134 return defaultImpl.broadcast(bits, s);
135 }
136
137 /* ============================================================================ */
138 public interface ShuffleIotaOperation<E, S extends VectorSpecies<E>> {
139 VectorShuffle<E> apply(int length, int start, int step, S s);
140 }
141
142 @HotSpotIntrinsicCandidate
143 public static
144 <E, S extends VectorSpecies<E>>
145 VectorShuffle<E> shuffleIota(Class<?> E, Class<?> ShuffleClass, S s, int length,
146 int start, int step, int wrap, ShuffleIotaOperation<E, S> defaultImpl) {
147 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
148 return defaultImpl.apply(length, start, step, s);
149 }
150
151 public interface ShuffleToVectorOperation<VM, Sh, E> {
152 VM apply(Sh s);
153 }
154
155 @HotSpotIntrinsicCandidate
156 public static
157 <VM ,Sh extends VectorShuffle<E>, E>
158 VM shuffleToVector(Class<?> VM, Class<?>E , Class<?> ShuffleClass, Sh s, int length,
159 ShuffleToVectorOperation<VM,Sh,E> defaultImpl) {
160 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
161 return defaultImpl.apply(s);
162 }
163
164 /* ============================================================================ */
165 public interface IndexOperation<V extends Vector<E>, E, S extends VectorSpecies<E>> {
166 V index(V v, int step, S s);
167 }
168
169 //FIXME @HotSpotIntrinsicCandidate
170 public static
171 <V extends Vector<E>, E, S extends VectorSpecies<E>>
172 V indexVector(Class<? extends V> vClass, Class<E> E, int length,
173 V v, int step, S s,
174 IndexOperation<V, E, S> defaultImpl) {
175 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
176 return defaultImpl.index(v, step, s);
177 }
178
179 /* ============================================================================ */
180
181 @HotSpotIntrinsicCandidate
182 public static
183 <V extends Vector<?>>
184 long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int length,
185 V v,
186 Function<V,Long> defaultImpl) {
187 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
188 return defaultImpl.apply(v);
189 }
190
191 /* ============================================================================ */
192
193 public interface VecExtractOp<V> {
194 long apply(V v1, int idx);
195 }
196
197 @HotSpotIntrinsicCandidate
198 public static
199 <V extends Vector<?>>
200 long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
201 V vec, int ix,
202 VecExtractOp<V> defaultImpl) {
203 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
204 return defaultImpl.apply(vec, ix);
205 }
206
207 /* ============================================================================ */
208
209 public interface VecInsertOp<V> {
210 V apply(V v1, int idx, long val);
211 }
212
213 @HotSpotIntrinsicCandidate
214 public static
215 <V extends Vector<?>>
216 V insert(Class<? extends V> vectorClass, Class<?> elementType, int vlen,
217 V vec, int ix, long val,
218 VecInsertOp<V> defaultImpl) {
219 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
220 return defaultImpl.apply(vec, ix, val);
221 }
222
223 /* ============================================================================ */
224
225 @HotSpotIntrinsicCandidate
226 public static
227 <VM>
228 VM unaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
229 VM vm,
230 Function<VM, VM> defaultImpl) {
231 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
232 return defaultImpl.apply(vm);
233 }
234
235 /* ============================================================================ */
236
237 @HotSpotIntrinsicCandidate
238 public static
239 <VM>
240 VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
241 VM vm1, VM vm2,
242 BiFunction<VM, VM, VM> defaultImpl) {
243 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
244 return defaultImpl.apply(vm1, vm2);
245 }
246
247 /* ============================================================================ */
248
249 public interface TernaryOperation<V> {
250 V apply(V v1, V v2, V v3);
251 }
252
253 @HotSpotIntrinsicCandidate
254 public static
255 <VM>
256 VM ternaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
257 VM vm1, VM vm2, VM vm3,
258 TernaryOperation<VM> defaultImpl) {
259 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
260 return defaultImpl.apply(vm1, vm2, vm3);
261 }
262
263 /* ============================================================================ */
264
265 // Memory operations
266
267 public interface LoadOperation<C, V, E, S extends VectorSpecies<E>> {
268 V load(C container, int index, S s);
269 }
270
271 @HotSpotIntrinsicCandidate
272 public static
273 <C, VM, E, S extends VectorSpecies<E>>
274 VM load(Class<? extends VM> vmClass, Class<E> E, int length,
275 Object base, long offset, // Unsafe addressing
276 C container, int index, S s, // Arguments for default implementation
277 LoadOperation<C, VM, E, S> defaultImpl) {
278 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
279 return defaultImpl.load(container, index, s);
280 }
281
282 /* ============================================================================ */
283
284 public interface LoadVectorOperationWithMap<C, V extends Vector<?>, E, S extends VectorSpecies<E>> {
285 V loadWithMap(C container, int index, int[] indexMap, int indexM, S s);
286 }
287
288 @HotSpotIntrinsicCandidate
289 public static
290 <C, V extends Vector<?>, W extends Vector<Integer>, E, S extends VectorSpecies<E>>
291 V loadWithMap(Class<?> vectorClass, Class<E> E, int length, Class<?> vectorIndexClass,
292 Object base, long offset, // Unsafe addressing
293 W index_vector,
294 C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
295 LoadVectorOperationWithMap<C, V, E, S> defaultImpl) {
296 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
297 return defaultImpl.loadWithMap(container, index, indexMap, indexM, s);
298 }
299
300 /* ============================================================================ */
301
302 public interface StoreVectorOperation<C, V extends Vector<?>> {
303 void store(C container, int index, V v);
304 }
305
306 @HotSpotIntrinsicCandidate
307 public static
308 <C, V extends Vector<?>>
309 void store(Class<?> vectorClass, Class<?> elementType, int length,
310 Object base, long offset, // Unsafe addressing
311 V v,
312 C container, int index, // Arguments for default implementation
313 StoreVectorOperation<C, V> defaultImpl) {
314 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
315 defaultImpl.store(container, index, v);
316 }
317
318 /* ============================================================================ */
319
320 public interface StoreVectorOperationWithMap<C, V extends Vector<?>> {
321 void storeWithMap(C container, int index, V v, int[] indexMap, int indexM);
322 }
323
324 @HotSpotIntrinsicCandidate
325 public static
326 <C, V extends Vector<?>, W extends Vector<Integer>>
327 void storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass,
328 Object base, long offset, // Unsafe addressing
329 W index_vector, V v,
330 C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
331 StoreVectorOperationWithMap<C, V> defaultImpl) {
332 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
333 defaultImpl.storeWithMap(container, index, v, indexMap, indexM);
334 }
335
336 /* ============================================================================ */
337
338 @HotSpotIntrinsicCandidate
339 public static
340 <VM>
341 boolean test(int cond, Class<?> vmClass, Class<?> elementType, int length,
342 VM vm1, VM vm2,
343 BiFunction<VM, VM, Boolean> defaultImpl) {
344 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
345 return defaultImpl.apply(vm1, vm2);
346 }
347
348 /* ============================================================================ */
349
350 public interface VectorCompareOp<V,M> {
351 M apply(int cond, V v1, V v2);
352 }
353
354 @HotSpotIntrinsicCandidate
355 public static <V extends Vector<E>,
356 M extends VectorMask<E>,
357 E>
358 M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
359 V v1, V v2,
360 VectorCompareOp<V,M> defaultImpl) {
361 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
362 return defaultImpl.apply(cond, v1, v2);
363 }
364
365 /* ============================================================================ */
366
367 public interface VectorRearrangeOp<V extends Vector<E>,
368 Sh extends VectorShuffle<E>,
369 E> {
370 V apply(V v1, Sh shuffle);
371 }
372
373 @HotSpotIntrinsicCandidate
374 public static
375 <V extends Vector<E>,
376 Sh extends VectorShuffle<E>,
377 E>
378 V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<?> elementType, int vlen,
379 V v1, Sh sh,
380 VectorRearrangeOp<V,Sh, E> defaultImpl) {
381 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
382 return defaultImpl.apply(v1, sh);
383 }
384
385 /* ============================================================================ */
386
387 public interface VectorBlendOp<V extends Vector<E>,
388 M extends VectorMask<E>,
389 E> {
390 V apply(V v1, V v2, M mask);
391 }
392
393 @HotSpotIntrinsicCandidate
394 public static
395 <V extends Vector<E>,
396 M extends VectorMask<E>,
397 E>
398 V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
399 V v1, V v2, M m,
400 VectorBlendOp<V,M, E> defaultImpl) {
401 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
402 return defaultImpl.apply(v1, v2, m);
403 }
404
405 /* ============================================================================ */
406
407 public interface VectorBroadcastIntOp<V extends Vector<?>> {
408 V apply(V v, int n);
409 }
410
411 @HotSpotIntrinsicCandidate
412 public static
413 <V extends Vector<?>>
414 V broadcastInt(int opr, Class<? extends V> vectorClass, Class<?> elementType, int length,
415 V v, int n,
416 VectorBroadcastIntOp<V> defaultImpl) {
417 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
418 return defaultImpl.apply(v, n);
419 }
420
421 /* ============================================================================ */
422
423 public interface VectorConvertOp<VOUT, VIN, S> {
424 VOUT apply(VIN v, S species);
425 }
426
427 // Users of this intrinsic assume that it respects
428 // REGISTER_ENDIAN, which is currently ByteOrder.LITTLE_ENDIAN.
429 // See javadoc for REGISTER_ENDIAN.
430
431 @HotSpotIntrinsicCandidate
432 public static <VOUT extends VectorPayload,
433 VIN extends VectorPayload,
434 S extends VectorSpecies<?>>
435 VOUT convert(int oprId,
436 Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
437 Class<?> toVectorClass, Class<?> toElementType, int toVLen,
438 VIN v, S s,
439 VectorConvertOp<VOUT, VIN, S> defaultImpl) {
440 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
441 return defaultImpl.apply(v, s);
442 }
443
444 /* ============================================================================ */
445
446 @HotSpotIntrinsicCandidate
447 public static <V> V maybeRebox(V v) {
448 // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
449 // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
450 U.loadFence();
451 return v;
452 }
453
454 /* ============================================================================ */
455
456 // query the JVM's supported vector sizes and types
457 public static native int getMaxLaneCount(Class<?> etype);
458
459 /* ============================================================================ */
460
461 public static boolean isNonCapturingLambda(Object o) {
462 return o.getClass().getDeclaredFields().length == 0;
463 }
464
465 /* ============================================================================ */
466
467 private static native int registerNatives();
468 }