1 package jdk.internal.vm.vector;
2
3 import jdk.internal.HotSpotIntrinsicCandidate;
4 import jdk.internal.misc.Unsafe;
5 import jdk.internal.vm.annotation.ForceInline;
6
7 import java.nio.Buffer;
8 import java.nio.ByteBuffer;
9 import java.util.Objects;
10 import java.util.function.*;
11
12 public class VectorSupport {
13 static {
14 registerNatives();
15 }
16
17 private static final Unsafe U = Unsafe.getUnsafe();
18
19 // Unary
20 public static final int VECTOR_OP_ABS = 0;
21 public static final int VECTOR_OP_NEG = 1;
22 public static final int VECTOR_OP_SQRT = 2;
23 public static final int VECTOR_OP_NOT = 3;
24
25 // Binary
26 public static final int VECTOR_OP_ADD = 4;
27 public static final int VECTOR_OP_SUB = 5;
28 public static final int VECTOR_OP_MUL = 6;
29 public static final int VECTOR_OP_DIV = 7;
30 public static final int VECTOR_OP_MIN = 8;
31 public static final int VECTOR_OP_MAX = 9;
32
33 public static final int VECTOR_OP_AND = 10;
34 public static final int VECTOR_OP_OR = 11;
35 public static final int VECTOR_OP_XOR = 12;
36
37 // Ternary
38 public static final int VECTOR_OP_FMA = 13;
39
40 // Broadcast int
41 public static final int VECTOR_OP_LSHIFT = 14;
42 public static final int VECTOR_OP_RSHIFT = 15;
43 public static final int VECTOR_OP_URSHIFT = 16;
44
45 public static final int VECTOR_OP_CAST = 17;
46 public static final int VECTOR_OP_REINTERPRET = 18;
47
48 // enum BoolTest
49 public static final int BT_eq = 0;
50 public static final int BT_ne = 4;
51 public static final int BT_le = 5;
52 public static final int BT_ge = 7;
53 public static final int BT_lt = 3;
54 public static final int BT_gt = 1;
55 public static final int BT_overflow = 2;
56 public static final int BT_no_overflow = 6;
57
58 // BasicType codes, for primitives only:
59 public static final int
60 T_FLOAT = 6,
61 T_DOUBLE = 7,
62 T_BYTE = 8,
63 T_SHORT = 9,
64 T_INT = 10,
65 T_LONG = 11;
66
67 /* ============================================================================ */
68
69 public static class VectorSpecies<E> {}
70
71 public static class VectorPayload {
72 private final Object payload; // array of primitives
73
74 public VectorPayload(Object payload) {
75 this.payload = payload;
76 }
77
78 protected final Object getPayload() {
79 return VectorSupport.maybeRebox(this).payload;
80 }
81 }
82
83 public static class Vector<E> extends VectorPayload {
84 public Vector(Object payload) {
85 super(payload);
86 }
87 }
88
89 public static class VectorShuffle<E> extends VectorPayload {
90 public VectorShuffle(Object payload) {
91 super(payload);
92 }
93 }
94 public static class VectorMask<E> extends VectorPayload {
95 public VectorMask(Object payload) {
96 super(payload);
97 }
98 }
99
100 /* ============================================================================ */
101 public interface BroadcastOperation<VM, E, S extends VectorSpecies<E>> {
102 VM broadcast(long l, S s);
103 }
104
105 @HotSpotIntrinsicCandidate
106 public static
107 <VM, E, S extends VectorSpecies<E>>
108 VM broadcastCoerced(Class<? extends VM> vmClass, Class<E> E, int length,
109 long bits, S s,
110 BroadcastOperation<VM, E, S> defaultImpl) {
111 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
112 return defaultImpl.broadcast(bits, s);
113 }
114
115 /* ============================================================================ */
116 public interface ShuffleIotaOperation<E> {
117 VectorShuffle<E> apply(int length, int start, int step);
118 }
119
120 @HotSpotIntrinsicCandidate
121 public static
122 <E>
123 VectorShuffle<E> shuffleIota(Class<?> E, Class<?> ShuffleClass, VectorSpecies<E> s, int length,
124 int start, int step, int wrap, ShuffleIotaOperation<E> defaultImpl) {
125 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
126 return defaultImpl.apply(length, start, step);
127 }
128
129 public interface ShuffleToVectorOperation<VM, Sh, E> {
130 VM apply(Sh s);
131 }
132
133 @HotSpotIntrinsicCandidate
134 public static
135 <VM ,Sh extends VectorShuffle<E>, E>
136 VM shuffleToVector(Class<?> VM, Class<?>E , Class<?> ShuffleClass, Sh s, int length,
137 ShuffleToVectorOperation<VM,Sh,E> defaultImpl) {
138 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
139 return defaultImpl.apply(s);
140 }
141
142 /* ============================================================================ */
143 public interface IndexOperation<V extends Vector<E>, E, S extends VectorSpecies<E>> {
144 V index(V v, int step, S s);
145 }
146
147 //FIXME @HotSpotIntrinsicCandidate
148 public static
149 <V extends Vector<E>, E, S extends VectorSpecies<E>>
150 V indexVector(Class<? extends V> vClass, Class<E> E, int length,
151 V v, int step, S s,
152 IndexOperation<V, E, S> defaultImpl) {
153 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
154 return defaultImpl.index(v, step, s);
155 }
156
157 /* ============================================================================ */
158
159 @HotSpotIntrinsicCandidate
160 public static
161 <V extends Vector<?>>
162 long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int length,
163 V v,
164 Function<V,Long> defaultImpl) {
165 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
166 return defaultImpl.apply(v);
167 }
168
169 /* ============================================================================ */
170
171 public interface VecExtractOp<V> {
172 long apply(V v1, int idx);
173 }
174
175 @HotSpotIntrinsicCandidate
176 public static
177 <V extends Vector<?>>
178 long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
179 V vec, int ix,
180 VecExtractOp<V> defaultImpl) {
181 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
182 return defaultImpl.apply(vec, ix);
183 }
184
185 /* ============================================================================ */
186
187 public interface VecInsertOp<V> {
188 V apply(V v1, int idx, long val);
189 }
190
191 @HotSpotIntrinsicCandidate
192 public static
193 <V extends Vector<?>>
194 V insert(Class<? extends V> vectorClass, Class<?> elementType, int vlen,
195 V vec, int ix, long val,
196 VecInsertOp<V> defaultImpl) {
197 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
198 return defaultImpl.apply(vec, ix, val);
199 }
200
201 /* ============================================================================ */
202
203 @HotSpotIntrinsicCandidate
204 public static
205 <VM>
206 VM unaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
207 VM vm,
208 Function<VM, VM> defaultImpl) {
209 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
210 return defaultImpl.apply(vm);
211 }
212
213 /* ============================================================================ */
214
215 @HotSpotIntrinsicCandidate
216 public static
217 <VM>
218 VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
219 VM vm1, VM vm2,
220 BiFunction<VM, VM, VM> defaultImpl) {
221 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
222 return defaultImpl.apply(vm1, vm2);
223 }
224
225 /* ============================================================================ */
226
227 public interface TernaryOperation<V> {
228 V apply(V v1, V v2, V v3);
229 }
230
231 @HotSpotIntrinsicCandidate
232 public static
233 <VM>
234 VM ternaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
235 VM vm1, VM vm2, VM vm3,
236 TernaryOperation<VM> defaultImpl) {
237 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
238 return defaultImpl.apply(vm1, vm2, vm3);
239 }
240
241 /* ============================================================================ */
242
243 // Memory operations
244
245 public interface LoadOperation<C, V, E, S extends VectorSpecies<E>> {
246 V load(C container, int index, S s);
247 }
248
249 @HotSpotIntrinsicCandidate
250 public static
251 <C, VM, E, S extends VectorSpecies<E>>
252 VM load(Class<? extends VM> vmClass, Class<E> E, int length,
253 Object base, long offset, // Unsafe addressing
254 C container, int index, S s, // Arguments for default implementation
255 LoadOperation<C, VM, E, S> defaultImpl) {
256 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
257 return defaultImpl.load(container, index, s);
258 }
259
260 /* ============================================================================ */
261
262 public interface LoadVectorOperationWithMap<C, V extends Vector<?>, E, S extends VectorSpecies<E>> {
263 V loadWithMap(C container, int index, int[] indexMap, int indexM, S s);
264 }
265
266 @HotSpotIntrinsicCandidate
267 public static
268 <C, V extends Vector<?>, W extends Vector<Integer>, E, S extends VectorSpecies<E>>
269 V loadWithMap(Class<?> vectorClass, Class<E> E, int length, Class<?> vectorIndexClass,
270 Object base, long offset, // Unsafe addressing
271 W index_vector,
272 C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
273 LoadVectorOperationWithMap<C, V, E, S> defaultImpl) {
274 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
275 return defaultImpl.loadWithMap(container, index, indexMap, indexM, s);
276 }
277
278 /* ============================================================================ */
279
280 public interface StoreVectorOperation<C, V extends Vector<?>> {
281 void store(C container, int index, V v);
282 }
283
284 @HotSpotIntrinsicCandidate
285 public static
286 <C, V extends Vector<?>>
287 void store(Class<?> vectorClass, Class<?> elementType, int length,
288 Object base, long offset, // Unsafe addressing
289 V v,
290 C container, int index, // Arguments for default implementation
291 StoreVectorOperation<C, V> defaultImpl) {
292 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
293 defaultImpl.store(container, index, v);
294 }
295
296 /* ============================================================================ */
297
298 public interface StoreVectorOperationWithMap<C, V extends Vector<?>> {
299 void storeWithMap(C container, int index, V v, int[] indexMap, int indexM);
300 }
301
302 @HotSpotIntrinsicCandidate
303 public static
304 <C, V extends Vector<?>, W extends Vector<Integer>>
305 void storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass,
306 Object base, long offset, // Unsafe addressing
307 W index_vector, V v,
308 C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
309 StoreVectorOperationWithMap<C, V> defaultImpl) {
310 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
311 defaultImpl.storeWithMap(container, index, v, indexMap, indexM);
312 }
313
314 /* ============================================================================ */
315
316 @HotSpotIntrinsicCandidate
317 public static
318 <VM>
319 boolean test(int cond, Class<?> vmClass, Class<?> elementType, int length,
320 VM vm1, VM vm2,
321 BiFunction<VM, VM, Boolean> defaultImpl) {
322 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
323 return defaultImpl.apply(vm1, vm2);
324 }
325
326 /* ============================================================================ */
327
328 public interface VectorCompareOp<V,M> {
329 M apply(int cond, V v1, V v2);
330 }
331
332 @HotSpotIntrinsicCandidate
333 public static <V extends Vector<E>,
334 M extends VectorMask<E>,
335 E>
336 M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
337 V v1, V v2,
338 VectorCompareOp<V,M> defaultImpl) {
339 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
340 return defaultImpl.apply(cond, v1, v2);
341 }
342
343 /* ============================================================================ */
344
345 public interface VectorRearrangeOp<V extends Vector<E>,
346 Sh extends VectorShuffle<E>,
347 E> {
348 V apply(V v1, Sh shuffle);
349 }
350
351 @HotSpotIntrinsicCandidate
352 public static
353 <V extends Vector<E>,
354 Sh extends VectorShuffle<E>,
355 E>
356 V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<?> elementType, int vlen,
357 V v1, Sh sh,
358 VectorRearrangeOp<V,Sh, E> defaultImpl) {
359 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
360 return defaultImpl.apply(v1, sh);
361 }
362
363 /* ============================================================================ */
364
365 public interface VectorBlendOp<V extends Vector<E>,
366 M extends VectorMask<E>,
367 E> {
368 V apply(V v1, V v2, M mask);
369 }
370
371 @HotSpotIntrinsicCandidate
372 public static
373 <V extends Vector<E>,
374 M extends VectorMask<E>,
375 E>
376 V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
377 V v1, V v2, M m,
378 VectorBlendOp<V,M, E> defaultImpl) {
379 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
380 return defaultImpl.apply(v1, v2, m);
381 }
382
383 /* ============================================================================ */
384
385 public interface VectorBroadcastIntOp<V extends Vector<?>> {
386 V apply(V v, int n);
387 }
388
389 @HotSpotIntrinsicCandidate
390 public static
391 <V extends Vector<?>>
392 V broadcastInt(int opr, Class<? extends V> vectorClass, Class<?> elementType, int length,
393 V v, int n,
394 VectorBroadcastIntOp<V> defaultImpl) {
395 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
396 return defaultImpl.apply(v, n);
397 }
398
399 /* ============================================================================ */
400
401 public interface VectorConvertOp<VOUT, VIN, S> {
402 VOUT apply(VIN v, S species);
403 }
404
405 // Users of this intrinsic assume that it respects
406 // REGISTER_ENDIAN, which is currently ByteOrder.LITTLE_ENDIAN.
407 // See javadoc for REGISTER_ENDIAN.
408
409 @HotSpotIntrinsicCandidate
410 public static <VOUT extends VectorPayload,
411 VIN extends VectorPayload,
412 S extends VectorSpecies<?>>
413 VOUT convert(int oprId,
414 Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
415 Class<?> toVectorClass, Class<?> toElementType, int toVLen,
416 VIN v, S s,
417 VectorConvertOp<VOUT, VIN, S> defaultImpl) {
418 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
419 return defaultImpl.apply(v, s);
420 }
421
422 /* ============================================================================ */
423
424 @HotSpotIntrinsicCandidate
425 public static <V> V maybeRebox(V v) {
426 // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
427 // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
428 U.loadFence();
429 return v;
430 }
431
432 /* ============================================================================ */
433
434 // query the JVM's supported vector sizes and types
435 public static native int getMaxLaneCount(Class<?> etype);
436
437 /* ============================================================================ */
438
439 public static boolean isNonCapturingLambda(Object o) {
440 return o.getClass().getDeclaredFields().length == 0;
441 }
442
443 /* ============================================================================ */
444
445 private static native int registerNatives();
446 }