1 package jdk.incubator.vector;
2
3 import jdk.internal.HotSpotIntrinsicCandidate;
4 import jdk.internal.misc.Unsafe;
5 import jdk.internal.vm.annotation.ForceInline;
6
7 import java.nio.Buffer;
8 import java.nio.ByteBuffer;
9 import java.util.Objects;
10 import java.util.function.*;
11
12 import jdk.incubator.vector.Vector;
13
14 /*non-public*/ class VectorIntrinsics {
15 static final Unsafe U = Unsafe.getUnsafe();
16
17 static final long BUFFER_ADDRESS
18 = U.objectFieldOffset(Buffer.class, "address");
19
20 // Buffer.limit
21 static final long BUFFER_LIMIT
22 = U.objectFieldOffset(Buffer.class, "limit");
23
24 // ByteBuffer.hb
25 static final long BYTE_BUFFER_HB
26 = U.objectFieldOffset(ByteBuffer.class, "hb");
27
28 // ByteBuffer.isReadOnly
29 static final long BYTE_BUFFER_IS_READ_ONLY
30 = U.objectFieldOffset(ByteBuffer.class, "isReadOnly");
31
32 // Unary
33 static final int VECTOR_OP_ABS = 0;
34 static final int VECTOR_OP_NEG = 1;
35 static final int VECTOR_OP_SQRT = 2;
36 static final int VECTOR_OP_NOT = 3;
37
38 // Binary
39 static final int VECTOR_OP_ADD = 4;
40 static final int VECTOR_OP_SUB = 5;
41 static final int VECTOR_OP_MUL = 6;
42 static final int VECTOR_OP_DIV = 7;
43 static final int VECTOR_OP_MIN = 8;
44 static final int VECTOR_OP_MAX = 9;
45
46 static final int VECTOR_OP_AND = 10;
47 static final int VECTOR_OP_OR = 11;
48 static final int VECTOR_OP_XOR = 12;
49
50 // Ternary
51 static final int VECTOR_OP_FMA = 13;
52
53 // Broadcast int
54 static final int VECTOR_OP_LSHIFT = 14;
55 static final int VECTOR_OP_RSHIFT = 15;
56 static final int VECTOR_OP_URSHIFT = 16;
57
58 // Math routines
59 static final int VECTOR_OP_TAN = 101;
60 static final int VECTOR_OP_TANH = 102;
61 static final int VECTOR_OP_SIN = 103;
62 static final int VECTOR_OP_SINH = 104;
63 static final int VECTOR_OP_COS = 105;
64 static final int VECTOR_OP_COSH = 106;
65 static final int VECTOR_OP_ASIN = 107;
66 static final int VECTOR_OP_ACOS = 108;
67 static final int VECTOR_OP_ATAN = 109;
68 static final int VECTOR_OP_ATAN2 = 110;
69 static final int VECTOR_OP_CBRT = 111;
70 static final int VECTOR_OP_LOG = 112;
71 static final int VECTOR_OP_LOG10 = 113;
72 static final int VECTOR_OP_LOG1P = 114;
73 static final int VECTOR_OP_POW = 115;
74 static final int VECTOR_OP_EXP = 116;
75 static final int VECTOR_OP_EXPM1 = 117;
76 static final int VECTOR_OP_HYPOT = 118;
77
78 // enum BoolTest
79 static final int BT_eq = 0;
80 static final int BT_ne = 4;
81 static final int BT_le = 5;
82 static final int BT_ge = 7;
83 static final int BT_lt = 3;
84 static final int BT_gt = 1;
85 static final int BT_overflow = 2;
86 static final int BT_no_overflow = 6;
87
88 /* ============================================================================ */
89 interface BroadcastOperation<V, E> {
90 V broadcast(long l, Vector.Species<E> s);
91 }
92
93 @HotSpotIntrinsicCandidate
94 static
95 <VM, E>
96 VM broadcastCoerced(Class<VM> vmClass, Class<?> E, int length,
97 long bits, Vector.Species<E> s,
98 BroadcastOperation<VM, E> defaultImpl) {
99 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
100 return defaultImpl.broadcast(bits, s);
101 }
102
103 /* ============================================================================ */
104
105 @HotSpotIntrinsicCandidate
106 static
107 <V extends Vector<?>>
108 long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int length,
109 V v,
110 Function<V,Long> defaultImpl) {
111 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
112 return defaultImpl.apply(v);
113 }
114
115 /* ============================================================================ */
116
117 interface VecExtractOp<V> {
118 long apply(V v1, int idx);
119 }
120
121 @HotSpotIntrinsicCandidate
122 static
123 <V extends Vector<?>>
124 long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
125 V vec, int ix,
126 VecExtractOp<V> defaultImpl) {
127 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
128 return defaultImpl.apply(vec, ix);
129 }
130
131 /* ============================================================================ */
132
133 interface VecInsertOp<V> {
134 V apply(V v1, int idx, long val);
135 }
136
137 @HotSpotIntrinsicCandidate
138 static <V extends Vector<?>>
139 V insert(Class<V> vectorClass, Class<?> elementType, int vlen,
140 V vec, int ix, long val,
141 VecInsertOp<V> defaultImpl) {
142 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
143 return defaultImpl.apply(vec, ix, val);
144 }
145
146 /* ============================================================================ */
147
148 @HotSpotIntrinsicCandidate
149 static
150 <VM>
151 VM unaryOp(int oprId, Class<VM> vmClass, Class<?> elementType, int length,
152 VM vm, /*Vector.Mask<E,S> m,*/
153 Function<VM, VM> defaultImpl) {
154 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
155 return defaultImpl.apply(vm);
156 }
157
158 /* ============================================================================ */
159
160 @HotSpotIntrinsicCandidate
161 static
162 <VM>
163 VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
164 VM vm1, VM vm2, /*Vector.Mask<E,S> m,*/
165 BiFunction<VM, VM, VM> defaultImpl) {
166 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
167 return defaultImpl.apply(vm1, vm2);
168 }
169
170 /* ============================================================================ */
171
172 interface TernaryOperation<V> {
173 V apply(V v1, V v2, V v3);
174 }
175
176 @HotSpotIntrinsicCandidate
177 static
178 <VM>
179 VM ternaryOp(int oprId, Class<VM> vmClass, Class<?> elementType, int length,
180 VM vm1, VM vm2, VM vm3, /*Vector.Mask<E,S> m,*/
181 TernaryOperation<VM> defaultImpl) {
182 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
183 return defaultImpl.apply(vm1, vm2, vm3);
184 }
185
186 /* ============================================================================ */
187
188 // Memory operations
189
190 interface LoadOperation<C, V, E> {
191 V load(C container, int index, Vector.Species<E> s);
192 }
193
194 @HotSpotIntrinsicCandidate
195 static
196 <C, VM, E>
197 VM load(Class<VM> vmClass, Class<?> E, int length,
198 Object base, long offset, // Unsafe addressing
199 // Vector.Mask<E,S> m,
200 C container, int index, Vector.Species<E> s, // Arguments for default implementation
201 LoadOperation<C, VM, E> defaultImpl) {
202 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
203 return defaultImpl.load(container, index, s);
204 }
205
206 /* ============================================================================ */
207
208 interface LoadVectorOperationWithMap<C, V extends Vector<?>, E> {
209 V loadWithMap(C container, int index, int[] indexMap, int indexM, Vector.Species<E> s);
210 }
211
212 @HotSpotIntrinsicCandidate
213 static
214 <C, V extends Vector<?>, W extends IntVector, E>
215 V loadWithMap(Class<?> vectorClass, Class<?> E, int length, Class<?> vectorIndexClass,
216 Object base, long offset, // Unsafe addressing
217 W index_vector,
218 C container, int index, int[] indexMap, int indexM, Vector.Species<E> s, // Arguments for default implementation
219 LoadVectorOperationWithMap<C, V, E> defaultImpl) {
220 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
221 return defaultImpl.loadWithMap(container, index, indexMap, indexM, s);
222 }
223
224 /* ============================================================================ */
225
226 interface StoreVectorOperation<C, V extends Vector<?>> {
227 void store(C container, int index, V v);
228 }
229
230 @HotSpotIntrinsicCandidate
231 static
232 <C, V extends Vector<?>>
233 void store(Class<?> vectorClass, Class<?> elementType, int length,
234 Object base, long offset, // Unsafe addressing
235 V v,
236 // Vector.Mask<E,S> m,
237 C container, int index, // Arguments for default implementation
238 StoreVectorOperation<C, V> defaultImpl) {
239 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
240 defaultImpl.store(container, index, v);
241 }
242
243 /* ============================================================================ */
244
245 interface StoreVectorOperationWithMap<C, V extends Vector<?>> {
246 void storeWithMap(C container, int index, V v, int[] indexMap, int indexM);
247 }
248
249 @HotSpotIntrinsicCandidate
250 static
251 <C, V extends Vector<?>, W extends IntVector>
252 void storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass,
253 Object base, long offset, // Unsafe addressing
254 W index_vector, V v,
255 C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
256 StoreVectorOperationWithMap<C, V> defaultImpl) {
257 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
258 defaultImpl.storeWithMap(container, index, v, indexMap, indexM);
259 }
260
261 /* ============================================================================ */
262
263 @HotSpotIntrinsicCandidate
264 static
265 <VM>
266 boolean test(int cond, Class<?> vmClass, Class<?> elementType, int length,
267 VM vm1, VM vm2,
268 BiFunction<VM, VM, Boolean> defaultImpl) {
269 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
270 return defaultImpl.apply(vm1, vm2);
271 }
272
273 /* ============================================================================ */
274
275 interface VectorCompareOp<V,M> {
276 M apply(V v1, V v2);
277 }
278
279 @HotSpotIntrinsicCandidate
280 static <V extends Vector<E>,
281 M extends Vector.Mask<E>,
282 E>
283 M compare(int cond, Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
284 V v1, V v2,
285 VectorCompareOp<V,M> defaultImpl) {
286 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
287 return defaultImpl.apply(v1, v2);
288 }
289
290 /* ============================================================================ */
291
292 interface VectorRearrangeOp<V extends Vector<E>,
293 Sh extends Vector.Shuffle<E>,
294 E> {
295 V apply(V v1, Sh shuffle);
296 }
297
298 @HotSpotIntrinsicCandidate
299 static
300 <V extends Vector<E>,
301 Sh extends Vector.Shuffle<E>,
302 E>
303 V rearrangeOp(Class<V> vectorClass, Class<Sh> shuffleClass, Class<?> elementType, int vlen,
304 V v1, Sh sh,
305 VectorRearrangeOp<V,Sh, E> defaultImpl) {
306 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
307 return defaultImpl.apply(v1, sh);
308 }
309
310 /* ============================================================================ */
311
312 interface VectorBlendOp<V extends Vector<E>,
313 M extends Vector.Mask<E>,
314 E> {
315 V apply(V v1, V v2, M mask);
316 }
317
318 @HotSpotIntrinsicCandidate
319 static
320 <V extends Vector<E>,
321 M extends Vector.Mask<E>,
322 E>
323 V blend(Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
324 V v1, V v2, M m,
325 VectorBlendOp<V,M, E> defaultImpl) {
326 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
327 return defaultImpl.apply(v1, v2, m);
328 }
329
330 /* ============================================================================ */
331
332 interface VectorBroadcastIntOp<V extends Vector<?>> {
333 V apply(V v, int i);
334 }
335
336 @HotSpotIntrinsicCandidate
337 static
338 <V extends Vector<?>>
339 V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int length,
340 V v, int i,
341 VectorBroadcastIntOp<V> defaultImpl) {
342 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
343 return defaultImpl.apply(v, i);
344 }
345
346 /* ============================================================================ */
347
348 interface VectorReinterpretOp<S, VIN, V> {
349 V apply(S species, VIN v);
350 }
351
352 @HotSpotIntrinsicCandidate
353 static
354 <S, VIN, V>
355 V reinterpret(Class<?> fromVectorClass,
356 Class<?> fromElementType, int fromVLen,
357 Class<?> toVectorClass,
358 Class<?> toElementType, int toVLen,
359 VIN v, S s,
360 VectorReinterpretOp<S, VIN, V> defaultImpl) {
361 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
362 return defaultImpl.apply(s, v);
363 }
364
365 /* ============================================================================ */
366
367 interface VectorCastOp<S, VIN, V> {
368 V apply(S species, VIN v);
369 }
370
371 @HotSpotIntrinsicCandidate
372 static
373 <S, VIN, V>
374 V cast(Class<?> fromVectorClass,
375 Class<?> fromElementType, int fromVLen,
376 Class<?> toVectorClass,
377 Class<?> toElementType, int toVLen,
378 VIN v, S s,
379 VectorCastOp<S, VIN, V> defaultImpl) {
380 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
381 return defaultImpl.apply(s, v);
382 }
383
384 /* ============================================================================ */
385
386 @HotSpotIntrinsicCandidate
387 static <V> V maybeRebox(V v) {
388 // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
389 // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
390 U.loadFence();
391 return v;
392 }
393
394 /* ============================================================================ */
395
396 static final int VECTOR_ACCESS_OOB_CHECK = Integer.getInteger("jdk.incubator.vector.VECTOR_ACCESS_OOB_CHECK", 2);
397
398 @ForceInline
399 static int checkIndex(int ix, int length, int vlen) {
400 switch (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK) {
401 case 0: return ix; // no range check
402 case 1: return Objects.checkFromIndexSize(ix, vlen, length);
403 case 2: return Objects.checkIndex(ix, length - (vlen - 1));
404 default: throw new InternalError();
405 }
406 }
407
408 @ForceInline
409 static IntVector checkIndex(IntVector vix, int length) {
410 switch (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK) {
411 case 0: return vix; // no range check
412 case 1: // fall-through
413 case 2:
414 if(vix.lessThan(0).anyTrue() || vix.greaterThanEq(length).anyTrue()) {
415 String msg = String.format("Range check failed: vector %s out of bounds for length %d", vix, length);
416 throw new ArrayIndexOutOfBoundsException(msg);
417 }
418 return vix;
419 default: throw new InternalError();
420 }
421 }
422
423 static boolean isNonCapturingLambda(Object o) {
424 return o.getClass().getDeclaredFields().length == 0;
425 }
426 }