1 package jdk.incubator.vector;
2
3 import jdk.internal.HotSpotIntrinsicCandidate;
4 import jdk.internal.misc.Unsafe;
5 import jdk.internal.vm.annotation.ForceInline;
6
7 import java.nio.Buffer;
8 import java.nio.ByteBuffer;
9 import java.util.Objects;
10 import java.util.function.*;
11
12 /*non-public*/ class VectorIntrinsics {
13 static final Unsafe U = Unsafe.getUnsafe();
14
15 static final long BUFFER_ADDRESS
16 = U.objectFieldOffset(Buffer.class, "address");
17
18 // Buffer.limit
19 static final long BUFFER_LIMIT
20 = U.objectFieldOffset(Buffer.class, "limit");
21
22 // ByteBuffer.hb
23 static final long BYTE_BUFFER_HB
24 = U.objectFieldOffset(ByteBuffer.class, "hb");
25
26 // ByteBuffer.isReadOnly
27 static final long BYTE_BUFFER_IS_READ_ONLY
28 = U.objectFieldOffset(ByteBuffer.class, "isReadOnly");
29
30 // Unary
31 static final int VECTOR_OP_ABS = 0;
32 static final int VECTOR_OP_NEG = 1;
33 static final int VECTOR_OP_SQRT = 2;
34 static final int VECTOR_OP_NOT = 3;
35
36 // Binary
37 static final int VECTOR_OP_ADD = 4;
38 static final int VECTOR_OP_SUB = 5;
39 static final int VECTOR_OP_MUL = 6;
40 static final int VECTOR_OP_DIV = 7;
41 static final int VECTOR_OP_MIN = 8;
42 static final int VECTOR_OP_MAX = 9;
43
44 static final int VECTOR_OP_AND = 10;
45 static final int VECTOR_OP_OR = 11;
46 static final int VECTOR_OP_XOR = 12;
47
48 // Ternary
49 static final int VECTOR_OP_FMA = 13;
50
51 // Broadcast int
52 static final int VECTOR_OP_LSHIFT = 14;
53 static final int VECTOR_OP_RSHIFT = 15;
54 static final int VECTOR_OP_URSHIFT = 16;
55
56 // Math routines
57 static final int VECTOR_OP_TAN = 101;
58 static final int VECTOR_OP_TANH = 102;
59 static final int VECTOR_OP_SIN = 103;
60 static final int VECTOR_OP_SINH = 104;
61 static final int VECTOR_OP_COS = 105;
62 static final int VECTOR_OP_COSH = 106;
63 static final int VECTOR_OP_ASIN = 107;
64 static final int VECTOR_OP_ACOS = 108;
65 static final int VECTOR_OP_ATAN = 109;
66 static final int VECTOR_OP_ATAN2 = 110;
67 static final int VECTOR_OP_CBRT = 111;
68 static final int VECTOR_OP_LOG = 112;
69 static final int VECTOR_OP_LOG10 = 113;
70 static final int VECTOR_OP_LOG1P = 114;
71 static final int VECTOR_OP_POW = 115;
72 static final int VECTOR_OP_EXP = 116;
73 static final int VECTOR_OP_EXPM1 = 117;
74 static final int VECTOR_OP_HYPOT = 118;
75
76 // Copied from open/src/hotspot/cpu/x86/assembler_x86.hpp
77 // enum Condition { // The x86 condition codes used for conditional jumps/moves.
78 static final int COND_zero = 0x4;
79 static final int COND_notZero = 0x5;
80 static final int COND_equal = 0x4;
81 static final int COND_notEqual = 0x5;
82 static final int COND_less = 0xc;
83 static final int COND_lessEqual = 0xe;
84 static final int COND_greater = 0xf;
85 static final int COND_greaterEqual = 0xd;
86 static final int COND_below = 0x2;
87 static final int COND_belowEqual = 0x6;
88 static final int COND_above = 0x7;
89 static final int COND_aboveEqual = 0x3;
90 static final int COND_overflow = 0x0;
91 static final int COND_noOverflow = 0x1;
92 static final int COND_carrySet = 0x2;
93 static final int COND_carryClear = 0x3;
94 static final int COND_negative = 0x8;
95 static final int COND_positive = 0x9;
96 static final int COND_parity = 0xa;
97 static final int COND_noParity = 0xb;
98
99
100 // enum BoolTest
101 static final int BT_eq = 0;
102 static final int BT_ne = 4;
103 static final int BT_le = 5;
104 static final int BT_ge = 7;
105 static final int BT_lt = 3;
106 static final int BT_gt = 1;
107 static final int BT_overflow = 2;
108 static final int BT_no_overflow = 6;
109
110 /* ============================================================================ */
111
112 @HotSpotIntrinsicCandidate
113 static
114 <VM>
115 VM broadcastCoerced(Class<VM> vmClass, Class<?> elementType, int length,
116 long bits,
117 LongFunction<VM> defaultImpl) {
118 return defaultImpl.apply(bits);
119 }
120
121 /* ============================================================================ */
122
123 @HotSpotIntrinsicCandidate
124 static
125 <V extends Vector<?,?>>
126 long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int length,
127 V v,
128 Function<V,Long> defaultImpl) {
129 return defaultImpl.apply(v);
130 }
131
132 /* ============================================================================ */
133
134 interface VecExtractOp<V> {
135 long apply(V v1, int idx);
136 }
137
138 @HotSpotIntrinsicCandidate
139 static
140 <V extends Vector<?,?>>
141 long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
142 V vec, int ix,
143 VecExtractOp<V> defaultImpl) {
144 return defaultImpl.apply(vec, ix);
145 }
146
147 /* ============================================================================ */
148
149 interface VecInsertOp<V> {
150 V apply(V v1, int idx, long val);
151 }
152
153 @HotSpotIntrinsicCandidate
154 static <V extends Vector<?,?>>
155 V insert(Class<V> vectorClass, Class<?> elementType, int vlen,
156 V vec, int ix, long val,
157 VecInsertOp<V> defaultImpl) {
158 return defaultImpl.apply(vec, ix, val);
159 }
160
161 /* ============================================================================ */
162
163 @HotSpotIntrinsicCandidate
164 static
165 <VM>
166 VM unaryOp(int oprId, Class<VM> vmClass, Class<?> elementType, int length,
167 VM vm, /*Vector.Mask<E,S> m,*/
168 Function<VM, VM> defaultImpl) {
169 return defaultImpl.apply(vm);
170 }
171
172 /* ============================================================================ */
173
174 @HotSpotIntrinsicCandidate
175 static
176 <VM>
177 VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
178 VM vm1, VM vm2, /*Vector.Mask<E,S> m,*/
179 BiFunction<VM, VM, VM> defaultImpl) {
180 return defaultImpl.apply(vm1, vm2);
181 }
182
183 /* ============================================================================ */
184
185 interface TernaryOperation<V> {
186 V apply(V v1, V v2, V v3);
187 }
188
189 @SuppressWarnings("unchecked")
190 @HotSpotIntrinsicCandidate
191 static
192 <VM>
193 VM ternaryOp(int oprId, Class<VM> vmClass, Class<?> elementType, int length,
194 VM vm1, VM vm2, VM vm3, /*Vector.Mask<E,S> m,*/
195 TernaryOperation<VM> defaultImpl) {
196 return defaultImpl.apply(vm1, vm2, vm3);
197 }
198
199 /* ============================================================================ */
200
201 // Memory operations
202
203 interface LoadVectorOperation<C, V extends Vector<?,?>> {
204 V load(C container, int index);
205 }
206
207 // @@@ Support endianness, pass in as argument to intrinsic
208 @HotSpotIntrinsicCandidate
209 static
210 <C, V extends Vector<?,?>>
211 V load(Class<?> vectorClass, Class<?> elementType, int length,
212 Object base, long offset, // Unsafe addressing
213 // Vector.Mask<E,S> m,
214 C container, int index, // Arguments for default implementation
215 LoadVectorOperation<C, V> defaultImpl) {
216 return defaultImpl.load(container, index);
217 }
218
219 interface StoreVectorOperation<C, V extends Vector<?,?>> {
220 void store(C container, int index, V v);
221 }
222
223 // @@@ Support endianness, pass in as argument to intrinsic
224 @HotSpotIntrinsicCandidate
225 static
226 <C, V extends Vector<?,?>>
227 void store(Class<?> vectorClass, Class<?> elementType, int length,
228 Object base, long offset, // Unsafe addressing
229 V v,
230 // Vector.Mask<E,S> m,
231 C container, int index, // Arguments for default implementation
232 StoreVectorOperation<C, V> defaultImpl) {
233 defaultImpl.store(container, index, v);
234 }
235
236 /* ============================================================================ */
237
238 @HotSpotIntrinsicCandidate
239 static
240 <VM>
241 boolean test(int cond, Class<?> vmClass, Class<?> elementType, int length,
242 VM vm1, VM vm2,
243 BiFunction<VM, VM, Boolean> defaultImpl) {
244 return defaultImpl.apply(vm1, vm2);
245 }
246
247 /* ============================================================================ */
248
249 interface VectorCompareOp<V,M> {
250 M apply(V v1, V v2);
251 }
252
253 @HotSpotIntrinsicCandidate
254 static <V extends Vector<E,S>,
255 M extends Vector.Mask<E,S>,
256 S extends Vector.Shape, E>
257 M compare(int cond, Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
258 V v1, V v2,
259 VectorCompareOp<V,M> defaultImpl) {
260 return defaultImpl.apply(v1, v2);
261 }
262
263 /* ============================================================================ */
264
265 interface VectorBlendOp<V extends Vector<E,S>,
266 M extends Vector.Mask<E,S>,
267 S extends Vector.Shape, E> {
268 V apply(V v1, V v2, M mask);
269 }
270
271 @HotSpotIntrinsicCandidate
272 static
273 <V extends Vector<E,S>,
274 M extends Vector.Mask<E,S>,
275 S extends Vector.Shape, E>
276 V blend(Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
277 V v1, V v2, M m,
278 VectorBlendOp<V,M,S,E> defaultImpl) {
279 return defaultImpl.apply(v1, v2, m);
280 }
281
282 /* ============================================================================ */
283
284 interface VectorBroadcastIntOp<V extends Vector<?,?>> {
285 V apply(V v, int i);
286 }
287
288 @HotSpotIntrinsicCandidate
289 static
290 <V extends Vector<?,?>>
291 V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int length,
292 V v, int i,
293 VectorBroadcastIntOp<V> defaultImpl) {
294 return defaultImpl.apply(v, i);
295 }
296
297 /* ============================================================================ */
298
299 interface VectorReinterpretOp<S, VIN, V> {
300 V apply(S species, VIN v);
301 }
302
303 @HotSpotIntrinsicCandidate
304 static
305 <S, VIN, V>
306 V reinterpret(Class<?> fromVectorClass,
307 Class<?> fromElementType, int fromVLen,
308 Class<?> toElementType, int toVLen,
309 VIN v, S s,
310 VectorReinterpretOp<S, VIN, V> defaultImpl) {
311 return defaultImpl.apply(s, v);
312 }
313
314 /* ============================================================================ */
315
316 interface VectorCastOp<S, VIN, V> {
317 V apply(S species, VIN v);
318 }
319
320 @HotSpotIntrinsicCandidate
321 static
322 <S, VIN, V>
323 V cast(Class<?> fromVectorClass,
324 Class<?> fromElementType, int fromVLen,
325 Class<?> toElementType, int toVLen,
326 VIN v, S s,
327 VectorCastOp<S, VIN, V> defaultImpl) {
328 return defaultImpl.apply(s, v);
329 }
330
331 /* ============================================================================ */
332
333 @HotSpotIntrinsicCandidate
334 static <V> V maybeRebox(V v) {
335 // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
336 // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
337 U.loadFence();
338 return v;
339 }
340
341 /* ============================================================================ */
342
343 static final int VECTOR_ACCESS_OOB_CHECK = Integer.getInteger("jdk.incubator.vector.VECTOR_ACCESS_OOB_CHECK", 2);
344
345 @ForceInline
346 static int checkIndex(int ix, int length, int vlen) {
347 switch (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK) {
348 case 0: return ix; // no range check
349 case 1: return Objects.checkFromIndexSize(ix, vlen, length);
350 case 2: return Objects.checkIndex(ix, length - (vlen - 1));
351 default: throw new InternalError();
352 }
353 }
354 }