1 package jdk.incubator.vector;
2
3 import jdk.internal.HotSpotIntrinsicCandidate;
4 import jdk.internal.misc.Unsafe;
5 import jdk.internal.vm.annotation.ForceInline;
6
7 import java.util.Objects;
8 import java.util.function.*;
9
10 /*non-public*/ class VectorIntrinsics {
11 private static final Unsafe U = Unsafe.getUnsafe();
12
13 // Unary
14 static final int VECTOR_OP_ABS = 0;
15 static final int VECTOR_OP_NEG = 1;
16 static final int VECTOR_OP_SQRT = 2;
17 static final int VECTOR_OP_NOT = 3;
18
19 // Binary
20 static final int VECTOR_OP_ADD = 4;
21 static final int VECTOR_OP_SUB = 5;
22 static final int VECTOR_OP_MUL = 6;
23 static final int VECTOR_OP_DIV = 7;
24 static final int VECTOR_OP_MIN = 8;
25 static final int VECTOR_OP_MAX = 9;
26
27 static final int VECTOR_OP_AND = 10;
28 static final int VECTOR_OP_OR = 11;
29 static final int VECTOR_OP_XOR = 12;
30
31 // Ternary
32 static final int VECTOR_OP_FMA = 13;
33
34 // Broadcast int
35 static final int VECTOR_OP_LSHIFT = 14;
36 static final int VECTOR_OP_RSHIFT = 15;
37 static final int VECTOR_OP_URSHIFT = 16;
38
39 // Copied from open/src/hotspot/cpu/x86/assembler_x86.hpp
40 // enum Condition { // The x86 condition codes used for conditional jumps/moves.
41 static final int COND_zero = 0x4;
42 static final int COND_notZero = 0x5;
43 static final int COND_equal = 0x4;
44 static final int COND_notEqual = 0x5;
45 static final int COND_less = 0xc;
46 static final int COND_lessEqual = 0xe;
47 static final int COND_greater = 0xf;
48 static final int COND_greaterEqual = 0xd;
49 static final int COND_below = 0x2;
50 static final int COND_belowEqual = 0x6;
51 static final int COND_above = 0x7;
52 static final int COND_aboveEqual = 0x3;
53 static final int COND_overflow = 0x0;
54 static final int COND_noOverflow = 0x1;
55 static final int COND_carrySet = 0x2;
56 static final int COND_carryClear = 0x3;
57 static final int COND_negative = 0x8;
58 static final int COND_positive = 0x9;
59 static final int COND_parity = 0xa;
60 static final int COND_noParity = 0xb;
61
62
63 // enum BoolTest
64 static final int BT_eq = 0;
65 static final int BT_ne = 4;
66 static final int BT_le = 5;
67 static final int BT_ge = 7;
68 static final int BT_lt = 3;
69 static final int BT_gt = 1;
70 static final int BT_overflow = 2;
71 static final int BT_no_overflow = 6;
72
73 /* ============================================================================ */
74
75 @HotSpotIntrinsicCandidate
76 static <V> V broadcastCoerced(Class<V> vectorClass, Class<?> elementType, int vlen,
77 long bits,
78 LongFunction<V> defaultImpl) {
79 return defaultImpl.apply(bits);
80 }
81
82 @HotSpotIntrinsicCandidate
83 static
84 <V extends Vector<?,?>>
85 long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int vlen,
86 V v,
87 Function<V,Long> defaultImpl) {
88 return defaultImpl.apply(v);
89 }
90
91 /* ============================================================================ */
92
93 @HotSpotIntrinsicCandidate
94 static <V> V unaryOp(int oprId, Class<V> vectorClass, Class<?> elementType, int vlen,
95 V v1, /*Vector.Mask<E,S> m,*/
96 Function<V,V> defaultImpl) {
97 return defaultImpl.apply(v1);
98 }
99
100 /* ============================================================================ */
101
102 @HotSpotIntrinsicCandidate
103 static <V> V binaryOp(int oprId, Class<? extends V> vectorClass, Class<?> elementType, int vlen,
104 V v1, V v2, /*Vector.Mask<E,S> m,*/
105 BiFunction<V,V,V> defaultImpl) {
106 return defaultImpl.apply(v1, v2);
107 }
108
109 /* ============================================================================ */
110
111 interface TernaryOperation<V> {
112 V apply(V v1, V v2, V v3);
113 }
114
115 @SuppressWarnings("unchecked")
116 @HotSpotIntrinsicCandidate
117 static <V> V ternaryOp(int oprId, Class<V> vectorClass, Class<?> elementType, int vlen,
118 V v1, V v2, V v3, /*Vector.Mask<E,S> m,*/
119 TernaryOperation<V> defaultImpl) {
120 return defaultImpl.apply(v1, v2, v3);
121 }
122
123 /* ============================================================================ */
124
125 // Memory operations
126
127 // FIXME: arrays are erased to Object
128
129 @HotSpotIntrinsicCandidate
130 static
131 <V extends Vector<?,?>>
132 V load(Class<?> vectorClass, Class<?> elementType, int vlen,
133 Object array, int index, /* Vector.Mask<E,S> m*/
134 BiFunction<Object, Integer, V> defaultImpl) {
135 return defaultImpl.apply(array, index);
136 }
137
138 interface StoreVectorOperation<V extends Vector<?,?>> {
139 void store(Object array, int index, V v);
140 }
141
142 @HotSpotIntrinsicCandidate
143 static
144 <V extends Vector<?,?>>
145 void store(Class<?> vectorClass, Class<?> elementType, int vlen,
146 Object array, int index, V v, /*Vector.Mask<E,S> m*/
147 StoreVectorOperation<V> defaultImpl) {
148 defaultImpl.store(array, index, v);
149 }
150
151 /* ============================================================================ */
152
153 @HotSpotIntrinsicCandidate
154 static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
155 V v1, V v2,
156 BiFunction<V, V, Boolean> defaultImpl) {
157 return defaultImpl.apply(v1, v2);
158 }
159
160 /* ============================================================================ */
161
162 interface VectorCompareOp<V,M> {
163 M apply(V v1, V v2);
164 }
165
166 @HotSpotIntrinsicCandidate
167 static <V extends Vector<E,S>,
168 M extends Vector.Mask<E,S>,
169 S extends Vector.Shape, E>
170 M compare(int cond, Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int vlen,
171 V v1, V v2,
172 VectorCompareOp<V,M> defaultImpl) {
173 return defaultImpl.apply(v1, v2);
174 }
175
176 /* ============================================================================ */
177
178 interface VectorBlendOp<V extends Vector<E,S>,
179 M extends Vector.Mask<E,S>,
180 S extends Vector.Shape, E> {
181 V apply(V v1, V v2, M mask);
182 }
183
184 @HotSpotIntrinsicCandidate
185 static
186 <V extends Vector<E,S>,
187 M extends Vector.Mask<E,S>,
188 S extends Vector.Shape, E>
189 V blend(Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int vlen,
190 V v1, V v2, M m,
191 VectorBlendOp<V,M,S,E> defaultImpl) {
192 return defaultImpl.apply(v1, v2, m);
193 }
194
195 /* ============================================================================ */
196
197 interface VectorBroadcastIntOp<V extends Vector<?,?>> {
198 V apply(V v, int i);
199 }
200
201 @HotSpotIntrinsicCandidate
202 static
203 <V extends Vector<?,?>>
204 V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int vlen,
205 V v, int i,
206 VectorBroadcastIntOp<V> defaultImpl) {
207 return defaultImpl.apply(v, i);
208 }
209
210 /* ============================================================================ */
211
212 interface VectorReinterpretOp<VT, VF> {
213 VT apply(VF v, Class<?> elementType);
214 }
215
216 @HotSpotIntrinsicCandidate
217 static
218 <VT, VF>
219 VT reinterpret(Class<VF> fromVectorClass, Class<?> fromElementType, int fromVLen,
220 Class<?> toElementType, int toVLen, VF v,
221 VectorReinterpretOp<VT,VF> defaultImpl) {
222 return defaultImpl.apply(v, toElementType);
223 }
224
225 /* ============================================================================ */
226
227 interface VectorCastOp<VT, VF> {
228 VT apply(VF v, Class<?> elementType);
229 }
230
231 @HotSpotIntrinsicCandidate
232 static
233 <VT, VF>
234 VT cast(Class<VF> fromVectorClass, Class<?> fromElementType, int fromVLen,
235 Class<?> toElementType, int toVLen, VF v,
236 VectorCastOp<VT,VF> defaultImpl) {
237 return defaultImpl.apply(v, toElementType);
238 }
239
240 /* ============================================================================ */
241
242 @HotSpotIntrinsicCandidate
243 static <V> V maybeRebox(V v) {
244 // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
245 // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
246 U.loadFence();
247 return v;
248 }
249
250 /* ============================================================================ */
251
252 static final int VECTOR_ACCESS_OOB_CHECK = Integer.getInteger("jdk.incubator.vector.VECTOR_ACCESS_OOB_CHECK", 2);
253
254 @ForceInline
255 static int checkIndex(int ix, int length, int vlen) {
256 switch (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK) {
257 case 0: return ix; // no range check
258 case 1: return Objects.checkFromIndexSize(ix, vlen, length);
259 case 2: return Objects.checkIndex(ix, length - (vlen - 1));
260 default: throw new InternalError();
261 }
262 }
263 }