1 package jdk.incubator.vector;
   2 
   3 import jdk.internal.HotSpotIntrinsicCandidate;
   4 import jdk.internal.misc.Unsafe;
   5 import jdk.internal.vm.annotation.ForceInline;
   6 
   7 import java.nio.Buffer;
   8 import java.nio.ByteBuffer;
   9 import java.util.Objects;
  10 import java.util.function.*;
  11 
  12 import jdk.incubator.vector.Vector;
  13 
  14 /*non-public*/ class VectorIntrinsics {
  15     static final Unsafe U = Unsafe.getUnsafe();
  16 
  17     static final long BUFFER_ADDRESS
  18             = U.objectFieldOffset(Buffer.class, "address");
  19 
  20     // Buffer.limit
  21     static final long BUFFER_LIMIT
  22             = U.objectFieldOffset(Buffer.class, "limit");
  23 
  24     // ByteBuffer.hb
  25     static final long BYTE_BUFFER_HB
  26             = U.objectFieldOffset(ByteBuffer.class, "hb");
  27 
  28     // ByteBuffer.isReadOnly
  29     static final long BYTE_BUFFER_IS_READ_ONLY
  30             = U.objectFieldOffset(ByteBuffer.class, "isReadOnly");
  31 
  32     // Unary
  33     static final int VECTOR_OP_ABS  = 0;
  34     static final int VECTOR_OP_NEG  = 1;
  35     static final int VECTOR_OP_SQRT = 2;
  36     static final int VECTOR_OP_NOT  = 3;
  37 
  38     // Binary
  39     static final int VECTOR_OP_ADD  = 4;
  40     static final int VECTOR_OP_SUB  = 5;
  41     static final int VECTOR_OP_MUL  = 6;
  42     static final int VECTOR_OP_DIV  = 7;
  43     static final int VECTOR_OP_MIN  = 8;
  44     static final int VECTOR_OP_MAX  = 9;
  45 
  46     static final int VECTOR_OP_AND  = 10;
  47     static final int VECTOR_OP_OR   = 11;
  48     static final int VECTOR_OP_XOR  = 12;
  49 
  50     // Ternary
  51     static final int VECTOR_OP_FMA  = 13;
  52 
  53     // Broadcast int
  54     static final int VECTOR_OP_LSHIFT  = 14;
  55     static final int VECTOR_OP_RSHIFT  = 15;
  56     static final int VECTOR_OP_URSHIFT = 16;
  57 
  58     // Math routines
  59     static final int VECTOR_OP_TAN = 101;
  60     static final int VECTOR_OP_TANH = 102;
  61     static final int VECTOR_OP_SIN = 103;
  62     static final int VECTOR_OP_SINH = 104;
  63     static final int VECTOR_OP_COS = 105;
  64     static final int VECTOR_OP_COSH = 106;
  65     static final int VECTOR_OP_ASIN = 107;
  66     static final int VECTOR_OP_ACOS = 108;
  67     static final int VECTOR_OP_ATAN = 109;
  68     static final int VECTOR_OP_ATAN2 = 110;
  69     static final int VECTOR_OP_CBRT = 111;
  70     static final int VECTOR_OP_LOG = 112;
  71     static final int VECTOR_OP_LOG10 = 113;
  72     static final int VECTOR_OP_LOG1P = 114;
  73     static final int VECTOR_OP_POW = 115;
  74     static final int VECTOR_OP_EXP = 116;
  75     static final int VECTOR_OP_EXPM1 = 117;
  76     static final int VECTOR_OP_HYPOT = 118;
  77 
  78     // enum BoolTest
  79     static final int BT_eq = 0;
  80     static final int BT_ne = 4;
  81     static final int BT_le = 5;
  82     static final int BT_ge = 7;
  83     static final int BT_lt = 3;
  84     static final int BT_gt = 1;
  85     static final int BT_overflow = 2;
  86     static final int BT_no_overflow = 6;
  87 
  88     /* ============================================================================ */
  89     interface BroadcastOperation<V, E> {
  90         V broadcast(long l, VectorSpecies<E> s);
  91     }
  92 
  93     @HotSpotIntrinsicCandidate
  94     static
  95     <VM, E>
  96     VM broadcastCoerced(Class<VM> vmClass, Class<?> E, int length,
  97                                   long bits, VectorSpecies<E> s,
  98                                   BroadcastOperation<VM, E> defaultImpl) {
  99         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 100         return defaultImpl.broadcast(bits, s);
 101     }
 102 
 103     /* ============================================================================ */
 104     interface ShuffleIotaOperation<E> {
 105         VectorShuffle<E> apply(int step, int length);
 106     }
 107 
 108     @HotSpotIntrinsicCandidate
 109     static
 110     <E>
 111     VectorShuffle<E> shuffleIota(Class<?> E, Class<?> ShuffleClass, VectorSpecies<E> s, int length,
 112                      int step, ShuffleIotaOperation<E> defaultImpl) {
 113        assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 114        return defaultImpl.apply(step, length);
 115     }
 116 
 117     interface ShuffleToVectorOperation<VM, Sh, E> {
 118        VM apply(Sh s);
 119     }
 120 
 121     @HotSpotIntrinsicCandidate 
 122     static 
 123     <VM ,Sh extends VectorShuffle<E>, E> 
 124     VM shuffleToVector(Class<?> VM, Class<?>E , Class<?> ShuffleClass, Sh s, int length,
 125                        ShuffleToVectorOperation<VM,Sh,E> defaultImpl) {
 126       assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 127       return defaultImpl.apply(s);
 128     }
 129 
 130     /* ============================================================================ */
 131 
 132     @HotSpotIntrinsicCandidate
 133     static
 134     <V extends Vector<?>>
 135     long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int length,
 136                           V v,
 137                           Function<V,Long> defaultImpl) {
 138         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 139         return defaultImpl.apply(v);
 140     }
 141 
 142     /* ============================================================================ */
 143 
 144     interface VecExtractOp<V> {
 145         long apply(V v1, int idx);
 146     }
 147 
 148     @HotSpotIntrinsicCandidate
 149     static
 150     <V extends Vector<?>>
 151     long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
 152                  V vec, int ix,
 153                  VecExtractOp<V> defaultImpl) {
 154         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 155         return defaultImpl.apply(vec, ix);
 156     }
 157 
 158     /* ============================================================================ */
 159 
 160     interface VecInsertOp<V> {
 161         V apply(V v1, int idx, long val);
 162     }
 163 
 164     @HotSpotIntrinsicCandidate
 165     static <V extends Vector<?>>
 166     V insert(Class<V> vectorClass, Class<?> elementType, int vlen,
 167                         V vec, int ix, long val,
 168                         VecInsertOp<V> defaultImpl) {
 169         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 170         return defaultImpl.apply(vec, ix, val);
 171     }
 172 
 173     /* ============================================================================ */
 174 
 175     @HotSpotIntrinsicCandidate
 176     static
 177     <VM>
 178     VM unaryOp(int oprId, Class<VM> vmClass, Class<?> elementType, int length,
 179                VM vm,
 180                Function<VM, VM> defaultImpl) {
 181         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 182         return defaultImpl.apply(vm);
 183     }
 184 
 185     /* ============================================================================ */
 186 
 187     @HotSpotIntrinsicCandidate
 188     static
 189     <VM>
 190     VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
 191                 VM vm1, VM vm2,
 192                 BiFunction<VM, VM, VM> defaultImpl) {
 193         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 194         return defaultImpl.apply(vm1, vm2);
 195     }
 196 
 197     /* ============================================================================ */
 198 
 199     interface TernaryOperation<V> {
 200         V apply(V v1, V v2, V v3);
 201     }
 202 
 203     @HotSpotIntrinsicCandidate
 204     static
 205     <VM>
 206     VM ternaryOp(int oprId, Class<VM> vmClass, Class<?> elementType, int length,
 207                  VM vm1, VM vm2, VM vm3,
 208                  TernaryOperation<VM> defaultImpl) {
 209         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 210         return defaultImpl.apply(vm1, vm2, vm3);
 211     }
 212 
 213     /* ============================================================================ */
 214 
 215     // Memory operations
 216 
 217     interface LoadOperation<C, V, E> {
 218         V load(C container, int index, VectorSpecies<E> s);
 219     }
 220 
 221     @HotSpotIntrinsicCandidate
 222     static
 223     <C, VM, E>
 224     VM load(Class<VM> vmClass, Class<?> E, int length,
 225            Object base, long offset,    // Unsafe addressing
 226            C container, int index, VectorSpecies<E> s,     // Arguments for default implementation
 227            LoadOperation<C, VM, E> defaultImpl) {
 228         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 229         return defaultImpl.load(container, index, s);
 230     }
 231 
 232     /* ============================================================================ */
 233 
 234     interface LoadVectorOperationWithMap<C, V extends Vector<?>, E> {
 235         V loadWithMap(C container, int index, int[] indexMap, int indexM, VectorSpecies<E> s);
 236     }
 237 
 238     @HotSpotIntrinsicCandidate
 239     static
 240     <C, V extends Vector<?>, W extends IntVector, E>
 241     V loadWithMap(Class<?> vectorClass, Class<?> E, int length, Class<?> vectorIndexClass,
 242                   Object base, long offset, // Unsafe addressing
 243                   W index_vector,
 244                   C container, int index, int[] indexMap, int indexM, VectorSpecies<E> s, // Arguments for default implementation
 245                   LoadVectorOperationWithMap<C, V, E> defaultImpl) {
 246         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 247         return defaultImpl.loadWithMap(container, index, indexMap, indexM, s);
 248     }
 249 
 250     /* ============================================================================ */
 251 
 252     interface StoreVectorOperation<C, V extends Vector<?>> {
 253         void store(C container, int index, V v);
 254     }
 255 
 256     @HotSpotIntrinsicCandidate
 257     static
 258     <C, V extends Vector<?>>
 259     void store(Class<?> vectorClass, Class<?> elementType, int length,
 260                Object base, long offset,    // Unsafe addressing
 261                V v,
 262                C container, int index,      // Arguments for default implementation
 263                StoreVectorOperation<C, V> defaultImpl) {
 264         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 265         defaultImpl.store(container, index, v);
 266     }
 267 
 268     /* ============================================================================ */
 269 
 270     interface StoreVectorOperationWithMap<C, V extends Vector<?>> {
 271         void storeWithMap(C container, int index, V v, int[] indexMap, int indexM);
 272     }
 273 
 274     @HotSpotIntrinsicCandidate
 275     static
 276     <C, V extends Vector<?>, W extends IntVector>
 277     void storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass,
 278                       Object base, long offset,    // Unsafe addressing
 279                       W index_vector, V v,
 280                       C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
 281                       StoreVectorOperationWithMap<C, V> defaultImpl) {
 282         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 283         defaultImpl.storeWithMap(container, index, v, indexMap, indexM);
 284     }
 285 
 286     /* ============================================================================ */
 287 
 288     @HotSpotIntrinsicCandidate
 289     static
 290     <VM>
 291     boolean test(int cond, Class<?> vmClass, Class<?> elementType, int length,
 292                  VM vm1, VM vm2,
 293                  BiFunction<VM, VM, Boolean> defaultImpl) {
 294         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 295         return defaultImpl.apply(vm1, vm2);
 296     }
 297 
 298     /* ============================================================================ */
 299 
 300     interface VectorCompareOp<V,M> {
 301         M apply(V v1, V v2);
 302     }
 303 
 304     @HotSpotIntrinsicCandidate
 305     static <V extends Vector<E>,
 306             M extends VectorMask<E>,
 307             E>
 308     M compare(int cond, Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
 309               V v1, V v2,
 310               VectorCompareOp<V,M> defaultImpl) {
 311         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 312         return defaultImpl.apply(v1, v2);
 313     }
 314 
 315     /* ============================================================================ */
 316 
 317     interface VectorRearrangeOp<V extends Vector<E>,
 318             Sh extends VectorShuffle<E>,
 319             E> {
 320         V apply(V v1, Sh shuffle);
 321     }
 322 
 323     @HotSpotIntrinsicCandidate
 324     static
 325     <V extends Vector<E>,
 326             Sh extends VectorShuffle<E>,
 327             E>
 328     V rearrangeOp(Class<V> vectorClass, Class<Sh> shuffleClass, Class<?> elementType, int vlen,
 329             V v1, Sh sh,
 330             VectorRearrangeOp<V,Sh, E> defaultImpl) {
 331         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 332         return defaultImpl.apply(v1, sh);
 333     }
 334 
 335     /* ============================================================================ */
 336 
 337     interface VectorBlendOp<V extends Vector<E>,
 338             M extends VectorMask<E>,
 339             E> {
 340         V apply(V v1, V v2, M mask);
 341     }
 342 
 343     @HotSpotIntrinsicCandidate
 344     static
 345     <V extends Vector<E>,
 346      M extends VectorMask<E>,
 347      E>
 348     V blend(Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
 349             V v1, V v2, M m,
 350             VectorBlendOp<V,M, E> defaultImpl) {
 351         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 352         return defaultImpl.apply(v1, v2, m);
 353     }
 354 
 355     /* ============================================================================ */
 356 
 357     interface VectorBroadcastIntOp<V extends Vector<?>> {
 358         V apply(V v, int i);
 359     }
 360 
 361     @HotSpotIntrinsicCandidate
 362     static
 363     <V extends Vector<?>>
 364     V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int length,
 365                    V v, int i,
 366                    VectorBroadcastIntOp<V> defaultImpl) {
 367         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 368         return defaultImpl.apply(v, i);
 369     }
 370 
 371     /* ============================================================================ */
 372 
 373     interface VectorReinterpretOp<S, VIN, V> {
 374         V apply(S species, VIN v);
 375     }
 376 
 377     @HotSpotIntrinsicCandidate
 378     static
 379     <S, VIN, V>
 380     V reinterpret(Class<?> fromVectorClass,
 381                   Class<?> fromElementType, int fromVLen,
 382                   Class<?> toVectorClass,
 383                   Class<?> toElementType, int toVLen,
 384                   VIN v, S s,
 385                   VectorReinterpretOp<S, VIN, V> defaultImpl) {
 386         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 387         return defaultImpl.apply(s, v);
 388     }
 389 
 390     /* ============================================================================ */
 391 
 392     interface VectorCastOp<S, VIN, V> {
 393         V apply(S species, VIN v);
 394     }
 395 
 396     @HotSpotIntrinsicCandidate
 397     static
 398     <S, VIN, V>
 399     V cast(Class<?> fromVectorClass,
 400            Class<?> fromElementType, int fromVLen,
 401            Class<?> toVectorClass,
 402            Class<?> toElementType, int toVLen,
 403            VIN v, S s,
 404            VectorCastOp<S, VIN, V> defaultImpl) {
 405         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 406         return defaultImpl.apply(s, v);
 407     }
 408 
 409     /* ============================================================================ */
 410 
 411     @HotSpotIntrinsicCandidate
 412     static <V> V maybeRebox(V v) {
 413         // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
 414         // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
 415         U.loadFence();
 416         return v;
 417     }
 418 
 419     /* ============================================================================ */
 420 
 421     static final int VECTOR_ACCESS_OOB_CHECK = Integer.getInteger("jdk.incubator.vector.VECTOR_ACCESS_OOB_CHECK", 2);
 422 
 423     @ForceInline
 424     static int checkIndex(int ix, int length, int vlen) {
 425         switch (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK) {
 426             case 0: return ix; // no range check
 427             case 1: return Objects.checkFromIndexSize(ix, vlen, length);
 428             case 2: return Objects.checkIndex(ix, length - (vlen - 1));
 429             default: throw new InternalError();
 430         }
 431     }
 432 
 433     @ForceInline
 434     static IntVector checkIndex(IntVector vix, int length) {
 435         switch (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK) {
 436             case 0: return vix; // no range check
 437             case 1: // fall-through
 438             case 2:
 439                 if(vix.lessThan(0).anyTrue() || vix.greaterThanEq(length).anyTrue()) {
 440                     String msg = String.format("Range check failed: vector %s out of bounds for length %d", vix, length);
 441                     throw new ArrayIndexOutOfBoundsException(msg);
 442                 }
 443                 return vix;
 444             default: throw new InternalError();
 445         }
 446     }
 447 
 448     static boolean isNonCapturingLambda(Object o) {
 449         return o.getClass().getDeclaredFields().length == 0;
 450     }
 451 }