1 package jdk.incubator.vector;
   2 
   3 import jdk.internal.HotSpotIntrinsicCandidate;
   4 import jdk.internal.misc.Unsafe;
   5 import jdk.internal.vm.annotation.ForceInline;
   6 
   7 import java.nio.Buffer;
   8 import java.nio.ByteBuffer;
   9 import java.util.Objects;
  10 import java.util.function.*;
  11 
  12 import jdk.incubator.vector.Vector;
  13 
  14 /*non-public*/ class VectorIntrinsics {
  15     static final Unsafe U = Unsafe.getUnsafe();
  16 
  17     static final long BUFFER_ADDRESS
  18             = U.objectFieldOffset(Buffer.class, "address");
  19 
  20     // Buffer.limit
  21     static final long BUFFER_LIMIT
  22             = U.objectFieldOffset(Buffer.class, "limit");
  23 
  24     // ByteBuffer.hb
  25     static final long BYTE_BUFFER_HB
  26             = U.objectFieldOffset(ByteBuffer.class, "hb");
  27 
  28     // ByteBuffer.isReadOnly
  29     static final long BYTE_BUFFER_IS_READ_ONLY
  30             = U.objectFieldOffset(ByteBuffer.class, "isReadOnly");
  31 
  32     // Unary
  33     static final int VECTOR_OP_ABS  = 0;
  34     static final int VECTOR_OP_NEG  = 1;
  35     static final int VECTOR_OP_SQRT = 2;
  36     static final int VECTOR_OP_NOT  = 3;
  37 
  38     // Binary
  39     static final int VECTOR_OP_ADD  = 4;
  40     static final int VECTOR_OP_SUB  = 5;
  41     static final int VECTOR_OP_MUL  = 6;
  42     static final int VECTOR_OP_DIV  = 7;
  43     static final int VECTOR_OP_MIN  = 8;
  44     static final int VECTOR_OP_MAX  = 9;
  45 
  46     static final int VECTOR_OP_AND  = 10;
  47     static final int VECTOR_OP_OR   = 11;
  48     static final int VECTOR_OP_XOR  = 12;
  49 
  50     // Ternary
  51     static final int VECTOR_OP_FMA  = 13;
  52 
  53     // Broadcast int
  54     static final int VECTOR_OP_LSHIFT  = 14;
  55     static final int VECTOR_OP_RSHIFT  = 15;
  56     static final int VECTOR_OP_URSHIFT = 16;
  57 
  58     // Math routines
  59     static final int VECTOR_OP_TAN = 101;
  60     static final int VECTOR_OP_TANH = 102;
  61     static final int VECTOR_OP_SIN = 103;
  62     static final int VECTOR_OP_SINH = 104;
  63     static final int VECTOR_OP_COS = 105;
  64     static final int VECTOR_OP_COSH = 106;
  65     static final int VECTOR_OP_ASIN = 107;
  66     static final int VECTOR_OP_ACOS = 108;
  67     static final int VECTOR_OP_ATAN = 109;
  68     static final int VECTOR_OP_ATAN2 = 110;
  69     static final int VECTOR_OP_CBRT = 111;
  70     static final int VECTOR_OP_LOG = 112;
  71     static final int VECTOR_OP_LOG10 = 113;
  72     static final int VECTOR_OP_LOG1P = 114;
  73     static final int VECTOR_OP_POW = 115;
  74     static final int VECTOR_OP_EXP = 116;
  75     static final int VECTOR_OP_EXPM1 = 117;
  76     static final int VECTOR_OP_HYPOT = 118;
  77 
  78     // enum BoolTest
  79     static final int BT_eq = 0;
  80     static final int BT_ne = 4;
  81     static final int BT_le = 5;
  82     static final int BT_ge = 7;
  83     static final int BT_lt = 3;
  84     static final int BT_gt = 1;
  85     static final int BT_overflow = 2;
  86     static final int BT_no_overflow = 6;
  87 
  88     /* ============================================================================ */
  89     interface BroadcastOperation<V, E> {
  90         V broadcast(long l, VectorSpecies<E> s);
  91     }
  92 
  93     @HotSpotIntrinsicCandidate
  94     static
  95     <VM, E>
  96     VM broadcastCoerced(Class<VM> vmClass, Class<?> E, int length,
  97                                   long bits, VectorSpecies<E> s,
  98                                   BroadcastOperation<VM, E> defaultImpl) {
  99         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 100         return defaultImpl.broadcast(bits, s);
 101     }
 102 
 103     /* ============================================================================ */



























 104 
 105     @HotSpotIntrinsicCandidate
 106     static
 107     <V extends Vector<?>>
 108     long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int length,
 109                           V v,
 110                           Function<V,Long> defaultImpl) {
 111         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 112         return defaultImpl.apply(v);
 113     }
 114 
 115     /* ============================================================================ */
 116 
 117     interface VecExtractOp<V> {
 118         long apply(V v1, int idx);
 119     }
 120 
 121     @HotSpotIntrinsicCandidate
 122     static
 123     <V extends Vector<?>>
 124     long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
 125                  V vec, int ix,
 126                  VecExtractOp<V> defaultImpl) {
 127         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 128         return defaultImpl.apply(vec, ix);
 129     }
 130 
 131     /* ============================================================================ */
 132 
 133     interface VecInsertOp<V> {
 134         V apply(V v1, int idx, long val);
 135     }
 136 
 137     @HotSpotIntrinsicCandidate
 138     static <V extends Vector<?>>
 139     V insert(Class<V> vectorClass, Class<?> elementType, int vlen,
 140                         V vec, int ix, long val,
 141                         VecInsertOp<V> defaultImpl) {
 142         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 143         return defaultImpl.apply(vec, ix, val);
 144     }
 145 
 146     /* ============================================================================ */
 147 
 148     @HotSpotIntrinsicCandidate
 149     static
 150     <VM>
 151     VM unaryOp(int oprId, Class<VM> vmClass, Class<?> elementType, int length,
 152                VM vm,
 153                Function<VM, VM> defaultImpl) {
 154         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 155         return defaultImpl.apply(vm);
 156     }
 157 
 158     /* ============================================================================ */
 159 
 160     @HotSpotIntrinsicCandidate
 161     static
 162     <VM>
 163     VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
 164                 VM vm1, VM vm2,
 165                 BiFunction<VM, VM, VM> defaultImpl) {
 166         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 167         return defaultImpl.apply(vm1, vm2);
 168     }
 169 
 170     /* ============================================================================ */
 171 
 172     interface TernaryOperation<V> {
 173         V apply(V v1, V v2, V v3);
 174     }
 175 
 176     @HotSpotIntrinsicCandidate
 177     static
 178     <VM>
 179     VM ternaryOp(int oprId, Class<VM> vmClass, Class<?> elementType, int length,
 180                  VM vm1, VM vm2, VM vm3,
 181                  TernaryOperation<VM> defaultImpl) {
 182         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 183         return defaultImpl.apply(vm1, vm2, vm3);
 184     }
 185 
 186     /* ============================================================================ */
 187 
 188     // Memory operations
 189 
 190     interface LoadOperation<C, V, E> {
 191         V load(C container, int index, VectorSpecies<E> s);
 192     }
 193 
 194     @HotSpotIntrinsicCandidate
 195     static
 196     <C, VM, E>
 197     VM load(Class<VM> vmClass, Class<?> E, int length,
 198            Object base, long offset,    // Unsafe addressing
 199            C container, int index, VectorSpecies<E> s,     // Arguments for default implementation
 200            LoadOperation<C, VM, E> defaultImpl) {
 201         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 202         return defaultImpl.load(container, index, s);
 203     }
 204 
 205     /* ============================================================================ */
 206 
 207     interface LoadVectorOperationWithMap<C, V extends Vector<?>, E> {
 208         V loadWithMap(C container, int index, int[] indexMap, int indexM, VectorSpecies<E> s);
 209     }
 210 
 211     @HotSpotIntrinsicCandidate
 212     static
 213     <C, V extends Vector<?>, W extends IntVector, E>
 214     V loadWithMap(Class<?> vectorClass, Class<?> E, int length, Class<?> vectorIndexClass,
 215                   Object base, long offset, // Unsafe addressing
 216                   W index_vector,
 217                   C container, int index, int[] indexMap, int indexM, VectorSpecies<E> s, // Arguments for default implementation
 218                   LoadVectorOperationWithMap<C, V, E> defaultImpl) {
 219         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 220         return defaultImpl.loadWithMap(container, index, indexMap, indexM, s);
 221     }
 222 
 223     /* ============================================================================ */
 224 
 225     interface StoreVectorOperation<C, V extends Vector<?>> {
 226         void store(C container, int index, V v);
 227     }
 228 
 229     @HotSpotIntrinsicCandidate
 230     static
 231     <C, V extends Vector<?>>
 232     void store(Class<?> vectorClass, Class<?> elementType, int length,
 233                Object base, long offset,    // Unsafe addressing
 234                V v,
 235                C container, int index,      // Arguments for default implementation
 236                StoreVectorOperation<C, V> defaultImpl) {
 237         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 238         defaultImpl.store(container, index, v);
 239     }
 240 
 241     /* ============================================================================ */
 242 
 243     interface StoreVectorOperationWithMap<C, V extends Vector<?>> {
 244         void storeWithMap(C container, int index, V v, int[] indexMap, int indexM);
 245     }
 246 
 247     @HotSpotIntrinsicCandidate
 248     static
 249     <C, V extends Vector<?>, W extends IntVector>
 250     void storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass,
 251                       Object base, long offset,    // Unsafe addressing
 252                       W index_vector, V v,
 253                       C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
 254                       StoreVectorOperationWithMap<C, V> defaultImpl) {
 255         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 256         defaultImpl.storeWithMap(container, index, v, indexMap, indexM);
 257     }
 258 
 259     /* ============================================================================ */
 260 
 261     @HotSpotIntrinsicCandidate
 262     static
 263     <VM>
 264     boolean test(int cond, Class<?> vmClass, Class<?> elementType, int length,
 265                  VM vm1, VM vm2,
 266                  BiFunction<VM, VM, Boolean> defaultImpl) {
 267         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 268         return defaultImpl.apply(vm1, vm2);
 269     }
 270 
 271     /* ============================================================================ */
 272 
 273     interface VectorCompareOp<V,M> {
 274         M apply(V v1, V v2);
 275     }
 276 
 277     @HotSpotIntrinsicCandidate
 278     static <V extends Vector<E>,
 279             M extends VectorMask<E>,
 280             E>
 281     M compare(int cond, Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
 282               V v1, V v2,
 283               VectorCompareOp<V,M> defaultImpl) {
 284         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 285         return defaultImpl.apply(v1, v2);
 286     }
 287 
 288     /* ============================================================================ */
 289 
 290     interface VectorRearrangeOp<V extends Vector<E>,
 291             Sh extends VectorShuffle<E>,
 292             E> {
 293         V apply(V v1, Sh shuffle);
 294     }
 295 
 296     @HotSpotIntrinsicCandidate
 297     static
 298     <V extends Vector<E>,
 299             Sh extends VectorShuffle<E>,
 300             E>
 301     V rearrangeOp(Class<V> vectorClass, Class<Sh> shuffleClass, Class<?> elementType, int vlen,
 302             V v1, Sh sh,
 303             VectorRearrangeOp<V,Sh, E> defaultImpl) {
 304         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 305         return defaultImpl.apply(v1, sh);
 306     }
 307 
 308     /* ============================================================================ */
 309 
 310     interface VectorBlendOp<V extends Vector<E>,
 311             M extends VectorMask<E>,
 312             E> {
 313         V apply(V v1, V v2, M mask);
 314     }
 315 
 316     @HotSpotIntrinsicCandidate
 317     static
 318     <V extends Vector<E>,
 319      M extends VectorMask<E>,
 320      E>
 321     V blend(Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
 322             V v1, V v2, M m,
 323             VectorBlendOp<V,M, E> defaultImpl) {
 324         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 325         return defaultImpl.apply(v1, v2, m);
 326     }
 327 
 328     /* ============================================================================ */
 329 
 330     interface VectorBroadcastIntOp<V extends Vector<?>> {
 331         V apply(V v, int i);
 332     }
 333 
 334     @HotSpotIntrinsicCandidate
 335     static
 336     <V extends Vector<?>>
 337     V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int length,
 338                    V v, int i,
 339                    VectorBroadcastIntOp<V> defaultImpl) {
 340         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 341         return defaultImpl.apply(v, i);
 342     }
 343 
 344     /* ============================================================================ */
 345 
 346     interface VectorReinterpretOp<S, VIN, V> {
 347         V apply(S species, VIN v);
 348     }
 349 
 350     @HotSpotIntrinsicCandidate
 351     static
 352     <S, VIN, V>
 353     V reinterpret(Class<?> fromVectorClass,
 354                   Class<?> fromElementType, int fromVLen,
 355                   Class<?> toVectorClass,
 356                   Class<?> toElementType, int toVLen,
 357                   VIN v, S s,
 358                   VectorReinterpretOp<S, VIN, V> defaultImpl) {
 359         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 360         return defaultImpl.apply(s, v);
 361     }
 362 
 363     /* ============================================================================ */
 364 
 365     interface VectorCastOp<S, VIN, V> {
 366         V apply(S species, VIN v);
 367     }
 368 
 369     @HotSpotIntrinsicCandidate
 370     static
 371     <S, VIN, V>
 372     V cast(Class<?> fromVectorClass,
 373            Class<?> fromElementType, int fromVLen,
 374            Class<?> toVectorClass,
 375            Class<?> toElementType, int toVLen,
 376            VIN v, S s,
 377            VectorCastOp<S, VIN, V> defaultImpl) {
 378         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
 379         return defaultImpl.apply(s, v);
 380     }
 381 
 382     /* ============================================================================ */
 383 
 384     @HotSpotIntrinsicCandidate
 385     static <V> V maybeRebox(V v) {
 386         // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
 387         // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
 388         U.loadFence();
 389         return v;
 390     }
 391 
 392     /* ============================================================================ */
 393 
 394     static final int VECTOR_ACCESS_OOB_CHECK = Integer.getInteger("jdk.incubator.vector.VECTOR_ACCESS_OOB_CHECK", 2);
 395 
 396     @ForceInline
 397     static int checkIndex(int ix, int length, int vlen) {
 398         switch (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK) {
 399             case 0: return ix; // no range check
 400             case 1: return Objects.checkFromIndexSize(ix, vlen, length);
 401             case 2: return Objects.checkIndex(ix, length - (vlen - 1));
 402             default: throw new InternalError();
 403         }
 404     }
 405 
 406     @ForceInline
 407     static IntVector checkIndex(IntVector vix, int length) {
 408         switch (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK) {
 409             case 0: return vix; // no range check
 410             case 1: // fall-through
 411             case 2:
 412                 if(vix.lessThan(0).anyTrue() || vix.greaterThanEq(length).anyTrue()) {
 413                     String msg = String.format("Range check failed: vector %s out of bounds for length %d", vix, length);
 414                     throw new ArrayIndexOutOfBoundsException(msg);
 415                 }
 416                 return vix;
 417             default: throw new InternalError();
 418         }
 419     }
 420 
 421     static boolean isNonCapturingLambda(Object o) {
 422         return o.getClass().getDeclaredFields().length == 0;
 423     }
 424 }
--- EOF ---