1 package jdk.incubator.vector;
   2 
   3 import jdk.internal.HotSpotIntrinsicCandidate;
   4 import jdk.internal.misc.Unsafe;
   5 import jdk.internal.vm.annotation.ForceInline;
   6 
   7 import java.nio.Buffer;
   8 import java.nio.ByteBuffer;
   9 import java.util.Objects;
  10 import java.util.function.*;
  11 
  12 /*non-public*/ class VectorIntrinsics {
  13     static final Unsafe U = Unsafe.getUnsafe();
  14 
  15     static final long BUFFER_ADDRESS
  16             = U.objectFieldOffset(Buffer.class, "address");
  17 
  18     // Buffer.limit
  19     static final long BUFFER_LIMIT
  20             = U.objectFieldOffset(Buffer.class, "limit");
  21 
  22     // ByteBuffer.hb
  23     static final long BYTE_BUFFER_HB
  24             = U.objectFieldOffset(ByteBuffer.class, "hb");
  25 
  26     // ByteBuffer.isReadOnly
  27     static final long BYTE_BUFFER_IS_READ_ONLY
  28             = U.objectFieldOffset(ByteBuffer.class, "isReadOnly");
  29 
  30     // Unary
  31     static final int VECTOR_OP_ABS  = 0;
  32     static final int VECTOR_OP_NEG  = 1;
  33     static final int VECTOR_OP_SQRT = 2;
  34     static final int VECTOR_OP_NOT  = 3;
  35 
  36     // Binary
  37     static final int VECTOR_OP_ADD  = 4;
  38     static final int VECTOR_OP_SUB  = 5;
  39     static final int VECTOR_OP_MUL  = 6;
  40     static final int VECTOR_OP_DIV  = 7;
  41     static final int VECTOR_OP_MIN  = 8;
  42     static final int VECTOR_OP_MAX  = 9;
  43 
  44     static final int VECTOR_OP_AND  = 10;
  45     static final int VECTOR_OP_OR   = 11;
  46     static final int VECTOR_OP_XOR  = 12;
  47 
  48     // Ternary
  49     static final int VECTOR_OP_FMA  = 13;
  50 
  51     // Broadcast int
  52     static final int VECTOR_OP_LSHIFT  = 14;
  53     static final int VECTOR_OP_RSHIFT  = 15;
  54     static final int VECTOR_OP_URSHIFT = 16;
  55 
  56     // Math routines
  57     static final int VECTOR_OP_TAN = 101;
  58     static final int VECTOR_OP_TANH = 102;
  59     static final int VECTOR_OP_SIN = 103;
  60     static final int VECTOR_OP_SINH = 104;
  61     static final int VECTOR_OP_COS = 105;
  62     static final int VECTOR_OP_COSH = 106;
  63     static final int VECTOR_OP_ASIN = 107;
  64     static final int VECTOR_OP_ACOS = 108;
  65     static final int VECTOR_OP_ATAN = 109;
  66     static final int VECTOR_OP_ATAN2 = 110;
  67     static final int VECTOR_OP_CBRT = 111;
  68     static final int VECTOR_OP_LOG = 112;
  69     static final int VECTOR_OP_LOG10 = 113;
  70     static final int VECTOR_OP_LOG1P = 114;
  71     static final int VECTOR_OP_POW = 115;
  72     static final int VECTOR_OP_EXP = 116;
  73     static final int VECTOR_OP_EXPM1 = 117;
  74     static final int VECTOR_OP_HYPOT = 118;
  75 
  76     // Copied from open/src/hotspot/cpu/x86/assembler_x86.hpp
  77     // enum Condition { // The x86 condition codes used for conditional jumps/moves.
  78     static final int COND_zero          = 0x4;
  79     static final int COND_notZero       = 0x5;
  80     static final int COND_equal         = 0x4;
  81     static final int COND_notEqual      = 0x5;
  82     static final int COND_less          = 0xc;
  83     static final int COND_lessEqual     = 0xe;
  84     static final int COND_greater       = 0xf;
  85     static final int COND_greaterEqual  = 0xd;
  86     static final int COND_below         = 0x2;
  87     static final int COND_belowEqual    = 0x6;
  88     static final int COND_above         = 0x7;
  89     static final int COND_aboveEqual    = 0x3;
  90     static final int COND_overflow      = 0x0;
  91     static final int COND_noOverflow    = 0x1;
  92     static final int COND_carrySet      = 0x2;
  93     static final int COND_carryClear    = 0x3;
  94     static final int COND_negative      = 0x8;
  95     static final int COND_positive      = 0x9;
  96     static final int COND_parity        = 0xa;
  97     static final int COND_noParity      = 0xb;
  98 
  99 
 100     // enum BoolTest
 101     static final int BT_eq = 0;
 102     static final int BT_ne = 4;
 103     static final int BT_le = 5;
 104     static final int BT_ge = 7;
 105     static final int BT_lt = 3;
 106     static final int BT_gt = 1;
 107     static final int BT_overflow = 2;
 108     static final int BT_no_overflow = 6;
 109 
 110     /* ============================================================================ */
 111 
 112     @HotSpotIntrinsicCandidate
 113     static
 114     <VM>
 115     VM broadcastCoerced(Class<VM> vmClass, Class<?> elementType, int length,
 116                                   long bits,
 117                                   LongFunction<VM> defaultImpl) {
 118         return defaultImpl.apply(bits);
 119     }
 120 
 121     /* ============================================================================ */
 122 
 123     @HotSpotIntrinsicCandidate
 124     static
 125     <V extends Vector<?,?>>
 126     long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int length,
 127                           V v,
 128                           Function<V,Long> defaultImpl) {
 129         return defaultImpl.apply(v);
 130     }
 131 
 132     /* ============================================================================ */
 133 
 134     interface VecExtractOp<V> {
 135         long apply(V v1, int idx);
 136     }
 137 
 138     @HotSpotIntrinsicCandidate
 139     static
 140     <V extends Vector<?,?>>
 141     long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
 142                  V vec, int ix,
 143                  VecExtractOp<V> defaultImpl) {
 144         return defaultImpl.apply(vec, ix);
 145     }
 146 
 147     /* ============================================================================ */
 148 
 149     interface VecInsertOp<V> {
 150         V apply(V v1, int idx, long val);
 151     }
 152 
 153     @HotSpotIntrinsicCandidate
 154     static <V extends Vector<?,?>>
 155     V insert(Class<V> vectorClass, Class<?> elementType, int vlen,
 156                         V vec, int ix, long val,
 157                         VecInsertOp<V> defaultImpl) {
 158         return defaultImpl.apply(vec, ix, val);
 159     }
 160 
 161     /* ============================================================================ */
 162 
 163     @HotSpotIntrinsicCandidate
 164     static
 165     <VM>
 166     VM unaryOp(int oprId, Class<VM> vmClass, Class<?> elementType, int length,
 167                VM vm, /*Vector.Mask<E,S> m,*/
 168                Function<VM, VM> defaultImpl) {
 169         return defaultImpl.apply(vm);
 170     }
 171 
 172     /* ============================================================================ */
 173 
 174     @HotSpotIntrinsicCandidate
 175     static
 176     <VM>
 177     VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
 178                 VM vm1, VM vm2, /*Vector.Mask<E,S> m,*/
 179                 BiFunction<VM, VM, VM> defaultImpl) {
 180         return defaultImpl.apply(vm1, vm2);
 181     }
 182 
 183     /* ============================================================================ */
 184 
 185     interface TernaryOperation<V> {
 186         V apply(V v1, V v2, V v3);
 187     }
 188 
 189     @SuppressWarnings("unchecked")
 190     @HotSpotIntrinsicCandidate
 191     static
 192     <VM>
 193     VM ternaryOp(int oprId, Class<VM> vmClass, Class<?> elementType, int length,
 194                  VM vm1, VM vm2, VM vm3, /*Vector.Mask<E,S> m,*/
 195                  TernaryOperation<VM> defaultImpl) {
 196         return defaultImpl.apply(vm1, vm2, vm3);
 197     }
 198 
 199     /* ============================================================================ */
 200 
 201     // Memory operations
 202 
 203     interface LoadVectorOperation<C, V extends Vector<?,?>> {
 204         V load(C container, int index);
 205     }
 206 
 207     // @@@ Support endianness, pass in as argument to intrinsic
 208     @HotSpotIntrinsicCandidate
 209     static
 210     <C, V extends Vector<?,?>>
 211     V load(Class<?> vectorClass, Class<?> elementType, int length,
 212            Object base, long offset,    // Unsafe addressing
 213            // Vector.Mask<E,S> m,
 214            C container, int index,      // Arguments for default implementation
 215            LoadVectorOperation<C, V> defaultImpl) {
 216         return defaultImpl.load(container, index);
 217     }
 218 
 219     interface StoreVectorOperation<C, V extends Vector<?,?>> {
 220         void store(C container, int index, V v);
 221     }
 222 
 223     // @@@ Support endianness, pass in as argument to intrinsic
 224     @HotSpotIntrinsicCandidate
 225     static
 226     <C, V extends Vector<?,?>>
 227     void store(Class<?> vectorClass, Class<?> elementType, int length,
 228                Object base, long offset,    // Unsafe addressing
 229                V v,
 230                // Vector.Mask<E,S> m,
 231                C container, int index,      // Arguments for default implementation
 232                StoreVectorOperation<C, V> defaultImpl) {
 233         defaultImpl.store(container, index, v);
 234     }
 235 
 236     /* ============================================================================ */
 237 
 238     @HotSpotIntrinsicCandidate
 239     static
 240     <VM>
 241     boolean test(int cond, Class<?> vmClass, Class<?> elementType, int length,
 242                  VM vm1, VM vm2,
 243                  BiFunction<VM, VM, Boolean> defaultImpl) {
 244         return defaultImpl.apply(vm1, vm2);
 245     }
 246 
 247     /* ============================================================================ */
 248 
 249     interface VectorCompareOp<V,M> {
 250         M apply(V v1, V v2);
 251     }
 252 
 253     @HotSpotIntrinsicCandidate
 254     static <V extends Vector<E,S>,
 255             M extends Vector.Mask<E,S>,
 256             S extends Vector.Shape, E>
 257     M compare(int cond, Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
 258               V v1, V v2,
 259               VectorCompareOp<V,M> defaultImpl) {
 260         return defaultImpl.apply(v1, v2);
 261     }
 262 
 263     /* ============================================================================ */
 264 
 265     interface VectorRearrangeOp<V extends Vector<E,S>,
 266                             Sh extends Vector.Shuffle<E,S>,
 267                             S extends Vector.Shape, E> {
 268         V apply(V v1, Sh shuffle);
 269     }
 270 
 271     @HotSpotIntrinsicCandidate
 272     static
 273     <V extends Vector<E,S>,
 274      Sh extends Vector.Shuffle<E,S>,
 275      S extends Vector.Shape, E>
 276     V rearrangeOp(Class<V> vectorClass, Class<Sh> shuffleClass, Class<?> elementType, int vlen,
 277             V v1, Sh sh,
 278             VectorRearrangeOp<V,Sh,S,E> defaultImpl) {
 279         return defaultImpl.apply(v1, sh);
 280     }
 281 
 282     /* ============================================================================ */
 283 
 284     interface VectorBlendOp<V extends Vector<E,S>,
 285                             M extends Vector.Mask<E,S>,
 286                             S extends Vector.Shape, E> {
 287         V apply(V v1, V v2, M mask);
 288     }
 289 
 290     @HotSpotIntrinsicCandidate
 291     static
 292     <V extends Vector<E,S>,
 293      M extends Vector.Mask<E,S>,
 294      S extends Vector.Shape, E>
 295     V blend(Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
 296             V v1, V v2, M m,
 297             VectorBlendOp<V,M,S,E> defaultImpl) {
 298         return defaultImpl.apply(v1, v2, m);
 299     }
 300 
 301     /* ============================================================================ */
 302 
 303     interface VectorBroadcastIntOp<V extends Vector<?,?>> {
 304         V apply(V v, int i);
 305     }
 306 
 307     @HotSpotIntrinsicCandidate
 308     static
 309     <V extends Vector<?,?>>
 310     V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int length,
 311                    V v, int i,
 312                    VectorBroadcastIntOp<V> defaultImpl) {
 313         return defaultImpl.apply(v, i);
 314     }
 315 
 316     /* ============================================================================ */
 317 
 318     interface VectorReinterpretOp<S, VIN, V> {
 319         V apply(S species, VIN v);
 320     }
 321 
 322     @HotSpotIntrinsicCandidate
 323     static
 324     <S, VIN, V>
 325     V reinterpret(Class<?> fromVectorClass,
 326                   Class<?> fromElementType, int fromVLen,
 327                   Class<?> toElementType, int toVLen,
 328                   VIN v, S s,
 329                   VectorReinterpretOp<S, VIN, V> defaultImpl) {
 330         return defaultImpl.apply(s, v);
 331     }
 332 
 333     /* ============================================================================ */
 334 
 335     interface VectorCastOp<S, VIN, V> {
 336         V apply(S species, VIN v);
 337     }
 338 
 339     @HotSpotIntrinsicCandidate
 340     static
 341     <S, VIN, V>
 342     V cast(Class<?> fromVectorClass,
 343            Class<?> fromElementType, int fromVLen,
 344            Class<?> toElementType, int toVLen,
 345            VIN v, S s,
 346            VectorCastOp<S, VIN, V> defaultImpl) {
 347         return defaultImpl.apply(s, v);
 348     }
 349 
 350     /* ============================================================================ */
 351 
 352     @HotSpotIntrinsicCandidate
 353     static <V> V maybeRebox(V v) {
 354         // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
 355         // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
 356         U.loadFence();
 357         return v;
 358     }
 359 
 360     /* ============================================================================ */
 361 
 362     static final int VECTOR_ACCESS_OOB_CHECK = Integer.getInteger("jdk.incubator.vector.VECTOR_ACCESS_OOB_CHECK", 2);
 363 
 364     @ForceInline
 365     static int checkIndex(int ix, int length, int vlen) {
 366         switch (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK) {
 367             case 0: return ix; // no range check
 368             case 1: return Objects.checkFromIndexSize(ix, vlen, length);
 369             case 2: return Objects.checkIndex(ix, length - (vlen - 1));
 370             default: throw new InternalError();
 371         }
 372     }
 373 }