/*
 * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.  Oracle designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Oracle in the LICENSE file that accompanied this code.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have
 * questions.
 */
package jdk.incubator.vector;

import java.nio.ByteBuffer;
import java.nio.ByteOrder;
#if[!byte]
import java.nio.$Type$Buffer;
#end[!byte]
import java.nio.ReadOnlyBufferException;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.IntUnaryOperator;

import jdk.internal.misc.Unsafe;
import jdk.internal.vm.annotation.ForceInline;
import jdk.internal.vm.annotation.Stable;

import static jdk.incubator.vector.VectorIntrinsics.*;
import static jdk.incubator.vector.VectorOperators.*;

#warn This file is preprocessed before being compiled

@SuppressWarnings("cast")  // warning: redundant cast
final class $vectortype$ extends $abstractvectortype$ {
    static final $Type$Species VSPECIES =
        ($Type$Species) $Type$Vector.SPECIES_$BITS$;

    static final VectorShape VSHAPE =
        VSPECIES.vectorShape();

    static final Class<$vectortype$> VCLASS = $vectortype$.class;

    static final int VSIZE = VSPECIES.vectorBitSize();

    static final int VLENGTH = VSPECIES.laneCount();

    static final Class<$Boxtype$> ETYPE = $type$.class;

    // The JVM expects to find the state here.
    private final $type$[] vec; // Don't access directly, use vec() instead.

    $vectortype$($type$[] v) {
        vec = v;
    }

    // For compatibility as $vectortype$::new,
    // stored into species.vectorFactory.
    $vectortype$(Object v) {
        this(($type$[]) v);
    }

    static final $vectortype$ ZERO = new $vectortype$(new $type$[VLENGTH]);
    static final $vectortype$ IOTA = new $vectortype$(VSPECIES.iotaArray());

    static {
        // Warm up a few species caches.
        // If we do this too much we will
        // get NPEs from bootstrap circularity.
        VSPECIES.dummyVector();
        VSPECIES.withLanes(LaneType.BYTE);
    }

    // Specialized extractors

    @ForceInline
    final @Override
    public $Type$Species vspecies() {
        // ISSUE:  This should probably be a @Stable
        // field inside AbstractVector, rather than
        // a megamorphic method.
        return VSPECIES;
    }

    @ForceInline
    @Override
    public final Class<$Boxtype$> elementType() { return $type$.class; }

    @ForceInline
    @Override
    public final int elementSize() { return $Boxtype$.SIZE; }

    @ForceInline
    @Override
    public final VectorShape shape() { return VSHAPE; }

    @ForceInline
    @Override
    public final int length() { return VLENGTH; }

    @ForceInline
    @Override
    public final int bitSize() { return VSIZE; }

    @ForceInline
    @Override
    public final int byteSize() { return VSIZE / Byte.SIZE; }

    /*package-private*/
    @ForceInline
    final @Override
    $type$[] vec() {
        return VectorIntrinsics.maybeRebox(this).vec;
    }

    // Virtualized constructors

    @Override
    @ForceInline
    public final $vectortype$ broadcast($type$ e) {
        return ($vectortype$) super.broadcastTemplate(e);  // specialize
    }

#if[!long]
    @Override
    @ForceInline
    public final $vectortype$ broadcast(long e) {
        return ($vectortype$) super.broadcastTemplate(e);  // specialize
    }
#end[!long]

    @Override
    @ForceInline
    $masktype$ maskFromArray(boolean[] bits) {
        return new $masktype$(bits);
    }

    @Override
    @ForceInline
    $shuffletype$ iotaShuffle() { return $shuffletype$.IOTA; }

    @ForceInline
    $shuffletype$ iotaShuffle(int start) {
        return ($shuffletype$)VectorIntrinsics.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, (val, l) -> new $shuffletype$(i -> (VectorIntrinsics.wrapToRange(i + val, l))));
    }

    @Override
    @ForceInline
    $shuffletype$ shuffleFromBytes(byte[] reorder) { return new $shuffletype$(reorder); }

    @Override
    @ForceInline
    $shuffletype$ shuffleFromArray(int[] indexes, int i) { return new $shuffletype$(indexes, i); }

    @Override
    @ForceInline
    $shuffletype$ shuffleFromOp(IntUnaryOperator fn) { return new $shuffletype$(fn); }

    // Make a vector of the same species but the given elements:
    @ForceInline
    final @Override
    $vectortype$ vectorFactory($type$[] vec) {
        return new $vectortype$(vec);
    }

    @ForceInline
    final @Override
    Byte$bits$Vector asByteVectorRaw() {
        return (Byte$bits$Vector) super.asByteVectorRawTemplate();  // specialize
    }

    @ForceInline
    final @Override
    AbstractVector<?> asVectorRaw(LaneType laneType) {
        return super.asVectorRawTemplate(laneType);  // specialize
    }

    // Unary operator

    final @Override
    $vectortype$ uOp(FUnOp f) {
        return ($vectortype$) super.uOpTemplate(f);  // specialize
    }

    @ForceInline
    final @Override
    $vectortype$ uOp(VectorMask<$Boxtype$> m, FUnOp f) {
        return ($vectortype$)
            super.uOpTemplate(($masktype$)m, f);  // specialize
    }

    // Binary operator

    @ForceInline
    final @Override
    $vectortype$ bOp(Vector<$Boxtype$> v, FBinOp f) {
        return ($vectortype$) super.bOpTemplate(($vectortype$)v, f);  // specialize
    }

    @ForceInline
    final @Override
    $vectortype$ bOp(Vector<$Boxtype$> v,
                     VectorMask<$Boxtype$> m, FBinOp f) {
        return ($vectortype$)
            super.bOpTemplate(($vectortype$)v, ($masktype$)m,
                              f);  // specialize
    }

    // Ternary operator

    @ForceInline
    final @Override
    $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, FTriOp f) {
        return ($vectortype$)
            super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
                              f);  // specialize
    }

    @ForceInline
    final @Override
    $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2,
                     VectorMask<$Boxtype$> m, FTriOp f) {
        return ($vectortype$)
            super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
                              ($masktype$)m, f);  // specialize
    }

    @ForceInline
    final @Override
    $type$ rOp($type$ v, FBinOp f) {
        return super.rOpTemplate(v, f);  // specialize
    }

    @Override
    @ForceInline
    public final <F>
    Vector<F> convertShape(VectorOperators.Conversion<$Boxtype$,F> conv,
                           VectorSpecies<F> rsp, int part) {
        return super.convertShapeTemplate(conv, rsp, part);  // specialize
    }

    @Override
    @ForceInline
    public final <F>
    Vector<F> reinterpretShape(VectorSpecies<F> toSpecies, int part) {
        return super.reinterpretShapeTemplate(toSpecies, part);  // specialize
    }

    // Specialized algebraic operations:

    // The following definition forces a specialized version of this
    // crucial method into the v-table of this class.  A call to add()
    // will inline to a call to lanewise(ADD,), at which point the JIT
    // intrinsic will have the opcode of ADD, plus all the metadata
    // for this particular class, enabling it to generate precise
    // code.
    //
    // There is probably no benefit to the JIT to specialize the
    // masked or broadcast versions of the lanewise method.

    @Override
    @ForceInline
    public $vectortype$ lanewise(Unary op) {
        return ($vectortype$) super.lanewiseTemplate(op);  // specialize
    }

    @Override
    @ForceInline
    public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v) {
        return ($vectortype$) super.lanewiseTemplate(op, v);  // specialize
    }

#if[!FP]
    /*package-private*/
    @Override
    @ForceInline $vectortype$
    lanewiseShift(VectorOperators.Binary op, int e) {
        return ($vectortype$) super.lanewiseShiftTemplate(op, e);  // specialize
    }
#end[!FP]

    /*package-private*/
    @Override
    @ForceInline
    public final
    $vectortype$
    lanewise(VectorOperators.Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2) {
        return ($vectortype$) super.lanewiseTemplate(op, v1, v2);  // specialize
    }

    @Override
    @ForceInline
    public final
    $vectortype$ addIndex(int scale) {
        return ($vectortype$) super.addIndexTemplate(scale);  // specialize
    }

    // Type specific horizontal reductions

    @Override
    @ForceInline
    public final $type$ reduceLanes(VectorOperators.Associative op) {
        return super.reduceLanesTemplate(op);  // specialized
    }

    @Override
    @ForceInline
    public final $type$ reduceLanes(VectorOperators.Associative op,
                                    VectorMask<$Boxtype$> m) {
        return super.reduceLanesTemplate(op, m);  // specialized
    }

    @Override
    @ForceInline
    public final long reduceLanesToLong(VectorOperators.Associative op) {
        return (long) super.reduceLanesTemplate(op);  // specialized
    }

    @Override
    @ForceInline
    public final long reduceLanesToLong(VectorOperators.Associative op,
                                        VectorMask<$Boxtype$> m) {
        return (long) super.reduceLanesTemplate(op, m);  // specialized
    }

    @Override
    @ForceInline
    public VectorShuffle<$Boxtype$> toShuffle() {
        $type$[] a = toArray();
        int[] sa = new int[a.length];
        for (int i = 0; i < a.length; i++) {
            sa[i] = (int) a[i];
        }
        return VectorShuffle.fromArray(VSPECIES, sa, 0);
    }

    // Specialized unary testing

    @Override
    @ForceInline
    public final $masktype$ test(Test op) {
        return super.testTemplate($masktype$.class, op);  // specialize
    }

    // Specialized comparisons

    @Override
    @ForceInline
    public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v) {
        return super.compareTemplate($masktype$.class, op, v);  // specialize
    }

    @Override
    @ForceInline
    public final $masktype$ compare(Comparison op, $type$ s) {
        return super.compareTemplate($masktype$.class, op, s);  // specialize
    }

#if[!long]
    @Override
    @ForceInline
    public final $masktype$ compare(Comparison op, long s) {
        return super.compareTemplate($masktype$.class, op, s);  // specialize
    }
#end[!long]

    @Override
    @ForceInline
    public $vectortype$ blend(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
        return ($vectortype$)
            super.blendTemplate($masktype$.class,
                                ($vectortype$) v,
                                ($masktype$) m);  // specialize
    }

    @Override
    @ForceInline
    public $vectortype$ slice(int origin, Vector<$Boxtype$> v) {
        return ($vectortype$) super.sliceTemplate(origin, v);  // specialize
    }

    @Override
    @ForceInline
    public $vectortype$ slice(int origin) {
       if ((origin < 0) || (origin >= VLENGTH)) {
         throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH);
       } else {
         $shuffletype$ Iota = ($shuffletype$)VectorShuffle.iota(VSPECIES, 0, 1, true);
         VectorMask<$Boxtype$> BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast(($type$)(VLENGTH-origin))));
         Iota = ($shuffletype$)VectorShuffle.iota(VSPECIES, origin, 1, true);
         return ZERO.blend(this.rearrange(Iota), BlendMask);
       }
    }

    @Override
    @ForceInline
    public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part) {
        return ($vectortype$) super.unsliceTemplate(origin, w, part);  // specialize
    }

    @Override
    @ForceInline
    public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part, VectorMask<$Boxtype$> m) {
        return ($vectortype$)
            super.unsliceTemplate($masktype$.class,
                                  origin, w, part,
                                  ($masktype$) m);  // specialize
    }

    @Override
    @ForceInline
    public $vectortype$ unslice(int origin) {
       if ((origin < 0) || (origin >= VLENGTH)) {
         throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH);
       } else {
         $shuffletype$ Iota = ($shuffletype$)VectorShuffle.iota(VSPECIES, 0, 1, true);
         VectorMask<$Boxtype$> BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast(($type$)(origin))));
         Iota = ($shuffletype$)VectorShuffle.iota(VSPECIES, -origin, 1, true);
         return ZERO.blend(this.rearrange(Iota), BlendMask);
       }
    }

    @Override
    @ForceInline
    public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s) {
        return ($vectortype$)
            super.rearrangeTemplate($shuffletype$.class,
                                    ($shuffletype$) s);  // specialize
    }

    @Override
    @ForceInline
    public $vectortype$ rearrange(VectorShuffle<$Boxtype$> shuffle,
                                  VectorMask<$Boxtype$> m) {
        return ($vectortype$)
            super.rearrangeTemplate($shuffletype$.class,
                                    ($shuffletype$) shuffle,
                                    ($masktype$) m);  // specialize
    }

    @Override
    @ForceInline
    public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s,
                                  Vector<$Boxtype$> v) {
        return ($vectortype$)
            super.rearrangeTemplate($shuffletype$.class,
                                    ($shuffletype$) s,
                                    ($vectortype$) v);  // specialize
    }

    @Override
    @ForceInline
    public $vectortype$ selectFrom(Vector<$Boxtype$> v) {
        return ($vectortype$)
            super.selectFromTemplate(($vectortype$) v);  // specialize
    }

    @Override
    @ForceInline
    public $vectortype$ selectFrom(Vector<$Boxtype$> v,
                                   VectorMask<$Boxtype$> m) {
        return ($vectortype$)
            super.selectFromTemplate(($vectortype$) v,
                                     ($masktype$) m);  // specialize
    }


#if[FP]
    @Override
    public $type$ lane(int i) {
        if (i < 0 || i >= VLENGTH) {
            throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
        }
        $bitstype$ bits = ($bitstype$) VectorIntrinsics.extract(
                                VCLASS, ETYPE, VLENGTH,
                                this, i,
                                (vec, ix) -> {
                                    $type$[] vecarr = vec.vec();
                                    return (long)$Type$.$type$To$Bitstype$Bits(vecarr[ix]);
                                });
        return $Type$.$bitstype$BitsTo$Fptype$(bits);
    }

    @Override
    public $vectortype$ withLane(int i, $type$ e) {
        if (i < 0 || i >= VLENGTH) {
            throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
        }
        return VectorIntrinsics.insert(
                                VCLASS, ETYPE, VLENGTH,
                                this, i, (long)$Type$.$type$To$Bitstype$Bits(e),
                                (v, ix, bits) -> {
                                    $type$[] res = v.vec().clone();
                                    res[ix] = $Type$.$bitstype$BitsTo$Type$(($bitstype$)bits);
                                    return v.vectorFactory(res);
                                });
    }
#else[FP]
    @Override
    public $type$ lane(int i) {
        if (i < 0 || i >= VLENGTH) {
            throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
        }
        return ($type$) VectorIntrinsics.extract(
                                VCLASS, ETYPE, VLENGTH,
                                this, i,
                                (vec, ix) -> {
                                    $type$[] vecarr = vec.vec();
                                    return (long)vecarr[ix];
                                });
    }

    @Override
    public $vectortype$ withLane(int i, $type$ e) {
        if (i < 0 || i >= VLENGTH) {
            throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
        }
        return VectorIntrinsics.insert(
                                VCLASS, ETYPE, VLENGTH,
                                this, i, (long)e,
                                (v, ix, bits) -> {
                                    $type$[] res = v.vec().clone();
                                    res[ix] = ($type$)bits;
                                    return v.vectorFactory(res);
                                });
    }
#end[FP]

    // Mask

    static final class $masktype$ extends AbstractMask<$Boxtype$> {

        private final boolean[] bits; // Don't access directly, use getBits() instead.

        public $masktype$(boolean[] bits) {
            this(bits, 0);
        }

        public $masktype$(boolean[] bits, int offset) {
            boolean[] a = new boolean[vspecies().laneCount()];
            for (int i = 0; i < a.length; i++) {
                a[i] = bits[offset + i];
            }
            this.bits = a;
        }

        public $masktype$(boolean val) {
            boolean[] bits = new boolean[vspecies().laneCount()];
            Arrays.fill(bits, val);
            this.bits = bits;
        }

        @ForceInline
        final @Override
        public $Type$Species vspecies() {
            // ISSUE:  This should probably be a @Stable
            // field inside AbstractMask, rather than
            // a megamorphic method.
            return VSPECIES;
        }

        boolean[] getBits() {
            return VectorIntrinsics.maybeRebox(this).bits;
        }

        @Override
        $masktype$ uOp(MUnOp f) {
            boolean[] res = new boolean[vspecies().laneCount()];
            boolean[] bits = getBits();
            for (int i = 0; i < res.length; i++) {
                res[i] = f.apply(i, bits[i]);
            }
            return new $masktype$(res);
        }

        @Override
        $masktype$ bOp(VectorMask<$Boxtype$> m, MBinOp f) {
            boolean[] res = new boolean[vspecies().laneCount()];
            boolean[] bits = getBits();
            boolean[] mbits = (($masktype$)m).getBits();
            for (int i = 0; i < res.length; i++) {
                res[i] = f.apply(i, bits[i], mbits[i]);
            }
            return new $masktype$(res);
        }

        @ForceInline
        @Override
        public final
        $vectortype$ toVector() {
            return ($vectortype$) super.toVectorTemplate();  // specialize
        }

        @Override
        @ForceInline
        public <E> VectorMask<E> cast(VectorSpecies<E> s) {
            AbstractSpecies<E> species = (AbstractSpecies<E>) s;
            if (length() != species.laneCount())
                throw new IllegalArgumentException("VectorMask length and species length differ");
            boolean[] maskArray = toArray();
            // enum-switches don't optimize properly JDK-8161245
            switch (species.laneType.switchKey) {
            case LaneType.SK_BYTE:
                return new Byte$bits$Vector.Byte$bits$Mask(maskArray).check(species);
            case LaneType.SK_SHORT:
                return new Short$bits$Vector.Short$bits$Mask(maskArray).check(species);
            case LaneType.SK_INT:
                return new Int$bits$Vector.Int$bits$Mask(maskArray).check(species);
            case LaneType.SK_LONG:
                return new Long$bits$Vector.Long$bits$Mask(maskArray).check(species);
            case LaneType.SK_FLOAT:
                return new Float$bits$Vector.Float$bits$Mask(maskArray).check(species);
            case LaneType.SK_DOUBLE:
                return new Double$bits$Vector.Double$bits$Mask(maskArray).check(species);
            }

            // Should not reach here.
            throw new AssertionError(species);
        }

        // Unary operations

        @Override
        @ForceInline
        public $masktype$ not() {
            return ($masktype$) VectorIntrinsics.unaryOp(
                                             VECTOR_OP_NOT, $masktype$.class, $bitstype$.class, VLENGTH,
                                             this,
                                             (m1) -> m1.uOp((i, a) -> !a));
        }

        // Binary operations

        @Override
        @ForceInline
        public $masktype$ and(VectorMask<$Boxtype$> mask) {
            Objects.requireNonNull(mask);
            $masktype$ m = ($masktype$)mask;
            return VectorIntrinsics.binaryOp(VECTOR_OP_AND, $masktype$.class, $bitstype$.class, VLENGTH,
                                             this, m,
                                             (m1, m2) -> m1.bOp(m2, (i, a, b) -> a & b));
        }

        @Override
        @ForceInline
        public $masktype$ or(VectorMask<$Boxtype$> mask) {
            Objects.requireNonNull(mask);
            $masktype$ m = ($masktype$)mask;
            return VectorIntrinsics.binaryOp(VECTOR_OP_OR, $masktype$.class, $bitstype$.class, VLENGTH,
                                             this, m,
                                             (m1, m2) -> m1.bOp(m2, (i, a, b) -> a | b));
        }

        // Reductions

        @Override
        @ForceInline
        public boolean anyTrue() {
            return VectorIntrinsics.test(BT_ne, $masktype$.class, $bitstype$.class, VLENGTH,
                                         this, this,
                                         (m, __) -> anyTrueHelper((($masktype$)m).getBits()));
        }

        @Override
        @ForceInline
        public boolean allTrue() {
            return VectorIntrinsics.test(BT_overflow, $masktype$.class, $bitstype$.class, VLENGTH,
                                         this, vspecies().maskAll(true),
                                         (m, __) -> allTrueHelper((($masktype$)m).getBits()));
        }

        /*package-private*/
        static $masktype$ maskAll(boolean bit) {
            return bit ? TRUE_MASK : FALSE_MASK;
        }
        static final $masktype$ TRUE_MASK = new $masktype$(true);
        static final $masktype$ FALSE_MASK = new $masktype$(false);
    }

    // Shuffle

    static final class $shuffletype$ extends AbstractShuffle<$Boxtype$> {
        $shuffletype$(byte[] reorder) {
            super(reorder);
        }

        public $shuffletype$(int[] reorder) {
            super(reorder);
        }

        public $shuffletype$(int[] reorder, int i) {
            super(reorder, i);
        }

        public $shuffletype$(IntUnaryOperator fn) {
            super(fn);
        }

        @Override
        public $Type$Species vspecies() {
            return VSPECIES;
        }

        static {
            // There must be enough bits in the shuffle lanes to encode
            // VLENGTH valid indexes and VLENGTH exceptional ones.
            assert(VLENGTH < Byte.MAX_VALUE);
            assert(Byte.MIN_VALUE <= -VLENGTH);
        }
        static final $shuffletype$ IOTA = new $shuffletype$(IDENTITY);

        @Override
        @ForceInline
        public $vectortype$ toVector() {
            return VectorIntrinsics.shuffleToVector(VCLASS, ETYPE, $shuffletype$.class, this, VLENGTH,
                                                    (s) -> (($vectortype$)(((AbstractShuffle<$Boxtype$>)(s)).toVectorTemplate())));
        }

        @Override
        @ForceInline
        public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
            AbstractSpecies<F> species = (AbstractSpecies<F>) s;
            if (length() != species.laneCount())
                throw new IllegalArgumentException("VectorShuffle length and species length differ");
            int[] shuffleArray = toArray();
            // enum-switches don't optimize properly JDK-8161245
            switch (species.laneType.switchKey) {
            case LaneType.SK_BYTE:
                return new Byte$bits$Vector.Byte$bits$Shuffle(shuffleArray).check(species);
            case LaneType.SK_SHORT:
                return new Short$bits$Vector.Short$bits$Shuffle(shuffleArray).check(species);
            case LaneType.SK_INT:
                return new Int$bits$Vector.Int$bits$Shuffle(shuffleArray).check(species);
            case LaneType.SK_LONG:
                return new Long$bits$Vector.Long$bits$Shuffle(shuffleArray).check(species);
            case LaneType.SK_FLOAT:
                return new Float$bits$Vector.Float$bits$Shuffle(shuffleArray).check(species);
            case LaneType.SK_DOUBLE:
                return new Double$bits$Vector.Double$bits$Shuffle(shuffleArray).check(species);
            }

            // Should not reach here.
            throw new AssertionError(species);
        }

        @Override
        public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) {
            $shuffletype$ s = ($shuffletype$) shuffle;
            byte[] r = new byte[reorder.length];
            for (int i = 0; i < reorder.length; i++) {
                int ssi = s.reorder[i];
                r[i] = this.reorder[ssi];  // throws on exceptional index
            }
            return new $shuffletype$(r);
        }
    }

    // ================================================

    // Specialized low-level memory operations.

    @ForceInline
    @Override
    final
    $abstractvectortype$ fromArray0($type$[] a, int offset) {
        return super.fromArray0Template(a, offset);  // specialize
    }

    @ForceInline
    @Override
    final
    $abstractvectortype$ fromByteArray0(byte[] a, int offset) {
        return super.fromByteArray0Template(a, offset);  // specialize
    }

    @ForceInline
    @Override
    final
    $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset) {
        return super.fromByteBuffer0Template(bb, offset);  // specialize
    }

    @ForceInline
    @Override
    final
    void intoArray0($type$[] a, int offset) {
        super.intoArray0Template(a, offset);  // specialize
    }

    @ForceInline
    @Override
    final
    void intoByteArray0(byte[] a, int offset) {
        super.intoByteArray0Template(a, offset);  // specialize
    }

    // End of specialized low-level memory operations.

    // ================================================

}