1 /*
   2  * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.nio.ByteBuffer;
  28 import java.nio.ByteOrder;
  29 #if[!byte]
  30 import java.nio.$Type$Buffer;
  31 #end[!byte]
  32 import java.nio.ReadOnlyBufferException;
  33 import java.util.Arrays;
  34 import java.util.Objects;
  35 import java.util.function.IntUnaryOperator;
  36 
  37 import jdk.internal.misc.Unsafe;
  38 import jdk.internal.vm.annotation.ForceInline;
  39 import static jdk.incubator.vector.VectorIntrinsics.*;
  40 
  41 @SuppressWarnings("cast")
  42 final class $vectortype$ extends $abstractvectortype$ {
  43     static final $Type$$bits$Species SPECIES = new $Type$$bits$Species();
  44 
  45     static final $vectortype$ ZERO = new $vectortype$();
  46 
  47     static final int LENGTH = SPECIES.length();
  48 
  49 #if[!byteOrShort]
  50     // Index vector species
  51     private static final IntVector.IntSpecies INDEX_SPEC;
  52     static {
  53 #if[longOrDouble64]
  54         INDEX_SPEC = (IntVector.IntSpecies) Species.of(int.class, Shape.S_64_BIT);
  55 #else[longOrDouble64]
  56         int bitSize = Vector.bitSizeForVectorLength(int.class, LENGTH);
  57         Vector.Shape shape = Shape.forBitSize(bitSize);
  58         INDEX_SPEC = (IntVector.IntSpecies) Species.of(int.class, shape);
  59 #end[longOrDouble64]
  60     }
  61 #end[!byteOrShort]
  62     private final $type$[] vec; // Don't access directly, use getElements() instead.
  63 
  64     private $type$[] getElements() {
  65         return VectorIntrinsics.maybeRebox(this).vec;
  66     }
  67 
  68     $vectortype$() {
  69         vec = new $type$[SPECIES.length()];
  70     }
  71 
  72     $vectortype$($type$[] v) {
  73         vec = v;
  74     }
  75 
  76     @Override
  77     public int length() { return LENGTH; }
  78 
  79     // Unary operator
  80 
  81     @Override
  82     $vectortype$ uOp(FUnOp f) {
  83         $type$[] vec = getElements();
  84         $type$[] res = new $type$[length()];
  85         for (int i = 0; i < length(); i++) {
  86             res[i] = f.apply(i, vec[i]);
  87         }
  88         return new $vectortype$(res);
  89     }
  90 
  91     @Override
  92     $vectortype$ uOp(Mask<$Boxtype$> o, FUnOp f) {
  93         $type$[] vec = getElements();
  94         $type$[] res = new $type$[length()];
  95         boolean[] mbits = (($masktype$)o).getBits();
  96         for (int i = 0; i < length(); i++) {
  97             res[i] = mbits[i] ? f.apply(i, vec[i]) : vec[i];
  98         }
  99         return new $vectortype$(res);
 100     }
 101 
 102     // Binary operator
 103 
 104     @Override
 105     $vectortype$ bOp(Vector<$Boxtype$> o, FBinOp f) {
 106         $type$[] res = new $type$[length()];
 107         $type$[] vec1 = this.getElements();
 108         $type$[] vec2 = (($vectortype$)o).getElements();
 109         for (int i = 0; i < length(); i++) {
 110             res[i] = f.apply(i, vec1[i], vec2[i]);
 111         }
 112         return new $vectortype$(res);
 113     }
 114 
 115     @Override
 116     $vectortype$ bOp(Vector<$Boxtype$> o1, Mask<$Boxtype$> o2, FBinOp f) {
 117         $type$[] res = new $type$[length()];
 118         $type$[] vec1 = this.getElements();
 119         $type$[] vec2 = (($vectortype$)o1).getElements();
 120         boolean[] mbits = (($masktype$)o2).getBits();
 121         for (int i = 0; i < length(); i++) {
 122             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i]) : vec1[i];
 123         }
 124         return new $vectortype$(res);
 125     }
 126 
 127     // Trinary operator
 128 
 129     @Override
 130     $vectortype$ tOp(Vector<$Boxtype$> o1, Vector<$Boxtype$> o2, FTriOp f) {
 131         $type$[] res = new $type$[length()];
 132         $type$[] vec1 = this.getElements();
 133         $type$[] vec2 = (($vectortype$)o1).getElements();
 134         $type$[] vec3 = (($vectortype$)o2).getElements();
 135         for (int i = 0; i < length(); i++) {
 136             res[i] = f.apply(i, vec1[i], vec2[i], vec3[i]);
 137         }
 138         return new $vectortype$(res);
 139     }
 140 
 141     @Override
 142     $vectortype$ tOp(Vector<$Boxtype$> o1, Vector<$Boxtype$> o2, Mask<$Boxtype$> o3, FTriOp f) {
 143         $type$[] res = new $type$[length()];
 144         $type$[] vec1 = getElements();
 145         $type$[] vec2 = (($vectortype$)o1).getElements();
 146         $type$[] vec3 = (($vectortype$)o2).getElements();
 147         boolean[] mbits = (($masktype$)o3).getBits();
 148         for (int i = 0; i < length(); i++) {
 149             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i], vec3[i]) : vec1[i];
 150         }
 151         return new $vectortype$(res);
 152     }
 153 
 154     @Override
 155     $type$ rOp($type$ v, FBinOp f) {
 156         $type$[] vec = getElements();
 157         for (int i = 0; i < length(); i++) {
 158             v = f.apply(i, v, vec[i]);
 159         }
 160         return v;
 161     }
 162 
 163     @Override
 164     @ForceInline
 165     public <F> Vector<F> cast(Species<F> s) {
 166         Objects.requireNonNull(s);
 167         if (s.length() != LENGTH)
 168             throw new IllegalArgumentException("Vector length this species length differ");
 169 
 170         return VectorIntrinsics.cast(
 171             $vectortype$.class,
 172             $type$.class, LENGTH,
 173             s.vectorType(),
 174             s.elementType(), LENGTH,
 175             this, s,
 176             (species, vector) -> vector.castDefault(species)
 177         );
 178     }
 179 
 180     @SuppressWarnings("unchecked")
 181     @ForceInline
 182     private <F> Vector<F> castDefault(Species<F> s) {
 183         int limit = s.length();
 184 
 185         Class<?> stype = s.elementType();
 186         if (stype == byte.class) {
 187             byte[] a = new byte[limit];
 188             for (int i = 0; i < limit; i++) {
 189                 a[i] = (byte) this.get(i);
 190             }
 191             return (Vector) ByteVector.fromArray((ByteVector.ByteSpecies) s, a, 0);
 192         } else if (stype == short.class) {
 193             short[] a = new short[limit];
 194             for (int i = 0; i < limit; i++) {
 195                 a[i] = (short) this.get(i);
 196             }
 197             return (Vector) ShortVector.fromArray((ShortVector.ShortSpecies) s, a, 0);
 198         } else if (stype == int.class) {
 199             int[] a = new int[limit];
 200             for (int i = 0; i < limit; i++) {
 201                 a[i] = (int) this.get(i);
 202             }
 203             return (Vector) IntVector.fromArray((IntVector.IntSpecies) s, a, 0);
 204         } else if (stype == long.class) {
 205             long[] a = new long[limit];
 206             for (int i = 0; i < limit; i++) {
 207                 a[i] = (long) this.get(i);
 208             }
 209             return (Vector) LongVector.fromArray((LongVector.LongSpecies) s, a, 0);
 210         } else if (stype == float.class) {
 211             float[] a = new float[limit];
 212             for (int i = 0; i < limit; i++) {
 213                 a[i] = (float) this.get(i);
 214             }
 215             return (Vector) FloatVector.fromArray((FloatVector.FloatSpecies) s, a, 0);
 216         } else if (stype == double.class) {
 217             double[] a = new double[limit];
 218             for (int i = 0; i < limit; i++) {
 219                 a[i] = (double) this.get(i);
 220             }
 221             return (Vector) DoubleVector.fromArray((DoubleVector.DoubleSpecies) s, a, 0);
 222         } else {
 223             throw new UnsupportedOperationException("Bad lane type for casting.");
 224         }
 225     }
 226 
 227     @Override
 228     @ForceInline
 229     @SuppressWarnings("unchecked")
 230     public <F> Vector<F> reinterpret(Species<F> s) {
 231         Objects.requireNonNull(s);
 232 
 233         if(s.elementType().equals($type$.class)) {
 234             return (Vector<F>) reshape((Species<$Boxtype$>)s);
 235         }
 236         if(s.bitSize() == bitSize()) {
 237             return reinterpretType(s);
 238         }
 239 
 240         return defaultReinterpret(s);
 241     }
 242 
 243     @ForceInline
 244     private <F> Vector<F> reinterpretType(Species<F> s) {
 245         Objects.requireNonNull(s);
 246 
 247         Class<?> stype = s.elementType();
 248         if (stype == byte.class) {
 249             return VectorIntrinsics.reinterpret(
 250                 $vectortype$.class,
 251                 $type$.class, LENGTH,
 252                 Byte$bits$Vector.class,
 253                 byte.class, Byte$bits$Vector.LENGTH,
 254                 this, s,
 255                 (species, vector) -> vector.defaultReinterpret(species)
 256             );
 257         } else if (stype == short.class) {
 258             return VectorIntrinsics.reinterpret(
 259                 $vectortype$.class,
 260                 $type$.class, LENGTH,
 261                 Short$bits$Vector.class,
 262                 short.class, Short$bits$Vector.LENGTH,
 263                 this, s,
 264                 (species, vector) -> vector.defaultReinterpret(species)
 265             );
 266         } else if (stype == int.class) {
 267             return VectorIntrinsics.reinterpret(
 268                 $vectortype$.class,
 269                 $type$.class, LENGTH,
 270                 Int$bits$Vector.class,
 271                 int.class, Int$bits$Vector.LENGTH,
 272                 this, s,
 273                 (species, vector) -> vector.defaultReinterpret(species)
 274             );
 275         } else if (stype == long.class) {
 276             return VectorIntrinsics.reinterpret(
 277                 $vectortype$.class,
 278                 $type$.class, LENGTH,
 279                 Long$bits$Vector.class,
 280                 long.class, Long$bits$Vector.LENGTH,
 281                 this, s,
 282                 (species, vector) -> vector.defaultReinterpret(species)
 283             );
 284         } else if (stype == float.class) {
 285             return VectorIntrinsics.reinterpret(
 286                 $vectortype$.class,
 287                 $type$.class, LENGTH,
 288                 Float$bits$Vector.class,
 289                 float.class, Float$bits$Vector.LENGTH,
 290                 this, s,
 291                 (species, vector) -> vector.defaultReinterpret(species)
 292             );
 293         } else if (stype == double.class) {
 294             return VectorIntrinsics.reinterpret(
 295                 $vectortype$.class,
 296                 $type$.class, LENGTH,
 297                 Double$bits$Vector.class,
 298                 double.class, Double$bits$Vector.LENGTH,
 299                 this, s,
 300                 (species, vector) -> vector.defaultReinterpret(species)
 301             );
 302         } else {
 303             throw new UnsupportedOperationException("Bad lane type for casting.");
 304         }
 305     }
 306 
 307     @Override
 308     @ForceInline
 309     public $abstractvectortype$ reshape(Species<$Boxtype$> s) {
 310         Objects.requireNonNull(s);
 311         if (s.bitSize() == 64 && (s instanceof $Type$64Vector.$Type$64Species)) {
 312             $Type$64Vector.$Type$64Species ts = ($Type$64Vector.$Type$64Species)s;
 313             return VectorIntrinsics.reinterpret(
 314                 $vectortype$.class,
 315                 $type$.class, LENGTH,
 316                 $Type$64Vector.class,
 317                 $type$.class, $Type$64Vector.LENGTH,
 318                 this, ts,
 319                 (species, vector) -> ($abstractvectortype$) vector.defaultReinterpret(species)
 320             );
 321         } else if (s.bitSize() == 128 && (s instanceof $Type$128Vector.$Type$128Species)) {
 322             $Type$128Vector.$Type$128Species ts = ($Type$128Vector.$Type$128Species)s;
 323             return VectorIntrinsics.reinterpret(
 324                 $vectortype$.class,
 325                 $type$.class, LENGTH,
 326                 $Type$128Vector.class,
 327                 $type$.class, $Type$128Vector.LENGTH,
 328                 this, ts,
 329                 (species, vector) -> ($abstractvectortype$) vector.defaultReinterpret(species)
 330             );
 331         } else if (s.bitSize() == 256 && (s instanceof $Type$256Vector.$Type$256Species)) {
 332             $Type$256Vector.$Type$256Species ts = ($Type$256Vector.$Type$256Species)s;
 333             return VectorIntrinsics.reinterpret(
 334                 $vectortype$.class,
 335                 $type$.class, LENGTH,
 336                 $Type$256Vector.class,
 337                 $type$.class, $Type$256Vector.LENGTH,
 338                 this, ts,
 339                 (species, vector) -> ($abstractvectortype$) vector.defaultReinterpret(species)
 340             );
 341         } else if (s.bitSize() == 512 && (s instanceof $Type$512Vector.$Type$512Species)) {
 342             $Type$512Vector.$Type$512Species ts = ($Type$512Vector.$Type$512Species)s;
 343             return VectorIntrinsics.reinterpret(
 344                 $vectortype$.class,
 345                 $type$.class, LENGTH,
 346                 $Type$512Vector.class,
 347                 $type$.class, $Type$512Vector.LENGTH,
 348                 this, ts,
 349                 (species, vector) -> ($abstractvectortype$) vector.defaultReinterpret(species)
 350             );
 351         } else if ((s.bitSize() > 0) && (s.bitSize() <= 2048)
 352                 && (s.bitSize() % 128 == 0) && (s instanceof $Type$MaxVector.$Type$MaxSpecies)) {
 353             $Type$MaxVector.$Type$MaxSpecies ts = ($Type$MaxVector.$Type$MaxSpecies)s;
 354             return VectorIntrinsics.reinterpret(
 355                 $vectortype$.class,
 356                 $type$.class, LENGTH,
 357                 $Type$MaxVector.class,
 358                 $type$.class, $Type$MaxVector.LENGTH,
 359                 this, ts,
 360                 (species, vector) -> ($abstractvectortype$) vector.defaultReinterpret(species)
 361             );
 362         } else {
 363             throw new InternalError("Unimplemented size");
 364         }
 365     }
 366 
 367     // Binary operations with scalars
 368 
 369     @Override
 370     @ForceInline
 371     public $abstractvectortype$ add($type$ o) {
 372         return add(SPECIES.broadcast(o));
 373     }
 374 
 375     @Override
 376     @ForceInline
 377     public $abstractvectortype$ add($type$ o, Mask<$Boxtype$> m) {
 378         return add(SPECIES.broadcast(o), m);
 379     }
 380 
 381     @Override
 382     @ForceInline
 383     public $abstractvectortype$ sub($type$ o) {
 384         return sub(SPECIES.broadcast(o));
 385     }
 386 
 387     @Override
 388     @ForceInline
 389     public $abstractvectortype$ sub($type$ o, Mask<$Boxtype$> m) {
 390         return sub(SPECIES.broadcast(o), m);
 391     }
 392 
 393     @Override
 394     @ForceInline
 395     public $abstractvectortype$ mul($type$ o) {
 396         return mul(SPECIES.broadcast(o));
 397     }
 398 
 399     @Override
 400     @ForceInline
 401     public $abstractvectortype$ mul($type$ o, Mask<$Boxtype$> m) {
 402         return mul(SPECIES.broadcast(o), m);
 403     }
 404 
 405     @Override
 406     @ForceInline
 407     public $abstractvectortype$ min($type$ o) {
 408         return min(SPECIES.broadcast(o));
 409     }
 410 
 411     @Override
 412     @ForceInline
 413     public $abstractvectortype$ max($type$ o) {
 414         return max(SPECIES.broadcast(o));
 415     }
 416 
 417     @Override
 418     @ForceInline
 419     public Mask<$Boxtype$> equal($type$ o) {
 420         return equal(SPECIES.broadcast(o));
 421     }
 422 
 423     @Override
 424     @ForceInline
 425     public Mask<$Boxtype$> notEqual($type$ o) {
 426         return notEqual(SPECIES.broadcast(o));
 427     }
 428 
 429     @Override
 430     @ForceInline
 431     public Mask<$Boxtype$> lessThan($type$ o) {
 432         return lessThan(SPECIES.broadcast(o));
 433     }
 434 
 435     @Override
 436     @ForceInline
 437     public Mask<$Boxtype$> lessThanEq($type$ o) {
 438         return lessThanEq(SPECIES.broadcast(o));
 439     }
 440 
 441     @Override
 442     @ForceInline
 443     public Mask<$Boxtype$> greaterThan($type$ o) {
 444         return greaterThan(SPECIES.broadcast(o));
 445     }
 446 
 447     @Override
 448     @ForceInline
 449     public Mask<$Boxtype$> greaterThanEq($type$ o) {
 450         return greaterThanEq(SPECIES.broadcast(o));
 451     }
 452 
 453     @Override
 454     @ForceInline
 455     public $abstractvectortype$ blend($type$ o, Mask<$Boxtype$> m) {
 456         return blend(SPECIES.broadcast(o), m);
 457     }
 458 
 459 #if[FP]
 460     @Override
 461     @ForceInline
 462     public $abstractvectortype$ div($type$ o) {
 463         return div(SPECIES.broadcast(o));
 464     }
 465 
 466     @Override
 467     @ForceInline
 468     public $abstractvectortype$ div($type$ o, Mask<$Boxtype$> m) {
 469         return div(SPECIES.broadcast(o), m);
 470     }
 471 
 472     @Override
 473     @ForceInline
 474     public $vectortype$ div(Vector<$Boxtype$> v, Mask<$Boxtype$> m) {
 475         return blend(div(v), m);
 476     }
 477 
 478     @Override
 479     @ForceInline
 480     public $abstractvectortype$ atan2($type$ o) {
 481         return atan2(SPECIES.broadcast(o));
 482     }
 483 
 484     @Override
 485     @ForceInline
 486     public $abstractvectortype$ atan2($type$ o, Mask<$Boxtype$> m) {
 487         return atan2(SPECIES.broadcast(o), m);
 488     }
 489 
 490     @Override
 491     @ForceInline
 492     public $abstractvectortype$ pow($type$ o) {
 493         return pow(SPECIES.broadcast(o));
 494     }
 495 
 496     @Override
 497     @ForceInline
 498     public $abstractvectortype$ pow($type$ o, Mask<$Boxtype$> m) {
 499         return pow(SPECIES.broadcast(o), m);
 500     }
 501 
 502     @Override
 503     @ForceInline
 504     public $abstractvectortype$ fma($type$ o1, $type$ o2) {
 505         return fma(SPECIES.broadcast(o1), SPECIES.broadcast(o2));
 506     }
 507 
 508     @Override
 509     @ForceInline
 510     public $abstractvectortype$ fma($type$ o1, $type$ o2, Mask<$Boxtype$> m) {
 511         return fma(SPECIES.broadcast(o1), SPECIES.broadcast(o2), m);
 512     }
 513 
 514     @Override
 515     @ForceInline
 516     public $abstractvectortype$ hypot($type$ o) {
 517         return hypot(SPECIES.broadcast(o));
 518     }
 519 
 520     @Override
 521     @ForceInline
 522     public $abstractvectortype$ hypot($type$ o, Mask<$Boxtype$> m) {
 523         return hypot(SPECIES.broadcast(o), m);
 524     }
 525 #end[FP]
 526 
 527 #if[BITWISE]
 528     @Override
 529     @ForceInline
 530     public $abstractvectortype$ and($type$ o) {
 531         return and(SPECIES.broadcast(o));
 532     }
 533 
 534     @Override
 535     @ForceInline
 536     public $abstractvectortype$ and($type$ o, Mask<$Boxtype$> m) {
 537         return and(SPECIES.broadcast(o), m);
 538     }
 539 
 540     @Override
 541     @ForceInline
 542     public $abstractvectortype$ or($type$ o) {
 543         return or(SPECIES.broadcast(o));
 544     }
 545 
 546     @Override
 547     @ForceInline
 548     public $abstractvectortype$ or($type$ o, Mask<$Boxtype$> m) {
 549         return or(SPECIES.broadcast(o), m);
 550     }
 551 
 552     @Override
 553     @ForceInline
 554     public $abstractvectortype$ xor($type$ o) {
 555         return xor(SPECIES.broadcast(o));
 556     }
 557 
 558     @Override
 559     @ForceInline
 560     public $abstractvectortype$ xor($type$ o, Mask<$Boxtype$> m) {
 561         return xor(SPECIES.broadcast(o), m);
 562     }
 563 
 564     @Override
 565     @ForceInline
 566     public $vectortype$ neg() {
 567         return ($vectortype$)zero(SPECIES).sub(this);
 568     }
 569 #end[BITWISE]
 570 
 571     // Unary operations
 572 
 573     @ForceInline
 574     @Override
 575     public $vectortype$ neg(Mask<$Boxtype$> m) {
 576         return blend(neg(), m);
 577     }
 578 
 579     @Override
 580     @ForceInline
 581     public $vectortype$ abs() {
 582         return VectorIntrinsics.unaryOp(
 583             VECTOR_OP_ABS, $vectortype$.class, $type$.class, LENGTH,
 584             this,
 585             v1 -> v1.uOp((i, a) -> ($type$) Math.abs(a)));
 586     }
 587 
 588     @ForceInline
 589     @Override
 590     public $vectortype$ abs(Mask<$Boxtype$> m) {
 591         return blend(abs(), m);
 592     }
 593 
 594 #if[FP]
 595     @Override
 596     @ForceInline
 597     public $vectortype$ neg() {
 598         return VectorIntrinsics.unaryOp(
 599             VECTOR_OP_NEG, $vectortype$.class, $type$.class, LENGTH,
 600             this,
 601             v1 -> v1.uOp((i, a) -> ($type$) -a));
 602     }
 603 
 604     @Override
 605     @ForceInline
 606     public $vectortype$ div(Vector<$Boxtype$> o) {
 607         Objects.requireNonNull(o);
 608         $vectortype$ v = ($vectortype$)o;
 609         return VectorIntrinsics.binaryOp(
 610             VECTOR_OP_DIV, $vectortype$.class, $type$.class, LENGTH,
 611             this, v,
 612             (v1, v2) -> v1.bOp(v2, (i, a, b) -> ($type$)(a / b)));
 613     }
 614 
 615     @Override
 616     @ForceInline
 617     public $vectortype$ sqrt() {
 618         return VectorIntrinsics.unaryOp(
 619             VECTOR_OP_SQRT, $vectortype$.class, $type$.class, LENGTH,
 620             this,
 621             v1 -> v1.uOp((i, a) -> ($type$) Math.sqrt((double) a)));
 622     }
 623 
 624     @Override
 625     @ForceInline
 626     public $vectortype$ exp() {
 627         return ($vectortype$) VectorIntrinsics.unaryOp(
 628             VECTOR_OP_EXP, $vectortype$.class, $type$.class, LENGTH,
 629             this,
 630             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.exp((double) a)));
 631     }
 632 
 633     @Override
 634     @ForceInline
 635     public $vectortype$ log1p() {
 636         return ($vectortype$) VectorIntrinsics.unaryOp(
 637             VECTOR_OP_LOG1P, $vectortype$.class, $type$.class, LENGTH,
 638             this,
 639             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.log1p((double) a)));
 640     }
 641 
 642     @Override
 643     @ForceInline
 644     public $vectortype$ log() {
 645         return ($vectortype$) VectorIntrinsics.unaryOp(
 646             VECTOR_OP_LOG, $vectortype$.class, $type$.class, LENGTH,
 647             this,
 648             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.log((double) a)));
 649     }
 650 
 651     @Override
 652     @ForceInline
 653     public $vectortype$ log10() {
 654         return ($vectortype$) VectorIntrinsics.unaryOp(
 655             VECTOR_OP_LOG10, $vectortype$.class, $type$.class, LENGTH,
 656             this,
 657             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.log10((double) a)));
 658     }
 659 
 660     @Override
 661     @ForceInline
 662     public $vectortype$ expm1() {
 663         return ($vectortype$) VectorIntrinsics.unaryOp(
 664             VECTOR_OP_EXPM1, $vectortype$.class, $type$.class, LENGTH,
 665             this,
 666             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.expm1((double) a)));
 667     }
 668 
 669     @Override
 670     @ForceInline
 671     public $vectortype$ cbrt() {
 672         return ($vectortype$) VectorIntrinsics.unaryOp(
 673             VECTOR_OP_CBRT, $vectortype$.class, $type$.class, LENGTH,
 674             this,
 675             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.cbrt((double) a)));
 676     }
 677 
 678     @Override
 679     @ForceInline
 680     public $vectortype$ sin() {
 681         return ($vectortype$) VectorIntrinsics.unaryOp(
 682             VECTOR_OP_SIN, $vectortype$.class, $type$.class, LENGTH,
 683             this,
 684             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.sin((double) a)));
 685     }
 686 
 687     @Override
 688     @ForceInline
 689     public $vectortype$ cos() {
 690         return ($vectortype$) VectorIntrinsics.unaryOp(
 691             VECTOR_OP_COS, $vectortype$.class, $type$.class, LENGTH,
 692             this,
 693             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.cos((double) a)));
 694     }
 695 
 696     @Override
 697     @ForceInline
 698     public $vectortype$ tan() {
 699         return ($vectortype$) VectorIntrinsics.unaryOp(
 700             VECTOR_OP_TAN, $vectortype$.class, $type$.class, LENGTH,
 701             this,
 702             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.tan((double) a)));
 703     }
 704 
 705     @Override
 706     @ForceInline
 707     public $vectortype$ asin() {
 708         return ($vectortype$) VectorIntrinsics.unaryOp(
 709             VECTOR_OP_ASIN, $vectortype$.class, $type$.class, LENGTH,
 710             this,
 711             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.asin((double) a)));
 712     }
 713 
 714     @Override
 715     @ForceInline
 716     public $vectortype$ acos() {
 717         return ($vectortype$) VectorIntrinsics.unaryOp(
 718             VECTOR_OP_ACOS, $vectortype$.class, $type$.class, LENGTH,
 719             this,
 720             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.acos((double) a)));
 721     }
 722 
 723     @Override
 724     @ForceInline
 725     public $vectortype$ atan() {
 726         return ($vectortype$) VectorIntrinsics.unaryOp(
 727             VECTOR_OP_ATAN, $vectortype$.class, $type$.class, LENGTH,
 728             this,
 729             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.atan((double) a)));
 730     }
 731 
 732     @Override
 733     @ForceInline
 734     public $vectortype$ sinh() {
 735         return ($vectortype$) VectorIntrinsics.unaryOp(
 736             VECTOR_OP_SINH, $vectortype$.class, $type$.class, LENGTH,
 737             this,
 738             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.sinh((double) a)));
 739     }
 740 
 741     @Override
 742     @ForceInline
 743     public $vectortype$ cosh() {
 744         return ($vectortype$) VectorIntrinsics.unaryOp(
 745             VECTOR_OP_COSH, $vectortype$.class, $type$.class, LENGTH,
 746             this,
 747             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.cosh((double) a)));
 748     }
 749 
 750     @Override
 751     @ForceInline
 752     public $vectortype$ tanh() {
 753         return ($vectortype$) VectorIntrinsics.unaryOp(
 754             VECTOR_OP_TANH, $vectortype$.class, $type$.class, LENGTH,
 755             this,
 756             v1 -> (($vectortype$)v1).uOp((i, a) -> ($type$) Math.tanh((double) a)));
 757     }
 758 
 759     @Override
 760     @ForceInline
 761     public $vectortype$ pow(Vector<$Boxtype$> o) {
 762         Objects.requireNonNull(o);
 763         $vectortype$ v = ($vectortype$)o;
 764         return ($vectortype$) VectorIntrinsics.binaryOp(
 765             VECTOR_OP_POW, $vectortype$.class, $type$.class, LENGTH,
 766             this, v,
 767             (v1, v2) -> (($vectortype$)v1).bOp(v2, (i, a, b) -> ($type$)(Math.pow(a,b))));
 768     }
 769 
 770     @Override
 771     @ForceInline
 772     public $vectortype$ hypot(Vector<$Boxtype$> o) {
 773         Objects.requireNonNull(o);
 774         $vectortype$ v = ($vectortype$)o;
 775         return ($vectortype$) VectorIntrinsics.binaryOp(
 776             VECTOR_OP_HYPOT, $vectortype$.class, $type$.class, LENGTH,
 777             this, v,
 778             (v1, v2) -> (($vectortype$)v1).bOp(v2, (i, a, b) -> ($type$)(Math.hypot(a,b))));
 779     }
 780 
 781     @Override
 782     @ForceInline
 783     public $vectortype$ atan2(Vector<$Boxtype$> o) {
 784         Objects.requireNonNull(o);
 785         $vectortype$ v = ($vectortype$)o;
 786         return ($vectortype$) VectorIntrinsics.binaryOp(
 787             VECTOR_OP_ATAN2, $vectortype$.class, $type$.class, LENGTH,
 788             this, v,
 789             (v1, v2) -> (($vectortype$)v1).bOp(v2, (i, a, b) -> ($type$)(Math.atan2(a,b))));
 790     }
 791 
 792 #end[FP]
 793 
 794 #if[BITWISE]
 795     @Override
 796     @ForceInline
 797     public $vectortype$ not() {
 798         return VectorIntrinsics.unaryOp(
 799             VECTOR_OP_NOT, $vectortype$.class, $type$.class, LENGTH,
 800             this,
 801             v1 -> v1.uOp((i, a) -> ($type$) ~a));
 802     }
 803 
 804     @ForceInline
 805     @Override
 806     public $vectortype$ not(Mask<$Boxtype$> m) {
 807         return blend(not(), m);
 808     }
 809 #end[BITWISE]
 810     // Binary operations
 811 
 812     @Override
 813     @ForceInline
 814     public $vectortype$ add(Vector<$Boxtype$> o) {
 815         Objects.requireNonNull(o);
 816         $vectortype$ v = ($vectortype$)o;
 817         return VectorIntrinsics.binaryOp(
 818             VECTOR_OP_ADD, $vectortype$.class, $type$.class, LENGTH,
 819             this, v,
 820             (v1, v2) -> v1.bOp(v2, (i, a, b) -> ($type$)(a + b)));
 821     }
 822 
 823     @Override
 824     @ForceInline
 825     public $vectortype$ add(Vector<$Boxtype$> v, Mask<$Boxtype$> m) {
 826         return blend(add(v), m);
 827     }
 828 
 829     @Override
 830     @ForceInline
 831     public $vectortype$ sub(Vector<$Boxtype$> o) {
 832         Objects.requireNonNull(o);
 833         $vectortype$ v = ($vectortype$)o;
 834         return VectorIntrinsics.binaryOp(
 835             VECTOR_OP_SUB, $vectortype$.class, $type$.class, LENGTH,
 836             this, v,
 837             (v1, v2) -> v1.bOp(v2, (i, a, b) -> ($type$)(a - b)));
 838     }
 839 
 840     @Override
 841     @ForceInline
 842     public $vectortype$ sub(Vector<$Boxtype$> v, Mask<$Boxtype$> m) {
 843         return blend(sub(v), m);
 844     }
 845 
 846     @Override
 847     @ForceInline
 848     public $vectortype$ mul(Vector<$Boxtype$> o) {
 849         Objects.requireNonNull(o);
 850         $vectortype$ v = ($vectortype$)o;
 851         return VectorIntrinsics.binaryOp(
 852             VECTOR_OP_MUL, $vectortype$.class, $type$.class, LENGTH,
 853             this, v,
 854             (v1, v2) -> v1.bOp(v2, (i, a, b) -> ($type$)(a * b)));
 855     }
 856 
 857     @Override
 858     @ForceInline
 859     public $vectortype$ mul(Vector<$Boxtype$> v, Mask<$Boxtype$> m) {
 860         return blend(mul(v), m);
 861     }
 862 
 863     @Override
 864     @ForceInline
 865     public $vectortype$ min(Vector<$Boxtype$> o) {
 866         Objects.requireNonNull(o);
 867         $vectortype$ v = ($vectortype$)o;
 868         return ($vectortype$) VectorIntrinsics.binaryOp(
 869             VECTOR_OP_MIN, $vectortype$.class, $type$.class, LENGTH,
 870             this, v,
 871             (v1, v2) -> v1.bOp(v2, (i, a, b) -> ($type$) Math.min(a, b)));
 872     }
 873 
 874     @Override
 875     @ForceInline
 876     public $vectortype$ min(Vector<$Boxtype$> v, Mask<$Boxtype$> m) {
 877         return blend(min(v), m);
 878     }
 879 
 880     @Override
 881     @ForceInline
 882     public $vectortype$ max(Vector<$Boxtype$> o) {
 883         Objects.requireNonNull(o);
 884         $vectortype$ v = ($vectortype$)o;
 885         return VectorIntrinsics.binaryOp(
 886             VECTOR_OP_MAX, $vectortype$.class, $type$.class, LENGTH,
 887             this, v,
 888             (v1, v2) -> v1.bOp(v2, (i, a, b) -> ($type$) Math.max(a, b)));
 889         }
 890 
 891     @Override
 892     @ForceInline
 893     public $vectortype$ max(Vector<$Boxtype$> v, Mask<$Boxtype$> m) {
 894         return blend(max(v), m);
 895     }
 896 
 897 #if[BITWISE]
 898     @Override
 899     @ForceInline
 900     public $vectortype$ and(Vector<$Boxtype$> o) {
 901         Objects.requireNonNull(o);
 902         $vectortype$ v = ($vectortype$)o;
 903         return VectorIntrinsics.binaryOp(
 904             VECTOR_OP_AND, $vectortype$.class, $type$.class, LENGTH,
 905             this, v,
 906             (v1, v2) -> v1.bOp(v2, (i, a, b) -> ($type$)(a & b)));
 907     }
 908 
 909     @Override
 910     @ForceInline
 911     public $vectortype$ or(Vector<$Boxtype$> o) {
 912         Objects.requireNonNull(o);
 913         $vectortype$ v = ($vectortype$)o;
 914         return VectorIntrinsics.binaryOp(
 915             VECTOR_OP_OR, $vectortype$.class, $type$.class, LENGTH,
 916             this, v,
 917             (v1, v2) -> v1.bOp(v2, (i, a, b) -> ($type$)(a | b)));
 918     }
 919 
 920     @Override
 921     @ForceInline
 922     public $vectortype$ xor(Vector<$Boxtype$> o) {
 923         Objects.requireNonNull(o);
 924         $vectortype$ v = ($vectortype$)o;
 925         return VectorIntrinsics.binaryOp(
 926             VECTOR_OP_XOR, $vectortype$.class, $type$.class, LENGTH,
 927             this, v,
 928             (v1, v2) -> v1.bOp(v2, (i, a, b) -> ($type$)(a ^ b)));
 929     }
 930 
 931     @Override
 932     @ForceInline
 933     public $vectortype$ and(Vector<$Boxtype$> v, Mask<$Boxtype$> m) {
 934         return blend(and(v), m);
 935     }
 936 
 937     @Override
 938     @ForceInline
 939     public $vectortype$ or(Vector<$Boxtype$> v, Mask<$Boxtype$> m) {
 940         return blend(or(v), m);
 941     }
 942 
 943     @Override
 944     @ForceInline
 945     public $vectortype$ xor(Vector<$Boxtype$> v, Mask<$Boxtype$> m) {
 946         return blend(xor(v), m);
 947     }
 948 #end[BITWISE]
 949 
 950 #if[byte]
 951     @Override
 952     @ForceInline
 953     public $vectortype$ shiftL(int s) {
 954         return VectorIntrinsics.broadcastInt(
 955             VECTOR_OP_LSHIFT, $vectortype$.class, $type$.class, LENGTH,
 956             this, s,
 957             (v, i) -> v.uOp((__, a) -> ($type$) (a << (i & 7))));
 958     }
 959 
 960     @Override
 961     @ForceInline
 962     public $vectortype$ shiftL(int s, Mask<$Boxtype$> m) {
 963         return blend(shiftL(s), m);
 964     }
 965 
 966     @Override
 967     @ForceInline
 968     public $vectortype$ shiftR(int s) {
 969         return VectorIntrinsics.broadcastInt(
 970             VECTOR_OP_URSHIFT, $vectortype$.class, $type$.class, LENGTH,
 971             this, s,
 972             (v, i) -> v.uOp((__, a) -> ($type$) ((a & 0xFF) >>> (i & 7))));
 973     }
 974 
 975     @Override
 976     @ForceInline
 977     public $vectortype$ shiftR(int s, Mask<$Boxtype$> m) {
 978         return blend(shiftR(s), m);
 979     }
 980 
 981     @Override
 982     @ForceInline
 983     public $vectortype$ aShiftR(int s) {
 984         return VectorIntrinsics.broadcastInt(
 985             VECTOR_OP_RSHIFT, $vectortype$.class, $type$.class, LENGTH,
 986             this, s,
 987             (v, i) -> v.uOp((__, a) -> ($type$) (a >> (i & 7))));
 988     }
 989 
 990     @Override
 991     @ForceInline
 992     public $vectortype$ aShiftR(int s, Mask<$Boxtype$> m) {
 993         return blend(aShiftR(s), m);
 994     }
 995 #end[byte]
 996 #if[short]
 997     @Override
 998     @ForceInline
 999     public $vectortype$ shiftL(int s) {
1000         return VectorIntrinsics.broadcastInt(
1001             VECTOR_OP_LSHIFT, $vectortype$.class, $type$.class, LENGTH,
1002             this, s,
1003             (v, i) -> v.uOp((__, a) -> ($type$) (a << (i & 15))));
1004     }
1005 
1006     @Override
1007     @ForceInline
1008     public $vectortype$ shiftL(int s, Mask<$Boxtype$> m) {
1009         return blend(shiftL(s), m);
1010     }
1011 
1012     @Override
1013     @ForceInline
1014     public $vectortype$ shiftR(int s) {
1015         return VectorIntrinsics.broadcastInt(
1016             VECTOR_OP_URSHIFT, $vectortype$.class, $type$.class, LENGTH,
1017             this, s,
1018             (v, i) -> v.uOp((__, a) -> ($type$) ((a & 0xFFFF) >>> (i & 15))));
1019     }
1020 
1021     @Override
1022     @ForceInline
1023     public $vectortype$ shiftR(int s, Mask<$Boxtype$> m) {
1024         return blend(shiftR(s), m);
1025     }
1026 
1027     @Override
1028     @ForceInline
1029     public $vectortype$ aShiftR(int s) {
1030         return VectorIntrinsics.broadcastInt(
1031             VECTOR_OP_RSHIFT, $vectortype$.class, $type$.class, LENGTH,
1032             this, s,
1033             (v, i) -> v.uOp((__, a) -> ($type$) (a >> (i & 15))));
1034     }
1035 
1036     @Override
1037     @ForceInline
1038     public $vectortype$ aShiftR(int s, Mask<$Boxtype$> m) {
1039         return blend(aShiftR(s), m);
1040     }
1041 #end[short]
1042 #if[intOrLong]
1043     @Override
1044     @ForceInline
1045     public $vectortype$ shiftL(int s) {
1046         return VectorIntrinsics.broadcastInt(
1047             VECTOR_OP_LSHIFT, $vectortype$.class, $type$.class, LENGTH,
1048             this, s,
1049             (v, i) -> v.uOp((__, a) -> ($type$) (a << i)));
1050     }
1051 
1052     @Override
1053     @ForceInline
1054     public $vectortype$ shiftL(int s, Mask<$Boxtype$> m) {
1055         return blend(shiftL(s), m);
1056     }
1057 
1058     @Override
1059     @ForceInline
1060     public $vectortype$ shiftR(int s) {
1061         return VectorIntrinsics.broadcastInt(
1062             VECTOR_OP_URSHIFT, $vectortype$.class, $type$.class, LENGTH,
1063             this, s,
1064             (v, i) -> v.uOp((__, a) -> ($type$) (a >>> i)));
1065     }
1066 
1067     @Override
1068     @ForceInline
1069     public $vectortype$ shiftR(int s, Mask<$Boxtype$> m) {
1070         return blend(shiftR(s), m);
1071     }
1072 
1073     @Override
1074     @ForceInline
1075     public $vectortype$ aShiftR(int s) {
1076         return VectorIntrinsics.broadcastInt(
1077             VECTOR_OP_RSHIFT, $vectortype$.class, $type$.class, LENGTH,
1078             this, s,
1079             (v, i) -> v.uOp((__, a) -> ($type$) (a >> i)));
1080     }
1081 
1082     @Override
1083     @ForceInline
1084     public $vectortype$ aShiftR(int s, Mask<$Boxtype$> m) {
1085         return blend(aShiftR(s), m);
1086     }
1087 
1088     @Override
1089     @ForceInline
1090     public $vectortype$ shiftL(Vector<$Boxtype$> s) {
1091         $vectortype$ shiftv = ($vectortype$)s;
1092         // As per shift specification for Java, mask the shift count.
1093         shiftv = shiftv.and(species().broadcast({#if[int]?0x1f:0x3f}));
1094         return VectorIntrinsics.binaryOp(
1095             VECTOR_OP_LSHIFT, $vectortype$.class, $type$.class, LENGTH,
1096             this, shiftv,
1097             (v1, v2) -> v1.bOp(v2,(i,a, b) -> ($type$) (a << b)));
1098     }
1099 
1100     @Override
1101     @ForceInline
1102     public $vectortype$ shiftR(Vector<$Boxtype$> s) {
1103         $vectortype$ shiftv = ($vectortype$)s;
1104         // As per shift specification for Java, mask the shift count.
1105         shiftv = shiftv.and(species().broadcast({#if[int]?0x1f:0x3f}));
1106         return VectorIntrinsics.binaryOp(
1107             VECTOR_OP_URSHIFT, $vectortype$.class, $type$.class, LENGTH,
1108             this, shiftv,
1109             (v1, v2) -> v1.bOp(v2,(i,a, b) -> ($type$) (a >>> b)));
1110     }
1111 
1112     @Override
1113     @ForceInline
1114     public $vectortype$ aShiftR(Vector<$Boxtype$> s) {
1115         $vectortype$ shiftv = ($vectortype$)s;
1116         // As per shift specification for Java, mask the shift count.
1117         shiftv = shiftv.and(species().broadcast({#if[int]?0x1f:0x3f}));
1118         return VectorIntrinsics.binaryOp(
1119             VECTOR_OP_RSHIFT, $vectortype$.class, $type$.class, LENGTH,
1120             this, shiftv,
1121             (v1, v2) -> v1.bOp(v2,(i,a, b) -> ($type$) (a >> b)));
1122     }
1123 #end[intOrLong]
1124     // Ternary operations
1125 
1126 #if[FP]
1127     @Override
1128     @ForceInline
1129     public $vectortype$ fma(Vector<$Boxtype$> o1, Vector<$Boxtype$> o2) {
1130         Objects.requireNonNull(o1);
1131         Objects.requireNonNull(o2);
1132         $vectortype$ v1 = ($vectortype$)o1;
1133         $vectortype$ v2 = ($vectortype$)o2;
1134         return VectorIntrinsics.ternaryOp(
1135             VECTOR_OP_FMA, $vectortype$.class, $type$.class, LENGTH,
1136             this, v1, v2,
1137             (w1, w2, w3) -> w1.tOp(w2, w3, (i, a, b, c) -> Math.fma(a, b, c)));
1138     }
1139 #end[FP]
1140 
1141     // Type specific horizontal reductions
1142 #if[BITWISE]
1143 
1144     @Override
1145     @ForceInline
1146     public $type$ addAll() {
1147         return ($type$) VectorIntrinsics.reductionCoerced(
1148             VECTOR_OP_ADD, $vectortype$.class, $type$.class, LENGTH,
1149             this,
1150             v -> (long) v.rOp(($type$) 0, (i, a, b) -> ($type$) (a + b)));
1151     }
1152 
1153     @Override
1154     @ForceInline
1155     public $type$ andAll() {
1156         return ($type$) VectorIntrinsics.reductionCoerced(
1157             VECTOR_OP_AND, $vectortype$.class, $type$.class, LENGTH,
1158             this,
1159             v -> (long) v.rOp(($type$) -1, (i, a, b) -> ($type$) (a & b)));
1160     }
1161 
1162     @Override
1163     @ForceInline
1164     public $type$ andAll(Mask<$Boxtype$> m) {
1165         return blend(SPECIES.broadcast(($type$) -1), m).andAll();
1166     }
1167 
1168     @Override
1169     @ForceInline
1170     public $type$ minAll() {
1171         return ($type$) VectorIntrinsics.reductionCoerced(
1172             VECTOR_OP_MIN, $vectortype$.class, $type$.class, LENGTH,
1173             this,
1174             v -> (long) v.rOp($Boxtype$.MAX_VALUE , (i, a, b) -> ($type$) Math.min(a, b)));
1175     }
1176 
1177     @Override
1178     @ForceInline
1179     public $type$ maxAll() {
1180         return ($type$) VectorIntrinsics.reductionCoerced(
1181             VECTOR_OP_MAX, $vectortype$.class, $type$.class, LENGTH,
1182             this,
1183             v -> (long) v.rOp($Boxtype$.MIN_VALUE , (i, a, b) -> ($type$) Math.max(a, b)));
1184     }
1185 
1186     @Override
1187     @ForceInline
1188     public $type$ mulAll() {
1189         return ($type$) VectorIntrinsics.reductionCoerced(
1190             VECTOR_OP_MUL, $vectortype$.class, $type$.class, LENGTH,
1191             this,
1192             v -> (long) v.rOp(($type$) 1, (i, a, b) -> ($type$) (a * b)));
1193     }
1194 
1195     @Override
1196     @ForceInline
1197     public $type$ orAll() {
1198         return ($type$) VectorIntrinsics.reductionCoerced(
1199             VECTOR_OP_OR, $vectortype$.class, $type$.class, LENGTH,
1200             this,
1201             v -> (long) v.rOp(($type$) 0, (i, a, b) -> ($type$) (a | b)));
1202     }
1203 
1204     @Override
1205     @ForceInline
1206     public $type$ orAll(Mask<$Boxtype$> m) {
1207         return blend(SPECIES.broadcast(($type$) 0), m).orAll();
1208     }
1209 
1210     @Override
1211     @ForceInline
1212     public $type$ xorAll() {
1213         return ($type$) VectorIntrinsics.reductionCoerced(
1214             VECTOR_OP_XOR, $vectortype$.class, $type$.class, LENGTH,
1215             this,
1216             v -> (long) v.rOp(($type$) 0, (i, a, b) -> ($type$) (a ^ b)));
1217     }
1218 
1219     @Override
1220     @ForceInline
1221     public $type$ xorAll(Mask<$Boxtype$> m) {
1222         return blend(SPECIES.broadcast(($type$) 0), m).xorAll();
1223     }
1224 #end[BITWISE]
1225 
1226 #if[FP]
1227     @Override
1228     @ForceInline
1229     public $type$ addAll() {
1230         $bitstype$ bits = ($bitstype$) VectorIntrinsics.reductionCoerced(
1231                                 VECTOR_OP_ADD, $vectortype$.class, $type$.class, LENGTH,
1232                                 this,
1233                                 v -> {
1234                                     $type$ r = v.rOp(($type$) 0, (i, a, b) -> ($type$) (a + b));
1235                                     return (long)$Type$.$type$To$Bitstype$Bits(r);
1236                                 });
1237         return $Type$.$bitstype$BitsTo$Fptype$(bits);
1238     }
1239 
1240     @Override
1241     @ForceInline
1242     public $type$ mulAll() {
1243         $bitstype$ bits = ($bitstype$) VectorIntrinsics.reductionCoerced(
1244                                 VECTOR_OP_MUL, $vectortype$.class, $type$.class, LENGTH,
1245                                 this,
1246                                 v -> {
1247                                     $type$ r = v.rOp(($type$) 1, (i, a, b) -> ($type$) (a * b));
1248                                     return (long)$Type$.$type$To$Bitstype$Bits(r);
1249                                 });
1250         return $Type$.$bitstype$BitsTo$Fptype$(bits);
1251     }
1252 
1253     @Override
1254     @ForceInline
1255     public $type$ minAll() {
1256         $bitstype$ bits = ($bitstype$) VectorIntrinsics.reductionCoerced(
1257                                 VECTOR_OP_MIN, $vectortype$.class, $type$.class, LENGTH,
1258                                 this,
1259                                 v -> {
1260                                     $type$ r = v.rOp($Boxtype$.MAX_VALUE , (i, a, b) -> ($type$) Math.min(a, b));
1261                                     return (long)$Type$.$type$To$Bitstype$Bits(r);
1262                                 });
1263         return $Type$.$bitstype$BitsTo$Fptype$(bits);
1264     }
1265 
1266     @Override
1267     @ForceInline
1268     public $type$ maxAll() {
1269         $bitstype$ bits = ($bitstype$) VectorIntrinsics.reductionCoerced(
1270                                 VECTOR_OP_MAX, $vectortype$.class, $type$.class, LENGTH,
1271                                 this,
1272                                 v -> {
1273                                     $type$ r = v.rOp($Boxtype$.MIN_VALUE , (i, a, b) -> ($type$) Math.max(a, b));
1274                                     return (long)$Type$.$type$To$Bitstype$Bits(r);
1275                                 });
1276         return $Type$.$bitstype$BitsTo$Fptype$(bits);
1277     }
1278 
1279 #end[FP]
1280 
1281     @Override
1282     @ForceInline
1283     public $type$ addAll(Mask<$Boxtype$> m) {
1284         return blend(SPECIES.broadcast(($type$) 0), m).addAll();
1285     }
1286 
1287     @Override
1288     @ForceInline
1289     public $type$ mulAll(Mask<$Boxtype$> m) {
1290         return blend(SPECIES.broadcast(($type$) 1), m).mulAll();
1291     }
1292 
1293     @Override
1294     @ForceInline
1295     public $type$ minAll(Mask<$Boxtype$> m) {
1296         return blend(SPECIES.broadcast($Boxtype$.MAX_VALUE), m).minAll();
1297     }
1298 
1299     @Override
1300     @ForceInline
1301     public $type$ maxAll(Mask<$Boxtype$> m) {
1302         return blend(SPECIES.broadcast($Boxtype$.MIN_VALUE), m).maxAll();
1303     }
1304 
1305     @Override
1306     @ForceInline
1307     public Shuffle<$Boxtype$> toShuffle() {
1308         $type$[] a = toArray();
1309         int[] sa = new int[a.length];
1310         for (int i = 0; i < a.length; i++) {
1311             sa[i] = (int) a[i];
1312         }
1313         return $abstractvectortype$.shuffleFromArray(SPECIES, sa, 0);
1314     }
1315 
1316     // Memory operations
1317 
1318     private static final int ARRAY_SHIFT         = 31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_$TYPE$_INDEX_SCALE);
1319     private static final int BOOLEAN_ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_BOOLEAN_INDEX_SCALE);
1320 
1321     @Override
1322     @ForceInline
1323     public void intoArray($type$[] a, int ix) {
1324         Objects.requireNonNull(a);
1325         ix = VectorIntrinsics.checkIndex(ix, a.length, LENGTH);
1326         VectorIntrinsics.store($vectortype$.class, $type$.class, LENGTH,
1327                                a, (((long) ix) << ARRAY_SHIFT) + Unsafe.ARRAY_$TYPE$_BASE_OFFSET,
1328                                this,
1329                                a, ix,
1330                                (arr, idx, v) -> v.forEach((i, e) -> arr[idx + i] = e));
1331     }
1332 
1333     @Override
1334     @ForceInline
1335     public final void intoArray($type$[] a, int ax, Mask<$Boxtype$> m) {
1336         $abstractvectortype$ oldVal = $abstractvectortype$.fromArray(SPECIES, a, ax);
1337         $abstractvectortype$ newVal = oldVal.blend(this, m);
1338         newVal.intoArray(a, ax);
1339     }
1340 #if[!byteOrShort]
1341     @Override
1342     @ForceInline
1343     public void intoArray($type$[] a, int ix, int[] b, int iy) {
1344 #if[longOrDouble64]
1345         this.intoArray(a, ix + b[iy]);
1346 #else[longOrDouble64]
1347         Objects.requireNonNull(a);
1348         Objects.requireNonNull(b);
1349 
1350         // Index vector: vix[0:n] = i -> ix + indexMap[iy + i]
1351         IntVector vix = IntVector.fromArray(INDEX_SPEC, b, iy).add(ix);
1352 
1353         vix = VectorIntrinsics.checkIndex(vix, a.length);
1354 
1355         VectorIntrinsics.storeWithMap($vectortype$.class, $type$.class, LENGTH, $vectorindextype$,
1356                                a, Unsafe.ARRAY_$TYPE$_BASE_OFFSET, vix,
1357                                this,
1358                                a, ix, b, iy,
1359                                (arr, idx, v, indexMap, idy) -> v.forEach((i, e) -> arr[idx+indexMap[idy+i]] = e));
1360 #end[longOrDouble64]
1361     }
1362 
1363      @Override
1364      @ForceInline
1365      public final void intoArray($type$[] a, int ax, Mask<$Boxtype$> m, int[] b, int iy) {
1366          // @@@ This can result in out of bounds errors for unset mask lanes
1367          $abstractvectortype$ oldVal = $abstractvectortype$.fromArray(SPECIES, a, ax, b, iy);
1368          $abstractvectortype$ newVal = oldVal.blend(this, m);
1369          newVal.intoArray(a, ax, b, iy);
1370      }
1371 #end[!byteOrShort]
1372 
1373     @Override
1374     @ForceInline
1375     public void intoByteArray(byte[] a, int ix) {
1376         Objects.requireNonNull(a);
1377         ix = VectorIntrinsics.checkIndex(ix, a.length, bitSize() / Byte.SIZE);
1378         VectorIntrinsics.store($vectortype$.class, $type$.class, LENGTH,
1379                                a, ((long) ix) + Unsafe.ARRAY_BYTE_BASE_OFFSET,
1380                                this,
1381                                a, ix,
1382                                (c, idx, v) -> {
1383                                    ByteBuffer bbc = ByteBuffer.wrap(c, idx, c.length - idx).order(ByteOrder.nativeOrder());
1384                                    $Type$Buffer tb = bbc{#if[byte]?;:.as$Type$Buffer();}
1385                                    v.forEach((i, e) -> tb.put(e));
1386                                });
1387     }
1388 
1389     @Override
1390     @ForceInline
1391     public final void intoByteArray(byte[] a, int ix, Mask<$Boxtype$> m) {
1392         $vectortype$ oldVal = ($vectortype$) $abstractvectortype$.fromByteArray(SPECIES, a, ix);
1393         $vectortype$ newVal = oldVal.blend(this, m);
1394         newVal.intoByteArray(a, ix);
1395     }
1396 
1397     @Override
1398     @ForceInline
1399     public void intoByteBuffer(ByteBuffer bb, int ix) {
1400         if (bb.order() != ByteOrder.nativeOrder()) {
1401             throw new IllegalArgumentException();
1402         }
1403         if (bb.isReadOnly()) {
1404             throw new ReadOnlyBufferException();
1405         }
1406         ix = VectorIntrinsics.checkIndex(ix, bb.limit(), bitSize() / Byte.SIZE);
1407         VectorIntrinsics.store($vectortype$.class, $type$.class, LENGTH,
1408                                U.getReference(bb, BYTE_BUFFER_HB), ix + U.getLong(bb, BUFFER_ADDRESS),
1409                                this,
1410                                bb, ix,
1411                                (c, idx, v) -> {
1412                                    ByteBuffer bbc = c.duplicate().position(idx).order(ByteOrder.nativeOrder());
1413                                    $Type$Buffer tb = bbc{#if[byte]?;:.as$Type$Buffer();}
1414                                    v.forEach((i, e) -> tb.put(e));
1415                                });
1416     }
1417 
1418     @Override
1419     @ForceInline
1420     public void intoByteBuffer(ByteBuffer bb, int ix, Mask<$Boxtype$> m) {
1421         $vectortype$ oldVal = ($vectortype$) $abstractvectortype$.fromByteBuffer(SPECIES, bb, ix);
1422         $vectortype$ newVal = oldVal.blend(this, m);
1423         newVal.intoByteBuffer(bb, ix);
1424     }
1425 
1426     //
1427 
1428     @Override
1429     public String toString() {
1430         return Arrays.toString(getElements());
1431     }
1432 
1433     @Override
1434     public boolean equals(Object o) {
1435         if (this == o) return true;
1436         if (o == null || this.getClass() != o.getClass()) return false;
1437 
1438         $vectortype$ that = ($vectortype$) o;
1439         return this.equal(that).allTrue();
1440     }
1441 
1442     @Override
1443     public int hashCode() {
1444         return Arrays.hashCode(vec);
1445     }
1446 
1447     // Binary test
1448 
1449     @Override
1450     $masktype$ bTest(Vector<$Boxtype$> o, FBinTest f) {
1451         $type$[] vec1 = getElements();
1452         $type$[] vec2 = (($vectortype$)o).getElements();
1453         boolean[] bits = new boolean[length()];
1454         for (int i = 0; i < length(); i++){
1455             bits[i] = f.apply(i, vec1[i], vec2[i]);
1456         }
1457         return new $masktype$(bits);
1458     }
1459 
1460     // Comparisons
1461 
1462     @Override
1463     @ForceInline
1464     public $masktype$ equal(Vector<$Boxtype$> o) {
1465         Objects.requireNonNull(o);
1466         $vectortype$ v = ($vectortype$)o;
1467 
1468         return VectorIntrinsics.compare(
1469             BT_eq, $vectortype$.class, $masktype$.class, $type$.class, LENGTH,
1470             this, v,
1471             (v1, v2) -> v1.bTest(v2, (i, a, b) -> a == b));
1472     }
1473 
1474     @Override
1475     @ForceInline
1476     public $masktype$ notEqual(Vector<$Boxtype$> o) {
1477         Objects.requireNonNull(o);
1478         $vectortype$ v = ($vectortype$)o;
1479 
1480         return VectorIntrinsics.compare(
1481             BT_ne, $vectortype$.class, $masktype$.class, $type$.class, LENGTH,
1482             this, v,
1483             (v1, v2) -> v1.bTest(v2, (i, a, b) -> a != b));
1484     }
1485 
1486     @Override
1487     @ForceInline
1488     public $masktype$ lessThan(Vector<$Boxtype$> o) {
1489         Objects.requireNonNull(o);
1490         $vectortype$ v = ($vectortype$)o;
1491 
1492         return VectorIntrinsics.compare(
1493             BT_lt, $vectortype$.class, $masktype$.class, $type$.class, LENGTH,
1494             this, v,
1495             (v1, v2) -> v1.bTest(v2, (i, a, b) -> a < b));
1496     }
1497 
1498     @Override
1499     @ForceInline
1500     public $masktype$ lessThanEq(Vector<$Boxtype$> o) {
1501         Objects.requireNonNull(o);
1502         $vectortype$ v = ($vectortype$)o;
1503 
1504         return VectorIntrinsics.compare(
1505             BT_le, $vectortype$.class, $masktype$.class, $type$.class, LENGTH,
1506             this, v,
1507             (v1, v2) -> v1.bTest(v2, (i, a, b) -> a <= b));
1508     }
1509 
1510     @Override
1511     @ForceInline
1512     public $masktype$ greaterThan(Vector<$Boxtype$> o) {
1513         Objects.requireNonNull(o);
1514         $vectortype$ v = ($vectortype$)o;
1515 
1516         return ($masktype$) VectorIntrinsics.compare(
1517             BT_gt, $vectortype$.class, $masktype$.class, $type$.class, LENGTH,
1518             this, v,
1519             (v1, v2) -> v1.bTest(v2, (i, a, b) -> a > b));
1520     }
1521 
1522     @Override
1523     @ForceInline
1524     public $masktype$ greaterThanEq(Vector<$Boxtype$> o) {
1525         Objects.requireNonNull(o);
1526         $vectortype$ v = ($vectortype$)o;
1527 
1528         return VectorIntrinsics.compare(
1529             BT_ge, $vectortype$.class, $masktype$.class, $type$.class, LENGTH,
1530             this, v,
1531             (v1, v2) -> v1.bTest(v2, (i, a, b) -> a >= b));
1532     }
1533 
1534     // Foreach
1535 
1536     @Override
1537     void forEach(FUnCon f) {
1538         $type$[] vec = getElements();
1539         for (int i = 0; i < length(); i++) {
1540             f.apply(i, vec[i]);
1541         }
1542     }
1543 
1544     @Override
1545     void forEach(Mask<$Boxtype$> o, FUnCon f) {
1546         boolean[] mbits = (($masktype$)o).getBits();
1547         forEach((i, a) -> {
1548             if (mbits[i]) { f.apply(i, a); }
1549         });
1550     }
1551 
1552 #if[FP]
1553     $bitsvectortype$ toBits() {
1554         $type$[] vec = getElements();
1555         $bitstype$[] res = new $bitstype$[this.species().length()];
1556         for(int i = 0; i < this.species().length(); i++){
1557             res[i] = $Type$.$type$To$Bitstype$Bits(vec[i]);
1558         }
1559         return new $bitsvectortype$(res);
1560     }
1561 #end[FP]
1562 
1563 #if[intOrLong]
1564     $fpvectortype$ toFP() {
1565         $type$[] vec = getElements();
1566         $fptype$[] res = new $fptype$[this.species().length()];
1567         for(int i = 0; i < this.species().length(); i++){
1568             res[i] = $Boxfptype$.$bitstype$BitsTo$Fptype$(vec[i]);
1569         }
1570         return new $fpvectortype$(res);
1571     }
1572 #end[intOrLong]
1573 
1574     @Override
1575     public $vectortype$ rotateEL(int j) {
1576         $type$[] vec = getElements();
1577         $type$[] res = new $type$[length()];
1578         for (int i = 0; i < length(); i++){
1579             res[(j + i) % length()] = vec[i];
1580         }
1581         return new $vectortype$(res);
1582     }
1583 
1584     @Override
1585     public $vectortype$ rotateER(int j) {
1586         $type$[] vec = getElements();
1587         $type$[] res = new $type$[length()];
1588         for (int i = 0; i < length(); i++){
1589             int z = i - j;
1590             if(j < 0) {
1591                 res[length() + z] = vec[i];
1592             } else {
1593                 res[z] = vec[i];
1594             }
1595         }
1596         return new $vectortype$(res);
1597     }
1598 
1599     @Override
1600     public $vectortype$ shiftEL(int j) {
1601         $type$[] vec = getElements();
1602         $type$[] res = new $type$[length()];
1603         for (int i = 0; i < length() - j; i++) {
1604             res[i] = vec[i + j];
1605         }
1606         return new $vectortype$(res);
1607     }
1608 
1609     @Override
1610     public $vectortype$ shiftER(int j) {
1611         $type$[] vec = getElements();
1612         $type$[] res = new $type$[length()];
1613         for (int i = 0; i < length() - j; i++){
1614             res[i + j] = vec[i];
1615         }
1616         return new $vectortype$(res);
1617     }
1618 
1619     @Override
1620     @ForceInline
1621     public $vectortype$ rearrange(Vector<$Boxtype$> v,
1622                                   Shuffle<$Boxtype$> s, Mask<$Boxtype$> m) {
1623         return this.rearrange(s).blend(v.rearrange(s), m);
1624     }
1625 
1626     @Override
1627     @ForceInline
1628     public $vectortype$ rearrange(Shuffle<$Boxtype$> o1) {
1629         Objects.requireNonNull(o1);
1630         $shuffletype$ s =  ($shuffletype$)o1;
1631 
1632         return VectorIntrinsics.rearrangeOp(
1633             $vectortype$.class, $shuffletype$.class, $type$.class, LENGTH,
1634             this, s,
1635             (v1, s_) -> v1.uOp((i, a) -> {
1636                 int ei = s_.getElement(i);
1637                 return v1.get(ei);
1638             }));
1639     }
1640 
1641     @Override
1642     @ForceInline
1643     public $vectortype$ blend(Vector<$Boxtype$> o1, Mask<$Boxtype$> o2) {
1644         Objects.requireNonNull(o1);
1645         Objects.requireNonNull(o2);
1646         $vectortype$ v = ($vectortype$)o1;
1647         $masktype$   m = ($masktype$)o2;
1648 
1649         return VectorIntrinsics.blend(
1650             $vectortype$.class, $masktype$.class, $type$.class, LENGTH,
1651             this, v, m,
1652             (v1, v2, m_) -> v1.bOp(v2, (i, a, b) -> m_.getElement(i) ? b : a));
1653     }
1654 
1655     // Accessors
1656 
1657 #if[FP]
1658     @Override
1659     public $type$ get(int i) {
1660         if (i < 0 || i >= LENGTH) {
1661             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + LENGTH);
1662         }
1663         $bitstype$ bits = ($bitstype$) VectorIntrinsics.extract(
1664                                 $vectortype$.class, $type$.class, LENGTH,
1665                                 this, i,
1666                                 (vec, ix) -> {
1667                                     $type$[] vecarr = vec.getElements();
1668                                     return (long)$Type$.$type$To$Bitstype$Bits(vecarr[ix]);
1669                                 });
1670         return $Type$.$bitstype$BitsTo$Fptype$(bits);
1671     }
1672 
1673     @Override
1674     public $vectortype$ with(int i, $type$ e) {
1675         if (i < 0 || i >= LENGTH) {
1676             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + LENGTH);
1677         }
1678         return VectorIntrinsics.insert(
1679                                 $vectortype$.class, $type$.class, LENGTH,
1680                                 this, i, (long)$Type$.$type$To$Bitstype$Bits(e),
1681                                 (v, ix, bits) -> {
1682                                     $type$[] res = v.getElements().clone();
1683                                     res[ix] = $Type$.$bitstype$BitsTo$Type$(($bitstype$)bits);
1684                                     return new $vectortype$(res);
1685                                 });
1686     }
1687 #else[FP]
1688     @Override
1689     public $type$ get(int i) {
1690         if (i < 0 || i >= LENGTH) {
1691             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + LENGTH);
1692         }
1693         return ($type$) VectorIntrinsics.extract(
1694                                 $vectortype$.class, $type$.class, LENGTH,
1695                                 this, i,
1696                                 (vec, ix) -> {
1697                                     $type$[] vecarr = vec.getElements();
1698                                     return (long)vecarr[ix];
1699                                 });
1700     }
1701 
1702     @Override
1703     public $vectortype$ with(int i, $type$ e) {
1704         if (i < 0 || i >= LENGTH) {
1705             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + LENGTH);
1706         }
1707         return VectorIntrinsics.insert(
1708                                 $vectortype$.class, $type$.class, LENGTH,
1709                                 this, i, (long)e,
1710                                 (v, ix, bits) -> {
1711                                     $type$[] res = v.getElements().clone();
1712                                     res[ix] = ($type$)bits;
1713                                     return new $vectortype$(res);
1714                                 });
1715     }
1716 #end[FP]
1717 
1718     // Mask
1719 
1720     static final class $masktype$ extends AbstractMask<$Boxtype$> {
1721         static final $masktype$ TRUE_MASK = new $masktype$(true);
1722         static final $masktype$ FALSE_MASK = new $masktype$(false);
1723 
1724         private final boolean[] bits; // Don't access directly, use getBits() instead.
1725 
1726         public $masktype$(boolean[] bits) {
1727             this(bits, 0);
1728         }
1729 
1730         public $masktype$(boolean[] bits, int offset) {
1731             boolean[] a = new boolean[species().length()];
1732             for (int i = 0; i < a.length; i++) {
1733                 a[i] = bits[offset + i];
1734             }
1735             this.bits = a;
1736         }
1737 
1738         public $masktype$(boolean val) {
1739             boolean[] bits = new boolean[species().length()];
1740             Arrays.fill(bits, val);
1741             this.bits = bits;
1742         }
1743 
1744         boolean[] getBits() {
1745             return VectorIntrinsics.maybeRebox(this).bits;
1746         }
1747 
1748         @Override
1749         $masktype$ uOp(MUnOp f) {
1750             boolean[] res = new boolean[species().length()];
1751             boolean[] bits = getBits();
1752             for (int i = 0; i < species().length(); i++) {
1753                 res[i] = f.apply(i, bits[i]);
1754             }
1755             return new $masktype$(res);
1756         }
1757 
1758         @Override
1759         $masktype$ bOp(Mask<$Boxtype$> o, MBinOp f) {
1760             boolean[] res = new boolean[species().length()];
1761             boolean[] bits = getBits();
1762             boolean[] mbits = (($masktype$)o).getBits();
1763             for (int i = 0; i < species().length(); i++) {
1764                 res[i] = f.apply(i, bits[i], mbits[i]);
1765             }
1766             return new $masktype$(res);
1767         }
1768 
1769         @Override
1770         public $Type$$bits$Species species() {
1771             return SPECIES;
1772         }
1773 
1774         @Override
1775         public $vectortype$ toVector() {
1776             $type$[] res = new $type$[species().length()];
1777             boolean[] bits = getBits();
1778             for (int i = 0; i < species().length(); i++) {
1779                 // -1 will result in the most significant bit being set in
1780                 // addition to some or all other bits
1781                 res[i] = ($type$) (bits[i] ? -1 : 0);
1782             }
1783             return new $vectortype$(res);
1784         }
1785 
1786         // Unary operations
1787 
1788         @Override
1789         @ForceInline
1790         public $masktype$ not() {
1791             return ($masktype$) VectorIntrinsics.unaryOp(
1792                                              VECTOR_OP_NOT, $masktype$.class, $bitstype$.class, LENGTH,
1793                                              this,
1794                                              (m1) -> m1.uOp((i, a) -> !a));
1795         }
1796 
1797         // Binary operations
1798 
1799         @Override
1800         @ForceInline
1801         public $masktype$ and(Mask<$Boxtype$> o) {
1802             Objects.requireNonNull(o);
1803             $masktype$ m = ($masktype$)o;
1804             return VectorIntrinsics.binaryOp(VECTOR_OP_AND, $masktype$.class, $bitstype$.class, LENGTH,
1805                                              this, m,
1806                                              (m1, m2) -> m1.bOp(m2, (i, a, b) -> a & b));
1807         }
1808 
1809         @Override
1810         @ForceInline
1811         public $masktype$ or(Mask<$Boxtype$> o) {
1812             Objects.requireNonNull(o);
1813             $masktype$ m = ($masktype$)o;
1814             return VectorIntrinsics.binaryOp(VECTOR_OP_OR, $masktype$.class, $bitstype$.class, LENGTH,
1815                                              this, m,
1816                                              (m1, m2) -> m1.bOp(m2, (i, a, b) -> a | b));
1817         }
1818 
1819         // Reductions
1820 
1821         @Override
1822         @ForceInline
1823         public boolean anyTrue() {
1824             return VectorIntrinsics.test(BT_ne, $masktype$.class, $bitstype$.class, LENGTH,
1825                                          this, this,
1826                                          (m, __) -> anyTrueHelper((($masktype$)m).getBits()));
1827         }
1828 
1829         @Override
1830         @ForceInline
1831         public boolean allTrue() {
1832             return VectorIntrinsics.test(BT_overflow, $masktype$.class, $bitstype$.class, LENGTH,
1833                                          this, $abstractvectortype$.maskAllTrue(species()),
1834                                          (m, __) -> allTrueHelper((($masktype$)m).getBits()));
1835         }
1836     }
1837 
1838     // Shuffle
1839 
1840     static final class $shuffletype$ extends AbstractShuffle<$Boxtype$> {
1841         $shuffletype$(byte[] reorder) {
1842             super(reorder);
1843         }
1844 
1845         public $shuffletype$(int[] reorder) {
1846             super(reorder);
1847         }
1848 
1849         public $shuffletype$(int[] reorder, int i) {
1850             super(reorder, i);
1851         }
1852 
1853         public $shuffletype$(IntUnaryOperator f) {
1854             super(f);
1855         }
1856 
1857         @Override
1858         public $Type$$bits$Species species() {
1859             return SPECIES;
1860         }
1861 
1862         @Override
1863         public $abstractvectortype$ toVector() {
1864             $type$[] va = new $type$[SPECIES.length()];
1865             for (int i = 0; i < va.length; i++) {
1866               va[i] = ($type$) getElement(i);
1867             }
1868             return $abstractvectortype$.fromArray(SPECIES, va, 0);
1869         }
1870 
1871         @Override
1872         public $shuffletype$ rearrange(Vector.Shuffle<$Boxtype$> o) {
1873             $shuffletype$ s = ($shuffletype$) o;
1874             byte[] r = new byte[reorder.length];
1875             for (int i = 0; i < reorder.length; i++) {
1876                 r[i] = reorder[s.reorder[i]];
1877             }
1878             return new $shuffletype$(r);
1879         }
1880     }
1881 
1882     // Species
1883 
1884     @Override
1885     public $Type$$bits$Species species() {
1886         return SPECIES;
1887     }
1888 
1889     static final class $Type$$bits$Species extends $Type$Species {
1890         static final int BIT_SIZE = Shape.$Shape$.bitSize();
1891 
1892         static final int LENGTH = BIT_SIZE / $Boxtype$.SIZE;
1893 
1894         @Override
1895         public String toString() {
1896            StringBuilder sb = new StringBuilder("Shape[");
1897            sb.append(bitSize()).append(" bits, ");
1898            sb.append(length()).append(" ").append($type$.class.getSimpleName()).append("s x ");
1899            sb.append(elementSize()).append(" bits");
1900            sb.append("]");
1901            return sb.toString();
1902         }
1903 
1904         @Override
1905         @ForceInline
1906         public int bitSize() {
1907             return BIT_SIZE;
1908         }
1909 
1910         @Override
1911         @ForceInline
1912         public int length() {
1913             return LENGTH;
1914         }
1915 
1916         @Override
1917         @ForceInline
1918         public Class<$Boxtype$> elementType() {
1919             return $type$.class;
1920         }
1921 
1922         @Override
1923         @ForceInline
1924         public Class<?> boxType() {
1925             return $vectortype$.class;
1926         }
1927 
1928         @Override
1929         @ForceInline
1930         public Class<?> maskType() {
1931             return $masktype$.class;
1932         }
1933 
1934         @Override
1935         @ForceInline
1936         public int elementSize() {
1937             return $Boxtype$.SIZE;
1938         }
1939 
1940         @Override
1941         @ForceInline
1942         @SuppressWarnings("unchecked")
1943         Class<?> vectorType() {
1944             return $vectortype$.class;
1945         }
1946 
1947         @Override
1948         @ForceInline
1949         public Shape shape() {
1950             return Shape.$Shape$;
1951         }
1952 
1953 #if[!byteOrShort]
1954        @Override
1955        IntVector.IntSpecies indexSpecies() {
1956           return INDEX_SPEC;
1957        }
1958 
1959 #end[!byteOrShort]
1960         @Override
1961         $vectortype$ op(FOp f) {
1962             $type$[] res = new $type$[length()];
1963             for (int i = 0; i < length(); i++) {
1964                 res[i] = f.apply(i);
1965             }
1966             return new $vectortype$(res);
1967         }
1968 
1969         @Override
1970         $vectortype$ op(Mask<$Boxtype$> o, FOp f) {
1971             $type$[] res = new $type$[length()];
1972             boolean[] mbits = (($masktype$)o).getBits();
1973             for (int i = 0; i < length(); i++) {
1974                 if (mbits[i]) {
1975                     res[i] = f.apply(i);
1976                 }
1977             }
1978             return new $vectortype$(res);
1979         }
1980 
1981         @Override
1982         $masktype$ opm(FOpm f) {
1983             boolean[] res = new boolean[length()];
1984             for (int i = 0; i < length(); i++) {
1985                 res[i] = (boolean)f.apply(i);
1986             }
1987             return new $masktype$(res);
1988         }
1989 
1990         // Factories
1991 
1992 #if[FP]
1993         @Override
1994         @ForceInline
1995         public $vectortype$ zero() {
1996             return VectorIntrinsics.broadcastCoerced($vectortype$.class, $type$.class, LENGTH,
1997                                                      $Type$.$type$To$Bitstype$Bits(0.0f), SPECIES, 
1998                                                      ((bits, s) -> (($Type$$bits$Species)s).op(i -> $Type$.$bitstype$BitsTo$Type$(($bitstype$)bits))));
1999         }
2000 
2001         @Override
2002         @ForceInline
2003         public $vectortype$ broadcast($type$ e) {
2004             return VectorIntrinsics.broadcastCoerced(
2005                 $vectortype$.class, $type$.class, LENGTH,
2006                 $Type$.$type$To$Bitstype$Bits(e), SPECIES,
2007                 ((bits, s) -> (($Type$$bits$Species)s).op(i -> $Type$.$bitstype$BitsTo$Type$(($bitstype$)bits))));
2008         }
2009 #end[FP]
2010 #if[BITWISE]
2011         @Override
2012         @ForceInline
2013         public $vectortype$ zero() {
2014             return VectorIntrinsics.broadcastCoerced($vectortype$.class, $type$.class, LENGTH,
2015                                                      0, SPECIES,
2016                                                      ((bits, s) -> (($Type$$bits$Species)s).op(i -> ($type$)bits)));
2017         }
2018 
2019         @Override
2020         @ForceInline
2021         public $vectortype$ broadcast($type$ e) {
2022             return VectorIntrinsics.broadcastCoerced(
2023                 $vectortype$.class, $type$.class, LENGTH,
2024                 e, SPECIES,
2025                 ((bits, s) -> (($Type$$bits$Species)s).op(i -> ($type$)bits)));
2026         }
2027 #end[BITWISE]
2028 
2029         @Override
2030         @ForceInline
2031         public $vectortype$ scalars($type$... es) {
2032             Objects.requireNonNull(es);
2033             int ix = VectorIntrinsics.checkIndex(0, es.length, LENGTH);
2034             return VectorIntrinsics.load($vectortype$.class, $type$.class, LENGTH,
2035                                          es, Unsafe.ARRAY_$TYPE$_BASE_OFFSET,
2036                                          es, ix, SPECIES,
2037                                          (c, idx, s) -> (($Type$$bits$Species)s).op(n -> c[idx + n]));
2038         }
2039 
2040         @Override
2041         @ForceInline
2042         public <E> $masktype$ cast(Mask<E> m) {
2043             if (m.length() != LENGTH)
2044                 throw new IllegalArgumentException("Mask length this species length differ");
2045             return new $masktype$(m.toArray());
2046         }
2047 
2048         @Override
2049         @ForceInline
2050         public <E> $shuffletype$ cast(Shuffle<E> s) {
2051             if (s.length() != LENGTH)
2052                 throw new IllegalArgumentException("Shuffle length this species length differ");
2053             return new $shuffletype$(s.toArray());
2054         }
2055     }
2056 }