// Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License version 2 only, as
// published by the Free Software Foundation.
//
// This code is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
// version 2 for more details (a copy is included in the LICENSE file that
// accompanied this code).
//
// You should have received a copy of the GNU General Public License version
// 2 along with this work; if not, write to the Free Software Foundation,
// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
//
// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
// or visit www.oracle.com if you need additional information or have any
// questions.
//
//

// This file is automatically generated by running "m4 aarch64_neon_ad.m4". Do not edit ----

// AArch64 NEON Architecture Description File

// ====================VECTOR INSTRUCTIONS==================================

// ------------------------------ Load/store/reinterpret -----------------------

// Load vector (16 bits)
instruct loadV2(vecD dst, memory mem)
%{
  predicate(n->as_LoadVector()->memory_size() == 2);
  match(Set dst (LoadVector mem));
  ins_cost(4 * INSN_COST);
  format %{ "ldrh   $dst,$mem\t# vector (16 bits)" %}
  ins_encode( aarch64_enc_ldrvH(dst, mem) );
  ins_pipe(vload_reg_mem64);
%}

// Store Vector (16 bits)
instruct storeV2(vecD src, memory mem)
%{
  predicate(n->as_StoreVector()->memory_size() == 2);
  match(Set mem (StoreVector mem src));
  ins_cost(4 * INSN_COST);
  format %{ "strh   $mem,$src\t# vector (16 bits)" %}
  ins_encode( aarch64_enc_strvH(src, mem) );
  ins_pipe(vstore_reg_mem64);
%}

instruct reinterpretD(vecD dst)
%{
  predicate(n->bottom_type()->is_vect()->length_in_bytes() == 8 &&
            n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 8);
  match(Set dst (VectorReinterpret dst));
  ins_cost(0);
  format %{ " # reinterpret $dst" %}
  ins_encode %{
    // empty
  %}
  ins_pipe(pipe_class_empty);
%}

instruct reinterpretX(vecX dst)
%{
  predicate(n->bottom_type()->is_vect()->length_in_bytes() == 16 &&
            n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 16);
  match(Set dst (VectorReinterpret dst));
  ins_cost(0);
  format %{ " # reinterpret $dst" %}
  ins_encode %{
    // empty
  %}
  ins_pipe(pipe_class_empty);
%}

instruct reinterpretD2X(vecX dst, vecD src)
%{
  predicate(n->bottom_type()->is_vect()->length_in_bytes() == 16 &&
            n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 8);
  match(Set dst (VectorReinterpret src));
  ins_cost(INSN_COST);
  format %{ " # reinterpret $dst,$src" %}
  ins_encode %{
    // If register is the same, then move is not needed.
    if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
      __ orr(as_FloatRegister($dst$$reg), __ T8B,
             as_FloatRegister($src$$reg),
             as_FloatRegister($src$$reg));
    }
  %}
  ins_pipe(vlogical64);
%}

instruct reinterpretX2D(vecD dst, vecX src)
%{
  predicate(n->bottom_type()->is_vect()->length_in_bytes() == 8 &&
            n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 16);
  match(Set dst (VectorReinterpret src));
  ins_cost(INSN_COST);
  format %{ " # reinterpret $dst,$src" %}
  ins_encode %{
    // If register is the same, then move is not needed.
    if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
      __ orr(as_FloatRegister($dst$$reg), __ T8B,
             as_FloatRegister($src$$reg),
             as_FloatRegister($src$$reg));
    }
  %}
  ins_pipe(vlogical64);
%}

// ------------------------------ Vector cast -------------------------------

instruct vcvt4Bto4S(vecD dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorCastB2X src));
  format %{ "sxtl  $dst, T8H, $src, T8B\t# convert 4B to 4S vector" %}
  ins_encode %{
    __ sxtl(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg), __ T8B);
  %}
  ins_pipe(pipe_class_default);
%}

instruct vcvt8Bto8S(vecX dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorCastB2X src));
  format %{ "sxtl  $dst, T8H, $src, T8B\t# convert 8B to 8S vector" %}
  ins_encode %{
    __ sxtl(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg), __ T8B);
  %}
  ins_pipe(pipe_class_default);
%}

instruct vcvt4Sto4B(vecD dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorCastS2X src));
  format %{ "xtn  $dst, T8B, $src, T8H\t# convert 4S to 4B vector" %}
  ins_encode %{
    __ xtn(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($src$$reg), __ T8H);
  %}
  ins_pipe(pipe_class_default);
%}

instruct vcvt8Sto8B(vecD dst, vecX src)
%{
  predicate(n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorCastS2X src));
  format %{ "xtn  $dst, T8B, $src, T8H\t# convert 8S to 8B vector" %}
  ins_encode %{
    __ xtn(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($src$$reg), __ T8H);
  %}
  ins_pipe(pipe_class_default);
%}

instruct vcvt4Sto4I(vecX dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorCastS2X src));
  format %{ "sxtl  $dst, T4S, $src, T4H\t# convert 4S to 4I vector" %}
  ins_encode %{
    __ sxtl(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src$$reg), __ T4H);
  %}
  ins_pipe(pipe_class_default);
%}

instruct vcvt4Ito4S(vecD dst, vecX src)
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorCastI2X src));
  format %{ "xtn  $dst, T4H, $src, T4S\t# convert 4I to 4S vector" %}
  ins_encode %{
    __ xtn(as_FloatRegister($dst$$reg), __ T4H, as_FloatRegister($src$$reg), __ T4S);
  %}
  ins_pipe(pipe_class_default);
%}

instruct vcvt2Ito2L(vecX dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (VectorCastI2X src));
  format %{ "sxtl  $dst, T2D, $src, T2S\t# convert 2I to 2L vector" %}
  ins_encode %{
    __ sxtl(as_FloatRegister($dst$$reg), __ T2D, as_FloatRegister($src$$reg), __ T2S);
  %}
  ins_pipe(pipe_class_default);
%}

instruct vcvt2Lto2I(vecD dst, vecX src)
%{
  predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorCastL2X src));
  format %{ "xtn  $dst, T2S, $src, T2D\t# convert 2L to 2I vector" %}
  ins_encode %{
    __ xtn(as_FloatRegister($dst$$reg), __ T2S, as_FloatRegister($src$$reg), __ T2D);
  %}
  ins_pipe(pipe_class_default);
%}

instruct vcvt4Bto4I(vecX dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorCastB2X src));
  format %{ "sxtl  $dst, T8H, $src, T8B\n\t"
            "sxtl  $dst, T4S, $dst, T4H\t# convert 4B to 4I vector"
  %}
  ins_encode %{
    __ sxtl(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg), __ T8B);
    __ sxtl(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($dst$$reg), __ T4H);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvt4Ito4B(vecD dst, vecX src)
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorCastI2X src));
  format %{ "xtn  $dst, T4H, $src, T4S\n\t"
            "xtn  $dst, T8B, $dst, T8H\t# convert 4I to 4B vector"
  %}
  ins_encode %{
    __ xtn(as_FloatRegister($dst$$reg), __ T4H, as_FloatRegister($src$$reg), __ T4S);
    __ xtn(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($dst$$reg), __ T8H);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvt4Bto4F(vecX dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorCastB2X src));
  format %{ "sxtl  $dst, T8H, $src, T8B\n\t"
            "sxtl  $dst, T4S, $dst, T4H\n\t"
            "scvtfv  T4S, $dst, $dst\t# convert 4B to 4F vector"
  %}
  ins_encode %{
    __ sxtl(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg), __ T8B);
    __ sxtl(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($dst$$reg), __ T4H);
    __ scvtfv(__ T4S, as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvt4Sto4F(vecX dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorCastS2X src));
  format %{ "sxtl    $dst, T4S, $src, T4H\n\t"
            "scvtfv  T4S, $dst, $dst\t# convert 4S to 4F vector"
  %}
  ins_encode %{
    __ sxtl(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src$$reg), __ T4H);
    __ scvtfv(__ T4S, as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvt2Ito2D(vecX dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorCastI2X src));
  format %{ "sxtl    $dst, T2D, $src, T2S\n\t"
            "scvtfv  T2D, $dst, $dst\t# convert 2I to 2D vector"
  %}
  ins_encode %{
    __ sxtl(as_FloatRegister($dst$$reg), __ T2D, as_FloatRegister($src$$reg), __ T2S);
    __ scvtfv(__ T2D, as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvt2Ito2F(vecD dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorCastI2X src));
  format %{ "scvtfv  T2S, $dst, $src\t# convert 2I to 2F vector" %}
  ins_encode %{
    __ scvtfv(__ T2S, as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_class_default);
%}

instruct vcvt4Ito4F(vecX dst, vecX src)
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorCastI2X src));
  format %{ "scvtfv  T4S, $dst, $src\t# convert 4I to 4F vector" %}
  ins_encode %{
    __ scvtfv(__ T4S, as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_class_default);
%}

instruct vcvt2Lto2D(vecX dst, vecX src)
%{
  predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorCastL2X src));
  format %{ "scvtfv  T2D, $dst, $src\t# convert 2L to 2D vector" %}
  ins_encode %{
    __ scvtfv(__ T2D, as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_class_default);
%}

instruct vcvt2Fto2D(vecX dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorCastF2X src));
  format %{ "fcvtl  $dst, T2D, $src, T2S\t# convert 2F to 2D vector" %}
  ins_encode %{
    __ fcvtl(as_FloatRegister($dst$$reg), __ T2D, as_FloatRegister($src$$reg), __ T2S);
  %}
  ins_pipe(pipe_class_default);
%}

instruct vcvt2Dto2F(vecD dst, vecX src)
%{
  predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorCastD2X src));
  format %{ "fcvtn  $dst, T2S, $src, T2D\t# convert 2D to 2F vector" %}
  ins_encode %{
    __ fcvtn(as_FloatRegister($dst$$reg), __ T2S, as_FloatRegister($src$$reg), __ T2D);
  %}
  ins_pipe(pipe_class_default);
%}

instruct vcvt2Lto2F(vecD dst, vecX src)
%{
  predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorCastL2X src));
  format %{ "scvtfv  T2D, $dst, $src\n\t"
            "fcvtn   $dst, T2S, $dst, T2D\t# convert 2L to 2F vector"
  %}
  ins_encode %{
    __ scvtfv(__ T2D, as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
    __ fcvtn(as_FloatRegister($dst$$reg), __ T2S, as_FloatRegister($dst$$reg), __ T2D);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Reduction -------------------------------

instruct reduce_add8B(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, vecD tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (AddReductionVI isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "addv  $tmp, T8B, $vsrc\n\t"
            "smov  $dst, $tmp, B, 0\n\t"
            "addw  $dst, $dst, $isrc\n\t"
            "sxtb  $dst, $dst\t# add reduction8B"
  %}
  ins_encode %{
    __ addv(as_FloatRegister($tmp$$reg), __ T8B, as_FloatRegister($vsrc$$reg));
    __ smov($dst$$Register, as_FloatRegister($tmp$$reg), __ B, 0);
    __ addw($dst$$Register, $dst$$Register, $isrc$$Register);
    __ sxtb($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_add16B(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (AddReductionVI isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "addv  $tmp, T16B, $vsrc\n\t"
            "smov  $dst, $tmp, B, 0\n\t"
            "addw  $dst, $dst, $isrc\n\t"
            "sxtb  $dst, $dst\t# add reduction16B"
  %}
  ins_encode %{
    __ addv(as_FloatRegister($tmp$$reg), __ T16B, as_FloatRegister($vsrc$$reg));
    __ smov($dst$$Register, as_FloatRegister($tmp$$reg), __ B, 0);
    __ addw($dst$$Register, $dst$$Register, $isrc$$Register);
    __ sxtb($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_add4S(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, vecD tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (AddReductionVI isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "addv  $tmp, T4H, $vsrc\n\t"
            "smov  $dst, $tmp, H, 0\n\t"
            "addw  $dst, $dst, $isrc\n\t"
            "sxth  $dst, $dst\t# add reduction4S"
  %}
  ins_encode %{
    __ addv(as_FloatRegister($tmp$$reg), __ T4H, as_FloatRegister($vsrc$$reg));
    __ smov($dst$$Register, as_FloatRegister($tmp$$reg), __ H, 0);
    __ addw($dst$$Register, $dst$$Register, $isrc$$Register);
    __ sxth($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_add8S(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (AddReductionVI isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "addv  $tmp, T8H, $vsrc\n\t"
            "smov  $dst, $tmp, H, 0\n\t"
            "addw  $dst, $dst, $isrc\n\t"
            "sxth  $dst, $dst\t# add reduction8S"
  %}
  ins_encode %{
    __ addv(as_FloatRegister($tmp$$reg), __ T8H, as_FloatRegister($vsrc$$reg));
    __ smov($dst$$Register, as_FloatRegister($tmp$$reg), __ H, 0);
    __ addw($dst$$Register, $dst$$Register, $isrc$$Register);
    __ sxth($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_add2L(iRegLNoSp dst, iRegL isrc, vecX vsrc, vecX tmp)
%{
  match(Set dst (AddReductionVL isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "addpd $tmp, $vsrc\n\t"
            "umov  $dst, $tmp, D, 0\n\t"
            "add   $dst, $isrc, $dst\t# add reduction2L"
  %}
  ins_encode %{
    __ addpd(as_FloatRegister($tmp$$reg), as_FloatRegister($vsrc$$reg));
    __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ D, 0);
    __ add($dst$$Register, $isrc$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_mul8B(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, vecD vtmp1, vecD vtmp2, iRegINoSp itmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (MulReductionVI isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP vtmp1, TEMP vtmp2, TEMP itmp);
  format %{ "ins   $vtmp1, S, $vsrc, 0, 1\n\t"
            "mulv  $vtmp1, T8B, $vtmp1, $vsrc\n\t"
            "ins   $vtmp2, H, $vtmp1, 0, 1\n\t"
            "mulv  $vtmp2, T8B, $vtmp2, $vtmp1\n\t"
            "umov  $itmp, $vtmp2, B, 0\n\t"
            "mulw  $dst, $itmp, $isrc\n\t"
            "sxtb  $dst, $dst\n\t"
            "umov  $itmp, $vtmp2, B, 1\n\t"
            "mulw  $dst, $itmp, $dst\n\t"
            "sxtb  $dst, $dst\t# mul reduction8B"
  %}
  ins_encode %{
    __ ins(as_FloatRegister($vtmp1$$reg), __ S,
           as_FloatRegister($vsrc$$reg), 0, 1);
    __ mulv(as_FloatRegister($vtmp1$$reg), __ T8B,
            as_FloatRegister($vtmp1$$reg), as_FloatRegister($vsrc$$reg));
    __ ins(as_FloatRegister($vtmp2$$reg), __ H,
           as_FloatRegister($vtmp1$$reg), 0, 1);
    __ mulv(as_FloatRegister($vtmp2$$reg), __ T8B,
            as_FloatRegister($vtmp2$$reg), as_FloatRegister($vtmp1$$reg));
    __ umov($itmp$$Register, as_FloatRegister($vtmp2$$reg), __ B, 0);
    __ mulw($dst$$Register, $itmp$$Register, $isrc$$Register);
    __ sxtb($dst$$Register, $dst$$Register);
    __ umov($itmp$$Register, as_FloatRegister($vtmp2$$reg), __ B, 1);
    __ mulw($dst$$Register, $itmp$$Register, $dst$$Register);
    __ sxtb($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_mul16B(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX vtmp1, vecX vtmp2, iRegINoSp itmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (MulReductionVI isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP vtmp1, TEMP vtmp2, TEMP itmp);
  format %{ "ins   $vtmp1, D, $vsrc, 0, 1\n\t"
            "mulv  $vtmp1, T8B, $vtmp1, $vsrc\n\t"
            "ins   $vtmp2, S, $vtmp1, 0, 1\n\t"
            "mulv  $vtmp1, T8B, $vtmp2, $vtmp1\n\t"
            "ins   $vtmp2, H, $vtmp1, 0, 1\n\t"
            "mulv  $vtmp2, T8B, $vtmp2, $vtmp1\n\t"
            "umov  $itmp, $vtmp2, B, 0\n\t"
            "mulw  $dst, $itmp, $isrc\n\t"
            "sxtb  $dst, $dst\n\t"
            "umov  $itmp, $vtmp2, B, 1\n\t"
            "mulw  $dst, $itmp, $dst\n\t"
            "sxtb  $dst, $dst\t# mul reduction16B"
  %}
  ins_encode %{
    __ ins(as_FloatRegister($vtmp1$$reg), __ D,
           as_FloatRegister($vsrc$$reg), 0, 1);
    __ mulv(as_FloatRegister($vtmp1$$reg), __ T8B,
            as_FloatRegister($vtmp1$$reg), as_FloatRegister($vsrc$$reg));
    __ ins(as_FloatRegister($vtmp2$$reg), __ S,
           as_FloatRegister($vtmp1$$reg), 0, 1);
    __ mulv(as_FloatRegister($vtmp1$$reg), __ T8B,
            as_FloatRegister($vtmp2$$reg), as_FloatRegister($vtmp1$$reg));
    __ ins(as_FloatRegister($vtmp2$$reg), __ H,
           as_FloatRegister($vtmp1$$reg), 0, 1);
    __ mulv(as_FloatRegister($vtmp2$$reg), __ T8B,
            as_FloatRegister($vtmp2$$reg), as_FloatRegister($vtmp1$$reg));
    __ umov($itmp$$Register, as_FloatRegister($vtmp2$$reg), __ B, 0);
    __ mulw($dst$$Register, $itmp$$Register, $isrc$$Register);
    __ sxtb($dst$$Register, $dst$$Register);
    __ umov($itmp$$Register, as_FloatRegister($vtmp2$$reg), __ B, 1);
    __ mulw($dst$$Register, $itmp$$Register, $dst$$Register);
    __ sxtb($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_mul4S(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, vecD vtmp, iRegINoSp itmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (MulReductionVI isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP vtmp, TEMP itmp);
  format %{ "ins   $vtmp, S, $vsrc, 0, 1\n\t"
            "mulv  $vtmp, T4H, $vtmp, $vsrc\n\t"
            "umov  $itmp, $vtmp, H, 0\n\t"
            "mulw  $dst, $itmp, $isrc\n\t"
            "sxth  $dst, $dst\n\t"
            "umov  $itmp, $vtmp, H, 1\n\t"
            "mulw  $dst, $itmp, $dst\n\t"
            "sxth  $dst, $dst\t# mul reduction4S"
  %}
  ins_encode %{
    __ ins(as_FloatRegister($vtmp$$reg), __ S,
           as_FloatRegister($vsrc$$reg), 0, 1);
    __ mulv(as_FloatRegister($vtmp$$reg), __ T4H,
            as_FloatRegister($vtmp$$reg), as_FloatRegister($vsrc$$reg));
    __ umov($itmp$$Register, as_FloatRegister($vtmp$$reg), __ H, 0);
    __ mulw($dst$$Register, $itmp$$Register, $isrc$$Register);
    __ sxth($dst$$Register, $dst$$Register);
    __ umov($itmp$$Register, as_FloatRegister($vtmp$$reg), __ H, 1);
    __ mulw($dst$$Register, $itmp$$Register, $dst$$Register);
    __ sxth($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_mul8S(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX vtmp1, vecX vtmp2, iRegINoSp itmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (MulReductionVI isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP vtmp1, TEMP vtmp2, TEMP itmp);
  format %{ "ins   $vtmp1, D, $vsrc, 0, 1\n\t"
            "mulv  $vtmp1, T4H, $vtmp1, $vsrc\n\t"
            "ins   $vtmp2, S, $vtmp1, 0, 1\n\t"
            "mulv  $vtmp2, T4H, $vtmp2, $vtmp1\n\t"
            "umov  $itmp, $vtmp2, H, 0\n\t"
            "mulw  $dst, $itmp, $isrc\n\t"
            "sxth  $dst, $dst\n\t"
            "umov  $itmp, $vtmp2, H, 1\n\t"
            "mulw  $dst, $itmp, $dst\n\t"
            "sxth  $dst, $dst\t# mul reduction8S"
  %}
  ins_encode %{
    __ ins(as_FloatRegister($vtmp1$$reg), __ D,
           as_FloatRegister($vsrc$$reg), 0, 1);
    __ mulv(as_FloatRegister($vtmp1$$reg), __ T4H,
            as_FloatRegister($vtmp1$$reg), as_FloatRegister($vsrc$$reg));
    __ ins(as_FloatRegister($vtmp2$$reg), __ S,
           as_FloatRegister($vtmp1$$reg), 0, 1);
    __ mulv(as_FloatRegister($vtmp2$$reg), __ T4H,
            as_FloatRegister($vtmp2$$reg), as_FloatRegister($vtmp1$$reg));
    __ umov($itmp$$Register, as_FloatRegister($vtmp2$$reg), __ H, 0);
    __ mulw($dst$$Register, $itmp$$Register, $isrc$$Register);
    __ sxth($dst$$Register, $dst$$Register);
    __ umov($itmp$$Register, as_FloatRegister($vtmp2$$reg), __ H, 1);
    __ mulw($dst$$Register, $itmp$$Register, $dst$$Register);
    __ sxth($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_mul2L(iRegLNoSp dst, iRegL isrc, vecX vsrc, iRegLNoSp tmp)
%{
  match(Set dst (MulReductionVL isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov  $tmp, $vsrc, D, 0\n\t"
            "mul   $dst, $isrc, $tmp\n\t"
            "umov  $tmp, $vsrc, D, 1\n\t"
            "mul   $dst, $dst, $tmp\t# mul reduction2L"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 0);
    __ mul($dst$$Register, $isrc$$Register, $tmp$$Register);
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 1);
    __ mul($dst$$Register, $dst$$Register, $tmp$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_max8B(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, vecD tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (MaxReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "smaxv $tmp, T8B, $vsrc\n\t"
            "smov  $dst, $tmp, B, 0\n\t"
            "cmpw  $dst, $isrc\n\t"
            "cselw $dst, $dst, $isrc GT\t# max reduction8B"
  %}
  ins_encode %{
    __ smaxv(as_FloatRegister($tmp$$reg), __ T8B, as_FloatRegister($vsrc$$reg));
    __ smov(as_Register($dst$$reg), as_FloatRegister($tmp$$reg), __ B, 0);
    __ cmpw(as_Register($dst$$reg), as_Register($isrc$$reg));
    __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($isrc$$reg), Assembler::GT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_max16B(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (MaxReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "smaxv $tmp, T16B, $vsrc\n\t"
            "smov  $dst, $tmp, B, 0\n\t"
            "cmpw  $dst, $isrc\n\t"
            "cselw $dst, $dst, $isrc GT\t# max reduction16B"
  %}
  ins_encode %{
    __ smaxv(as_FloatRegister($tmp$$reg), __ T16B, as_FloatRegister($vsrc$$reg));
    __ smov(as_Register($dst$$reg), as_FloatRegister($tmp$$reg), __ B, 0);
    __ cmpw(as_Register($dst$$reg), as_Register($isrc$$reg));
    __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($isrc$$reg), Assembler::GT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_max4S(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, vecD tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (MaxReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "smaxv $tmp, T4H, $vsrc\n\t"
            "smov  $dst, $tmp, H, 0\n\t"
            "cmpw  $dst, $isrc\n\t"
            "cselw $dst, $dst, $isrc GT\t# max reduction4S"
  %}
  ins_encode %{
    __ smaxv(as_FloatRegister($tmp$$reg), __ T4H, as_FloatRegister($vsrc$$reg));
    __ smov(as_Register($dst$$reg), as_FloatRegister($tmp$$reg), __ H, 0);
    __ cmpw(as_Register($dst$$reg), as_Register($isrc$$reg));
    __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($isrc$$reg), Assembler::GT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_max8S(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (MaxReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "smaxv $tmp, T8H, $vsrc\n\t"
            "smov  $dst, $tmp, H, 0\n\t"
            "cmpw  $dst, $isrc\n\t"
            "cselw $dst, $dst, $isrc GT\t# max reduction8S"
  %}
  ins_encode %{
    __ smaxv(as_FloatRegister($tmp$$reg), __ T8H, as_FloatRegister($vsrc$$reg));
    __ smov(as_Register($dst$$reg), as_FloatRegister($tmp$$reg), __ H, 0);
    __ cmpw(as_Register($dst$$reg), as_Register($isrc$$reg));
    __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($isrc$$reg), Assembler::GT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_max4I(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (MaxReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "smaxv $tmp, T4S, $vsrc\n\t"
            "umov  $dst, $tmp, S, 0\n\t"
            "cmpw  $dst, $isrc\n\t"
            "cselw $dst, $dst, $isrc GT\t# max reduction4I"
  %}
  ins_encode %{
    __ smaxv(as_FloatRegister($tmp$$reg), __ T4S, as_FloatRegister($vsrc$$reg));
    __ umov(as_Register($dst$$reg), as_FloatRegister($tmp$$reg), __ S, 0);
    __ cmpw(as_Register($dst$$reg), as_Register($isrc$$reg));
    __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($isrc$$reg), Assembler::GT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_min8B(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, vecD tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (MinReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "sminv $tmp, T8B, $vsrc\n\t"
            "smov  $dst, $tmp, B, 0\n\t"
            "cmpw  $dst, $isrc\n\t"
            "cselw $dst, $dst, $isrc LT\t# min reduction8B"
  %}
  ins_encode %{
    __ sminv(as_FloatRegister($tmp$$reg), __ T8B, as_FloatRegister($vsrc$$reg));
    __ smov(as_Register($dst$$reg), as_FloatRegister($tmp$$reg), __ B, 0);
    __ cmpw(as_Register($dst$$reg), as_Register($isrc$$reg));
    __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($isrc$$reg), Assembler::LT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_min16B(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (MinReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "sminv $tmp, T16B, $vsrc\n\t"
            "smov  $dst, $tmp, B, 0\n\t"
            "cmpw  $dst, $isrc\n\t"
            "cselw $dst, $dst, $isrc LT\t# min reduction16B"
  %}
  ins_encode %{
    __ sminv(as_FloatRegister($tmp$$reg), __ T16B, as_FloatRegister($vsrc$$reg));
    __ smov(as_Register($dst$$reg), as_FloatRegister($tmp$$reg), __ B, 0);
    __ cmpw(as_Register($dst$$reg), as_Register($isrc$$reg));
    __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($isrc$$reg), Assembler::LT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_min4S(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, vecD tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (MinReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "sminv $tmp, T4H, $vsrc\n\t"
            "smov  $dst, $tmp, H, 0\n\t"
            "cmpw  $dst, $isrc\n\t"
            "cselw $dst, $dst, $isrc LT\t# min reduction4S"
  %}
  ins_encode %{
    __ sminv(as_FloatRegister($tmp$$reg), __ T4H, as_FloatRegister($vsrc$$reg));
    __ smov(as_Register($dst$$reg), as_FloatRegister($tmp$$reg), __ H, 0);
    __ cmpw(as_Register($dst$$reg), as_Register($isrc$$reg));
    __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($isrc$$reg), Assembler::LT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_min8S(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (MinReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "sminv $tmp, T8H, $vsrc\n\t"
            "smov  $dst, $tmp, H, 0\n\t"
            "cmpw  $dst, $isrc\n\t"
            "cselw $dst, $dst, $isrc LT\t# min reduction8S"
  %}
  ins_encode %{
    __ sminv(as_FloatRegister($tmp$$reg), __ T8H, as_FloatRegister($vsrc$$reg));
    __ smov(as_Register($dst$$reg), as_FloatRegister($tmp$$reg), __ H, 0);
    __ cmpw(as_Register($dst$$reg), as_Register($isrc$$reg));
    __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($isrc$$reg), Assembler::LT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_min4I(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, vecX tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (MinReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "sminv $tmp, T4S, $vsrc\n\t"
            "umov  $dst, $tmp, S, 0\n\t"
            "cmpw  $dst, $isrc\n\t"
            "cselw $dst, $dst, $isrc LT\t# min reduction4I"
  %}
  ins_encode %{
    __ sminv(as_FloatRegister($tmp$$reg), __ T4S, as_FloatRegister($vsrc$$reg));
    __ umov(as_Register($dst$$reg), as_FloatRegister($tmp$$reg), __ S, 0);
    __ cmpw(as_Register($dst$$reg), as_Register($isrc$$reg));
    __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($isrc$$reg), Assembler::LT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_max2I(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, vecX tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (MaxReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "dup   $tmp, T2D, $vsrc\n\t"
            "smaxv $tmp, T4S, $tmp\n\t"
            "umov  $dst, $tmp, S, 0\n\t"
            "cmpw  $dst, $isrc\n\t"
            "cselw $dst, $dst, $isrc GT\t# max reduction2I"
  %}
  ins_encode %{
    __ dup(as_FloatRegister($tmp$$reg), __ T2D, as_FloatRegister($vsrc$$reg));
    __ smaxv(as_FloatRegister($tmp$$reg), __ T4S, as_FloatRegister($tmp$$reg));
    __ umov(as_Register($dst$$reg), as_FloatRegister($tmp$$reg), __ S, 0);
    __ cmpw(as_Register($dst$$reg), as_Register($isrc$$reg));
    __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($isrc$$reg), Assembler::GT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_min2I(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, vecX tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (MinReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "dup   $tmp, T2D, $vsrc\n\t"
            "sminv $tmp, T4S, $tmp\n\t"
            "umov  $dst, $tmp, S, 0\n\t"
            "cmpw  $dst, $isrc\n\t"
            "cselw $dst, $dst, $isrc LT\t# min reduction2I"
  %}
  ins_encode %{
    __ dup(as_FloatRegister($tmp$$reg), __ T2D, as_FloatRegister($vsrc$$reg));
    __ sminv(as_FloatRegister($tmp$$reg), __ T4S, as_FloatRegister($tmp$$reg));
    __ umov(as_Register($dst$$reg), as_FloatRegister($tmp$$reg), __ S, 0);
    __ cmpw(as_Register($dst$$reg), as_Register($isrc$$reg));
    __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($isrc$$reg), Assembler::LT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_max2L(iRegLNoSp dst, iRegL isrc, vecX vsrc, iRegLNoSp tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (MaxReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "umov  $tmp, $vsrc, D, 0\n\t"
            "cmp   $isrc,$tmp\n\t"
            "csel  $dst, $isrc, $tmp GT\n\t"
            "umov  $tmp, $vsrc, D, 1\n\t"
            "cmp   $dst, $tmp\n\t"
            "csel  $dst, $dst, $tmp GT\t# max reduction2L"
  %}
  ins_encode %{
    __ umov(as_Register($tmp$$reg), as_FloatRegister($vsrc$$reg), __ D, 0);
    __ cmp(as_Register($isrc$$reg), as_Register($tmp$$reg));
    __ csel(as_Register($dst$$reg), as_Register($isrc$$reg), as_Register($tmp$$reg), Assembler::GT);
    __ umov(as_Register($tmp$$reg), as_FloatRegister($vsrc$$reg), __ D, 1);
    __ cmp(as_Register($dst$$reg), as_Register($tmp$$reg));
    __ csel(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($tmp$$reg), Assembler::GT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_min2L(iRegLNoSp dst, iRegL isrc, vecX vsrc, iRegLNoSp tmp, rFlagsReg cr)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (MinReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "umov  $tmp, $vsrc, D, 0\n\t"
            "cmp   $isrc,$tmp\n\t"
            "csel  $dst, $isrc, $tmp LT\n\t"
            "umov  $tmp, $vsrc, D, 1\n\t"
            "cmp   $dst, $tmp\n\t"
            "csel  $dst, $dst, $tmp LT\t# min reduction2L"
  %}
  ins_encode %{
    __ umov(as_Register($tmp$$reg), as_FloatRegister($vsrc$$reg), __ D, 0);
    __ cmp(as_Register($isrc$$reg), as_Register($tmp$$reg));
    __ csel(as_Register($dst$$reg), as_Register($isrc$$reg), as_Register($tmp$$reg), Assembler::LT);
    __ umov(as_Register($tmp$$reg), as_FloatRegister($vsrc$$reg), __ D, 1);
    __ cmp(as_Register($dst$$reg), as_Register($tmp$$reg));
    __ csel(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($tmp$$reg), Assembler::LT);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_and8B(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (AndReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, S, 0\n\t"
            "umov   $dst, $vsrc, S, 1\n\t"
            "andw   $dst, $dst, $tmp\n\t"
            "andw   $dst, $dst, $dst, LSR #16\n\t"
            "andw   $dst, $dst, $dst, LSR #8\n\t"
            "andw   $dst, $isrc, $dst\n\t"
            "sxtb   $dst, $dst\t# and reduction8B"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ S, 1);
    __ andw($dst$$Register, $dst$$Register, $tmp$$Register);
    __ andw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 16);
    __ andw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 8);
    __ andw($dst$$Register, $isrc$$Register, $dst$$Register);
    __ sxtb($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orr8B(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (OrReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, S, 0\n\t"
            "umov   $dst, $vsrc, S, 1\n\t"
            "orrw   $dst, $dst, $tmp\n\t"
            "orrw   $dst, $dst, $dst, LSR #16\n\t"
            "orrw   $dst, $dst, $dst, LSR #8\n\t"
            "orrw   $dst, $isrc, $dst\n\t"
            "sxtb   $dst, $dst\t# orr reduction8B"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ S, 1);
    __ orrw($dst$$Register, $dst$$Register, $tmp$$Register);
    __ orrw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 16);
    __ orrw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 8);
    __ orrw($dst$$Register, $isrc$$Register, $dst$$Register);
    __ sxtb($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_eor8B(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (XorReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, S, 0\n\t"
            "umov   $dst, $vsrc, S, 1\n\t"
            "eorw   $dst, $dst, $tmp\n\t"
            "eorw   $dst, $dst, $dst, LSR #16\n\t"
            "eorw   $dst, $dst, $dst, LSR #8\n\t"
            "eorw   $dst, $isrc, $dst\n\t"
            "sxtb   $dst, $dst\t# eor reduction8B"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ S, 1);
    __ eorw($dst$$Register, $dst$$Register, $tmp$$Register);
    __ eorw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 16);
    __ eorw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 8);
    __ eorw($dst$$Register, $isrc$$Register, $dst$$Register);
    __ sxtb($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_and16B(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (AndReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, D, 0\n\t"
            "umov   $dst, $vsrc, D, 1\n\t"
            "andr   $dst, $dst, $tmp\n\t"
            "andr   $dst, $dst, $dst, LSR #32\n\t"
            "andw   $dst, $dst, $dst, LSR #16\n\t"
            "andw   $dst, $dst, $dst, LSR #8\n\t"
            "andw   $dst, $isrc, $dst\n\t"
            "sxtb   $dst, $dst\t# and reduction16B"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ D, 1);
    __ andr($dst$$Register, $dst$$Register, $tmp$$Register);
    __ andr($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 32);
    __ andw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 16);
    __ andw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 8);
    __ andw($dst$$Register, $isrc$$Register, $dst$$Register);
    __ sxtb($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orr16B(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (OrReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, D, 0\n\t"
            "umov   $dst, $vsrc, D, 1\n\t"
            "orr    $dst, $dst, $tmp\n\t"
            "orr    $dst, $dst, $dst, LSR #32\n\t"
            "orrw   $dst, $dst, $dst, LSR #16\n\t"
            "orrw   $dst, $dst, $dst, LSR #8\n\t"
            "orrw   $dst, $isrc, $dst\n\t"
            "sxtb   $dst, $dst\t# orr reduction16B"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ D, 1);
    __ orr ($dst$$Register, $dst$$Register, $tmp$$Register);
    __ orr ($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 32);
    __ orrw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 16);
    __ orrw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 8);
    __ orrw($dst$$Register, $isrc$$Register, $dst$$Register);
    __ sxtb($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_eor16B(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (XorReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, D, 0\n\t"
            "umov   $dst, $vsrc, D, 1\n\t"
            "eor    $dst, $dst, $tmp\n\t"
            "eor    $dst, $dst, $dst, LSR #32\n\t"
            "eorw   $dst, $dst, $dst, LSR #16\n\t"
            "eorw   $dst, $dst, $dst, LSR #8\n\t"
            "eorw   $dst, $isrc, $dst\n\t"
            "sxtb   $dst, $dst\t# eor reduction16B"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ D, 1);
    __ eor ($dst$$Register, $dst$$Register, $tmp$$Register);
    __ eor ($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 32);
    __ eorw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 16);
    __ eorw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 8);
    __ eorw($dst$$Register, $isrc$$Register, $dst$$Register);
    __ sxtb($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_and4S(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (AndReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, S, 0\n\t"
            "umov   $dst, $vsrc, S, 1\n\t"
            "andw   $dst, $dst, $tmp\n\t"
            "andw   $dst, $dst, $dst, LSR #16\n\t"
            "andw   $dst, $isrc, $dst\n\t"
            "sxth   $dst, $dst\t# and reduction4S"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ S, 1);
    __ andw($dst$$Register, $dst$$Register, $tmp$$Register);
    __ andw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 16);
    __ andw($dst$$Register, $isrc$$Register, $dst$$Register);
    __ sxth($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orr4S(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (OrReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, S, 0\n\t"
            "umov   $dst, $vsrc, S, 1\n\t"
            "orrw   $dst, $dst, $tmp\n\t"
            "orrw   $dst, $dst, $dst, LSR #16\n\t"
            "orrw   $dst, $isrc, $dst\n\t"
            "sxth   $dst, $dst\t# orr reduction4S"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ S, 1);
    __ orrw($dst$$Register, $dst$$Register, $tmp$$Register);
    __ orrw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 16);
    __ orrw($dst$$Register, $isrc$$Register, $dst$$Register);
    __ sxth($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_eor4S(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (XorReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, S, 0\n\t"
            "umov   $dst, $vsrc, S, 1\n\t"
            "eorw   $dst, $dst, $tmp\n\t"
            "eorw   $dst, $dst, $dst, LSR #16\n\t"
            "eorw   $dst, $isrc, $dst\n\t"
            "sxth   $dst, $dst\t# eor reduction4S"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ S, 1);
    __ eorw($dst$$Register, $dst$$Register, $tmp$$Register);
    __ eorw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 16);
    __ eorw($dst$$Register, $isrc$$Register, $dst$$Register);
    __ sxth($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_and8S(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (AndReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, D, 0\n\t"
            "umov   $dst, $vsrc, D, 1\n\t"
            "andr   $dst, $dst, $tmp\n\t"
            "andr   $dst, $dst, $dst, LSR #32\n\t"
            "andw   $dst, $dst, $dst, LSR #16\n\t"
            "andw   $dst, $isrc, $dst\n\t"
            "sxth   $dst, $dst\t# and reduction8S"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ D, 1);
    __ andr($dst$$Register, $dst$$Register, $tmp$$Register);
    __ andr($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 32);
    __ andw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 16);
    __ andw($dst$$Register, $isrc$$Register, $dst$$Register);
    __ sxth($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orr8S(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (OrReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, D, 0\n\t"
            "umov   $dst, $vsrc, D, 1\n\t"
            "orr    $dst, $dst, $tmp\n\t"
            "orr    $dst, $dst, $dst, LSR #32\n\t"
            "orrw   $dst, $dst, $dst, LSR #16\n\t"
            "orrw   $dst, $isrc, $dst\n\t"
            "sxth   $dst, $dst\t# orr reduction8S"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ D, 1);
    __ orr ($dst$$Register, $dst$$Register, $tmp$$Register);
    __ orr ($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 32);
    __ orrw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 16);
    __ orrw($dst$$Register, $isrc$$Register, $dst$$Register);
    __ sxth($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_eor8S(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (XorReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, D, 0\n\t"
            "umov   $dst, $vsrc, D, 1\n\t"
            "eor    $dst, $dst, $tmp\n\t"
            "eor    $dst, $dst, $dst, LSR #32\n\t"
            "eorw   $dst, $dst, $dst, LSR #16\n\t"
            "eorw   $dst, $isrc, $dst\n\t"
            "sxth   $dst, $dst\t# eor reduction8S"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ D, 1);
    __ eor ($dst$$Register, $dst$$Register, $tmp$$Register);
    __ eor ($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 32);
    __ eorw($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 16);
    __ eorw($dst$$Register, $isrc$$Register, $dst$$Register);
    __ sxth($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_and2I(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (AndReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov  $tmp, $vsrc, S, 0\n\t"
            "andw  $dst, $tmp, $isrc\n\t"
            "umov  $tmp, $vsrc, S, 1\n\t"
            "andw  $dst, $tmp, $dst\t# and reduction2I"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 0);
    __ andw($dst$$Register, $tmp$$Register, $isrc$$Register);
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 1);
    __ andw($dst$$Register, $tmp$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orr2I(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (OrReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov  $tmp, $vsrc, S, 0\n\t"
            "orrw  $dst, $tmp, $isrc\n\t"
            "umov  $tmp, $vsrc, S, 1\n\t"
            "orrw  $dst, $tmp, $dst\t# orr reduction2I"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 0);
    __ orrw($dst$$Register, $tmp$$Register, $isrc$$Register);
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 1);
    __ orrw($dst$$Register, $tmp$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_eor2I(iRegINoSp dst, iRegIorL2I isrc, vecD vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (XorReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov  $tmp, $vsrc, S, 0\n\t"
            "eorw  $dst, $tmp, $isrc\n\t"
            "umov  $tmp, $vsrc, S, 1\n\t"
            "eorw  $dst, $tmp, $dst\t# eor reduction2I"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 0);
    __ eorw($dst$$Register, $tmp$$Register, $isrc$$Register);
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ S, 1);
    __ eorw($dst$$Register, $tmp$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_and4I(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (AndReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, D, 0\n\t"
            "umov   $dst, $vsrc, D, 1\n\t"
            "andr   $dst, $dst, $tmp\n\t"
            "andr   $dst, $dst, $dst, LSR #32\n\t"
            "andw   $dst, $isrc, $dst\t# and reduction4I"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ D, 1);
    __ andr($dst$$Register, $dst$$Register, $tmp$$Register);
    __ andr($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 32);
    __ andw($dst$$Register, $isrc$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orr4I(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (OrReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, D, 0\n\t"
            "umov   $dst, $vsrc, D, 1\n\t"
            "orr    $dst, $dst, $tmp\n\t"
            "orr    $dst, $dst, $dst, LSR #32\n\t"
            "orrw   $dst, $isrc, $dst\t# orr reduction4I"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ D, 1);
    __ orr ($dst$$Register, $dst$$Register, $tmp$$Register);
    __ orr ($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 32);
    __ orrw($dst$$Register, $isrc$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_eor4I(iRegINoSp dst, iRegIorL2I isrc, vecX vsrc, iRegINoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (XorReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov   $tmp, $vsrc, D, 0\n\t"
            "umov   $dst, $vsrc, D, 1\n\t"
            "eor    $dst, $dst, $tmp\n\t"
            "eor    $dst, $dst, $dst, LSR #32\n\t"
            "eorw   $dst, $isrc, $dst\t# eor reduction4I"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 0);
    __ umov($dst$$Register, as_FloatRegister($vsrc$$reg), __ D, 1);
    __ eor ($dst$$Register, $dst$$Register, $tmp$$Register);
    __ eor ($dst$$Register, $dst$$Register, $dst$$Register, Assembler::LSR, 32);
    __ eorw($dst$$Register, $isrc$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_and2L(iRegLNoSp dst, iRegL isrc, vecX vsrc, iRegLNoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (AndReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov  $tmp, $vsrc, D, 0\n\t"
            "andr  $dst, $isrc, $tmp\n\t"
            "umov  $tmp, $vsrc, D, 1\n\t"
            "andr  $dst, $dst, $tmp\t# and reduction2L"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 0);
    __ andr($dst$$Register, $isrc$$Register, $tmp$$Register);
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 1);
    __ andr($dst$$Register, $dst$$Register, $tmp$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orr2L(iRegLNoSp dst, iRegL isrc, vecX vsrc, iRegLNoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (OrReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov  $tmp, $vsrc, D, 0\n\t"
            "orr   $dst, $isrc, $tmp\n\t"
            "umov  $tmp, $vsrc, D, 1\n\t"
            "orr   $dst, $dst, $tmp\t# orr reduction2L"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 0);
    __ orr ($dst$$Register, $isrc$$Register, $tmp$$Register);
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 1);
    __ orr ($dst$$Register, $dst$$Register, $tmp$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_eor2L(iRegLNoSp dst, iRegL isrc, vecX vsrc, iRegLNoSp tmp)
%{
  predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (XorReductionV isrc vsrc));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "umov  $tmp, $vsrc, D, 0\n\t"
            "eor   $dst, $isrc, $tmp\n\t"
            "umov  $tmp, $vsrc, D, 1\n\t"
            "eor   $dst, $dst, $tmp\t# eor reduction2L"
  %}
  ins_encode %{
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 0);
    __ eor ($dst$$Register, $isrc$$Register, $tmp$$Register);
    __ umov($tmp$$Register, as_FloatRegister($vsrc$$reg), __ D, 1);
    __ eor ($dst$$Register, $dst$$Register, $tmp$$Register);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector insert ---------------------------------

instruct insert8B(vecD dst, vecD src, iRegIorL2I val, immI idx)
%{
  predicate(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(INSN_COST);
  format %{ "orr    $dst, T8B, $src, $src\n\t"
            "mov    $dst, T8B, $idx, $val\t# insert into vector(8B)" %}
  ins_encode %{
    if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
      __ orr(as_FloatRegister($dst$$reg), __ T8B,
             as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    }
    __ mov(as_FloatRegister($dst$$reg), __ T8B, $idx$$constant, $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct insert16B(vecX dst, vecX src, iRegIorL2I val, immI idx)
%{
  predicate(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(INSN_COST);
  format %{ "orr    $dst, T16B, $src, $src\n\t"
            "mov    $dst, T16B, $idx, $val\t# insert into vector(16B)" %}
  ins_encode %{
    if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
      __ orr(as_FloatRegister($dst$$reg), __ T16B,
             as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    }
    __ mov(as_FloatRegister($dst$$reg), __ T16B, $idx$$constant, $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct insert4S(vecD dst, vecD src, iRegIorL2I val, immI idx)
%{
  predicate(n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(INSN_COST);
  format %{ "orr    $dst, T8B, $src, $src\n\t"
            "mov    $dst, T4H, $idx, $val\t# insert into vector(4S)" %}
  ins_encode %{
    if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
      __ orr(as_FloatRegister($dst$$reg), __ T8B,
             as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    }
    __ mov(as_FloatRegister($dst$$reg), __ T4H, $idx$$constant, $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct insert8S(vecX dst, vecX src, iRegIorL2I val, immI idx)
%{
  predicate(n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(INSN_COST);
  format %{ "orr    $dst, T16B, $src, $src\n\t"
            "mov    $dst, T8H, $idx, $val\t# insert into vector(8S)" %}
  ins_encode %{
    if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
      __ orr(as_FloatRegister($dst$$reg), __ T16B,
             as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    }
    __ mov(as_FloatRegister($dst$$reg), __ T8H, $idx$$constant, $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct insert2I(vecD dst, vecD src, iRegIorL2I val, immI idx)
%{
  predicate(n->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(INSN_COST);
  format %{ "orr    $dst, T8B, $src, $src\n\t"
            "mov    $dst, T2S, $idx, $val\t# insert into vector(2I)" %}
  ins_encode %{
    if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
      __ orr(as_FloatRegister($dst$$reg), __ T8B,
             as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    }
    __ mov(as_FloatRegister($dst$$reg), __ T2S, $idx$$constant, $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct insert4I(vecX dst, vecX src, iRegIorL2I val, immI idx)
%{
  predicate(n->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(INSN_COST);
  format %{ "orr    $dst, T16B, $src, $src\n\t"
            "mov    $dst, T4S, $idx, $val\t# insert into vector(4I)" %}
  ins_encode %{
    if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
      __ orr(as_FloatRegister($dst$$reg), __ T16B,
             as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    }
    __ mov(as_FloatRegister($dst$$reg), __ T4S, $idx$$constant, $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct insert2L(vecX dst, vecX src, iRegL val, immI idx)
%{
  predicate(n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(INSN_COST);
  format %{ "orr    $dst, T16B, $src, $src\n\t"
            "mov    $dst, T2D, $idx, $val\t# insert into vector(2L)" %}
  ins_encode %{
    if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
      __ orr(as_FloatRegister($dst$$reg), __ T16B,
             as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    }
    __ mov(as_FloatRegister($dst$$reg), __ T2D, $idx$$constant, $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct insert2F(vecD dst, vecD src, vRegF val, immI idx)
%{
  predicate(n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "orr    $dst, T8B, $src, $src\n\t"
            "ins    $dst, S, $val, $idx, 0\t# insert into vector(2F)" %}
  ins_encode %{
    __ orr(as_FloatRegister($dst$$reg), __ T8B,
           as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    __ ins(as_FloatRegister($dst$$reg), __ S,
           as_FloatRegister($val$$reg), $idx$$constant, 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct insert4F(vecX dst, vecX src, vRegF val, immI idx)
%{
  predicate(n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "orr    $dst, T16B, $src, $src\n\t"
            "ins    $dst, S, $val, $idx, 0\t# insert into vector(4F)" %}
  ins_encode %{
    __ orr(as_FloatRegister($dst$$reg), __ T16B,
           as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    __ ins(as_FloatRegister($dst$$reg), __ S,
           as_FloatRegister($val$$reg), $idx$$constant, 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct insert2D(vecX dst, vecX src, vRegD val, immI idx)
%{
  predicate(n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "orr    $dst, T16B, $src, $src\n\t"
            "ins    $dst, D, $val, $idx, 0\t# insert into vector(2D)" %}
  ins_encode %{
    __ orr(as_FloatRegister($dst$$reg), __ T16B,
           as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    __ ins(as_FloatRegister($dst$$reg), __ D,
           as_FloatRegister($val$$reg), $idx$$constant, 0);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector extract ---------------------------------

instruct extract8B(iRegINoSp dst, vecD src, immI idx)
%{
  predicate(n->in(1)->bottom_type()->is_vect()->length() == 8);
  match(Set dst (ExtractB src idx));
  ins_cost(INSN_COST);
  format %{ "smov    $dst, $src, B, $idx\t# extract from vector(8B)" %}
  ins_encode %{
    __ smov($dst$$Register, as_FloatRegister($src$$reg), __ B, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

instruct extract16B(iRegINoSp dst, vecX src, immI idx)
%{
  predicate(n->in(1)->bottom_type()->is_vect()->length() == 16);
  match(Set dst (ExtractB src idx));
  ins_cost(INSN_COST);
  format %{ "smov    $dst, $src, B, $idx\t# extract from vector(16B)" %}
  ins_encode %{
    __ smov($dst$$Register, as_FloatRegister($src$$reg), __ B, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

instruct extract4S(iRegINoSp dst, vecD src, immI idx)
%{
  predicate(n->in(1)->bottom_type()->is_vect()->length() == 4);
  match(Set dst (ExtractS src idx));
  ins_cost(INSN_COST);
  format %{ "smov    $dst, $src, H, $idx\t# extract from vector(4S)" %}
  ins_encode %{
    __ smov($dst$$Register, as_FloatRegister($src$$reg), __ H, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

instruct extract8S(iRegINoSp dst, vecX src, immI idx)
%{
  predicate(n->in(1)->bottom_type()->is_vect()->length() == 8);
  match(Set dst (ExtractS src idx));
  ins_cost(INSN_COST);
  format %{ "smov    $dst, $src, H, $idx\t# extract from vector(8S)" %}
  ins_encode %{
    __ smov($dst$$Register, as_FloatRegister($src$$reg), __ H, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

instruct extract2I(iRegINoSp dst, vecD src, immI idx)
%{
  predicate(n->in(1)->bottom_type()->is_vect()->length() == 2);
  match(Set dst (ExtractI src idx));
  ins_cost(INSN_COST);
  format %{ "umov    $dst, $src, S, $idx\t# extract from vector(2I)" %}
  ins_encode %{
    __ umov($dst$$Register, as_FloatRegister($src$$reg), __ S, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

instruct extract4I(iRegINoSp dst, vecX src, immI idx)
%{
  predicate(n->in(1)->bottom_type()->is_vect()->length() == 4);
  match(Set dst (ExtractI src idx));
  ins_cost(INSN_COST);
  format %{ "umov    $dst, $src, S, $idx\t# extract from vector(4I)" %}
  ins_encode %{
    __ umov($dst$$Register, as_FloatRegister($src$$reg), __ S, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

instruct extract2L(iRegLNoSp dst, vecX src, immI idx)
%{
  predicate(n->in(1)->bottom_type()->is_vect()->length() == 2);
  match(Set dst (ExtractL src idx));
  ins_cost(INSN_COST);
  format %{ "umov    $dst, $src, D, $idx\t# extract from vector(2L)" %}
  ins_encode %{
    __ umov($dst$$Register, as_FloatRegister($src$$reg), __ D, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

instruct extract2F(vRegF dst, vecD src, immI idx)
%{
  predicate(n->in(1)->bottom_type()->is_vect()->length() == 2);
  match(Set dst (ExtractF src idx));
  ins_cost(INSN_COST);
  format %{ "ins   $dst, S, $src, 0, $idx\t# extract from vector(2F)" %}
  ins_encode %{
    __ ins(as_FloatRegister($dst$$reg), __ S,
           as_FloatRegister($src$$reg), 0, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

instruct extract4F(vRegF dst, vecX src, immI idx)
%{
  predicate(n->in(1)->bottom_type()->is_vect()->length() == 4);
  match(Set dst (ExtractF src idx));
  ins_cost(INSN_COST);
  format %{ "ins   $dst, S, $src, 0, $idx\t# extract from vector(4F)" %}
  ins_encode %{
    __ ins(as_FloatRegister($dst$$reg), __ S,
           as_FloatRegister($src$$reg), 0, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

instruct extract2D(vRegD dst, vecX src, immI idx)
%{
  predicate(n->in(1)->bottom_type()->is_vect()->length() == 2);
  match(Set dst (ExtractD src idx));
  ins_cost(INSN_COST);
  format %{ "ins   $dst, D, $src, 0, $idx\t# extract from vector(2D)" %}
  ins_encode %{
    __ ins(as_FloatRegister($dst$$reg), __ D,
           as_FloatRegister($src$$reg), 0, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

// ------------------------------ Vector comparison ---------------------------------

instruct vcmeq8B(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\t# vector cmp (8B)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T8B,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmeq16B(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 16 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\t# vector cmp (16B)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T16B,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmeq4S(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\t# vector cmp (4S)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T4H,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmeq8S(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\t# vector cmp (8S)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T8H,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmeq2I(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\t# vector cmp (2I)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmeq4I(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\t# vector cmp (4I)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmeq2L(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\t# vector cmp (2L)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmeq2F(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmeq  $dst, $src1, $src2\t# vector cmp (2F)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmeq(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmeq4F(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmeq  $dst, $src1, $src2\t# vector cmp (4F)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmeq(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmeq2D(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmeq  $dst, $src1, $src2\t# vector cmp (2D)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmeq(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmgt8B(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src1, $src2\t# vector cmp (8B)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T8B,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmgt16B(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 16 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src1, $src2\t# vector cmp (16B)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T16B,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmgt4S(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src1, $src2\t# vector cmp (4S)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T4H,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmgt8S(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src1, $src2\t# vector cmp (8S)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T8H,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmgt2I(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src1, $src2\t# vector cmp (2I)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmgt4I(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src1, $src2\t# vector cmp (4I)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmgt2L(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src1, $src2\t# vector cmp (2L)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmgt2F(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmgt  $dst, $src1, $src2\t# vector cmp (2F)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmgt(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmgt4F(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmgt  $dst, $src1, $src2\t# vector cmp (4F)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmgt(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmgt2D(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmgt  $dst, $src1, $src2\t# vector cmp (2D)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmgt(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmge8B(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src1, $src2\t# vector cmp (8B)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T8B,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmge16B(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 16 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src1, $src2\t# vector cmp (16B)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T16B,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmge4S(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src1, $src2\t# vector cmp (4S)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T4H,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmge8S(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src1, $src2\t# vector cmp (8S)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T8H,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmge2I(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src1, $src2\t# vector cmp (2I)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmge4I(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src1, $src2\t# vector cmp (4I)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmge2L(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src1, $src2\t# vector cmp (2L)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmge2F(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmge  $dst, $src1, $src2\t# vector cmp (2F)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmge(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmge4F(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmge  $dst, $src1, $src2\t# vector cmp (4F)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmge(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmge2D(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmge  $dst, $src1, $src2\t# vector cmp (2D)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmge(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmne8B(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\n\t# vector cmp (8B)"
            "not   $dst, $dst\t" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T8B,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ notr(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcmne16B(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 16 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\n\t# vector cmp (16B)"
            "not   $dst, $dst\t" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T16B,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ notr(as_FloatRegister($dst$$reg), __ T16B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcmne4S(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\n\t# vector cmp (4S)"
            "not   $dst, $dst\t" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T4H,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ notr(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcmne8S(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\n\t# vector cmp (8S)"
            "not   $dst, $dst\t" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T8H,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ notr(as_FloatRegister($dst$$reg), __ T16B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcmne2I(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\n\t# vector cmp (2I)"
            "not   $dst, $dst\t" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ notr(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcmne4I(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\n\t# vector cmp (4I)"
            "not   $dst, $dst\t" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ notr(as_FloatRegister($dst$$reg), __ T16B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcmne2L(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmeq  $dst, $src1, $src2\n\t# vector cmp (2L)"
            "not   $dst, $dst\t" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmeq(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ notr(as_FloatRegister($dst$$reg), __ T16B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcmne2F(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmeq  $dst, $src1, $src2\n\t# vector cmp (2F)"
            "not   $dst, $dst\t" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmeq(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ notr(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcmne4F(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmeq  $dst, $src1, $src2\n\t# vector cmp (4F)"
            "not   $dst, $dst\t" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmeq(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ notr(as_FloatRegister($dst$$reg), __ T16B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcmne2D(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmeq  $dst, $src1, $src2\n\t# vector cmp (2D)"
            "not   $dst, $dst\t" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmeq(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ notr(as_FloatRegister($dst$$reg), __ T16B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcmlt8B(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src2, $src1\t# vector cmp (8B)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T8B,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmlt16B(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 16 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src2, $src1\t# vector cmp (16B)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T16B,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmlt4S(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src2, $src1\t# vector cmp (4S)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T4H,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmlt8S(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src2, $src1\t# vector cmp (8S)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T8H,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmlt2I(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src2, $src1\t# vector cmp (2I)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmlt4I(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src2, $src1\t# vector cmp (4I)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmlt2L(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmgt  $dst, $src2, $src1\t# vector cmp (2L)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmlt2F(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmgt  $dst, $src2, $src1\t# vector cmp (2F)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmgt(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmlt4F(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmgt  $dst, $src2, $src1\t# vector cmp (4F)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmgt(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmlt2D(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmgt  $dst, $src2, $src1\t# vector cmp (2D)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmgt(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmle8B(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src2, $src1\t# vector cmp (8B)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T8B,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmle16B(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 16 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src2, $src1\t# vector cmp (16B)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T16B,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmle4S(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src2, $src1\t# vector cmp (4S)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T4H,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmle8S(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src2, $src1\t# vector cmp (8S)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T8H,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmle2I(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src2, $src1\t# vector cmp (2I)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmle4I(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src2, $src1\t# vector cmp (4I)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmle2L(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "cmge  $dst, $src2, $src1\t# vector cmp (2L)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ cmge(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmle2F(vecD dst, vecD src1, vecD src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmge  $dst, $src2, $src1\t# vector cmp (2F)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmge(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vcmle4F(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmge  $dst, $src2, $src1\t# vector cmp (4F)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmge(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vcmle2D(vecX dst, vecX src1, vecX src2, immI cond)
%{
  predicate(n->as_Vector()->length() == 2 &&
            n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
            n->in(1)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "fcmge  $dst, $src2, $src1\t# vector cmp (2D)" %}
  ins_cost(INSN_COST);
  ins_encode %{
    __ fcmge(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop128);
%}

// ------------------------------ Vector mul -----------------------------------

instruct vmul2L(vecX dst, vecX src1, vecX src2, iRegLNoSp tmp1, iRegLNoSp tmp2)
%{
  predicate(n->as_Vector()->length() == 2);
  match(Set dst (MulVL src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP tmp1, TEMP tmp2);
  format %{ "umov   $tmp1, $src1, D, 0\n\t"
            "umov   $tmp2, $src2, D, 0\n\t"
            "mul    $tmp2, $tmp2, $tmp1\n\t"
            "mov    $dst,  T2D,   0, $tmp2\t# insert into vector(2L)\n\t"
            "umov   $tmp1, $src1, D, 1\n\t"
            "umov   $tmp2, $src2, D, 1\n\t"
            "mul    $tmp2, $tmp2, $tmp1\n\t"
            "mov    $dst,  T2D,   1, $tmp2\t# insert into vector(2L)\n\t"
  %}
  ins_encode %{
    __ umov($tmp1$$Register, as_FloatRegister($src1$$reg), __ D, 0);
    __ umov($tmp2$$Register, as_FloatRegister($src2$$reg), __ D, 0);
    __ mul(as_Register($tmp2$$reg), as_Register($tmp2$$reg), as_Register($tmp1$$reg));
    __ mov(as_FloatRegister($dst$$reg), __ T2D, 0, $tmp2$$Register);
    __ umov($tmp1$$Register, as_FloatRegister($src1$$reg), __ D, 1);
    __ umov($tmp2$$Register, as_FloatRegister($src2$$reg), __ D, 1);
    __ mul(as_Register($tmp2$$reg), as_Register($tmp2$$reg), as_Register($tmp1$$reg));
    __ mov(as_FloatRegister($dst$$reg), __ T2D, 1, $tmp2$$Register);
  %}
  ins_pipe(pipe_slow);
%}

// --------------------------------- Vector not --------------------------------

instruct vnot2I(vecD dst, vecD src, immI_M1 m1)
%{
  predicate(n->as_Vector()->length_in_bytes() == 8);
  match(Set dst (XorV src (ReplicateB m1)));
  match(Set dst (XorV src (ReplicateS m1)));
  match(Set dst (XorV src (ReplicateI m1)));
  ins_cost(INSN_COST);
  format %{ "not  $dst, $src\t# vector (8B)" %}
  ins_encode %{
    __ notr(as_FloatRegister($dst$$reg), __ T8B,
            as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_class_default);
%}

instruct vnot4I(vecX dst, vecX src, immI_M1 m1)
%{
  predicate(n->as_Vector()->length_in_bytes() == 16);
  match(Set dst (XorV src (ReplicateB m1)));
  match(Set dst (XorV src (ReplicateS m1)));
  match(Set dst (XorV src (ReplicateI m1)));
  ins_cost(INSN_COST);
  format %{ "not  $dst, $src\t# vector (16B)" %}
  ins_encode %{
    __ notr(as_FloatRegister($dst$$reg), __ T16B,
            as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_class_default);
%}

instruct vnot2L(vecX dst, vecX src, immL_M1 m1)
%{
  predicate(n->as_Vector()->length_in_bytes() == 16);
  match(Set dst (XorV src (ReplicateL m1)));
  ins_cost(INSN_COST);
  format %{ "not  $dst, $src\t# vector (16B)" %}
  ins_encode %{
    __ notr(as_FloatRegister($dst$$reg), __ T16B,
            as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_class_default);
%}

// ------------------------------ Vector max/min -------------------------------

instruct vmax8B(vecD dst, vecD src1, vecD src2)
%{
  predicate((n->as_Vector()->length() == 4 || n->as_Vector()->length() == 8) &&
             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (MaxV src1 src2));
  ins_cost(INSN_COST);
  format %{ "maxv  $dst, $src1, $src2\t# vector (8B)" %}
  ins_encode %{
    __ maxv(as_FloatRegister($dst$$reg), __ T8B,
            as_FloatRegister($src1$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vmax16B(vecX dst, vecX src1, vecX src2)
%{
  predicate(n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (MaxV src1 src2));
  ins_cost(INSN_COST);
  format %{ "maxv  $dst, $src1, $src2\t# vector (16B)" %}
  ins_encode %{
    __ maxv(as_FloatRegister($dst$$reg), __ T16B,
            as_FloatRegister($src1$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vmax4S(vecD dst, vecD src1, vecD src2)
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (MaxV src1 src2));
  ins_cost(INSN_COST);
  format %{ "maxv  $dst, $src1, $src2\t# vector (4S)" %}
  ins_encode %{
    __ maxv(as_FloatRegister($dst$$reg), __ T4H,
            as_FloatRegister($src1$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vmax8S(vecX dst, vecX src1, vecX src2)
%{
  predicate(n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (MaxV src1 src2));
  ins_cost(INSN_COST);
  format %{ "maxv  $dst, $src1, $src2\t# vector (8S)" %}
  ins_encode %{
    __ maxv(as_FloatRegister($dst$$reg), __ T8H,
            as_FloatRegister($src1$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vmax2I(vecD dst, vecD src1, vecD src2)
%{
  predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (MaxV src1 src2));
  ins_cost(INSN_COST);
  format %{ "maxv  $dst, $src1, $src2\t# vector (2I)" %}
  ins_encode %{
    __ maxv(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src1$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vmax4I(vecX dst, vecX src1, vecX src2)
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (MaxV src1 src2));
  ins_cost(INSN_COST);
  format %{ "maxv  $dst, $src1, $src2\t# vector (4I)" %}
  ins_encode %{
    __ maxv(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src1$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vmin8B(vecD dst, vecD src1, vecD src2)
%{
  predicate((n->as_Vector()->length() == 4 || n->as_Vector()->length() == 8) &&
             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (MinV src1 src2));
  ins_cost(INSN_COST);
  format %{ "minv  $dst, $src1, $src2\t# vector (8B)" %}
  ins_encode %{
    __ minv(as_FloatRegister($dst$$reg), __ T8B,
            as_FloatRegister($src1$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vmin16B(vecX dst, vecX src1, vecX src2)
%{
  predicate(n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (MinV src1 src2));
  ins_cost(INSN_COST);
  format %{ "minv  $dst, $src1, $src2\t# vector (16B)" %}
  ins_encode %{
    __ minv(as_FloatRegister($dst$$reg), __ T16B,
            as_FloatRegister($src1$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vmin4S(vecD dst, vecD src1, vecD src2)
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (MinV src1 src2));
  ins_cost(INSN_COST);
  format %{ "minv  $dst, $src1, $src2\t# vector (4S)" %}
  ins_encode %{
    __ minv(as_FloatRegister($dst$$reg), __ T4H,
            as_FloatRegister($src1$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vmin8S(vecX dst, vecX src1, vecX src2)
%{
  predicate(n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (MinV src1 src2));
  ins_cost(INSN_COST);
  format %{ "minv  $dst, $src1, $src2\t# vector (8S)" %}
  ins_encode %{
    __ minv(as_FloatRegister($dst$$reg), __ T8H,
            as_FloatRegister($src1$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vmin2I(vecD dst, vecD src1, vecD src2)
%{
  predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (MinV src1 src2));
  ins_cost(INSN_COST);
  format %{ "minv  $dst, $src1, $src2\t# vector (2I)" %}
  ins_encode %{
    __ minv(as_FloatRegister($dst$$reg), __ T2S,
            as_FloatRegister($src1$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop64);
%}

instruct vmin4I(vecX dst, vecX src1, vecX src2)
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (MinV src1 src2));
  ins_cost(INSN_COST);
  format %{ "minv  $dst, $src1, $src2\t# vector (4I)" %}
  ins_encode %{
    __ minv(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($src1$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}


instruct vmax2L(vecX dst, vecX src1, vecX src2)
%{
  predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (MaxV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP dst);
  format %{ "cmgt  $dst, $src1, $src2\t# vector (2L)\n\t"
            "bsl   $dst, $src1, $src2\t# vector (16B)" %}
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ bsl(as_FloatRegister($dst$$reg), __ T16B,
           as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(vdop128);
%}

instruct vmin2L(vecX dst, vecX src1, vecX src2)
%{
  predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (MinV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP dst);
  format %{ "cmgt  $dst, $src1, $src2\t# vector (2L)\n\t"
            "bsl   $dst, $src2, $src1\t# vector (16B)" %}
  ins_encode %{
    __ cmgt(as_FloatRegister($dst$$reg), __ T2D,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ bsl(as_FloatRegister($dst$$reg), __ T16B,
           as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vdop128);
%}

// --------------------------------- blend (bsl) ----------------------------

instruct vbsl8B(vecD dst, vecD src1, vecD src2)
%{
  predicate(n->as_Vector()->length_in_bytes() == 8);
  match(Set dst (VectorBlend (Binary src1 src2) dst));
  ins_cost(INSN_COST);
  format %{ "bsl  $dst, $src2, $src1\t# vector (8B)" %}
  ins_encode %{
    __ bsl(as_FloatRegister($dst$$reg), __ T8B,
           as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vlogical64);
%}

instruct vbsl16B(vecX dst, vecX src1, vecX src2)
%{
  predicate(n->as_Vector()->length_in_bytes() == 16);
  match(Set dst (VectorBlend (Binary src1 src2) dst));
  ins_cost(INSN_COST);
  format %{ "bsl  $dst, $src2, $src1\t# vector (16B)" %}
  ins_encode %{
    __ bsl(as_FloatRegister($dst$$reg), __ T16B,
           as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(vlogical128);
%}

// --------------------------------- Load/store Mask ----------------------------

instruct loadmask8B(vecD dst, vecD src  )
%{
  predicate(n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorLoadMask src ));
  ins_cost(INSN_COST);
  format %{ "negr  $dst, $src\t# load mask (8B to 8B)" %}
  ins_encode %{
    __ negr(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_class_default);
%}

instruct loadmask16B(vecX dst, vecX src  )
%{
  predicate(n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorLoadMask src ));
  ins_cost(INSN_COST);
  format %{ "negr  $dst, $src\t# load mask (16B to 16B)" %}
  ins_encode %{
    __ negr(as_FloatRegister($dst$$reg), __ T16B, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_class_default);
%}

instruct storemask8B(vecD dst, vecD src , immI_1 size)
%{
  predicate(n->as_Vector()->length() == 8);
  match(Set dst (VectorStoreMask src size));
  ins_cost(INSN_COST);
  format %{ "negr  $dst, $src\t# store mask (8B to 8B)" %}
  ins_encode %{
    __ negr(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_class_default);
%}

instruct storemask16B(vecX dst, vecX src , immI_1 size)
%{
  predicate(n->as_Vector()->length() == 16);
  match(Set dst (VectorStoreMask src size));
  ins_cost(INSN_COST);
  format %{ "negr  $dst, $src\t# store mask (16B to 16B)" %}
  ins_encode %{
    __ negr(as_FloatRegister($dst$$reg), __ T16B, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_class_default);
%}

instruct loadmask4S(vecD dst, vecD src  )
%{
  predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorLoadMask src ));
  ins_cost(INSN_COST);
  format %{ "uxtl  $dst, $src\n\t"
            "negr  $dst, $dst\t# load mask (4B to 4H)" %}
  ins_encode %{
    __ uxtl(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg), __ T8B);
    __ negr(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct loadmask8S(vecX dst, vecD src  )
%{
  predicate(n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorLoadMask src ));
  ins_cost(INSN_COST);
  format %{ "uxtl  $dst, $src\n\t"
            "negr  $dst, $dst\t# load mask (8B to 8H)" %}
  ins_encode %{
    __ uxtl(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg), __ T8B);
    __ negr(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct storemask4S(vecD dst, vecD src , immI_2 size)
%{
  predicate(n->as_Vector()->length() == 4);
  match(Set dst (VectorStoreMask src size));
  ins_cost(INSN_COST);
  format %{ "xtn  $dst, $src\n\t"
            "negr  $dst, $dst\t# store mask (4H to 4B)" %}
  ins_encode %{
    __ xtn(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($src$$reg), __ T8H);
    __ negr(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct storemask8S(vecD dst, vecX src , immI_2 size)
%{
  predicate(n->as_Vector()->length() == 8);
  match(Set dst (VectorStoreMask src size));
  ins_cost(INSN_COST);
  format %{ "xtn  $dst, $src\n\t"
            "negr  $dst, $dst\t# store mask (8H to 8B)" %}
  ins_encode %{
    __ xtn(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($src$$reg), __ T8H);
    __ negr(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct loadmask2I(vecD dst, vecD src  )
%{
  predicate(n->as_Vector()->length() == 2 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
  match(Set dst (VectorLoadMask src ));
  ins_cost(INSN_COST);
  format %{ "uxtl  $dst, $src\t# 2B to 2H\n\t"
            "uxtl  $dst, $dst\t# 2H to 2S\n\t"
            "negr   $dst, $dst\t# load mask (2B to 2S)" %}
  ins_encode %{
    __ uxtl(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg), __ T8B);
    __ uxtl(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($dst$$reg), __ T4H);
    __ negr(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct loadmask4I(vecX dst, vecD src  )
%{
  predicate(n->as_Vector()->length() == 4 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
  match(Set dst (VectorLoadMask src ));
  ins_cost(INSN_COST);
  format %{ "uxtl  $dst, $src\t# 4B to 4H\n\t"
            "uxtl  $dst, $dst\t# 4H to 4S\n\t"
            "negr   $dst, $dst\t# load mask (4B to 4S)" %}
  ins_encode %{
    __ uxtl(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg), __ T8B);
    __ uxtl(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($dst$$reg), __ T4H);
    __ negr(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct storemask2I(vecD dst, vecD src , immI_4 size)
%{
  predicate(n->as_Vector()->length() == 2);
  match(Set dst (VectorStoreMask src size));
  ins_cost(INSN_COST);
  format %{ "xtn  $dst, $src\t# 2S to 2H\n\t"
            "xtn  $dst, $dst\t# 2H to 2B\n\t"
            "negr   $dst, $dst\t# store mask (2S to 2B)" %}
  ins_encode %{
    __ xtn(as_FloatRegister($dst$$reg), __ T4H, as_FloatRegister($src$$reg), __ T4S);
    __ xtn(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($dst$$reg), __ T8H);
    __ negr(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct storemask4I(vecD dst, vecX src , immI_4 size)
%{
  predicate(n->as_Vector()->length() == 4);
  match(Set dst (VectorStoreMask src size));
  ins_cost(INSN_COST);
  format %{ "xtn  $dst, $src\t# 4S to 4H\n\t"
            "xtn  $dst, $dst\t# 4H to 4B\n\t"
            "negr   $dst, $dst\t# store mask (4S to 4B)" %}
  ins_encode %{
    __ xtn(as_FloatRegister($dst$$reg), __ T4H, as_FloatRegister($src$$reg), __ T4S);
    __ xtn(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($dst$$reg), __ T8H);
    __ negr(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct loadmask2L(vecX dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 2 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
  match(Set dst (VectorLoadMask src));
  ins_cost(INSN_COST);
  format %{ "uxtl  $dst, $src\t# 2B to 2S\n\t"
            "uxtl  $dst, $dst\t# 2S to 2I\n\t"
            "uxtl  $dst, $dst\t# 2I to 2L\n\t"
            "neg   $dst, $dst\t# load mask (2B to 2L)" %}
  ins_encode %{
    __ uxtl(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg), __ T8B);
    __ uxtl(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($dst$$reg), __ T4H);
    __ uxtl(as_FloatRegister($dst$$reg), __ T2D, as_FloatRegister($dst$$reg), __ T2S);
    __ negr(as_FloatRegister($dst$$reg), __ T2D, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct storemask2L(vecD dst, vecX src, immI_8 size)
%{
  predicate(n->as_Vector()->length() == 2);
  match(Set dst (VectorStoreMask src size));
  ins_cost(INSN_COST);
  format %{ "xtn  $dst, $src\t# 2L to 2I\n\t"
            "xtn  $dst, $dst\t# 2I to 2S\n\t"
            "xtn  $dst, $dst\t# 2S to 2B\n\t"
            "neg  $dst, $dst\t# store mask (2L to 2B)" %}
  ins_encode %{
    __ xtn(as_FloatRegister($dst$$reg), __ T2S, as_FloatRegister($src$$reg), __ T2D);
    __ xtn(as_FloatRegister($dst$$reg), __ T4H, as_FloatRegister($dst$$reg), __ T4S);
    __ xtn(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($dst$$reg), __ T8H);
    __ negr(as_FloatRegister($dst$$reg), __ T8B, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

//-------------------------------- LOAD_IOTA_INDICES----------------------------------

instruct loadcon8B(vecD dst, immI0 src)
%{
  predicate((n->as_Vector()->length() == 2 || n->as_Vector()->length() == 4 ||
             n->as_Vector()->length() == 8) &&
             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorLoadConst src));
  ins_cost(INSN_COST);
  format %{ "ldr $dst, CONSTANT_MEMORY\t# load iota indices" %}
  ins_encode %{
    __ lea(rscratch1, ExternalAddress(StubRoutines::aarch64::vector_iota_indices()));
    __ ldrd(as_FloatRegister($dst$$reg), rscratch1);
  %}
  ins_pipe(pipe_class_memory);
%}

instruct loadcon16B(vecX dst, immI0 src)
%{
  predicate(n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorLoadConst src));
  ins_cost(INSN_COST);
  format %{ "ldr $dst, CONSTANT_MEMORY\t# load iota indices" %}
  ins_encode %{
    __ lea(rscratch1, ExternalAddress(StubRoutines::aarch64::vector_iota_indices()));
    __ ldrq(as_FloatRegister($dst$$reg), rscratch1);
  %}
  ins_pipe(pipe_class_memory);
%}

//-------------------------------- LOAD_SHUFFLE ----------------------------------

instruct loadshuffle8B(vecD dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorLoadShuffle src));
  ins_cost(INSN_COST);
  format %{ "mov  $dst, $src\t# get 8B shuffle" %}
  ins_encode %{
    __ orr(as_FloatRegister($dst$$reg), __ T8B,
           as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_class_default);
%}

instruct loadshuffle16B(vecX dst, vecX src)
%{
  predicate(n->as_Vector()->length() == 16 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorLoadShuffle src));
  ins_cost(INSN_COST);
  format %{ "mov  $dst, $src\t# get 16B shuffle" %}
  ins_encode %{
    __ orr(as_FloatRegister($dst$$reg), __ T16B,
           as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_class_default);
%}

instruct loadshuffle4S(vecD dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorLoadShuffle src));
  ins_cost(INSN_COST);
  format %{ "uxtl  $dst, $src\t# 4B to 4H" %}
  ins_encode %{
    __ uxtl(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg), __ T8B);
  %}
  ins_pipe(pipe_class_default);
%}

instruct loadshuffle8S(vecX dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorLoadShuffle src));
  ins_cost(INSN_COST);
  format %{ "uxtl  $dst, $src\t# 8B to 8H" %}
  ins_encode %{
    __ uxtl(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg), __ T8B);
  %}
  ins_pipe(pipe_class_default);
%}

instruct loadshuffle4I(vecX dst, vecD src)
%{
  predicate(n->as_Vector()->length() == 4 &&
           (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
            n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
  match(Set dst (VectorLoadShuffle src));
  ins_cost(INSN_COST);
  format %{ "uxtl  $dst, $src\t# 4B to 4H \n\t"
            "uxtl  $dst, $dst\t# 4H to 4S" %}
  ins_encode %{
    __ uxtl(as_FloatRegister($dst$$reg), __ T8H, as_FloatRegister($src$$reg), __ T8B);
    __ uxtl(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($dst$$reg), __ T4H);
  %}
  ins_pipe(pipe_slow);
%}

//-------------------------------- Rearrange -------------------------------------
// Here is an example that rearranges a NEON vector with 4 ints:
// Rearrange V1 int[a0, a1, a2, a3] to V2 int[a2, a3, a0, a1]
//   1. Get the indices of V1 and store them as Vi byte[0, 1, 2, 3].
//   2. Convert Vi byte[0, 1, 2, 3] to the indices of V2 and also store them as Vi byte[2, 3, 0, 1].
//   3. Unsigned extend Long Vi from byte[2, 3, 0, 1] to int[2, 3, 0, 1].
//   4. Multiply Vi int[2, 3, 0, 1] with constant int[0x04040404, 0x04040404, 0x04040404, 0x04040404]
//      and get tbl base Vm int[0x08080808, 0x0c0c0c0c, 0x00000000, 0x04040404].
//   5. Add Vm with constant int[0x03020100, 0x03020100, 0x03020100, 0x03020100]
//      and get tbl index Vm int[0x0b0a0908, 0x0f0e0d0c, 0x03020100, 0x07060504]
//   6. Use Vm as index register, and use V1 as table register.
//      Then get V2 as the result by tbl NEON instructions.
// Notes:
//   Step 1 matches VectorLoadConst.
//   Step 3 matches VectorLoadShuffle.
//   Step 4, 5, 6 match VectorRearrange.
//   For VectorRearrange short/int, the reason why such complex calculation is
//   required is because NEON tbl supports bytes table only, so for short/int, we
//   need to lookup 2/4 bytes as a group. For VectorRearrange long, we use bsl
//   to implement rearrange.

instruct rearrange8B(vecD dst, vecD src, vecD shuffle)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorRearrange src shuffle));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "tbl $dst, {$dst}, $shuffle\t# rearrange 8B" %}
  ins_encode %{
    __ tbl(as_FloatRegister($dst$$reg), __ T8B,
           as_FloatRegister($src$$reg), 1, as_FloatRegister($shuffle$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct rearrange16B(vecX dst, vecX src, vecX shuffle)
%{
  predicate(n->as_Vector()->length() == 16 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorRearrange src shuffle));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "tbl $dst, {$dst}, $shuffle\t# rearrange 16B" %}
  ins_encode %{
    __ tbl(as_FloatRegister($dst$$reg), __ T16B,
           as_FloatRegister($src$$reg), 1, as_FloatRegister($shuffle$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct rearrange4S(vecD dst, vecD src, vecD shuffle, vecD tmp0, vecD tmp1)
%{
  predicate(n->as_Vector()->length() == 4 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorRearrange src shuffle));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp0, TEMP tmp1);
  format %{ "mov   $tmp0, CONSTANT\t# constant 0x0202020202020202\n\t"
            "mov   $tmp1, CONSTANT\t# constant 0x0100010001000100\n\t"
            "mulv  $dst, T4H, $shuffle, $tmp0\n\t"
            "addv  $dst, T8B, $dst, $tmp1\n\t"
            "tbl   $dst, {$src}, $dst\t# rearrange 4S" %}
  ins_encode %{
    __ mov(as_FloatRegister($tmp0$$reg), __ T8B, 0x02);
    __ mov(as_FloatRegister($tmp1$$reg), __ T4H, 0x0100);
    __ mulv(as_FloatRegister($dst$$reg), __ T4H,
            as_FloatRegister($shuffle$$reg), as_FloatRegister($tmp0$$reg));
    __ addv(as_FloatRegister($dst$$reg), __ T8B,
            as_FloatRegister($dst$$reg), as_FloatRegister($tmp1$$reg));
    __ tbl(as_FloatRegister($dst$$reg), __ T8B,
           as_FloatRegister($src$$reg), 1, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct rearrange8S(vecX dst, vecX src, vecX shuffle, vecX tmp0, vecX tmp1)
%{
  predicate(n->as_Vector()->length() == 8 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorRearrange src shuffle));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp0, TEMP tmp1);
  format %{ "mov   $tmp0, CONSTANT\t# constant 0x0202020202020202\n\t"
            "mov   $tmp1, CONSTANT\t# constant 0x0100010001000100\n\t"
            "mulv  $dst, T8H, $shuffle, $tmp0\n\t"
            "addv  $dst, T16B, $dst, $tmp1\n\t"
            "tbl   $dst, {$src}, $dst\t# rearrange 8S" %}
  ins_encode %{
    __ mov(as_FloatRegister($tmp0$$reg), __ T16B, 0x02);
    __ mov(as_FloatRegister($tmp1$$reg), __ T8H, 0x0100);
    __ mulv(as_FloatRegister($dst$$reg), __ T8H,
            as_FloatRegister($shuffle$$reg), as_FloatRegister($tmp0$$reg));
    __ addv(as_FloatRegister($dst$$reg), __ T16B,
            as_FloatRegister($dst$$reg), as_FloatRegister($tmp1$$reg));
    __ tbl(as_FloatRegister($dst$$reg), __ T16B,
           as_FloatRegister($src$$reg), 1, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct rearrange4I(vecX dst, vecX src, vecX shuffle, vecX tmp0, vecX tmp1)
%{
  predicate(n->as_Vector()->length() == 4 &&
           (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
            n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
  match(Set dst (VectorRearrange src shuffle));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP tmp0, TEMP tmp1);
  format %{ "mov   $tmp0, CONSTANT\t# constant 0x0404040404040404\n\t"
            "mov   $tmp1, CONSTANT\t# constant 0x0302010003020100\n\t"
            "mulv  $dst, T8H, $shuffle, $tmp0\n\t"
            "addv  $dst, T16B, $dst, $tmp1\n\t"
            "tbl   $dst, {$src}, $dst\t# rearrange 4I" %}
  ins_encode %{
    __ mov(as_FloatRegister($tmp0$$reg), __ T16B, 0x04);
    __ mov(as_FloatRegister($tmp1$$reg), __ T4S, 0x03020100);
    __ mulv(as_FloatRegister($dst$$reg), __ T4S,
            as_FloatRegister($shuffle$$reg), as_FloatRegister($tmp0$$reg));
    __ addv(as_FloatRegister($dst$$reg), __ T16B,
            as_FloatRegister($dst$$reg), as_FloatRegister($tmp1$$reg));
    __ tbl(as_FloatRegister($dst$$reg), __ T16B,
           as_FloatRegister($src$$reg), 1, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

//-------------------------------- Anytrue/alltrue -----------------------------

instruct anytrue_in_mask8B(iRegINoSp dst, vecD src1, vecD src2, vecD tmp, rFlagsReg cr)
%{
  predicate(static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
  match(Set dst (VectorTest src1 src2 ));
  ins_cost(INSN_COST);
  effect(TEMP tmp, KILL cr);
  format %{ "addv  $tmp, T8B, $src1\t# src1 and src2 are the same\n\t"
            "umov  $dst, $tmp, B, 0\n\t"
            "cmp   $dst, 0\n\t"
            "cset  $dst" %}
  ins_encode %{
    __ addv(as_FloatRegister($tmp$$reg), __ T8B, as_FloatRegister($src1$$reg));
    __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ B, 0);
    __ cmpw($dst$$Register, zr);
    __ csetw($dst$$Register, Assembler::NE);
  %}
  ins_pipe(pipe_slow);
%}

instruct anytrue_in_mask16B(iRegINoSp dst, vecX src1, vecX src2, vecX tmp, rFlagsReg cr)
%{
  predicate(static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
  match(Set dst (VectorTest src1 src2 ));
  ins_cost(INSN_COST);
  effect(TEMP tmp, KILL cr);
  format %{ "addv  $tmp, T16B, $src1\t# src1 and src2 are the same\n\t"
            "umov  $dst, $tmp, B, 0\n\t"
            "cmp   $dst, 0\n\t"
            "cset  $dst" %}
  ins_encode %{
    __ addv(as_FloatRegister($tmp$$reg), __ T16B, as_FloatRegister($src1$$reg));
    __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ B, 0);
    __ cmpw($dst$$Register, zr);
    __ csetw($dst$$Register, Assembler::NE);
  %}
  ins_pipe(pipe_slow);
%}

instruct alltrue_in_mask8B(iRegINoSp dst, vecD src1, vecD src2, vecD tmp, rFlagsReg cr)
%{
  predicate(static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
  match(Set dst (VectorTest src1 src2 ));
  ins_cost(INSN_COST);
  effect(TEMP tmp, KILL cr);
  format %{ "andr  $tmp, T8B, $src1, $src2\t# src2 is maskAllTrue\n\t"
            "notr  $tmp, T8B, $tmp\n\t"
            "addv  $tmp, T8B, $tmp\n\t"
            "umov  $dst, $tmp, B, 0\n\t"
            "cmp   $dst, 0\n\t"
            "cset  $dst" %}
  ins_encode %{
    __ andr(as_FloatRegister($tmp$$reg), __ T8B,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ notr(as_FloatRegister($tmp$$reg), __ T8B, as_FloatRegister($tmp$$reg));
    __ addv(as_FloatRegister($tmp$$reg), __ T8B, as_FloatRegister($tmp$$reg));
    __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ B, 0);
    __ cmpw($dst$$Register, zr);
    __ csetw($dst$$Register, Assembler::EQ);
  %}
  ins_pipe(pipe_slow);
%}

instruct alltrue_in_mask16B(iRegINoSp dst, vecX src1, vecX src2, vecX tmp, rFlagsReg cr)
%{
  predicate(static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
  match(Set dst (VectorTest src1 src2 ));
  ins_cost(INSN_COST);
  effect(TEMP tmp, KILL cr);
  format %{ "andr  $tmp, T16B, $src1, $src2\t# src2 is maskAllTrue\n\t"
            "notr  $tmp, T16B, $tmp\n\t"
            "addv  $tmp, T16B, $tmp\n\t"
            "umov  $dst, $tmp, B, 0\n\t"
            "cmp   $dst, 0\n\t"
            "cset  $dst" %}
  ins_encode %{
    __ andr(as_FloatRegister($tmp$$reg), __ T16B,
            as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
    __ notr(as_FloatRegister($tmp$$reg), __ T16B, as_FloatRegister($tmp$$reg));
    __ addv(as_FloatRegister($tmp$$reg), __ T16B, as_FloatRegister($tmp$$reg));
    __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ B, 0);
    __ cmpw($dst$$Register, zr);
    __ csetw($dst$$Register, Assembler::EQ);
  %}
  ins_pipe(pipe_slow);
%}