//
// Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, Arm Ltd. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License version 2 only, as
// published by the Free Software Foundation.
//
// This code is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
// version 2 for more details (a copy is included in the LICENSE file that
// accompanied this code).
//
// You should have received a copy of the GNU General Public License version
// 2 along with this work; if not, write to the Free Software Foundation,
// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
//
// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
// or visit www.oracle.com if you need additional information or have any
// questions.
//
//

// This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ----

// AArch64 SVE Architecture Description File


// 4 bit signed offset -- for predicated load/store

operand vmemA_immIOffset4()
%{
  predicate(Address::offset_ok_for_sve_immed(n->get_int(), 4,
            Matcher::scalable_vector_reg_size(T_BYTE)));
  match(ConI);

  op_cost(0);
  format %{ %}
  interface(CONST_INTER);
%}

operand vmemA_immLOffset4()
%{
  predicate(Address::offset_ok_for_sve_immed(n->get_long(), 4,
            Matcher::scalable_vector_reg_size(T_BYTE)));
  match(ConL);

  op_cost(0);
  format %{ %}
  interface(CONST_INTER);
%}


operand vmemA_indOffI4(iRegP reg, vmemA_immIOffset4 off)
%{
  constraint(ALLOC_IN_RC(ptr_reg));
  match(AddP reg off);
  op_cost(0);
  format %{ "[$reg, $off, MUL VL]" %}
  interface(MEMORY_INTER) %{
    base($reg);
    index(0xffffffff);
    scale(0x0);
    disp($off);
  %}
%}

operand vmemA_indOffL4(iRegP reg, vmemA_immLOffset4 off)
%{
  constraint(ALLOC_IN_RC(ptr_reg));
  match(AddP reg off);
  op_cost(0);
  format %{ "[$reg, $off, MUL VL]" %}
  interface(MEMORY_INTER) %{
    base($reg);
    index(0xffffffff);
    scale(0x0);
    disp($off);
  %}
%}

opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);

source_hpp %{
  bool op_sve_supported(int opcode);
%}

source %{

  static inline BasicType vector_element_basic_type(const MachNode* n) {
    const TypeVect* vt = n->bottom_type()->is_vect();
    return vt->element_basic_type();
  }

  static inline BasicType vector_element_basic_type(const MachNode* use, const MachOper* opnd) {
    int def_idx = use->operand_index(opnd);
    Node* def = use->in(def_idx);
    const TypeVect* vt = def->bottom_type()->is_vect();
    return vt->element_basic_type();
  }

  typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
                                                             PRegister Pg, const Address &adr);

  // Predicated load/store, with optional ptrue to all elements of given predicate register.
  static void loadStoreA_predicate(C2_MacroAssembler masm, bool is_store,
                                   FloatRegister reg, PRegister pg, BasicType bt,
                                   int opcode, Register base, int index, int size, int disp) {
    sve_mem_insn_predicate insn;
    Assembler::SIMD_RegVariant type;
    int esize = type2aelembytes(bt);
    if (index == -1) {
      assert(size == 0, "unsupported address mode: scale size = %d", size);
      switch(esize) {
      case 1:
        insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
        type = Assembler::B;
        break;
      case 2:
        insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
        type = Assembler::H;
        break;
      case 4:
        insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
        type = Assembler::S;
        break;
      case 8:
        insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
        type = Assembler::D;
        break;
      default:
        assert(false, "unsupported");
        ShouldNotReachHere();
      }
      (masm.*insn)(reg, type, pg, Address(base, disp / Matcher::scalable_vector_reg_size(T_BYTE)));
    } else {
      assert(false, "unimplemented");
      ShouldNotReachHere();
    }
  }

  bool op_sve_supported(int opcode) {
    switch (opcode) {
      case Op_MulAddVS2VI:
        // No multiply reduction instructions
      case Op_MulReductionVD:
      case Op_MulReductionVF:
      case Op_MulReductionVI:
      case Op_MulReductionVL:
        // Others
      case Op_Extract:
      case Op_ExtractB:
      case Op_ExtractC:
      case Op_ExtractD:
      case Op_ExtractF:
      case Op_ExtractI:
      case Op_ExtractL:
      case Op_ExtractS:
      case Op_ExtractUB:
        return false;
      default:
        return true;
    }
  }

%}

definitions %{
  int_def SVE_COST             (200, 200);
%}




// All SVE instructions

// sve vector load/store

// Use predicated vector load/store
instruct loadVA(vecA dst, vmemA mem) %{
  predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16);
  match(Set dst (LoadVector mem));
  ins_cost(SVE_COST);
  format %{ "sve_ldr $dst, $mem\t # vector (sve)" %}
  ins_encode %{
    FloatRegister dst_reg = as_FloatRegister($dst$$reg);
    loadStoreA_predicate(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
                         vector_element_basic_type(this), $mem->opcode(),
                         as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

instruct storeVA(vecA src, vmemA mem) %{
  predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16);
  match(Set mem (StoreVector mem src));
  ins_cost(SVE_COST);
  format %{ "sve_str $mem, $src\t # vector (sve)" %}
  ins_encode %{
    FloatRegister src_reg = as_FloatRegister($src$$reg);
    loadStoreA_predicate(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
                         vector_element_basic_type(this, $src), $mem->opcode(),
                         as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}


// sve abs

instruct vabsAB(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (AbsVB src));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst, $src\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst$$reg), __ B,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsAS(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (AbsVS src));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst, $src\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst$$reg), __ H,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsAI(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (AbsVI src));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsAL(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (AbsVL src));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsAF(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (AbsVF src));
  ins_cost(SVE_COST);
  format %{ "sve_fabs $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fabs(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsAD(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (AbsVD src));
  ins_cost(SVE_COST);
  format %{ "sve_fabs $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fabs(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve add

instruct vaddAB(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (AddVB src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddAS(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (AddVS src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddAI(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (AddVI src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddAL(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (AddVL src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddAF(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (AddVF src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fadd(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddAD(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (AddVD src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fadd(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve and

instruct vandA(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (AndV src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_and  $dst, $src1, $src2\t# vector (sve)" %}
  ins_encode %{
    __ sve_and(as_FloatRegister($dst$$reg),
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve or

instruct vorA(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (OrV src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_orr  $dst, $src1, $src2\t# vector (sve)" %}
  ins_encode %{
    __ sve_orr(as_FloatRegister($dst$$reg),
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve xor

instruct vxorA(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (XorV src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_eor  $dst, $src1, $src2\t# vector (sve)" %}
  ins_encode %{
    __ sve_eor(as_FloatRegister($dst$$reg),
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve float div

instruct vdivAF(vecA dst_src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (DivVF dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vdivAD(vecA dst_src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (DivVD dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve max

instruct vmaxAF(vecA dst_src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst_src1 (MaxV dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fmax $dst_src1, $dst_src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmax(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaxAD(vecA dst_src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst_src1 (MaxV dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fmax $dst_src1, $dst_src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmax(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vminAF(vecA dst_src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst_src1 (MinV dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fmin $dst_src1, $dst_src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmin(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vminAD(vecA dst_src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst_src1 (MinV dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fmin $dst_src1, $dst_src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmin(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve fmla

// dst_src1 = dst_src1 + src2 * src3
instruct vfmlaAF(vecA dst_src1, vecA src2, vecA src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vfmlaAD(vecA dst_src1, vecA src2, vecA src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve fmls

// dst_src1 = dst_src1 + -src2 * src3
// dst_src1 = dst_src1 + src2 * -src3
instruct vfmlsAF(vecA dst_src1, vecA src2, vecA src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3)));
  match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + -src2 * src3
// dst_src1 = dst_src1 + src2 * -src3
instruct vfmlsAD(vecA dst_src1, vecA src2, vecA src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (FmaVD dst_src1 (Binary (NegVD src2) src3)));
  match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve fnmla

// dst_src1 = -dst_src1 + -src2 * src3
// dst_src1 = -dst_src1 + src2 * -src3
instruct vfnmlaAF(vecA dst_src1, vecA src2, vecA src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary (NegVF src2) src3)));
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = -dst_src1 + -src2 * src3
// dst_src1 = -dst_src1 + src2 * -src3
instruct vfnmlaAD(vecA dst_src1, vecA src2, vecA src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary (NegVD src2) src3)));
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve fnmls

// dst_src1 = -dst_src1 + src2 * src3
instruct vfnmlsAF(vecA dst_src1, vecA src2, vecA src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = -dst_src1 + src2 * src3
instruct vfnmlsAD(vecA dst_src1, vecA src2, vecA src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve mla

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaAB(vecA dst_src1, vecA src2, vecA src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaAS(vecA dst_src1, vecA src2, vecA src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaAI(vecA dst_src1, vecA src2, vecA src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaAL(vecA dst_src1, vecA src2, vecA src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve mls

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsAB(vecA dst_src1, vecA src2, vecA src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsAS(vecA dst_src1, vecA src2, vecA src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsAI(vecA dst_src1, vecA src2, vecA src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsAL(vecA dst_src1, vecA src2, vecA src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}


// sve mul

instruct vmulAB(vecA dst_src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst_src1 (MulVB dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulAS(vecA dst_src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst_src1 (MulVS dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulAI(vecA dst_src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (MulVI dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulAL(vecA dst_src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (MulVL dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulAF(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (MulVF src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmul(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulAD(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (MulVD src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmul(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve fneg

instruct vnegAF(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (NegVF src));
  ins_cost(SVE_COST);
  format %{ "sve_fneg $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fneg(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegAD(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (NegVD src));
  ins_cost(SVE_COST);
  format %{ "sve_fneg $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fneg(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve popcount vector

instruct vpopcountAI(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (PopCountVI src));
  format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t"  %}
  ins_encode %{
     __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve add reduction

instruct reduce_addAI(iRegINoSp dst, iRegIorL2I src1, vecA src2, vRegD tmp) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16 &&
            (n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT));
  match(Set dst (AddReductionVI src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_uaddv $tmp, $src2\t# vector (sve) (S)\n\t"
            "umov  $dst, $tmp, S, 0\n\t"
            "addw  $dst, $dst, $src1\t # add reduction S" %}
  ins_encode %{
    __ sve_uaddv(as_FloatRegister($tmp$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
    __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ S, 0);
    __ addw($dst$$Register, $dst$$Register, $src1$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addAL(iRegLNoSp dst, iRegL src1, vecA src2, vRegD tmp) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16 &&
            (n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG));
  match(Set dst (AddReductionVL src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_uaddv $tmp, $src2\t# vector (sve) (D)\n\t"
            "umov  $dst, $tmp, D, 0\n\t"
            "add  $dst, $dst, $src1\t # add reduction D" %}
  ins_encode %{
    __ sve_uaddv(as_FloatRegister($tmp$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
    __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ D, 0);
    __ add($dst$$Register, $dst$$Register, $src1$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addAF(vRegF src1_dst, vecA src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set src1_dst (AddReductionVF src1_dst src2));
  ins_cost(SVE_COST);
  format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addAD(vRegD src1_dst, vecA src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set src1_dst (AddReductionVD src1_dst src2));
  ins_cost(SVE_COST);
  format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve max reduction

instruct reduce_maxAF(vRegF dst, vRegF src1, vecA src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set dst (MaxReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_fmaxv $dst, $src2 # vector (sve) (S)\n\t"
            "fmaxs $dst, $dst, $src1\t # max reduction F" %}
  ins_encode %{
    __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
    __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxAD(vRegD dst, vRegD src1, vecA src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set dst (MaxReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_fmaxv $dst, $src2 # vector (sve) (S)\n\t"
            "fmaxs $dst, $dst, $src1\t # max reduction D" %}
  ins_encode %{
    __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
    __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve min reduction

instruct reduce_minAF(vRegF dst, vRegF src1, vecA src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set dst (MinReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_fminv $dst, $src2 # vector (sve) (S)\n\t"
            "fmins $dst, $dst, $src1\t # min reduction F" %}
  ins_encode %{
    __ sve_fminv(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
    __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minAD(vRegD dst, vRegD src1, vecA src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set dst (MinReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_fminv $dst, $src2 # vector (sve) (S)\n\t"
            "fmins $dst, $dst, $src1\t # min reduction D" %}
  ins_encode %{
    __ sve_fminv(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
    __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve vector Math.rint, floor, ceil

instruct vroundAD(vecA dst, vecA src, immI rmode) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (RoundDoubleModeV src rmode));
  format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
  ins_encode %{
    switch ($rmode$$constant) {
      case RoundDoubleModeNode::rmode_rint:
        __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
             ptrue, as_FloatRegister($src$$reg));
        break;
      case RoundDoubleModeNode::rmode_floor:
        __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
             ptrue, as_FloatRegister($src$$reg));
        break;
      case RoundDoubleModeNode::rmode_ceil:
        __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
             ptrue, as_FloatRegister($src$$reg));
        break;
    }
  %}
  ins_pipe(pipe_slow);
%}

// sve replicate

instruct replicateAB(vecA dst, iRegIorL2I src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (ReplicateB src));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $src\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateAS(vecA dst, iRegIorL2I src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (ReplicateS src));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $src\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateAI(vecA dst, iRegIorL2I src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (ReplicateI src));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateAL(vecA dst, iRegL src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (ReplicateL src));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}


instruct replicateAB_imm8(vecA dst, immI8 con) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (ReplicateB con));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $con\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ B, $con$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateAS_imm8(vecA dst, immI8_shift8 con) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (ReplicateS con));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $con\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ H, $con$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateAI_imm8(vecA dst, immI8_shift8 con) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (ReplicateI con));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $con\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ S, $con$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateAL_imm8(vecA dst, immL8_shift8 con) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (ReplicateL con));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $con\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ D, $con$$constant);
  %}
  ins_pipe(pipe_slow);
%}


instruct replicateAF(vecA dst, vRegF src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (ReplicateF src));
  ins_cost(SVE_COST);
  format %{ "sve_cpy  $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateAD(vecA dst, vRegD src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (ReplicateD src));
  ins_cost(SVE_COST);
  format %{ "sve_cpy  $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve shift

instruct vasrAB(vecA dst, vecA shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (RShiftVB dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst$$reg), __ B,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrAS(vecA dst, vecA shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (RShiftVS dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst$$reg), __ H,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrAI(vecA dst, vecA shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (RShiftVI dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrAL(vecA dst, vecA shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (RShiftVL dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslAB(vecA dst, vecA shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (LShiftVB dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslAS(vecA dst, vecA shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (LShiftVS dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslAI(vecA dst, vecA shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (LShiftVI dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslAL(vecA dst, vecA shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (LShiftVL dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrAB(vecA dst, vecA shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (URShiftVB dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrAS(vecA dst, vecA shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (URShiftVS dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrAI(vecA dst, vecA shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (URShiftVI dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrAL(vecA dst, vecA shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (URShiftVL dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrAB_imm(vecA dst, vecA src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (RShiftVB src shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    if (con >= 8) con = 7;
    __ sve_asr(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrAS_imm(vecA dst, vecA src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (RShiftVS src shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    if (con >= 16) con = 15;
    __ sve_asr(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrAI_imm(vecA dst, vecA src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (RShiftVI src shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_asr(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrAL_imm(vecA dst, vecA src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (RShiftVL src shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_asr(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrAB_imm(vecA dst, vecA src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (URShiftVB src shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    if (con >= 8) {
      __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrAS_imm(vecA dst, vecA src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (URShiftVS src shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    if (con >= 8) {
      __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrAI_imm(vecA dst, vecA src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (URShiftVI src shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrAL_imm(vecA dst, vecA src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (URShiftVL src shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslAB_imm(vecA dst, vecA src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (LShiftVB src shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con >= 8) {
      __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslAS_imm(vecA dst, vecA src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (LShiftVS src shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con >= 8) {
      __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslAI_imm(vecA dst, vecA src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (LShiftVI src shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslAL_imm(vecA dst, vecA src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (LShiftVL src shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntAB(vecA dst, iRegIorL2I cnt) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE));
  match(Set dst (LShiftCntV cnt));
  match(Set dst (RShiftCntV cnt));
  format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (B)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($cnt$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntAS(vecA dst, iRegIorL2I cnt) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
            (n->bottom_type()->is_vect()->element_basic_type() == T_CHAR)));
  match(Set dst (LShiftCntV cnt));
  match(Set dst (RShiftCntV cnt));
  format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (H)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($cnt$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntAI(vecA dst, iRegIorL2I cnt) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
  match(Set dst (LShiftCntV cnt));
  match(Set dst (RShiftCntV cnt));
  format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (S)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($cnt$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntAL(vecA dst, iRegIorL2I cnt) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
  match(Set dst (LShiftCntV cnt));
  match(Set dst (RShiftCntV cnt));
  format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (D)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($cnt$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve sqrt

instruct vsqrtAF(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (SqrtVF src));
  ins_cost(SVE_COST);
  format %{ "sve_fsqrt $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fsqrt(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsqrtAD(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (SqrtVD src));
  ins_cost(SVE_COST);
  format %{ "sve_fsqrt $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fsqrt(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// sve sub

instruct vsubAB(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (SubVB src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubAS(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (SubVS src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubAI(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (SubVI src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubAL(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (SubVL src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubAF(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (SubVF src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fsub(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubAD(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (SubVD src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fsub(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}