//
// Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, Arm Ltd. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License version 2 only, as
// published by the Free Software Foundation.
//
// This code is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
// version 2 for more details (a copy is included in the LICENSE file that
// accompanied this code).
//
// You should have received a copy of the GNU General Public License version
// 2 along with this work; if not, write to the Free Software Foundation,
// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
//
// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
// or visit www.oracle.com if you need additional information or have any
// questions.
//
//

dnl Generate the warning
// This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ----
dnl

// AArch64 SVE Architecture Description File

dnl
dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET($1,            $2,       $3     )
dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET(imm_type_abbr, imm_type, imm_len)
define(`OPERAND_VMEMORYA_IMMEDIATE_OFFSET', `
operand vmemA_imm$1Offset$3()
%{
  predicate(Address::offset_ok_for_sve_immed(n->get_$2(), $3,
            Matcher::scalable_vector_reg_size(T_BYTE)));
  match(Con$1);

  op_cost(0);
  format %{ %}
  interface(CONST_INTER);
%}')
dnl
// 4 bit signed offset -- for predicated load/store
OPERAND_VMEMORYA_IMMEDIATE_OFFSET(I, int,  4)
OPERAND_VMEMORYA_IMMEDIATE_OFFSET(L, long, 4)
dnl
dnl OPERAND_VMEMORYA_INDIRECT_OFFSET($1,            $2     )
dnl OPERAND_VMEMORYA_INDIRECT_OFFSET(imm_type_abbr, imm_len)
define(`OPERAND_VMEMORYA_INDIRECT_OFFSET', `
operand vmemA_indOff$1$2(iRegP reg, vmemA_imm$1Offset$2 off)
%{
  constraint(ALLOC_IN_RC(ptr_reg));
  match(AddP reg off);
  op_cost(0);
  format %{ "[$reg, $off, MUL VL]" %}
  interface(MEMORY_INTER) %{
    base($reg);
    `index'(0xffffffff);
    scale(0x0);
    disp($off);
  %}
%}')
dnl
OPERAND_VMEMORYA_INDIRECT_OFFSET(I, 4)
OPERAND_VMEMORYA_INDIRECT_OFFSET(L, 4)

opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);

source_hpp %{
  bool op_sve_supported(int opcode);
%}

source %{

  static inline BasicType vector_element_basic_type(const MachNode* n) {
    const TypeVect* vt = n->bottom_type()->is_vect();
    return vt->element_basic_type();
  }

  static inline BasicType vector_element_basic_type(const MachNode* use, const MachOper* opnd) {
    int def_idx = use->operand_index(opnd);
    Node* def = use->in(def_idx);
    const TypeVect* vt = def->bottom_type()->is_vect();
    return vt->element_basic_type();
  }

  typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
                                                             PRegister Pg, const Address &adr);

  // Predicated load/store, with optional ptrue to all elements of given predicate register.
  static void loadStoreA_predicate(C2_MacroAssembler masm, bool is_store,
                                   FloatRegister reg, PRegister pg, BasicType bt,
                                   int opcode, Register base, int index, int size, int disp) {
    sve_mem_insn_predicate insn;
    Assembler::SIMD_RegVariant type;
    int esize = type2aelembytes(bt);
    if (index == -1) {
      assert(size == 0, "unsupported address mode: scale size = %d", size);
      switch(esize) {
      case 1:
        insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
        type = Assembler::B;
        break;
      case 2:
        insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
        type = Assembler::H;
        break;
      case 4:
        insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
        type = Assembler::S;
        break;
      case 8:
        insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
        type = Assembler::D;
        break;
      default:
        assert(false, "unsupported");
        ShouldNotReachHere();
      }
      (masm.*insn)(reg, type, pg, Address(base, disp / Matcher::scalable_vector_reg_size(T_BYTE)));
    } else {
      assert(false, "unimplemented");
      ShouldNotReachHere();
    }
  }

  bool op_sve_supported(int opcode) {
    switch (opcode) {
      case Op_MulAddVS2VI:
        // No multiply reduction instructions
      case Op_MulReductionVD:
      case Op_MulReductionVF:
      case Op_MulReductionVI:
      case Op_MulReductionVL:
        // Others
      case Op_Extract:
      case Op_ExtractB:
      case Op_ExtractC:
      case Op_ExtractD:
      case Op_ExtractF:
      case Op_ExtractI:
      case Op_ExtractL:
      case Op_ExtractS:
      case Op_ExtractUB:
        return false;
      default:
        return true;
    }
  }

%}

definitions %{
  int_def SVE_COST             (200, 200);
%}


dnl
dnl ELEMENT_SHORT_CHART($1, $2)
dnl ELEMENT_SHORT_CHART(etype, node)
define(`ELEMENT_SHORT_CHAR',`ifelse(`$1', `T_SHORT',
  `($2->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
            ($2->bottom_type()->is_vect()->element_basic_type() == T_CHAR))',
   `($2->bottom_type()->is_vect()->element_basic_type() == $1)')')
dnl

// All SVE instructions

// sve vector load/store

// Use predicated vector load/store
instruct loadVA(vecA dst, vmemA mem) %{
  predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16);
  match(Set dst (LoadVector mem));
  ins_cost(SVE_COST);
  format %{ "sve_ldr $dst, $mem\t # vector (sve)" %}
  ins_encode %{
    FloatRegister dst_reg = as_FloatRegister($dst$$reg);
    loadStoreA_predicate(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
                         vector_element_basic_type(this), $mem->opcode(),
                         as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

instruct storeVA(vecA src, vmemA mem) %{
  predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16);
  match(Set mem (StoreVector mem src));
  ins_cost(SVE_COST);
  format %{ "sve_str $mem, $src\t # vector (sve)" %}
  ins_encode %{
    FloatRegister src_reg = as_FloatRegister($src$$reg);
    loadStoreA_predicate(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
                         vector_element_basic_type(this, $src), $mem->opcode(),
                         as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

dnl
dnl UNARY_OP_TRUE_PREDICATE_ETYPE($1,        $2,      $3,           $4,   $5,          %6  )
dnl UNARY_OP_TRUE_PREDICATE_ETYPE(insn_name, op_name, element_type, size, min_vec_len, insn)
define(`UNARY_OP_TRUE_PREDICATE_ETYPE', `
instruct $1(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= $5 &&
            n->bottom_type()->is_vect()->element_basic_type() == $3);
  match(Set dst ($2 src));
  ins_cost(SVE_COST);
  format %{ "$6 $dst, $src\t# vector (sve) ($4)" %}
  ins_encode %{
    __ $6(as_FloatRegister($dst$$reg), __ $4,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl

// sve abs
UNARY_OP_TRUE_PREDICATE_ETYPE(vabsAB, AbsVB, T_BYTE,   B, 16, sve_abs)
UNARY_OP_TRUE_PREDICATE_ETYPE(vabsAS, AbsVS, T_SHORT,  H, 8,  sve_abs)
UNARY_OP_TRUE_PREDICATE_ETYPE(vabsAI, AbsVI, T_INT,    S, 4,  sve_abs)
UNARY_OP_TRUE_PREDICATE_ETYPE(vabsAL, AbsVL, T_LONG,   D, 2,  sve_abs)
UNARY_OP_TRUE_PREDICATE_ETYPE(vabsAF, AbsVF, T_FLOAT,  S, 4,  sve_fabs)
UNARY_OP_TRUE_PREDICATE_ETYPE(vabsAD, AbsVD, T_DOUBLE, D, 2,  sve_fabs)
dnl
dnl BINARY_OP_UNPREDICATED($1,        $2       $3,   $4           $5  )
dnl BINARY_OP_UNPREDICATED(insn_name, op_name, size, min_vec_len, insn)
define(`BINARY_OP_UNPREDICATED', `
instruct $1(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= $4);
  match(Set dst ($2 src1 src2));
  ins_cost(SVE_COST);
  format %{ "$5 $dst, $src1, $src2\t # vector (sve) ($3)" %}
  ins_encode %{
    __ $5(as_FloatRegister($dst$$reg), __ $3,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl

// sve add
BINARY_OP_UNPREDICATED(vaddAB, AddVB, B, 16, sve_add)
BINARY_OP_UNPREDICATED(vaddAS, AddVS, H, 8,  sve_add)
BINARY_OP_UNPREDICATED(vaddAI, AddVI, S, 4,  sve_add)
BINARY_OP_UNPREDICATED(vaddAL, AddVL, D, 2,  sve_add)
BINARY_OP_UNPREDICATED(vaddAF, AddVF, S, 4,  sve_fadd)
BINARY_OP_UNPREDICATED(vaddAD, AddVD, D, 2,  sve_fadd)
dnl
dnl BINARY_OP_UNSIZED($1,        $2,      $3,          $4  )
dnl BINARY_OP_UNSIZED(insn_name, op_name, min_vec_len, insn)
define(`BINARY_OP_UNSIZED', `
instruct $1(vecA dst, vecA src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= $3);
  match(Set dst ($2 src1 src2));
  ins_cost(SVE_COST);
  format %{ "$4  $dst, $src1, $src2\t# vector (sve)" %}
  ins_encode %{
    __ $4(as_FloatRegister($dst$$reg),
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl

// sve and
BINARY_OP_UNSIZED(vandA, AndV, 16, sve_and)

// sve or
BINARY_OP_UNSIZED(vorA, OrV, 16, sve_orr)

// sve xor
BINARY_OP_UNSIZED(vxorA, XorV, 16, sve_eor)
dnl
dnl VDIVF($1,          $2  , $3         )
dnl VDIVF(name_suffix, size, min_vec_len)
define(`VDIVF', `
instruct vdivA$1(vecA dst_src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= $3);
  match(Set dst_src1 (DivV$1 dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) ($2)" %}
  ins_encode %{
    __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ $2,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl

// sve float div
VDIVF(F, S, 4)
VDIVF(D, D, 2)

dnl
dnl BINARY_OP_TRUE_PREDICATE_ETYPE($1,        $2,      $3,           $4,   $5,          $6  )
dnl BINARY_OP_TRUE_PREDICATE_ETYPE(insn_name, op_name, element_type, size, min_vec_len, insn)
define(`BINARY_OP_TRUE_PREDICATE_ETYPE', `
instruct $1(vecA dst_src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= $5 &&
            n->bottom_type()->is_vect()->element_basic_type() == $3);
  match(Set dst_src1 ($2 dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "$6 $dst_src1, $dst_src1, $src2\t # vector (sve) ($4)" %}
  ins_encode %{
    __ $6(as_FloatRegister($dst_src1$$reg), __ $4,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// sve max
BINARY_OP_TRUE_PREDICATE_ETYPE(vmaxAF, MaxV, T_FLOAT,  S, 4,  sve_fmax)
BINARY_OP_TRUE_PREDICATE_ETYPE(vmaxAD, MaxV, T_DOUBLE, D, 2,  sve_fmax)
BINARY_OP_TRUE_PREDICATE_ETYPE(vminAF, MinV, T_FLOAT,  S, 4,  sve_fmin)
BINARY_OP_TRUE_PREDICATE_ETYPE(vminAD, MinV, T_DOUBLE, D, 2,  sve_fmin)

dnl
dnl VFMLA($1           $2    $3         )
dnl VFMLA(name_suffix, size, min_vec_len)
define(`VFMLA', `
// dst_src1 = dst_src1 + src2 * src3
instruct vfmlaA$1(vecA dst_src1, vecA src2, vecA src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= $3);
  match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
  ins_encode %{
    __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ $2,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// sve fmla
VFMLA(F, S, 4)
VFMLA(D, D, 2)

dnl
dnl VFMLS($1           $2    $3         )
dnl VFMLS(name_suffix, size, min_vec_len)
define(`VFMLS', `
// dst_src1 = dst_src1 + -src2 * src3
// dst_src1 = dst_src1 + src2 * -src3
instruct vfmlsA$1(vecA dst_src1, vecA src2, vecA src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= $3);
  match(Set dst_src1 (FmaV$1 dst_src1 (Binary (NegV$1 src2) src3)));
  match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 (NegV$1 src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
  ins_encode %{
    __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ $2,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// sve fmls
VFMLS(F, S, 4)
VFMLS(D, D, 2)

dnl
dnl VFNMLA($1           $2    $3         )
dnl VFNMLA(name_suffix, size, min_vec_len)
define(`VFNMLA', `
// dst_src1 = -dst_src1 + -src2 * src3
// dst_src1 = -dst_src1 + src2 * -src3
instruct vfnmlaA$1(vecA dst_src1, vecA src2, vecA src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= $3);
  match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary (NegV$1 src2) src3)));
  match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary src2 (NegV$1 src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
  ins_encode %{
    __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ $2,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// sve fnmla
VFNMLA(F, S, 4)
VFNMLA(D, D, 2)

dnl
dnl VFNMLS($1           $2    $3         )
dnl VFNMLS(name_suffix, size, min_vec_len)
define(`VFNMLS', `
// dst_src1 = -dst_src1 + src2 * src3
instruct vfnmlsA$1(vecA dst_src1, vecA src2, vecA src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= $3);
  match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
  ins_encode %{
    __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ $2,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// sve fnmls
VFNMLS(F, S, 4)
VFNMLS(D, D, 2)

dnl
dnl VMLA($1           $2    $3         )
dnl VMLA(name_suffix, size, min_vec_len)
define(`VMLA', `
// dst_src1 = dst_src1 + src2 * src3
instruct vmlaA$1(vecA dst_src1, vecA src2, vecA src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= $3);
  match(Set dst_src1 (AddV$1 dst_src1 (MulV$1 src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) ($2)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ $2,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// sve mla
VMLA(B, B, 16)
VMLA(S, H, 8)
VMLA(I, S, 4)
VMLA(L, D, 2)

dnl
dnl VMLS($1           $2    $3         )
dnl VMLS(name_suffix, size, min_vec_len)
define(`VMLS', `
// dst_src1 = dst_src1 - src2 * src3
instruct vmlsA$1(vecA dst_src1, vecA src2, vecA src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= $3);
  match(Set dst_src1 (SubV$1 dst_src1 (MulV$1 src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) ($2)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ $2,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// sve mls
VMLS(B, B, 16)
VMLS(S, H, 8)
VMLS(I, S, 4)
VMLS(L, D, 2)

dnl
dnl BINARY_OP_TRUE_PREDICATE($1,        $2,      $3,   $4,          $5  )
dnl BINARY_OP_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
define(`BINARY_OP_TRUE_PREDICATE', `
instruct $1(vecA dst_src1, vecA src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= $4);
  match(Set dst_src1 ($2 dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "$5 $dst_src1, $dst_src1, $src2\t # vector (sve) ($3)" %}
  ins_encode %{
    __ $5(as_FloatRegister($dst_src1$$reg), __ $3,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl

// sve mul
BINARY_OP_TRUE_PREDICATE(vmulAB, MulVB, B, 16, sve_mul)
BINARY_OP_TRUE_PREDICATE(vmulAS, MulVS, H, 8,  sve_mul)
BINARY_OP_TRUE_PREDICATE(vmulAI, MulVI, S, 4,  sve_mul)
BINARY_OP_TRUE_PREDICATE(vmulAL, MulVL, D, 2,  sve_mul)
BINARY_OP_UNPREDICATED(vmulAF, MulVF, S, 4, sve_fmul)
BINARY_OP_UNPREDICATED(vmulAD, MulVD, D, 2, sve_fmul)

dnl
dnl UNARY_OP_TRUE_PREDICATE($1,        $2,      $3,   $4,            $5  )
dnl UNARY_OP_TRUE_PREDICATE(insn_name, op_name, size, min_vec_bytes, insn)
define(`UNARY_OP_TRUE_PREDICATE', `
instruct $1(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= $4);
  match(Set dst ($2 src));
  ins_cost(SVE_COST);
  format %{ "$5 $dst, $src\t# vector (sve) ($3)" %}
  ins_encode %{
    __ $5(as_FloatRegister($dst$$reg), __ $3,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// sve fneg
UNARY_OP_TRUE_PREDICATE(vnegAF, NegVF, S, 16, sve_fneg)
UNARY_OP_TRUE_PREDICATE(vnegAD, NegVD, D, 16, sve_fneg)

// sve popcount vector

instruct vpopcountAI(vecA dst, vecA src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (PopCountVI src));
  format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t"  %}
  ins_encode %{
     __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

dnl
dnl REDUCE_ADD($1,        $2,      $3,      $4,      $5,   $6,        $7   )
dnl REDUCE_ADD(insn_name, op_name, reg_dst, reg_src, size, elem_type, insn1)
define(`REDUCE_ADD', `
instruct $1($3 dst, $4 src1, vecA src2, vRegD tmp) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16 &&
            ELEMENT_SHORT_CHAR($6, n->in(2)));
  match(Set dst ($2 src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_uaddv $tmp, $src2\t# vector (sve) ($5)\n\t"
            "umov  $dst, $tmp, $5, 0\n\t"
            "$7  $dst, $dst, $src1\t # add reduction $5" %}
  ins_encode %{
    __ sve_uaddv(as_FloatRegister($tmp$$reg), __ $5,
         ptrue, as_FloatRegister($src2$$reg));
    __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ $5, 0);
    __ $7($dst$$Register, $dst$$Register, $src1$$Register);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REDUCE_ADDF($1,        $2,      $3,      $4  )
dnl REDUCE_ADDF(insn_name, op_name, reg_dst, size)
define(`REDUCE_ADDF', `
instruct $1($3 src1_dst, vecA src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set src1_dst ($2 src1_dst src2));
  ins_cost(SVE_COST);
  format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) ($4)" %}
  ins_encode %{
    __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
// sve add reduction
REDUCE_ADD(reduce_addAI, AddReductionVI, iRegINoSp, iRegIorL2I, S, T_INT, addw)
REDUCE_ADD(reduce_addAL, AddReductionVL, iRegLNoSp, iRegL, D, T_LONG, add)
REDUCE_ADDF(reduce_addAF, AddReductionVF, vRegF, S)
REDUCE_ADDF(reduce_addAD, AddReductionVD, vRegD, D)

dnl
dnl REDUCE_FMINMAX($1,      $2,          $3,           $4,   $5         )
dnl REDUCE_FMINMAX(min_max, name_suffix, element_type, size, reg_src_dst)
define(`REDUCE_FMINMAX', `
instruct reduce_$1A$2($5 dst, $5 src1, vecA src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_f$1v $dst, $src2 # vector (sve) (S)\n\t"
            "f$1s $dst, $dst, $src1\t # $1 reduction $2" %}
  ins_encode %{
    __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4,
         ptrue, as_FloatRegister($src2$$reg));
    __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl
// sve max reduction
REDUCE_FMINMAX(max, F, T_FLOAT,  S, vRegF)
REDUCE_FMINMAX(max, D, T_DOUBLE, D, vRegD)

// sve min reduction
REDUCE_FMINMAX(min, F, T_FLOAT,  S, vRegF)
REDUCE_FMINMAX(min, D, T_DOUBLE, D, vRegD)

// sve vector Math.rint, floor, ceil

instruct vroundAD(vecA dst, vecA src, immI rmode) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (RoundDoubleModeV src rmode));
  format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
  ins_encode %{
    switch ($rmode$$constant) {
      case RoundDoubleModeNode::rmode_rint:
        __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
             ptrue, as_FloatRegister($src$$reg));
        break;
      case RoundDoubleModeNode::rmode_floor:
        __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
             ptrue, as_FloatRegister($src$$reg));
        break;
      case RoundDoubleModeNode::rmode_ceil:
        __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
             ptrue, as_FloatRegister($src$$reg));
        break;
    }
  %}
  ins_pipe(pipe_slow);
%}
dnl
dnl REPLICATE($1,        $2,      $3,      $4,   $5         )
dnl REPLICATE(insn_name, op_name, reg_src, size, min_vec_len)
define(`REPLICATE', `
instruct $1(vecA dst, $3 src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= $5);
  match(Set dst ($2 src));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $src\t# vector (sve) ($4)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ $4, as_Register($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REPLICATE_IMM8($1,        $2,      $3,       $4,   $5         )
dnl REPLICATE_IMM8(insn_name, op_name, imm_type, size, min_vec_len)
define(`REPLICATE_IMM8', `
instruct $1(vecA dst, $3 con) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= $5);
  match(Set dst ($2 con));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $con\t# vector (sve) ($4)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ $4, $con$$constant);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl FREPLICATE($1,        $2,      $3,      $4,   $5         )
dnl FREPLICATE(insn_name, op_name, reg_src, size, min_vec_len)
define(`FREPLICATE', `
instruct $1(vecA dst, $3 src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= $5);
  match(Set dst ($2 src));
  ins_cost(SVE_COST);
  format %{ "sve_cpy  $dst, $src\t# vector (sve) ($4)" %}
  ins_encode %{
    __ sve_cpy(as_FloatRegister($dst$$reg), __ $4,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl

// sve replicate
REPLICATE(replicateAB, ReplicateB, iRegIorL2I, B, 16)
REPLICATE(replicateAS, ReplicateS, iRegIorL2I, H, 8)
REPLICATE(replicateAI, ReplicateI, iRegIorL2I, S, 4)
REPLICATE(replicateAL, ReplicateL, iRegL,      D, 2)

REPLICATE_IMM8(replicateAB_imm8, ReplicateB, immI8,        B, 16)
REPLICATE_IMM8(replicateAS_imm8, ReplicateS, immI8_shift8, H, 8)
REPLICATE_IMM8(replicateAI_imm8, ReplicateI, immI8_shift8, S, 4)
REPLICATE_IMM8(replicateAL_imm8, ReplicateL, immL8_shift8, D, 2)

FREPLICATE(replicateAF, ReplicateF, vRegF, S, 4)
FREPLICATE(replicateAD, ReplicateD, vRegD, D, 2)
dnl
dnl VSHIFT_TRUE_PREDICATE($1,        $2,      $3,   $4,          $5  )
dnl VSHIFT_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
define(`VSHIFT_TRUE_PREDICATE', `
instruct $1(vecA dst, vecA shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= $4);
  match(Set dst ($2 dst shift));
  ins_cost(SVE_COST);
  format %{ "$5 $dst, $dst, $shift\t# vector (sve) ($3)" %}
  ins_encode %{
    __ $5(as_FloatRegister($dst$$reg), __ $3,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VSHIFT_IMM_UNPREDICATE($1,        $2,      $3,   $4,          $5  )
dnl VSHIFT_IMM_UNPREDICATE(insn_name, op_name, size, min_vec_len, insn)
define(`VSHIFT_IMM_UNPREDICATE', `
instruct $1(vecA dst, vecA src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= $4);
  match(Set dst ($2 src shift));
  ins_cost(SVE_COST);
  format %{ "$5 $dst, $src, $shift\t# vector (sve) ($3)" %}
  ins_encode %{
    int con = (int)$shift$$constant;dnl
ifelse(eval(index(`$1', `vasr') == 0 || index(`$1', `vlsr') == 0), 1, `
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }')dnl
ifelse(eval(index(`$1', `vasr') == 0), 1, `ifelse(eval(index(`$3', `B') == 0), 1, `
    if (con >= 8) con = 7;')ifelse(eval(index(`$3', `H') == 0), 1, `
    if (con >= 16) con = 15;')')dnl
ifelse(eval((index(`$1', `vlsl') == 0 || index(`$1', `vlsr') == 0) && (index(`$3', `B') == 0 || index(`$3', `H') == 0)), 1, `
    if (con >= 8) {
      __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }')
    __ $5(as_FloatRegister($dst$$reg), __ $3,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VSHIFT_COUNT($1,        $2,   $3,          $4  )
dnl VSHIFT_COUNT(insn_name, size, min_vec_len, type)
define(`VSHIFT_COUNT', `
instruct $1(vecA dst, iRegIorL2I cnt) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= $3 &&
            ELEMENT_SHORT_CHAR($4, n));
  match(Set dst (LShiftCntV cnt));
  match(Set dst (RShiftCntV cnt));
  format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) ($2)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ $2, as_Register($cnt$$reg));
  %}
  ins_pipe(pipe_slow);
%}')dnl

// sve shift
VSHIFT_TRUE_PREDICATE(vasrAB, RShiftVB,  B, 16, sve_asr)
VSHIFT_TRUE_PREDICATE(vasrAS, RShiftVS,  H,  8, sve_asr)
VSHIFT_TRUE_PREDICATE(vasrAI, RShiftVI,  S,  4, sve_asr)
VSHIFT_TRUE_PREDICATE(vasrAL, RShiftVL,  D,  2, sve_asr)
VSHIFT_TRUE_PREDICATE(vlslAB, LShiftVB,  B, 16, sve_lsl)
VSHIFT_TRUE_PREDICATE(vlslAS, LShiftVS,  H,  8, sve_lsl)
VSHIFT_TRUE_PREDICATE(vlslAI, LShiftVI,  S,  4, sve_lsl)
VSHIFT_TRUE_PREDICATE(vlslAL, LShiftVL,  D,  2, sve_lsl)
VSHIFT_TRUE_PREDICATE(vlsrAB, URShiftVB, B, 16, sve_lsr)
VSHIFT_TRUE_PREDICATE(vlsrAS, URShiftVS, H,  8, sve_lsr)
VSHIFT_TRUE_PREDICATE(vlsrAI, URShiftVI, S,  4, sve_lsr)
VSHIFT_TRUE_PREDICATE(vlsrAL, URShiftVL, D,  2, sve_lsr)
VSHIFT_IMM_UNPREDICATE(vasrAB_imm, RShiftVB,  B, 16, sve_asr)
VSHIFT_IMM_UNPREDICATE(vasrAS_imm, RShiftVS,  H,  8, sve_asr)
VSHIFT_IMM_UNPREDICATE(vasrAI_imm, RShiftVI,  S,  4, sve_asr)
VSHIFT_IMM_UNPREDICATE(vasrAL_imm, RShiftVL,  D,  2, sve_asr)
VSHIFT_IMM_UNPREDICATE(vlsrAB_imm, URShiftVB, B, 16, sve_lsr)
VSHIFT_IMM_UNPREDICATE(vlsrAS_imm, URShiftVS, H,  8, sve_lsr)
VSHIFT_IMM_UNPREDICATE(vlsrAI_imm, URShiftVI, S,  4, sve_lsr)
VSHIFT_IMM_UNPREDICATE(vlsrAL_imm, URShiftVL, D,  2, sve_lsr)
VSHIFT_IMM_UNPREDICATE(vlslAB_imm, LShiftVB,  B, 16, sve_lsl)
VSHIFT_IMM_UNPREDICATE(vlslAS_imm, LShiftVS,  H,  8, sve_lsl)
VSHIFT_IMM_UNPREDICATE(vlslAI_imm, LShiftVI,  S,  4, sve_lsl)
VSHIFT_IMM_UNPREDICATE(vlslAL_imm, LShiftVL,  D,  2, sve_lsl)
VSHIFT_COUNT(vshiftcntAB, B, 16, T_BYTE)
VSHIFT_COUNT(vshiftcntAS, H,  8, T_SHORT)
VSHIFT_COUNT(vshiftcntAI, S,  4, T_INT)
VSHIFT_COUNT(vshiftcntAL, D,  2, T_LONG)

// sve sqrt
UNARY_OP_TRUE_PREDICATE(vsqrtAF, SqrtVF, S, 16, sve_fsqrt)
UNARY_OP_TRUE_PREDICATE(vsqrtAD, SqrtVD, D, 16, sve_fsqrt)

// sve sub
BINARY_OP_UNPREDICATED(vsubAB, SubVB, B, 16, sve_sub)
BINARY_OP_UNPREDICATED(vsubAS, SubVS, H, 8, sve_sub)
BINARY_OP_UNPREDICATED(vsubAI, SubVI, S, 4, sve_sub)
BINARY_OP_UNPREDICATED(vsubAL, SubVL, D, 2, sve_sub)
BINARY_OP_UNPREDICATED(vsubAF, SubVF, S, 4, sve_fsub)
BINARY_OP_UNPREDICATED(vsubAD, SubVD, D, 2, sve_fsub)