< prev index next >

src/cpu/aarch64/vm/aarch64.ad

Print this page
rev 8499 : 8086087: aarch64: add support for 64 bit vectors
Summary: Support 64 bit vectors
Reviewed-by: duke

@@ -863,10 +863,46 @@
     V29, V29_H,
     V30, V30_H,
     V31, V31_H
 );
 
+// Class for all 64bit vector registers
+reg_class vectord_reg(
+    V0, V0_H,
+    V1, V1_H,
+    V2, V2_H,
+    V3, V3_H,
+    V4, V4_H,
+    V5, V5_H,
+    V6, V6_H,
+    V7, V7_H,
+    V8, V8_H,
+    V9, V9_H,
+    V10, V10_H,
+    V11, V11_H,
+    V12, V12_H,
+    V13, V13_H,
+    V14, V14_H,
+    V15, V15_H,
+    V16, V16_H,
+    V17, V17_H,
+    V18, V18_H,
+    V19, V19_H,
+    V20, V20_H,
+    V21, V21_H,
+    V22, V22_H,
+    V23, V23_H,
+    V24, V24_H,
+    V25, V25_H,
+    V26, V26_H,
+    V27, V27_H,
+    V28, V28_H,
+    V29, V29_H,
+    V30, V30_H,
+    V31, V31_H
+);
+
 // Class for all 128bit vector registers
 reg_class vectorx_reg(
     V0, V0_H, V0_J, V0_K,
     V1, V1_H, V1_J, V1_K,
     V2, V2_H, V2_J, V2_K,

@@ -2131,21 +2167,25 @@
     return 0;            // Self copy, no move.
   }
 
   if (bottom_type()->isa_vect() != NULL) {
     uint len = 4;
+    uint ireg = ideal_reg();
+    assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
     if (cbuf) {
       MacroAssembler _masm(cbuf);
-      uint ireg = ideal_reg();
       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
-      assert(ireg == Op_VecX, "sanity");
       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
         // stack->stack
         int src_offset = ra_->reg2offset(src_lo);
         int dst_offset = ra_->reg2offset(dst_lo);
         assert((src_offset & 7) && (dst_offset & 7), "unaligned stack offset");
         len = 8;
+        if (ireg == Op_VecD) {
+          __ ldr(rscratch1, Address(sp, src_offset));
+          __ str(rscratch1, Address(sp, dst_offset));
+        } else {
         if (src_offset < 512) {
           __ ldp(rscratch1, rscratch2, Address(sp, src_offset));
         } else {
           __ ldr(rscratch1, Address(sp, src_offset));
           __ ldr(rscratch2, Address(sp, src_offset+4));

@@ -2156,28 +2196,36 @@
         } else {
           __ str(rscratch1, Address(sp, dst_offset));
           __ str(rscratch2, Address(sp, dst_offset+4));
           len += 4;
         }
+        }
       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
-        __ orr(as_FloatRegister(Matcher::_regEncode[dst_lo]), __ T16B,
+        __ orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
+               ireg == Op_VecD ? __ T8B : __ T16B,
                as_FloatRegister(Matcher::_regEncode[src_lo]),
                as_FloatRegister(Matcher::_regEncode[src_lo]));
       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
-        __ str(as_FloatRegister(Matcher::_regEncode[src_lo]), __ Q,
+        __ str(as_FloatRegister(Matcher::_regEncode[src_lo]),
+               ireg == Op_VecD ? __ D : __ Q,
                Address(sp, ra_->reg2offset(dst_lo)));
       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
-        __ ldr(as_FloatRegister(Matcher::_regEncode[dst_lo]), __ Q,
+        __ ldr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
+               ireg == Op_VecD ? __ D : __ Q,
                Address(sp, ra_->reg2offset(src_lo)));
       } else {
         ShouldNotReachHere();
       }
     } else if (st) {
       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
         // stack->stack
         int src_offset = ra_->reg2offset(src_lo);
         int dst_offset = ra_->reg2offset(dst_lo);
+        if (ireg == Op_VecD) {
+          st->print("ldr  rscratch1, [sp, #%d]", src_offset);
+          st->print("str  rscratch1, [sp, #%d]", dst_offset);
+        } else {
         if (src_offset < 512) {
           st->print("ldp  rscratch1, rscratch2, [sp, #%d]", src_offset);
         } else {
           st->print("ldr  rscratch1, [sp, #%d]", src_offset);
           st->print("\nldr  rscratch2, [sp, #%d]", src_offset+4);

@@ -2186,10 +2234,11 @@
           st->print("\nstp  rscratch1, rscratch2, [sp, #%d]", dst_offset);
         } else {
           st->print("\nstr  rscratch1, [sp, #%d]", dst_offset);
           st->print("\nstr  rscratch2, [sp, #%d]", dst_offset+4);
         }
+        }
         st->print("\t# vector spill, stack to stack");
       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
         st->print("mov  %s, %s\t# vector spill, reg to reg",
                    Matcher::regName[dst_lo], Matcher::regName[src_lo]);
       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {

@@ -2636,21 +2685,24 @@
 // Limits on vector size (number of elements) loaded into vector.
 const int Matcher::max_vector_size(const BasicType bt) {
   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 }
 const int Matcher::min_vector_size(const BasicType bt) {
-  //return (type2aelembytes(bt) == 1) ? 4 : 2;
-  // For the moment, only support 1 vector size, 128 bits
-  return max_vector_size(bt);
+  return (type2aelembytes(bt) == 1) ? 4 : 2;
 }
 
 // Vector ideal reg.
 const int Matcher::vector_ideal_reg(int len) {
-  return Op_VecX;
+  switch(len) {
+    case  4:
+    case  8: return Op_VecD;
+    case 16: return Op_VecX;
+  }
+  ShouldNotReachHere();
+  return 0;
 }
 
-// Only lowest bits of xmm reg are used for vector shift count.
 const int Matcher::vector_shift_count_ideal_reg(int size) {
   return Op_VecX;
 }
 
 // AES support not yet implemented

@@ -2658,13 +2710,11 @@
   return false;
 }
 
 // x86 supports misaligned vectors store/load.
 const bool Matcher::misaligned_vectors_ok() {
-  // TODO fixme
-  // return !AlignVector; // can be changed by flag
-  return false;
+  return !AlignVector; // can be changed by flag
 }
 
 // false => size gets scaled to BytesPerLong, ok.
 const bool Matcher::init_array_count_is_in_bytes = false;
 

@@ -3071,17 +3121,17 @@
     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
   %}
 
-  enc_class aarch64_enc_ldrvS(vecX dst, memory mem) %{
+  enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
   %}
 
-  enc_class aarch64_enc_ldrvD(vecX dst, memory mem) %{
+  enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
   %}
 

@@ -3157,17 +3207,17 @@
     FloatRegister src_reg = as_FloatRegister($src$$reg);
     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
   %}
 
-  enc_class aarch64_enc_strvS(vecX src, memory mem) %{
+  enc_class aarch64_enc_strvS(vecD src, memory mem) %{
     FloatRegister src_reg = as_FloatRegister($src$$reg);
     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
   %}
 
-  enc_class aarch64_enc_strvD(vecX src, memory mem) %{
+  enc_class aarch64_enc_strvD(vecD src, memory mem) %{
     FloatRegister src_reg = as_FloatRegister($src$$reg);
     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
   %}
 

@@ -5185,10 +5235,20 @@
   op_cost(0);
   format %{ %}
   interface(REG_INTER);
 %}
 
+operand vecD()
+%{
+  constraint(ALLOC_IN_RC(vectord_reg));
+  match(VecD);
+
+  op_cost(0);
+  format %{ %}
+  interface(REG_INTER);
+%}
+
 operand vecX()
 %{
   constraint(ALLOC_IN_RC(vectorx_reg));
   match(VecX);
 

@@ -13192,22 +13252,22 @@
 %}
 
 // ====================VECTOR INSTRUCTIONS=====================================
 
 // Load vector (32 bits)
-instruct loadV4(vecX dst, vmem mem)
+instruct loadV4(vecD dst, vmem mem)
 %{
   predicate(n->as_LoadVector()->memory_size() == 4);
   match(Set dst (LoadVector mem));
   ins_cost(4 * INSN_COST);
   format %{ "ldrs   $dst,$mem\t# vector (32 bits)" %}
   ins_encode( aarch64_enc_ldrvS(dst, mem) );
   ins_pipe(pipe_class_memory);
 %}
 
 // Load vector (64 bits)
-instruct loadV8(vecX dst, vmem mem)
+instruct loadV8(vecD dst, vmem mem)
 %{
   predicate(n->as_LoadVector()->memory_size() == 8);
   match(Set dst (LoadVector mem));
   ins_cost(4 * INSN_COST);
   format %{ "ldrd   $dst,$mem\t# vector (64 bits)" %}

@@ -13225,22 +13285,22 @@
   ins_encode( aarch64_enc_ldrvQ(dst, mem) );
   ins_pipe(pipe_class_memory);
 %}
 
 // Store Vector (32 bits)
-instruct storeV4(vecX src, vmem mem)
+instruct storeV4(vecD src, vmem mem)
 %{
   predicate(n->as_StoreVector()->memory_size() == 4);
   match(Set mem (StoreVector mem src));
   ins_cost(4 * INSN_COST);
   format %{ "strs   $mem,$src\t# vector (32 bits)" %}
   ins_encode( aarch64_enc_strvS(src, mem) );
   ins_pipe(pipe_class_memory);
 %}
 
 // Store Vector (64 bits)
-instruct storeV8(vecX src, vmem mem)
+instruct storeV8(vecD src, vmem mem)
 %{
   predicate(n->as_StoreVector()->memory_size() == 8);
   match(Set mem (StoreVector mem src));
   ins_cost(4 * INSN_COST);
   format %{ "strd   $mem,$src\t# vector (64 bits)" %}

@@ -13257,67 +13317,149 @@
   format %{ "strq   $mem,$src\t# vector (128 bits)" %}
   ins_encode( aarch64_enc_strvQ(src, mem) );
   ins_pipe(pipe_class_memory);
 %}
 
+instruct replicate8B(vecD dst, iRegIorL2I src)
+%{
+  predicate(n->as_Vector()->length() == 4 ||
+            n->as_Vector()->length() == 8);
+  match(Set dst (ReplicateB src));
+  ins_cost(INSN_COST);
+  format %{ "dup  $dst, $src\t# vector (8B)" %}
+  ins_encode %{
+    __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($src$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct replicate16B(vecX dst, iRegIorL2I src)
 %{
+  predicate(n->as_Vector()->length() == 16);
   match(Set dst (ReplicateB src));
   ins_cost(INSN_COST);
   format %{ "dup  $dst, $src\t# vector (16B)" %}
   ins_encode %{
     __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($src$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct replicate8B_imm(vecD dst, immI con)
+%{
+  predicate(n->as_Vector()->length() == 4 ||
+            n->as_Vector()->length() == 8);
+  match(Set dst (ReplicateB con));
+  ins_cost(INSN_COST);
+  format %{ "movi  $dst, $con\t# vector(8B)" %}
+  ins_encode %{
+    __ mov(as_FloatRegister($dst$$reg), __ T8B, $con$$constant);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct replicate16B_imm(vecX dst, immI con)
 %{
+  predicate(n->as_Vector()->length() == 16);
   match(Set dst (ReplicateB con));
   ins_cost(INSN_COST);
   format %{ "movi  $dst, $con\t# vector(16B)" %}
   ins_encode %{
     __ mov(as_FloatRegister($dst$$reg), __ T16B, $con$$constant);
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct replicate4S(vecD dst, iRegIorL2I src)
+%{
+  predicate(n->as_Vector()->length() == 2 ||
+            n->as_Vector()->length() == 4);
+  match(Set dst (ReplicateS src));
+  ins_cost(INSN_COST);
+  format %{ "dup  $dst, $src\t# vector (4S)" %}
+  ins_encode %{
+    __ dup(as_FloatRegister($dst$$reg), __ T4H, as_Register($src$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct replicate8S(vecX dst, iRegIorL2I src)
 %{
+  predicate(n->as_Vector()->length() == 8);
   match(Set dst (ReplicateS src));
   ins_cost(INSN_COST);
   format %{ "dup  $dst, $src\t# vector (8S)" %}
   ins_encode %{
     __ dup(as_FloatRegister($dst$$reg), __ T8H, as_Register($src$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct replicate4S_imm(vecD dst, immI con)
+%{
+  predicate(n->as_Vector()->length() == 2 ||
+            n->as_Vector()->length() == 4);
+  match(Set dst (ReplicateS con));
+  ins_cost(INSN_COST);
+  format %{ "movi  $dst, $con\t# vector(4H)" %}
+  ins_encode %{
+    __ mov(as_FloatRegister($dst$$reg), __ T4H, $con$$constant);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct replicate8S_imm(vecX dst, immI con)
 %{
+  predicate(n->as_Vector()->length() == 8);
   match(Set dst (ReplicateS con));
   ins_cost(INSN_COST);
   format %{ "movi  $dst, $con\t# vector(8H)" %}
   ins_encode %{
     __ mov(as_FloatRegister($dst$$reg), __ T8H, $con$$constant);
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct replicate2I(vecD dst, iRegIorL2I src)
+%{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (ReplicateI src));
+  ins_cost(INSN_COST);
+  format %{ "dup  $dst, $src\t# vector (2I)" %}
+  ins_encode %{
+    __ dup(as_FloatRegister($dst$$reg), __ T2S, as_Register($src$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct replicate4I(vecX dst, iRegIorL2I src)
 %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (ReplicateI src));
   ins_cost(INSN_COST);
   format %{ "dup  $dst, $src\t# vector (4I)" %}
   ins_encode %{
     __ dup(as_FloatRegister($dst$$reg), __ T4S, as_Register($src$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct replicate2I_imm(vecD dst, immI con)
+%{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (ReplicateI con));
+  ins_cost(INSN_COST);
+  format %{ "movi  $dst, $con\t# vector(2I)" %}
+  ins_encode %{
+    __ mov(as_FloatRegister($dst$$reg), __ T2S, $con$$constant);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct replicate4I_imm(vecX dst, immI con)
 %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (ReplicateI con));
   ins_cost(INSN_COST);
   format %{ "movi  $dst, $con\t# vector(4I)" %}
   ins_encode %{
     __ mov(as_FloatRegister($dst$$reg), __ T4S, $con$$constant);

@@ -13325,10 +13467,11 @@
   ins_pipe(pipe_class_default);
 %}
 
 instruct replicate2L(vecX dst, iRegL src)
 %{
+  predicate(n->as_Vector()->length() == 2);
   match(Set dst (ReplicateL src));
   ins_cost(INSN_COST);
   format %{ "dup  $dst, $src\t# vector (2L)" %}
   ins_encode %{
     __ dup(as_FloatRegister($dst$$reg), __ T2D, as_Register($src$$reg));

@@ -13336,10 +13479,11 @@
   ins_pipe(pipe_class_default);
 %}
 
 instruct replicate2L_zero(vecX dst, immI0 zero)
 %{
+  predicate(n->as_Vector()->length() == 2);
   match(Set dst (ReplicateI zero));
   ins_cost(INSN_COST);
   format %{ "movi  $dst, $zero\t# vector(4I)" %}
   ins_encode %{
     __ eor(as_FloatRegister($dst$$reg), __ T16B,

@@ -13347,12 +13491,26 @@
            as_FloatRegister($dst$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct replicate2F(vecD dst, vRegF src)
+%{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (ReplicateF src));
+  ins_cost(INSN_COST);
+  format %{ "dup  $dst, $src\t# vector (2F)" %}
+  ins_encode %{
+    __ dup(as_FloatRegister($dst$$reg), __ T2S,
+           as_FloatRegister($src$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct replicate4F(vecX dst, vRegF src)
 %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (ReplicateF src));
   ins_cost(INSN_COST);
   format %{ "dup  $dst, $src\t# vector (4F)" %}
   ins_encode %{
     __ dup(as_FloatRegister($dst$$reg), __ T4S,

@@ -13361,10 +13519,11 @@
   ins_pipe(pipe_class_default);
 %}
 
 instruct replicate2D(vecX dst, vRegD src)
 %{
+  predicate(n->as_Vector()->length() == 2);
   match(Set dst (ReplicateD src));
   ins_cost(INSN_COST);
   format %{ "dup  $dst, $src\t# vector (2D)" %}
   ins_encode %{
     __ dup(as_FloatRegister($dst$$reg), __ T2D,

@@ -13373,10 +13532,29 @@
   ins_pipe(pipe_class_default);
 %}
 
 // ====================REDUCTION ARITHMETIC====================================
 
+instruct reduce_add2I(iRegINoSp dst, iRegIorL2I src1, vecD src2, iRegI tmp, iRegI tmp2)
+%{
+  match(Set dst (AddReductionVI src1 src2));
+  ins_cost(INSN_COST);
+  effect(TEMP tmp, TEMP tmp2);
+  format %{ "umov  $tmp, $src2, S, 0\n\t"
+            "umov  $tmp2, $src2, S, 1\n\t"
+            "addw  $dst, $src1, $tmp\n\t"
+            "addw  $dst, $dst, $tmp2\t add reduction2i"
+  %}
+  ins_encode %{
+    __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 0);
+    __ umov($tmp2$$Register, as_FloatRegister($src2$$reg), __ S, 1);
+    __ addw($dst$$Register, $src1$$Register, $tmp$$Register);
+    __ addw($dst$$Register, $dst$$Register, $tmp2$$Register);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct reduce_add4I(iRegINoSp dst, iRegIorL2I src1, vecX src2, vecX tmp, iRegI tmp2)
 %{
   match(Set dst (AddReductionVI src1 src2));
   ins_cost(INSN_COST);
   effect(TEMP tmp, TEMP tmp2);

@@ -13391,10 +13569,29 @@
     __ addw($dst$$Register, $tmp2$$Register, $src1$$Register);
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct reduce_mul2I(iRegINoSp dst, iRegIorL2I src1, vecD src2, iRegI tmp)
+%{
+  match(Set dst (MulReductionVI src1 src2));
+  ins_cost(INSN_COST);
+  effect(TEMP tmp, TEMP dst);
+  format %{ "umov  $tmp, $src2, S, 0\n\t"
+            "mul   $dst, $tmp, $src1\n\t"
+            "umov  $tmp, $src2, S, 1\n\t"
+            "mul   $dst, $tmp, $dst\t mul reduction2i\n\t"
+  %}
+  ins_encode %{
+    __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 0);
+    __ mul($dst$$Register, $tmp$$Register, $src1$$Register);
+    __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 1);
+    __ mul($dst$$Register, $tmp$$Register, $dst$$Register);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct reduce_mul4I(iRegINoSp dst, iRegIorL2I src1, vecX src2, vecX tmp, iRegI tmp2)
 %{
   match(Set dst (MulReductionVI src1 src2));
   ins_cost(INSN_COST);
   effect(TEMP tmp, TEMP tmp2, TEMP dst);

@@ -13416,10 +13613,30 @@
     __ mul($dst$$Register, $tmp2$$Register, $dst$$Register);
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct reduce_add2F(vRegF dst, vRegF src1, vecD src2, vecD tmp)
+%{
+  match(Set dst (AddReductionVF src1 src2));
+  ins_cost(INSN_COST);
+  effect(TEMP tmp, TEMP dst);
+  format %{ "fadds $dst, $src1, $src2\n\t"
+            "ins   $tmp, S, $src2, 0, 1\n\t"
+            "fadds $dst, $dst, $tmp\t add reduction2f"
+  %}
+  ins_encode %{
+    __ fadds(as_FloatRegister($dst$$reg),
+             as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
+    __ ins(as_FloatRegister($tmp$$reg), __ S,
+           as_FloatRegister($src2$$reg), 0, 1);
+    __ fadds(as_FloatRegister($dst$$reg),
+             as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct reduce_add4F(vRegF dst, vRegF src1, vecX src2, vecX tmp)
 %{
   match(Set dst (AddReductionVF src1 src2));
   ins_cost(INSN_COST);
   effect(TEMP tmp, TEMP dst);

@@ -13448,10 +13665,30 @@
              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct reduce_mul2F(vRegF dst, vRegF src1, vecD src2, vecD tmp)
+%{
+  match(Set dst (MulReductionVF src1 src2));
+  ins_cost(INSN_COST);
+  effect(TEMP tmp, TEMP dst);
+  format %{ "fmuls $dst, $src1, $src2\n\t"
+            "ins   $tmp, S, $src2, 0, 1\n\t"
+            "fmuls $dst, $dst, $tmp\t add reduction4f"
+  %}
+  ins_encode %{
+    __ fmuls(as_FloatRegister($dst$$reg),
+             as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
+    __ ins(as_FloatRegister($tmp$$reg), __ S,
+           as_FloatRegister($src2$$reg), 0, 1);
+    __ fmuls(as_FloatRegister($dst$$reg),
+             as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct reduce_mul4F(vRegF dst, vRegF src1, vecX src2, vecX tmp)
 %{
   match(Set dst (MulReductionVF src1 src2));
   ins_cost(INSN_COST);
   effect(TEMP tmp, TEMP dst);

@@ -13524,12 +13761,28 @@
 
 // ====================VECTOR ARITHMETIC=======================================
 
 // --------------------------------- ADD --------------------------------------
 
+instruct vadd8B(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length() == 4 ||
+            n->as_Vector()->length() == 8);
+  match(Set dst (AddVB src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "addv  $dst,$src1,$src2\t# vector (8B)" %}
+  ins_encode %{
+    __ addv(as_FloatRegister($dst$$reg), __ T8B,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vadd16B(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 16);
   match(Set dst (AddVB src1 src2));
   ins_cost(INSN_COST);
   format %{ "addv  $dst,$src1,$src2\t# vector (16B)" %}
   ins_encode %{
     __ addv(as_FloatRegister($dst$$reg), __ T16B,

@@ -13537,12 +13790,28 @@
             as_FloatRegister($src2$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vadd4S(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length() == 2 ||
+            n->as_Vector()->length() == 4);
+  match(Set dst (AddVS src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "addv  $dst,$src1,$src2\t# vector (4H)" %}
+  ins_encode %{
+    __ addv(as_FloatRegister($dst$$reg), __ T4H,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vadd8S(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 8);
   match(Set dst (AddVS src1 src2));
   ins_cost(INSN_COST);
   format %{ "addv  $dst,$src1,$src2\t# vector (8H)" %}
   ins_encode %{
     __ addv(as_FloatRegister($dst$$reg), __ T8H,

@@ -13550,12 +13819,27 @@
             as_FloatRegister($src2$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vadd2I(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (AddVI src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "addv  $dst,$src1,$src2\t# vector (2S)" %}
+  ins_encode %{
+    __ addv(as_FloatRegister($dst$$reg), __ T2S,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vadd4I(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (AddVI src1 src2));
   ins_cost(INSN_COST);
   format %{ "addv  $dst,$src1,$src2\t# vector (4S)" %}
   ins_encode %{
     __ addv(as_FloatRegister($dst$$reg), __ T4S,

@@ -13565,10 +13849,11 @@
   ins_pipe(pipe_class_default);
 %}
 
 instruct vadd2L(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 2);
   match(Set dst (AddVL src1 src2));
   ins_cost(INSN_COST);
   format %{ "addv  $dst,$src1,$src2\t# vector (2L)" %}
   ins_encode %{
     __ addv(as_FloatRegister($dst$$reg), __ T2D,

@@ -13576,12 +13861,27 @@
             as_FloatRegister($src2$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vadd2F(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (AddVF src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "fadd  $dst,$src1,$src2\t# vector (2S)" %}
+  ins_encode %{
+    __ fadd(as_FloatRegister($dst$$reg), __ T2S,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vadd4F(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (AddVF src1 src2));
   ins_cost(INSN_COST);
   format %{ "fadd  $dst,$src1,$src2\t# vector (4S)" %}
   ins_encode %{
     __ fadd(as_FloatRegister($dst$$reg), __ T4S,

@@ -13604,12 +13904,28 @@
   ins_pipe(pipe_class_default);
 %}
 
 // --------------------------------- SUB --------------------------------------
 
+instruct vsub8B(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length() == 4 ||
+            n->as_Vector()->length() == 8);
+  match(Set dst (SubVB src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "subv  $dst,$src1,$src2\t# vector (8B)" %}
+  ins_encode %{
+    __ subv(as_FloatRegister($dst$$reg), __ T8B,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsub16B(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 16);
   match(Set dst (SubVB src1 src2));
   ins_cost(INSN_COST);
   format %{ "subv  $dst,$src1,$src2\t# vector (16B)" %}
   ins_encode %{
     __ subv(as_FloatRegister($dst$$reg), __ T16B,

@@ -13617,12 +13933,28 @@
             as_FloatRegister($src2$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsub4S(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length() == 2 ||
+            n->as_Vector()->length() == 4);
+  match(Set dst (SubVS src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "subv  $dst,$src1,$src2\t# vector (4H)" %}
+  ins_encode %{
+    __ subv(as_FloatRegister($dst$$reg), __ T4H,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsub8S(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 8);
   match(Set dst (SubVS src1 src2));
   ins_cost(INSN_COST);
   format %{ "subv  $dst,$src1,$src2\t# vector (8H)" %}
   ins_encode %{
     __ subv(as_FloatRegister($dst$$reg), __ T8H,

@@ -13630,12 +13962,27 @@
             as_FloatRegister($src2$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsub2I(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (SubVI src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "subv  $dst,$src1,$src2\t# vector (2S)" %}
+  ins_encode %{
+    __ subv(as_FloatRegister($dst$$reg), __ T2S,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsub4I(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (SubVI src1 src2));
   ins_cost(INSN_COST);
   format %{ "subv  $dst,$src1,$src2\t# vector (4S)" %}
   ins_encode %{
     __ subv(as_FloatRegister($dst$$reg), __ T4S,

@@ -13645,10 +13992,11 @@
   ins_pipe(pipe_class_default);
 %}
 
 instruct vsub2L(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 2);
   match(Set dst (SubVL src1 src2));
   ins_cost(INSN_COST);
   format %{ "subv  $dst,$src1,$src2\t# vector (2L)" %}
   ins_encode %{
     __ subv(as_FloatRegister($dst$$reg), __ T2D,

@@ -13656,12 +14004,27 @@
             as_FloatRegister($src2$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsub2F(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (AddVF src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "fsub  $dst,$src1,$src2\t# vector (2S)" %}
+  ins_encode %{
+    __ fsub(as_FloatRegister($dst$$reg), __ T2S,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsub4F(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (SubVF src1 src2));
   ins_cost(INSN_COST);
   format %{ "fsub  $dst,$src1,$src2\t# vector (4S)" %}
   ins_encode %{
     __ fsub(as_FloatRegister($dst$$reg), __ T4S,

@@ -13671,10 +14034,11 @@
   ins_pipe(pipe_class_default);
 %}
 
 instruct vsub2D(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 2);
   match(Set dst (SubVD src1 src2));
   ins_cost(INSN_COST);
   format %{ "fsub  $dst,$src1,$src2\t# vector (2D)" %}
   ins_encode %{
     __ fsub(as_FloatRegister($dst$$reg), __ T2D,

@@ -13684,12 +14048,28 @@
   ins_pipe(pipe_class_default);
 %}
 
 // --------------------------------- MUL --------------------------------------
 
+instruct vmul4S(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length() == 2 ||
+            n->as_Vector()->length() == 4);
+  match(Set dst (MulVS src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "mulv  $dst,$src1,$src2\t# vector (4H)" %}
+  ins_encode %{
+    __ mulv(as_FloatRegister($dst$$reg), __ T4H,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vmul8S(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 8);
   match(Set dst (MulVS src1 src2));
   ins_cost(INSN_COST);
   format %{ "mulv  $dst,$src1,$src2\t# vector (8H)" %}
   ins_encode %{
     __ mulv(as_FloatRegister($dst$$reg), __ T8H,

@@ -13697,12 +14077,27 @@
             as_FloatRegister($src2$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vmul2I(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (MulVI src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "mulv  $dst,$src1,$src2\t# vector (2S)" %}
+  ins_encode %{
+    __ mulv(as_FloatRegister($dst$$reg), __ T2S,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vmul4I(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (MulVI src1 src2));
   ins_cost(INSN_COST);
   format %{ "mulv  $dst,$src1,$src2\t# vector (4S)" %}
   ins_encode %{
     __ mulv(as_FloatRegister($dst$$reg), __ T4S,

@@ -13710,12 +14105,27 @@
             as_FloatRegister($src2$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vmul2F(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (MulVF src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "fmul  $dst,$src1,$src2\t# vector (2S)" %}
+  ins_encode %{
+    __ fmul(as_FloatRegister($dst$$reg), __ T2S,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vmul4F(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (MulVF src1 src2));
   ins_cost(INSN_COST);
   format %{ "fmul  $dst,$src1,$src2\t# vector (4S)" %}
   ins_encode %{
     __ fmul(as_FloatRegister($dst$$reg), __ T4S,

@@ -13725,10 +14135,11 @@
   ins_pipe(pipe_class_default);
 %}
 
 instruct vmul2D(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 2);
   match(Set dst (MulVD src1 src2));
   ins_cost(INSN_COST);
   format %{ "fmul  $dst,$src1,$src2\t# vector (2D)" %}
   ins_encode %{
     __ fmul(as_FloatRegister($dst$$reg), __ T2D,

@@ -13738,12 +14149,27 @@
   ins_pipe(pipe_class_default);
 %}
 
 // --------------------------------- DIV --------------------------------------
 
+instruct vdiv2F(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (DivVF src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "fdiv  $dst,$src1,$src2\t# vector (2S)" %}
+  ins_encode %{
+    __ fdiv(as_FloatRegister($dst$$reg), __ T2S,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vdiv4F(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (DivVF src1 src2));
   ins_cost(INSN_COST);
   format %{ "fdiv  $dst,$src1,$src2\t# vector (4S)" %}
   ins_encode %{
     __ fdiv(as_FloatRegister($dst$$reg), __ T4S,

@@ -13753,10 +14179,11 @@
   ins_pipe(pipe_class_default);
 %}
 
 instruct vdiv2D(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length() == 2);
   match(Set dst (DivVD src1 src2));
   ins_cost(INSN_COST);
   format %{ "fdiv  $dst,$src1,$src2\t# vector (2D)" %}
   ins_encode %{
     __ fdiv(as_FloatRegister($dst$$reg), __ T2D,

@@ -13766,12 +14193,28 @@
   ins_pipe(pipe_class_default);
 %}
 
 // --------------------------------- AND --------------------------------------
 
+instruct vand8B(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length_in_bytes() == 4 ||
+            n->as_Vector()->length_in_bytes() == 8);
+  match(Set dst (AndV src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "and  $dst,$src1,$src2\t# vector (8B)" %}
+  ins_encode %{
+    __ andr(as_FloatRegister($dst$$reg), __ T8B,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vand16B(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length_in_bytes() == 16);
   match(Set dst (AndV src1 src2));
   ins_cost(INSN_COST);
   format %{ "and  $dst,$src1,$src2\t# vector (16B)" %}
   ins_encode %{
     __ andr(as_FloatRegister($dst$$reg), __ T16B,

@@ -13781,12 +14224,28 @@
   ins_pipe(pipe_class_default);
 %}
 
 // --------------------------------- OR ---------------------------------------
 
+instruct vor8B(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length_in_bytes() == 4 ||
+            n->as_Vector()->length_in_bytes() == 8);
+  match(Set dst (OrV src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "and  $dst,$src1,$src2\t# vector (8B)" %}
+  ins_encode %{
+    __ orr(as_FloatRegister($dst$$reg), __ T8B,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vor16B(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length_in_bytes() == 16);
   match(Set dst (OrV src1 src2));
   ins_cost(INSN_COST);
   format %{ "orr  $dst,$src1,$src2\t# vector (16B)" %}
   ins_encode %{
     __ orr(as_FloatRegister($dst$$reg), __ T16B,

@@ -13796,12 +14255,28 @@
   ins_pipe(pipe_class_default);
 %}
 
 // --------------------------------- XOR --------------------------------------
 
+instruct vxor8B(vecD dst, vecD src1, vecD src2)
+%{
+  predicate(n->as_Vector()->length_in_bytes() == 4 ||
+            n->as_Vector()->length_in_bytes() == 8);
+  match(Set dst (XorV src1 src2));
+  ins_cost(INSN_COST);
+  format %{ "xor  $dst,$src1,$src2\t# vector (8B)" %}
+  ins_encode %{
+    __ eor(as_FloatRegister($dst$$reg), __ T8B,
+            as_FloatRegister($src1$$reg),
+            as_FloatRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vxor16B(vecX dst, vecX src1, vecX src2)
 %{
+  predicate(n->as_Vector()->length_in_bytes() == 16);
   match(Set dst (XorV src1 src2));
   ins_cost(INSN_COST);
   format %{ "xor  $dst,$src1,$src2\t# vector (16B)" %}
   ins_encode %{
     __ eor(as_FloatRegister($dst$$reg), __ T16B,

@@ -13831,11 +14306,27 @@
     __ negr(as_FloatRegister($dst$$reg), __ T16B, as_FloatRegister($dst$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsll8B(vecD dst, vecD src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 4 ||
+            n->as_Vector()->length() == 8);
+  match(Set dst (LShiftVB src shift));
+  match(Set dst (RShiftVB src shift));
+  ins_cost(INSN_COST);
+  format %{ "sshl  $dst,$src,$shift\t# vector (8B)" %}
+  ins_encode %{
+    __ sshl(as_FloatRegister($dst$$reg), __ T8B,
+            as_FloatRegister($src$$reg),
+            as_FloatRegister($shift$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsll16B(vecX dst, vecX src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 16);
   match(Set dst (LShiftVB src shift));
   match(Set dst (RShiftVB src shift));
   ins_cost(INSN_COST);
   format %{ "sshl  $dst,$src,$shift\t# vector (16B)" %}
   ins_encode %{

@@ -13844,11 +14335,26 @@
             as_FloatRegister($shift$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsrl8B(vecD dst, vecD src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 4 ||
+            n->as_Vector()->length() == 8);
+  match(Set dst (URShiftVB src shift));
+  ins_cost(INSN_COST);
+  format %{ "ushl  $dst,$src,$shift\t# vector (8B)" %}
+  ins_encode %{
+    __ ushl(as_FloatRegister($dst$$reg), __ T8B,
+            as_FloatRegister($src$$reg),
+            as_FloatRegister($shift$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsrl16B(vecX dst, vecX src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 16);
   match(Set dst (URShiftVB src shift));
   ins_cost(INSN_COST);
   format %{ "ushl  $dst,$src,$shift\t# vector (16B)" %}
   ins_encode %{
     __ ushl(as_FloatRegister($dst$$reg), __ T16B,

@@ -13856,11 +14362,32 @@
             as_FloatRegister($shift$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
+  predicate(n->as_Vector()->length() == 4 ||
+            n->as_Vector()->length() == 8);
+  match(Set dst (LShiftVB src shift));
+  ins_cost(INSN_COST);
+  format %{ "shl    $dst, $src, $shift\t# vector (8B)" %}
+  ins_encode %{
+    int sh = (int)$shift$$constant & 31;
+    if (sh >= 8) {
+      __ eor(as_FloatRegister($dst$$reg), __ T8B,
+             as_FloatRegister($src$$reg),
+             as_FloatRegister($src$$reg));
+    } else {
+      __ shl(as_FloatRegister($dst$$reg), __ T8B,
+             as_FloatRegister($src$$reg), sh);
+    }
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
+  predicate(n->as_Vector()->length() == 16);
   match(Set dst (LShiftVB src shift));
   ins_cost(INSN_COST);
   format %{ "shl    $dst, $src, $shift\t# vector (16B)" %}
   ins_encode %{
     int sh = (int)$shift$$constant & 31;

@@ -13874,11 +14401,28 @@
     }
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{
+  predicate(n->as_Vector()->length() == 4 ||
+            n->as_Vector()->length() == 8);
+  match(Set dst (RShiftVB src shift));
+  ins_cost(INSN_COST);
+  format %{ "sshr    $dst, $src, $shift\t# vector (8B)" %}
+  ins_encode %{
+    int sh = (int)$shift$$constant & 31;
+    if (sh >= 8) sh = 7;
+    sh = -sh & 7;
+    __ sshr(as_FloatRegister($dst$$reg), __ T8B,
+           as_FloatRegister($src$$reg), sh);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
+  predicate(n->as_Vector()->length() == 16);
   match(Set dst (RShiftVB src shift));
   ins_cost(INSN_COST);
   format %{ "sshr    $dst, $src, $shift\t# vector (16B)" %}
   ins_encode %{
     int sh = (int)$shift$$constant & 31;

@@ -13888,11 +14432,32 @@
            as_FloatRegister($src$$reg), sh);
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{
+  predicate(n->as_Vector()->length() == 4 ||
+            n->as_Vector()->length() == 8);
+  match(Set dst (URShiftVB src shift));
+  ins_cost(INSN_COST);
+  format %{ "ushr    $dst, $src, $shift\t# vector (8B)" %}
+  ins_encode %{
+    int sh = (int)$shift$$constant & 31;
+    if (sh >= 8) {
+      __ eor(as_FloatRegister($dst$$reg), __ T8B,
+             as_FloatRegister($src$$reg),
+             as_FloatRegister($src$$reg));
+    } else {
+      __ ushr(as_FloatRegister($dst$$reg), __ T8B,
+             as_FloatRegister($src$$reg), -sh & 7);
+    }
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsrl16B_imm(vecX dst, vecX src, immI shift) %{
+  predicate(n->as_Vector()->length() == 16);
   match(Set dst (URShiftVB src shift));
   ins_cost(INSN_COST);
   format %{ "ushr    $dst, $src, $shift\t# vector (16B)" %}
   ins_encode %{
     int sh = (int)$shift$$constant & 31;

@@ -13906,11 +14471,27 @@
     }
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsll4S(vecD dst, vecD src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 2 ||
+            n->as_Vector()->length() == 4);
+  match(Set dst (LShiftVS src shift));
+  match(Set dst (RShiftVS src shift));
+  ins_cost(INSN_COST);
+  format %{ "sshl  $dst,$src,$shift\t# vector (4H)" %}
+  ins_encode %{
+    __ sshl(as_FloatRegister($dst$$reg), __ T4H,
+            as_FloatRegister($src$$reg),
+            as_FloatRegister($shift$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsll8S(vecX dst, vecX src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 8);
   match(Set dst (LShiftVS src shift));
   match(Set dst (RShiftVS src shift));
   ins_cost(INSN_COST);
   format %{ "sshl  $dst,$src,$shift\t# vector (8H)" %}
   ins_encode %{

@@ -13919,11 +14500,26 @@
             as_FloatRegister($shift$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsrl4S(vecD dst, vecD src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 2 ||
+            n->as_Vector()->length() == 4);
+  match(Set dst (URShiftVS src shift));
+  ins_cost(INSN_COST);
+  format %{ "ushl  $dst,$src,$shift\t# vector (4H)" %}
+  ins_encode %{
+    __ ushl(as_FloatRegister($dst$$reg), __ T4H,
+            as_FloatRegister($src$$reg),
+            as_FloatRegister($shift$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsrl8S(vecX dst, vecX src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 8);
   match(Set dst (URShiftVS src shift));
   ins_cost(INSN_COST);
   format %{ "ushl  $dst,$src,$shift\t# vector (8H)" %}
   ins_encode %{
     __ ushl(as_FloatRegister($dst$$reg), __ T8H,

@@ -13931,11 +14527,32 @@
             as_FloatRegister($shift$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
+  predicate(n->as_Vector()->length() == 2 ||
+            n->as_Vector()->length() == 4);
+  match(Set dst (LShiftVS src shift));
+  ins_cost(INSN_COST);
+  format %{ "shl    $dst, $src, $shift\t# vector (4H)" %}
+  ins_encode %{
+    int sh = (int)$shift$$constant & 31;
+    if (sh >= 16) {
+      __ eor(as_FloatRegister($dst$$reg), __ T8B,
+             as_FloatRegister($src$$reg),
+             as_FloatRegister($src$$reg));
+    } else {
+      __ shl(as_FloatRegister($dst$$reg), __ T4H,
+             as_FloatRegister($src$$reg), sh);
+    }
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
+  predicate(n->as_Vector()->length() == 8);
   match(Set dst (LShiftVS src shift));
   ins_cost(INSN_COST);
   format %{ "shl    $dst, $src, $shift\t# vector (8H)" %}
   ins_encode %{
     int sh = (int)$shift$$constant & 31;

@@ -13949,11 +14566,28 @@
     }
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{
+  predicate(n->as_Vector()->length() == 2 ||
+            n->as_Vector()->length() == 4);
+  match(Set dst (RShiftVS src shift));
+  ins_cost(INSN_COST);
+  format %{ "sshr    $dst, $src, $shift\t# vector (4H)" %}
+  ins_encode %{
+    int sh = (int)$shift$$constant & 31;
+    if (sh >= 16) sh = 15;
+    sh = -sh & 15;
+    __ sshr(as_FloatRegister($dst$$reg), __ T4H,
+           as_FloatRegister($src$$reg), sh);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
+  predicate(n->as_Vector()->length() == 8);
   match(Set dst (RShiftVS src shift));
   ins_cost(INSN_COST);
   format %{ "sshr    $dst, $src, $shift\t# vector (8H)" %}
   ins_encode %{
     int sh = (int)$shift$$constant & 31;

@@ -13963,11 +14597,32 @@
            as_FloatRegister($src$$reg), sh);
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{
+  predicate(n->as_Vector()->length() == 2 ||
+            n->as_Vector()->length() == 4);
+  match(Set dst (URShiftVS src shift));
+  ins_cost(INSN_COST);
+  format %{ "ushr    $dst, $src, $shift\t# vector (4H)" %}
+  ins_encode %{
+    int sh = (int)$shift$$constant & 31;
+    if (sh >= 16) {
+      __ eor(as_FloatRegister($dst$$reg), __ T8B,
+             as_FloatRegister($src$$reg),
+             as_FloatRegister($src$$reg));
+    } else {
+      __ ushr(as_FloatRegister($dst$$reg), __ T4H,
+             as_FloatRegister($src$$reg), -sh & 15);
+    }
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsrl8S_imm(vecX dst, vecX src, immI shift) %{
+  predicate(n->as_Vector()->length() == 8);
   match(Set dst (URShiftVS src shift));
   ins_cost(INSN_COST);
   format %{ "ushr    $dst, $src, $shift\t# vector (8H)" %}
   ins_encode %{
     int sh = (int)$shift$$constant & 31;

@@ -13981,11 +14636,26 @@
     }
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsll2I(vecD dst, vecD src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (LShiftVI src shift));
+  match(Set dst (RShiftVI src shift));
+  ins_cost(INSN_COST);
+  format %{ "sshl  $dst,$src,$shift\t# vector (2S)" %}
+  ins_encode %{
+    __ sshl(as_FloatRegister($dst$$reg), __ T2S,
+            as_FloatRegister($src$$reg),
+            as_FloatRegister($shift$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsll4I(vecX dst, vecX src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (LShiftVI src shift));
   match(Set dst (RShiftVI src shift));
   ins_cost(INSN_COST);
   format %{ "sshl  $dst,$src,$shift\t# vector (4S)" %}
   ins_encode %{

@@ -13994,11 +14664,25 @@
             as_FloatRegister($shift$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsrl2I(vecD dst, vecD src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (URShiftVI src shift));
+  ins_cost(INSN_COST);
+  format %{ "ushl  $dst,$src,$shift\t# vector (2S)" %}
+  ins_encode %{
+    __ ushl(as_FloatRegister($dst$$reg), __ T2S,
+            as_FloatRegister($src$$reg),
+            as_FloatRegister($shift$$reg));
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsrl4I(vecX dst, vecX src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (URShiftVI src shift));
   ins_cost(INSN_COST);
   format %{ "ushl  $dst,$src,$shift\t# vector (4S)" %}
   ins_encode %{
     __ ushl(as_FloatRegister($dst$$reg), __ T4S,

@@ -14006,11 +14690,25 @@
             as_FloatRegister($shift$$reg));
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (LShiftVI src shift));
+  ins_cost(INSN_COST);
+  format %{ "shl    $dst, $src, $shift\t# vector (2S)" %}
+  ins_encode %{
+    __ shl(as_FloatRegister($dst$$reg), __ T2S,
+           as_FloatRegister($src$$reg),
+           (int)$shift$$constant & 31);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (LShiftVI src shift));
   ins_cost(INSN_COST);
   format %{ "shl    $dst, $src, $shift\t# vector (4S)" %}
   ins_encode %{
     __ shl(as_FloatRegister($dst$$reg), __ T4S,

@@ -14018,11 +14716,25 @@
            (int)$shift$$constant & 31);
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (RShiftVI src shift));
+  ins_cost(INSN_COST);
+  format %{ "sshr    $dst, $src, $shift\t# vector (2S)" %}
+  ins_encode %{
+    __ sshr(as_FloatRegister($dst$$reg), __ T2S,
+            as_FloatRegister($src$$reg),
+            -(int)$shift$$constant & 31);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (RShiftVI src shift));
   ins_cost(INSN_COST);
   format %{ "sshr    $dst, $src, $shift\t# vector (4S)" %}
   ins_encode %{
     __ sshr(as_FloatRegister($dst$$reg), __ T4S,

@@ -14030,11 +14742,25 @@
             -(int)$shift$$constant & 31);
   %}
   ins_pipe(pipe_class_default);
 %}
 
+instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{
+  predicate(n->as_Vector()->length() == 2);
+  match(Set dst (URShiftVI src shift));
+  ins_cost(INSN_COST);
+  format %{ "ushr    $dst, $src, $shift\t# vector (2S)" %}
+  ins_encode %{
+    __ ushr(as_FloatRegister($dst$$reg), __ T2S,
+            as_FloatRegister($src$$reg),
+            -(int)$shift$$constant & 31);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
 instruct vsrl4I_imm(vecX dst, vecX src, immI shift) %{
+  predicate(n->as_Vector()->length() == 4);
   match(Set dst (URShiftVI src shift));
   ins_cost(INSN_COST);
   format %{ "ushr    $dst, $src, $shift\t# vector (4S)" %}
   ins_encode %{
     __ ushr(as_FloatRegister($dst$$reg), __ T4S,

@@ -14043,10 +14769,11 @@
   %}
   ins_pipe(pipe_class_default);
 %}
 
 instruct vsll2L(vecX dst, vecX src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 2);
   match(Set dst (LShiftVL src shift));
   match(Set dst (RShiftVL src shift));
   ins_cost(INSN_COST);
   format %{ "sshl  $dst,$src,$shift\t# vector (2D)" %}
   ins_encode %{

@@ -14056,10 +14783,11 @@
   %}
   ins_pipe(pipe_class_default);
 %}
 
 instruct vsrl2L(vecX dst, vecX src, vecX shift) %{
+  predicate(n->as_Vector()->length() == 2);
   match(Set dst (URShiftVL src shift));
   ins_cost(INSN_COST);
   format %{ "ushl  $dst,$src,$shift\t# vector (2D)" %}
   ins_encode %{
     __ ushl(as_FloatRegister($dst$$reg), __ T2D,

@@ -14068,10 +14796,11 @@
   %}
   ins_pipe(pipe_class_default);
 %}
 
 instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
+  predicate(n->as_Vector()->length() == 2);
   match(Set dst (LShiftVL src shift));
   ins_cost(INSN_COST);
   format %{ "shl    $dst, $src, $shift\t# vector (2D)" %}
   ins_encode %{
     __ shl(as_FloatRegister($dst$$reg), __ T2D,

@@ -14080,10 +14809,11 @@
   %}
   ins_pipe(pipe_class_default);
 %}
 
 instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{
+  predicate(n->as_Vector()->length() == 2);
   match(Set dst (RShiftVL src shift));
   ins_cost(INSN_COST);
   format %{ "sshr    $dst, $src, $shift\t# vector (2D)" %}
   ins_encode %{
     __ sshr(as_FloatRegister($dst$$reg), __ T2D,

@@ -14092,10 +14822,11 @@
   %}
   ins_pipe(pipe_class_default);
 %}
 
 instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{
+  predicate(n->as_Vector()->length() == 2);
   match(Set dst (URShiftVL src shift));
   ins_cost(INSN_COST);
   format %{ "ushr    $dst, $src, $shift\t# vector (2D)" %}
   ins_encode %{
     __ ushr(as_FloatRegister($dst$$reg), __ T2D,
< prev index next >