< prev index next >
src/hotspot/cpu/aarch64/aarch64.ad
Print this page
rev 52603 : 8213134: AArch64: vector shift failed with MaxVectorSize=8
Summary: add vshiftcnt instructions for vector64 and add vsra/vsrl instructions to AArch64 backend.
To detect shift failures, MaxVectorSize options are added to jtreg test cases.
Reviewed-by: duke
@@ -2125,11 +2125,17 @@
ShouldNotReachHere();
return 0;
}
const uint Matcher::vector_shift_count_ideal_reg(int size) {
- return Op_VecX;
+ assert(MaxVectorSize >= size, "Length isn't supported");
+ switch(size) {
+ case 8: return Op_VecD;
+ case 16: return Op_VecX;
+ }
+ ShouldNotReachHere();
+ return 0;
}
// AES support not yet implemented
const bool Matcher::pass_original_key_for_aes() {
return false;
@@ -16516,36 +16522,36 @@
%}
ins_pipe(vlogical128);
%}
// ------------------------------ Shift ---------------------------------------
-
-instruct vshiftcntL(vecX dst, iRegIorL2I cnt) %{
+instruct vshiftcnt8B(vecD dst, iRegIorL2I cnt) %{
+ predicate(n->as_Vector()->length_in_bytes() == 8);
match(Set dst (LShiftCntV cnt));
- format %{ "dup $dst, $cnt\t# shift count (vecX)" %}
+ match(Set dst (RShiftCntV cnt));
+ format %{ "dup $dst, $cnt\t# shift count vector (8B)" %}
ins_encode %{
- __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($cnt$$reg));
+ __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($cnt$$reg));
%}
- ins_pipe(vdup_reg_reg128);
+ ins_pipe(vdup_reg_reg64);
%}
-// Right shifts on aarch64 SIMD are implemented as left shift by -ve amount
-instruct vshiftcntR(vecX dst, iRegIorL2I cnt) %{
+instruct vshiftcnt16B(vecX dst, iRegIorL2I cnt) %{
+ predicate(n->as_Vector()->length_in_bytes() == 16);
+ match(Set dst (LShiftCntV cnt));
match(Set dst (RShiftCntV cnt));
- format %{ "dup $dst, $cnt\t# shift count (vecX)\n\tneg $dst, $dst\t T16B" %}
+ format %{ "dup $dst, $cnt\t# shift count vector (16B)" %}
ins_encode %{
__ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($cnt$$reg));
- __ negr(as_FloatRegister($dst$$reg), __ T16B, as_FloatRegister($dst$$reg));
%}
ins_pipe(vdup_reg_reg128);
%}
-instruct vsll8B(vecD dst, vecD src, vecX shift) %{
+instruct vsll8B(vecD dst, vecD src, vecD shift) %{
predicate(n->as_Vector()->length() == 4 ||
n->as_Vector()->length() == 8);
match(Set dst (LShiftVB src shift));
- match(Set dst (RShiftVB src shift));
ins_cost(INSN_COST);
format %{ "sshl $dst,$src,$shift\t# vector (8B)" %}
ins_encode %{
__ sshl(as_FloatRegister($dst$$reg), __ T8B,
as_FloatRegister($src$$reg),
@@ -16555,44 +16561,107 @@
%}
instruct vsll16B(vecX dst, vecX src, vecX shift) %{
predicate(n->as_Vector()->length() == 16);
match(Set dst (LShiftVB src shift));
- match(Set dst (RShiftVB src shift));
ins_cost(INSN_COST);
format %{ "sshl $dst,$src,$shift\t# vector (16B)" %}
ins_encode %{
__ sshl(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($src$$reg),
as_FloatRegister($shift$$reg));
%}
ins_pipe(vshift128);
%}
-instruct vsrl8B(vecD dst, vecD src, vecX shift) %{
+// Right shifts with vector shift count on aarch64 SIMD are implemented
+// as left shift by negative shift count.
+// There are two cases for vector shift count.
+//
+// Case 1: The vector shift count is from replication.
+// | |
+// LoadVector RShiftCntV
+// | /
+// RShiftVI
+// Note: In inner loop, multiple neg instructions are used, which can be
+// moved to outer loop and merge into one neg instruction.
+//
+// Case 2: The vector shift count is from loading.
+// This case isn't supported by middle-end now. But it's supported by
+// panama/vectorIntrinsics(JEP 338: Vector API).
+// | |
+// LoadVector LoadVector
+// | /
+// RShiftVI
+//
+
+instruct vsra8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
+ predicate(n->as_Vector()->length() == 4 ||
+ n->as_Vector()->length() == 8);
+ match(Set dst (RShiftVB src shift));
+ ins_cost(INSN_COST);
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "sshl $dst,$src,$tmp\t# vector (8B)" %}
+ ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T8B,
+ as_FloatRegister($shift$$reg));
+ __ sshl(as_FloatRegister($dst$$reg), __ T8B,
+ as_FloatRegister($src$$reg),
+ as_FloatRegister($tmp$$reg));
+ %}
+ ins_pipe(vshift64);
+%}
+
+instruct vsra16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
+ predicate(n->as_Vector()->length() == 16);
+ match(Set dst (RShiftVB src shift));
+ ins_cost(INSN_COST);
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "sshl $dst,$src,$tmp\t# vector (16B)" %}
+ ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T16B,
+ as_FloatRegister($shift$$reg));
+ __ sshl(as_FloatRegister($dst$$reg), __ T16B,
+ as_FloatRegister($src$$reg),
+ as_FloatRegister($tmp$$reg));
+ %}
+ ins_pipe(vshift128);
+%}
+
+instruct vsrl8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
predicate(n->as_Vector()->length() == 4 ||
n->as_Vector()->length() == 8);
match(Set dst (URShiftVB src shift));
ins_cost(INSN_COST);
- format %{ "ushl $dst,$src,$shift\t# vector (8B)" %}
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "ushl $dst,$src,$tmp\t# vector (8B)" %}
ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T8B,
+ as_FloatRegister($shift$$reg));
__ ushl(as_FloatRegister($dst$$reg), __ T8B,
as_FloatRegister($src$$reg),
- as_FloatRegister($shift$$reg));
+ as_FloatRegister($tmp$$reg));
%}
ins_pipe(vshift64);
%}
-instruct vsrl16B(vecX dst, vecX src, vecX shift) %{
+instruct vsrl16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
predicate(n->as_Vector()->length() == 16);
match(Set dst (URShiftVB src shift));
ins_cost(INSN_COST);
- format %{ "ushl $dst,$src,$shift\t# vector (16B)" %}
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "ushl $dst,$src,$tmp\t# vector (16B)" %}
ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T16B,
+ as_FloatRegister($shift$$reg));
__ ushl(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($src$$reg),
- as_FloatRegister($shift$$reg));
+ as_FloatRegister($tmp$$reg));
%}
ins_pipe(vshift128);
%}
instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
@@ -16700,15 +16769,14 @@
}
%}
ins_pipe(vshift128_imm);
%}
-instruct vsll4S(vecD dst, vecD src, vecX shift) %{
+instruct vsll4S(vecD dst, vecD src, vecD shift) %{
predicate(n->as_Vector()->length() == 2 ||
n->as_Vector()->length() == 4);
match(Set dst (LShiftVS src shift));
- match(Set dst (RShiftVS src shift));
ins_cost(INSN_COST);
format %{ "sshl $dst,$src,$shift\t# vector (4H)" %}
ins_encode %{
__ sshl(as_FloatRegister($dst$$reg), __ T4H,
as_FloatRegister($src$$reg),
@@ -16718,44 +16786,86 @@
%}
instruct vsll8S(vecX dst, vecX src, vecX shift) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (LShiftVS src shift));
- match(Set dst (RShiftVS src shift));
ins_cost(INSN_COST);
format %{ "sshl $dst,$src,$shift\t# vector (8H)" %}
ins_encode %{
__ sshl(as_FloatRegister($dst$$reg), __ T8H,
as_FloatRegister($src$$reg),
as_FloatRegister($shift$$reg));
%}
ins_pipe(vshift128);
%}
-instruct vsrl4S(vecD dst, vecD src, vecX shift) %{
+instruct vsra4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
+ predicate(n->as_Vector()->length() == 2 ||
+ n->as_Vector()->length() == 4);
+ match(Set dst (RShiftVS src shift));
+ ins_cost(INSN_COST);
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "sshl $dst,$src,$tmp\t# vector (4H)" %}
+ ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T8B,
+ as_FloatRegister($shift$$reg));
+ __ sshl(as_FloatRegister($dst$$reg), __ T4H,
+ as_FloatRegister($src$$reg),
+ as_FloatRegister($tmp$$reg));
+ %}
+ ins_pipe(vshift64);
+%}
+
+instruct vsra8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
+ predicate(n->as_Vector()->length() == 8);
+ match(Set dst (RShiftVS src shift));
+ ins_cost(INSN_COST);
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "sshl $dst,$src,$tmp\t# vector (8H)" %}
+ ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T16B,
+ as_FloatRegister($shift$$reg));
+ __ sshl(as_FloatRegister($dst$$reg), __ T8H,
+ as_FloatRegister($src$$reg),
+ as_FloatRegister($tmp$$reg));
+ %}
+ ins_pipe(vshift128);
+%}
+
+instruct vsrl4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
predicate(n->as_Vector()->length() == 2 ||
n->as_Vector()->length() == 4);
match(Set dst (URShiftVS src shift));
ins_cost(INSN_COST);
- format %{ "ushl $dst,$src,$shift\t# vector (4H)" %}
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "ushl $dst,$src,$tmp\t# vector (4H)" %}
ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T8B,
+ as_FloatRegister($shift$$reg));
__ ushl(as_FloatRegister($dst$$reg), __ T4H,
as_FloatRegister($src$$reg),
- as_FloatRegister($shift$$reg));
+ as_FloatRegister($tmp$$reg));
%}
ins_pipe(vshift64);
%}
-instruct vsrl8S(vecX dst, vecX src, vecX shift) %{
+instruct vsrl8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (URShiftVS src shift));
ins_cost(INSN_COST);
- format %{ "ushl $dst,$src,$shift\t# vector (8H)" %}
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "ushl $dst,$src,$tmp\t# vector (8H)" %}
ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T16B,
+ as_FloatRegister($shift$$reg));
__ ushl(as_FloatRegister($dst$$reg), __ T8H,
as_FloatRegister($src$$reg),
- as_FloatRegister($shift$$reg));
+ as_FloatRegister($tmp$$reg));
%}
ins_pipe(vshift128);
%}
instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
@@ -16863,14 +16973,13 @@
}
%}
ins_pipe(vshift128_imm);
%}
-instruct vsll2I(vecD dst, vecD src, vecX shift) %{
+instruct vsll2I(vecD dst, vecD src, vecD shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (LShiftVI src shift));
- match(Set dst (RShiftVI src shift));
ins_cost(INSN_COST);
format %{ "sshl $dst,$src,$shift\t# vector (2S)" %}
ins_encode %{
__ sshl(as_FloatRegister($dst$$reg), __ T2S,
as_FloatRegister($src$$reg),
@@ -16880,43 +16989,84 @@
%}
instruct vsll4I(vecX dst, vecX src, vecX shift) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (LShiftVI src shift));
- match(Set dst (RShiftVI src shift));
ins_cost(INSN_COST);
format %{ "sshl $dst,$src,$shift\t# vector (4S)" %}
ins_encode %{
__ sshl(as_FloatRegister($dst$$reg), __ T4S,
as_FloatRegister($src$$reg),
as_FloatRegister($shift$$reg));
%}
ins_pipe(vshift128);
%}
-instruct vsrl2I(vecD dst, vecD src, vecX shift) %{
+instruct vsra2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
+ predicate(n->as_Vector()->length() == 2);
+ match(Set dst (RShiftVI src shift));
+ ins_cost(INSN_COST);
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "sshl $dst,$src,$tmp\t# vector (2S)" %}
+ ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T8B,
+ as_FloatRegister($shift$$reg));
+ __ sshl(as_FloatRegister($dst$$reg), __ T2S,
+ as_FloatRegister($src$$reg),
+ as_FloatRegister($tmp$$reg));
+ %}
+ ins_pipe(vshift64);
+%}
+
+instruct vsra4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
+ predicate(n->as_Vector()->length() == 4);
+ match(Set dst (RShiftVI src shift));
+ ins_cost(INSN_COST);
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "sshl $dst,$src,$tmp\t# vector (4S)" %}
+ ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T16B,
+ as_FloatRegister($shift$$reg));
+ __ sshl(as_FloatRegister($dst$$reg), __ T4S,
+ as_FloatRegister($src$$reg),
+ as_FloatRegister($tmp$$reg));
+ %}
+ ins_pipe(vshift128);
+%}
+
+instruct vsrl2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (URShiftVI src shift));
ins_cost(INSN_COST);
- format %{ "ushl $dst,$src,$shift\t# vector (2S)" %}
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "ushl $dst,$src,$tmp\t# vector (2S)" %}
ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T8B,
+ as_FloatRegister($shift$$reg));
__ ushl(as_FloatRegister($dst$$reg), __ T2S,
as_FloatRegister($src$$reg),
- as_FloatRegister($shift$$reg));
+ as_FloatRegister($tmp$$reg));
%}
ins_pipe(vshift64);
%}
-instruct vsrl4I(vecX dst, vecX src, vecX shift) %{
+instruct vsrl4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (URShiftVI src shift));
ins_cost(INSN_COST);
- format %{ "ushl $dst,$src,$shift\t# vector (4S)" %}
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "ushl $dst,$src,$tmp\t# vector (4S)" %}
ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T16B,
+ as_FloatRegister($shift$$reg));
__ ushl(as_FloatRegister($dst$$reg), __ T4S,
as_FloatRegister($src$$reg),
- as_FloatRegister($shift$$reg));
+ as_FloatRegister($tmp$$reg));
%}
ins_pipe(vshift128);
%}
instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
@@ -16998,30 +17148,50 @@
%}
instruct vsll2L(vecX dst, vecX src, vecX shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (LShiftVL src shift));
- match(Set dst (RShiftVL src shift));
ins_cost(INSN_COST);
format %{ "sshl $dst,$src,$shift\t# vector (2D)" %}
ins_encode %{
__ sshl(as_FloatRegister($dst$$reg), __ T2D,
as_FloatRegister($src$$reg),
as_FloatRegister($shift$$reg));
%}
ins_pipe(vshift128);
%}
-instruct vsrl2L(vecX dst, vecX src, vecX shift) %{
+instruct vsra2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
+ predicate(n->as_Vector()->length() == 2);
+ match(Set dst (RShiftVL src shift));
+ ins_cost(INSN_COST);
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "sshl $dst,$src,$tmp\t# vector (2D)" %}
+ ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T16B,
+ as_FloatRegister($shift$$reg));
+ __ sshl(as_FloatRegister($dst$$reg), __ T2D,
+ as_FloatRegister($src$$reg),
+ as_FloatRegister($tmp$$reg));
+ %}
+ ins_pipe(vshift128);
+%}
+
+instruct vsrl2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (URShiftVL src shift));
ins_cost(INSN_COST);
- format %{ "ushl $dst,$src,$shift\t# vector (2D)" %}
+ effect(TEMP tmp);
+ format %{ "negr $tmp,$shift\t"
+ "ushl $dst,$src,$tmp\t# vector (2D)" %}
ins_encode %{
+ __ negr(as_FloatRegister($tmp$$reg), __ T16B,
+ as_FloatRegister($shift$$reg));
__ ushl(as_FloatRegister($dst$$reg), __ T2D,
as_FloatRegister($src$$reg),
- as_FloatRegister($shift$$reg));
+ as_FloatRegister($tmp$$reg));
%}
ins_pipe(vshift128);
%}
instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
< prev index next >