--- old/src/hotspot/share/opto/vectornode.cpp 2020-04-02 14:11:59.000000000 +0300 +++ new/src/hotspot/share/opto/vectornode.cpp 2020-04-02 14:11:58.000000000 +0300 @@ -24,6 +24,7 @@ #include "precompiled.hpp" #include "memory/allocation.inline.hpp" #include "opto/connode.hpp" +#include "opto/subnode.hpp" #include "opto/vectornode.hpp" #include "utilities/powerOfTwo.hpp" @@ -117,12 +118,51 @@ case Op_AbsL: assert(bt == T_LONG, "must be"); return Op_AbsVL; + case Op_MinI: + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: return Op_MinV; + default: ShouldNotReachHere(); return 0; + } + case Op_MinL: + assert(bt == T_LONG, "must be"); + return Op_MinV; + case Op_MinF: + assert(bt == T_FLOAT, "must be"); + return Op_MinV; + case Op_MinD: + assert(bt == T_DOUBLE, "must be"); + return Op_MinV; + case Op_MaxI: + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: return Op_MaxV; + default: ShouldNotReachHere(); return 0; + } + case Op_MaxL: + assert(bt == T_LONG, "must be"); + return Op_MaxV; + case Op_MaxF: + assert(bt == T_FLOAT, "must be"); + return Op_MaxV; + case Op_MaxD: + assert(bt == T_DOUBLE, "must be"); + return Op_MaxV; case Op_AbsF: assert(bt == T_FLOAT, "must be"); return Op_AbsVF; case Op_AbsD: assert(bt == T_DOUBLE, "must be"); return Op_AbsVD; + case Op_NegI: + assert(bt == T_INT, "must be"); + return Op_NegVI; case Op_NegF: assert(bt == T_FLOAT, "must be"); return Op_NegVF; @@ -138,6 +178,8 @@ case Op_SqrtD: assert(bt == T_DOUBLE, "must be"); return Op_SqrtVD; + case Op_Not: + return Op_NotV; case Op_PopCountI: if (bt == T_INT) { return Op_PopCountVI; @@ -169,6 +211,12 @@ case Op_RShiftL: assert(bt == T_LONG, "must be"); return Op_RShiftVL; + case Op_URShiftB: + assert(bt == T_BYTE, "must be"); + return Op_URShiftVB; + case Op_URShiftS: + assert(bt == T_SHORT, "must be"); + return Op_URShiftVS; case Op_URShiftI: switch (bt) { case T_BOOLEAN:return Op_URShiftVB; @@ -194,18 +242,6 @@ case Op_XorI: case Op_XorL: return Op_XorV; - case Op_MinF: - assert(bt == T_FLOAT, "must be"); - return Op_MinV; - case Op_MinD: - assert(bt == T_DOUBLE, "must be"); - return Op_MinV; - case Op_MaxF: - assert(bt == T_FLOAT, "must be"); - return Op_MaxV; - case Op_MaxD: - assert(bt == T_DOUBLE, "must be"); - return Op_MaxV; case Op_LoadB: case Op_LoadUB: @@ -232,6 +268,29 @@ } } +int VectorNode::replicate_opcode(BasicType bt) { + switch(bt) { + case T_BOOLEAN: + case T_BYTE: + return Op_ReplicateB; + case T_SHORT: + case T_CHAR: + return Op_ReplicateS; + case T_INT: + return Op_ReplicateI; + case T_LONG: + return Op_ReplicateL; + case T_FLOAT: + return Op_ReplicateF; + case T_DOUBLE: + return Op_ReplicateD; + default: + break; + } + + return 0; +} + // Also used to check if the code generator // supports the vector operation. bool VectorNode::implemented(int opc, uint vlen, BasicType bt) { @@ -284,6 +343,38 @@ } } +bool VectorNode::is_vshift(Node* n) { + switch (n->Opcode()) { + case Op_LShiftVB: + case Op_LShiftVS: + case Op_LShiftVI: + case Op_LShiftVL: + + case Op_RShiftVB: + case Op_RShiftVS: + case Op_RShiftVI: + case Op_RShiftVL: + + case Op_URShiftVB: + case Op_URShiftVS: + case Op_URShiftVI: + case Op_URShiftVL: + return true; + default: + return false; + } +} + +bool VectorNode::is_vshift_cnt(Node* n) { + switch (n->Opcode()) { + case Op_LShiftCntV: + case Op_RShiftCntV: + return true; + default: + return false; + } +} + // Check if input is loop invariant vector. bool VectorNode::is_invariant_vector(Node* n) { // Only Replicate vector nodes are loop invariant for now. @@ -350,12 +441,10 @@ } } -// Return the vector version of a scalar operation node. -VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt) { - const TypeVect* vt = TypeVect::make(bt, vlen); - int vopc = VectorNode::opcode(opc, bt); +// Make a vector node for binary operation +VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, const TypeVect* vt) { // This method should not be called for unimplemented vectors. - guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]); + guarantee(vopc > 0, "vopc must be > 0"); switch (vopc) { case Op_AddVB: return new AddVBNode(n1, n2, vt); case Op_AddVS: return new AddVSNode(n1, n2, vt); @@ -381,13 +470,18 @@ case Op_DivVF: return new DivVFNode(n1, n2, vt); case Op_DivVD: return new DivVDNode(n1, n2, vt); + case Op_MinV: return new MinVNode(n1, n2, vt); + case Op_MaxV: return new MaxVNode(n1, n2, vt); + + case Op_AbsV: return new AbsVNode(n1, vt); + case Op_AbsVF: return new AbsVFNode(n1, vt); + case Op_AbsVD: return new AbsVDNode(n1, vt); case Op_AbsVB: return new AbsVBNode(n1, vt); case Op_AbsVS: return new AbsVSNode(n1, vt); case Op_AbsVI: return new AbsVINode(n1, vt); case Op_AbsVL: return new AbsVLNode(n1, vt); - case Op_AbsVF: return new AbsVFNode(n1, vt); - case Op_AbsVD: return new AbsVDNode(n1, vt); + case Op_NegVI: return new NegVINode(n1, vt); case Op_NegVF: return new NegVFNode(n1, vt); case Op_NegVD: return new NegVDNode(n1, vt); @@ -395,6 +489,7 @@ case Op_SqrtVD: return new SqrtVDNode(n1, vt); case Op_PopCountVI: return new PopCountVINode(n1, vt); + case Op_NotV: return new NotVNode(n1, vt); case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt); case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt); @@ -411,13 +506,15 @@ case Op_URShiftVI: return new URShiftVINode(n1, n2, vt); case Op_URShiftVL: return new URShiftVLNode(n1, n2, vt); + // Variable shift left + case Op_VLShiftV: return new VLShiftVNode(n1, n2, vt); + case Op_VRShiftV: return new VRShiftVNode(n1, n2, vt); + case Op_VURShiftV: return new VURShiftVNode(n1, n2, vt); + case Op_AndV: return new AndVNode(n1, n2, vt); case Op_OrV: return new OrVNode (n1, n2, vt); case Op_XorV: return new XorVNode(n1, n2, vt); - case Op_MinV: return new MinVNode(n1, n2, vt); - case Op_MaxV: return new MaxVNode(n1, n2, vt); - case Op_RoundDoubleModeV: return new RoundDoubleModeVNode(n1, n2, vt); case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt); @@ -427,11 +524,19 @@ } } -VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) { +// Return the vector version of a scalar binary operation node. +VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt) { const TypeVect* vt = TypeVect::make(bt, vlen); int vopc = VectorNode::opcode(opc, bt); // This method should not be called for unimplemented vectors. guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]); + return make(vopc, n1, n2, vt); +} + +// Make a vector node for ternary operation +VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, Node* n3, const TypeVect* vt) { + // This method should not be called for unimplemented vectors. + guarantee(vopc > 0, "vopc must be > 0"); switch (vopc) { case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt); case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt); @@ -441,6 +546,15 @@ } } +// Return the vector version of a scalar ternary operation node. +VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) { + const TypeVect* vt = TypeVect::make(bt, vlen); + int vopc = VectorNode::opcode(opc, bt); + // This method should not be called for unimplemented vectors. + guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]); + return make(vopc, n1, n2, n3, vt); +} + // Scalar promotion VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t) { BasicType bt = opd_t->array_element_basic_type(); @@ -467,21 +581,22 @@ } } -VectorNode* VectorNode::shift_count(Node* shift, Node* cnt, uint vlen, BasicType bt) { - assert(VectorNode::is_shift(shift), "sanity"); +VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt) { // Match shift count type with shift vector type. const TypeVect* vt = TypeVect::make(bt, vlen); - switch (shift->Opcode()) { + switch (opc) { case Op_LShiftI: case Op_LShiftL: return new LShiftCntVNode(cnt, vt); case Op_RShiftI: case Op_RShiftL: + case Op_URShiftB: + case Op_URShiftS: case Op_URShiftI: case Op_URShiftL: return new RShiftCntVNode(cnt, vt); default: - fatal("Missed vector creation for '%s'", NodeClassNames[shift->Opcode()]); + fatal("Missed vector creation for '%s'", NodeClassNames[opc]); return NULL; } } @@ -595,6 +710,30 @@ return new StoreVectorNode(ctl, mem, adr, atyp, val); } +int ExtractNode::opcode(BasicType bt) { + switch (bt) { + case T_BOOLEAN: + return Op_ExtractUB; + case T_BYTE: + return Op_ExtractB; + case T_CHAR: + return Op_ExtractC; + case T_SHORT: + return Op_ExtractS; + case T_INT: + return Op_ExtractI; + case T_LONG: + return Op_ExtractL; + case T_FLOAT: + return Op_ExtractF; + case T_DOUBLE: + return Op_ExtractD; + default: + fatal("Type '%s' is not supported for vectors", type2name(bt)); + return 0; + } +} + // Extract a scalar element of vector. Node* ExtractNode::make(Node* v, uint position, BasicType bt) { assert((int)position < Matcher::max_vector_size(bt), "pos in range"); @@ -626,8 +765,16 @@ int vopc = opc; switch (opc) { case Op_AddI: - assert(bt == T_INT, "must be"); - vopc = Op_AddReductionVI; + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_AddReductionVI; + break; + default: ShouldNotReachHere(); return 0; + } break; case Op_AddL: assert(bt == T_LONG, "must be"); @@ -642,8 +789,16 @@ vopc = Op_AddReductionVD; break; case Op_MulI: - assert(bt == T_INT, "must be"); - vopc = Op_MulReductionVI; + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_MulReductionVI; + break; + default: ShouldNotReachHere(); return 0; + } break; case Op_MulL: assert(bt == T_LONG, "must be"); @@ -657,6 +812,22 @@ assert(bt == T_DOUBLE, "must be"); vopc = Op_MulReductionVD; break; + case Op_MinI: + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_MinReductionV; + break; + default: ShouldNotReachHere(); return 0; + } + break; + case Op_MinL: + assert(bt == T_LONG, "must be"); + vopc = Op_MinReductionV; + break; case Op_MinF: assert(bt == T_FLOAT, "must be"); vopc = Op_MinReductionV; @@ -665,6 +836,22 @@ assert(bt == T_DOUBLE, "must be"); vopc = Op_MinReductionV; break; + case Op_MaxI: + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_MaxReductionV; + break; + default: ShouldNotReachHere(); return 0; + } + break; + case Op_MaxL: + assert(bt == T_LONG, "must be"); + vopc = Op_MaxReductionV; + break; case Op_MaxF: assert(bt == T_FLOAT, "must be"); vopc = Op_MaxReductionV; @@ -674,24 +861,48 @@ vopc = Op_MaxReductionV; break; case Op_AndI: - assert(bt == T_INT, "must be"); - vopc = Op_AndReductionV; + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_AndReductionV; + break; + default: ShouldNotReachHere(); return 0; + } break; case Op_AndL: assert(bt == T_LONG, "must be"); vopc = Op_AndReductionV; break; case Op_OrI: - assert(bt == T_INT, "must be"); - vopc = Op_OrReductionV; + switch(bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_OrReductionV; + break; + default: ShouldNotReachHere(); return 0; + } break; case Op_OrL: assert(bt == T_LONG, "must be"); vopc = Op_OrReductionV; break; case Op_XorI: - assert(bt == T_INT, "must be"); - vopc = Op_XorReductionV; + switch(bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_XorReductionV; + break; + default: ShouldNotReachHere(); return 0; + } break; case Op_XorL: assert(bt == T_LONG, "must be"); @@ -731,6 +942,102 @@ } } +VectorCastNode* VectorCastNode::make(int vopc, Node* n1, BasicType bt, uint vlen) { + const TypeVect* vt = TypeVect::make(bt, vlen); + switch (vopc) { + case Op_VectorCastB2X: return new VectorCastB2XNode(n1, vt); + case Op_VectorCastS2X: return new VectorCastS2XNode(n1, vt); + case Op_VectorCastI2X: return new VectorCastI2XNode(n1, vt); + case Op_VectorCastL2X: return new VectorCastL2XNode(n1, vt); + case Op_VectorCastF2X: return new VectorCastF2XNode(n1, vt); + case Op_VectorCastD2X: return new VectorCastD2XNode(n1, vt); + default: fatal("unknown node: %s", NodeClassNames[vopc]); + } + return NULL; +} + +int VectorCastNode::opcode(BasicType bt) { + switch (bt) { + case T_BYTE: return Op_VectorCastB2X; + case T_SHORT: return Op_VectorCastS2X; + case T_INT: return Op_VectorCastI2X; + case T_LONG: return Op_VectorCastL2X; + case T_FLOAT: return Op_VectorCastF2X; + case T_DOUBLE: return Op_VectorCastD2X; + default: Unimplemented(); + } + return 0; +} + +Node* ReductionNode::make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt) { + int vopc = opcode(opc, bt); + guarantee(vopc != opc, "Vector reduction for '%s' is not implemented", NodeClassNames[opc]); + + switch (vopc) { + case Op_AndReductionV: + switch (bt) { + case T_BYTE: + case T_SHORT: + case T_INT: + return gvn.makecon(TypeInt::MINUS_1); + case T_LONG: + return gvn.makecon(TypeLong::MINUS_1); + default: + fatal("Missed vector creation for '%s' as the basic type is not correct.", NodeClassNames[vopc]); + return NULL; + } + break; + case Op_AddReductionVI: // fallthrough + case Op_AddReductionVL: // fallthrough + case Op_AddReductionVF: // fallthrough + case Op_AddReductionVD: + case Op_OrReductionV: + case Op_XorReductionV: + return gvn.zerocon(bt); + case Op_MulReductionVI: + return gvn.makecon(TypeInt::ONE); + case Op_MulReductionVL: + return gvn.makecon(TypeLong::ONE); + case Op_MulReductionVF: + return gvn.makecon(TypeF::ONE); + case Op_MulReductionVD: + return gvn.makecon(TypeD::ONE); + case Op_MinReductionV: + switch (bt) { + case T_BYTE: + case T_SHORT: + case T_INT: + return gvn.makecon(TypeInt::MAX); + case T_LONG: + return gvn.makecon(TypeLong::MAX); + case T_FLOAT: + return gvn.makecon(TypeF::POS_INF); + case T_DOUBLE: + return gvn.makecon(TypeD::POS_INF); + default: Unimplemented(); return NULL; + } + break; + case Op_MaxReductionV: + switch (bt) { + case T_BYTE: + case T_SHORT: + case T_INT: + return gvn.makecon(TypeInt::MIN); + case T_LONG: + return gvn.makecon(TypeLong::MIN); + case T_FLOAT: + return gvn.makecon(TypeF::NEG_INF); + case T_DOUBLE: + return gvn.makecon(TypeD::NEG_INF); + default: Unimplemented(); return NULL; + } + break; + default: + fatal("Missed vector creation for '%s'", NodeClassNames[vopc]); + return NULL; + } +} + bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) { if (is_java_primitive(bt) && (vlen > 1) && is_power_of_2(vlen) && @@ -740,3 +1047,26 @@ } return false; } + +#ifndef PRODUCT +void VectorMaskCmpNode::dump_spec(outputStream *st) const { + st->print(" %d #", _predicate); _type->dump_on(st); +} +#endif // PRODUCT + +Node* VectorReinterpretNode::Identity(PhaseGVN *phase) { + Node* n = in(1); + if (n->Opcode() == Op_VectorReinterpret) { + if (Type::cmp(bottom_type(), n->in(1)->bottom_type()) == 0) { + return n->in(1); + } + } + return this; +} + +Node* VectorInsertNode::make(Node* vec, Node* new_val, int position) { + assert(position < (int)vec->bottom_type()->is_vect()->length(), "pos in range"); + ConINode* pos = ConINode::make(position); + return new VectorInsertNode(vec, new_val, pos, vec->bottom_type()->is_vect()); +} +