--- old/src/hotspot/share/adlc/forms.cpp 2020-08-07 00:44:06.000000000 +0300 +++ new/src/hotspot/share/adlc/forms.cpp 2020-08-07 00:44:06.000000000 +0300 @@ -268,6 +268,7 @@ if( strcmp(opType,"LoadRange")==0 ) return Form::idealI; if( strcmp(opType,"LoadS")==0 ) return Form::idealS; if( strcmp(opType,"LoadVector")==0 ) return Form::idealV; + if( strcmp(opType,"LoadVectorGather")==0 ) return Form::idealV; assert( strcmp(opType,"Load") != 0, "Must type Loads" ); return Form::none; } @@ -284,6 +285,7 @@ if( strcmp(opType,"StoreN")==0) return Form::idealN; if( strcmp(opType,"StoreNKlass")==0) return Form::idealNKlass; if( strcmp(opType,"StoreVector")==0 ) return Form::idealV; + if( strcmp(opType,"StoreVectorScatter")==0 ) return Form::idealV; assert( strcmp(opType,"Store") != 0, "Must type Stores" ); return Form::none; } --- old/src/hotspot/share/adlc/formssel.cpp 2020-08-07 00:44:07.000000000 +0300 +++ new/src/hotspot/share/adlc/formssel.cpp 2020-08-07 00:44:06.000000000 +0300 @@ -3482,7 +3482,7 @@ "StoreB","StoreC","Store" ,"StoreFP", "LoadI", "LoadL", "LoadP" ,"LoadN", "LoadD" ,"LoadF" , "LoadB" , "LoadUB", "LoadUS" ,"LoadS" ,"Load" , - "StoreVector", "LoadVector", + "StoreVector", "LoadVector", "LoadVectorGather", "StoreVectorScatter", "LoadRange", "LoadKlass", "LoadNKlass", "LoadL_unaligned", "LoadD_unaligned", "LoadPLocked", "StorePConditional", "StoreIConditional", "StoreLConditional", @@ -3799,6 +3799,7 @@ "MaxV", "MinV", "MulI","MulL","MulF","MulD", "MulVB","MulVS","MulVI","MulVL","MulVF","MulVD", + "MinV","MaxV", "OrI","OrL", "OrV", "XorI","XorL", @@ -4147,8 +4148,9 @@ "MulVB","MulVS","MulVI","MulVL","MulVF","MulVD", "CMoveVD", "CMoveVF", "DivVF","DivVD", + "MinV","MaxV", "AbsVB","AbsVS","AbsVI","AbsVL","AbsVF","AbsVD", - "NegVF","NegVD", + "NegVF","NegVD","NegVI", "SqrtVD","SqrtVF", "AndV" ,"XorV" ,"OrV", "MaxV", "MinV", @@ -4164,7 +4166,12 @@ "RShiftVB","RShiftVS","RShiftVI","RShiftVL", "URShiftVB","URShiftVS","URShiftVI","URShiftVL", "ReplicateB","ReplicateS","ReplicateI","ReplicateL","ReplicateF","ReplicateD", - "RoundDoubleModeV","LoadVector","StoreVector", + "RoundDoubleModeV","LoadVector","StoreVector","LoadVectorGather", "StoreVectorScatter", + "VectorTest", "VectorLoadMask", "VectorStoreMask", "VectorBlend", "VectorInsert", + "VectorRearrange","VectorLoadShuffle", "VectorLoadConst", + "VectorCastB2X", "VectorCastS2X", "VectorCastI2X", + "VectorCastL2X", "VectorCastF2X", "VectorCastD2X", + "VectorMaskWrapper", "VectorMaskCmp", "VectorReinterpret", "FmaVD", "FmaVF","PopCountVI", // Next are not supported currently. "PackB","PackS","PackI","PackL","PackF","PackD","Pack2L","Pack2D", --- old/src/hotspot/share/ci/ciField.cpp 2020-08-07 00:44:07.000000000 +0300 +++ new/src/hotspot/share/ci/ciField.cpp 2020-08-07 00:44:07.000000000 +0300 @@ -221,6 +221,7 @@ // Even if general trusting is disabled, trust system-built closures in these packages. if (holder->is_in_package("java/lang/invoke") || holder->is_in_package("sun/invoke") || holder->is_in_package("jdk/internal/foreign") || holder->is_in_package("jdk/incubator/foreign") || + holder->is_in_package("jdk/internal/vm/vector") || holder->is_in_package("jdk/incubator/vector") || holder->is_in_package("java/lang")) return true; // Trust hidden classes and VM unsafe anonymous classes. They are created via --- old/src/hotspot/share/ci/ciMethod.cpp 2020-08-07 00:44:07.000000000 +0300 +++ new/src/hotspot/share/ci/ciMethod.cpp 2020-08-07 00:44:07.000000000 +0300 @@ -1351,6 +1351,11 @@ return false; } +bool ciMethod::is_vector_method() const { + return (holder() == ciEnv::current()->vector_VectorSupport_klass()) && + (intrinsic_id() != vmIntrinsics::_none); +} + BCEscapeAnalyzer *ciMethod::get_bcea() { #ifdef COMPILER2 if (_bcea == NULL) { --- old/src/hotspot/share/ci/ciMethod.hpp 2020-08-07 00:44:08.000000000 +0300 +++ new/src/hotspot/share/ci/ciMethod.hpp 2020-08-07 00:44:08.000000000 +0300 @@ -356,6 +356,7 @@ bool has_reserved_stack_access() const { return _has_reserved_stack_access; } bool is_boxing_method() const; bool is_unboxing_method() const; + bool is_vector_method() const; bool is_object_initializer() const; bool can_be_statically_bound(ciInstanceKlass* context) const; --- old/src/hotspot/share/classfile/javaClasses.cpp 2020-08-07 00:44:08.000000000 +0300 +++ new/src/hotspot/share/classfile/javaClasses.cpp 2020-08-07 00:44:08.000000000 +0300 @@ -4533,6 +4533,30 @@ } #endif +int vector_VectorPayload::_payload_offset; + +#define VECTORPAYLOAD_FIELDS_DO(macro) \ + macro(_payload_offset, k, "payload", object_signature, false) + +void vector_VectorPayload::compute_offsets() { + InstanceKlass* k = SystemDictionary::vector_VectorPayload_klass(); + VECTORPAYLOAD_FIELDS_DO(FIELD_COMPUTE_OFFSET); +} + +#if INCLUDE_CDS +void vector_VectorPayload::serialize_offsets(SerializeClosure* f) { + VECTORPAYLOAD_FIELDS_DO(FIELD_SERIALIZE_OFFSET); +} +#endif + +void vector_VectorPayload::set_payload(oop o, oop val) { + o->obj_field_put(_payload_offset, val); +} + +bool vector_VectorPayload::is_instance(oop obj) { + return obj != NULL && is_subclass(obj->klass()); +} + int java_lang_Integer_IntegerCache::_static_cache_offset; int java_lang_Long_LongCache::_static_cache_offset; int java_lang_Character_CharacterCache::_static_cache_offset; --- old/src/hotspot/share/classfile/javaClasses.hpp 2020-08-07 00:44:09.000000000 +0300 +++ new/src/hotspot/share/classfile/javaClasses.hpp 2020-08-07 00:44:08.000000000 +0300 @@ -74,6 +74,7 @@ f(java_util_concurrent_locks_AbstractOwnableSynchronizer) \ f(jdk_internal_misc_UnsafeConstants) \ f(java_lang_boxing_object) \ + f(vector_VectorPayload) \ //end #define BASIC_JAVA_CLASSES_DO(f) \ @@ -1557,6 +1558,24 @@ static void serialize_offsets(SerializeClosure* f) { } }; +// Interface to jdk.internal.vm.vector.VectorSupport.VectorPayload objects + +class vector_VectorPayload : AllStatic { + private: + static int _payload_offset; + public: + static void set_payload(oop o, oop val); + + static void compute_offsets(); + static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN; + + // Testers + static bool is_subclass(Klass* klass) { + return klass->is_subclass_of(SystemDictionary::vector_VectorPayload_klass()); + } + static bool is_instance(oop obj); +}; + class java_lang_Integer : AllStatic { public: static jint value(oop obj); --- old/src/hotspot/share/classfile/modules.cpp 2020-08-07 00:44:09.000000000 +0300 +++ new/src/hotspot/share/classfile/modules.cpp 2020-08-07 00:44:09.000000000 +0300 @@ -448,6 +448,24 @@ if (h_loader.is_null() && !ClassLoader::has_jrt_entry()) { ClassLoader::add_to_exploded_build_list(module_symbol, CHECK); } + +#ifdef COMPILER2 + // Special handling of jdk.incubator.vector + if (strcmp(module_name, "jdk.incubator.vector") == 0) { + if (FLAG_IS_DEFAULT(EnableVectorSupport)) { + FLAG_SET_DEFAULT(EnableVectorSupport, true); + } + if (EnableVectorSupport && FLAG_IS_DEFAULT(EnableVectorReboxing)) { + FLAG_SET_DEFAULT(EnableVectorReboxing, true); + } + if (EnableVectorSupport && EnableVectorReboxing && FLAG_IS_DEFAULT(EnableVectorAggressiveReboxing)) { + FLAG_SET_DEFAULT(EnableVectorAggressiveReboxing, true); + } + log_info(compilation)("EnableVectorSupport=%s", (EnableVectorSupport ? "true" : "false")); + log_info(compilation)("EnableVectorReboxing=%s", (EnableVectorReboxing ? "true" : "false")); + log_info(compilation)("EnableVectorAggressiveReboxing=%s", (EnableVectorAggressiveReboxing ? "true" : "false")); + } +#endif // COMPILER2 } void Modules::set_bootloader_unnamed_module(jobject module, TRAPS) { --- old/src/hotspot/share/classfile/systemDictionary.hpp 2020-08-07 00:44:10.000000000 +0300 +++ new/src/hotspot/share/classfile/systemDictionary.hpp 2020-08-07 00:44:09.000000000 +0300 @@ -270,6 +270,13 @@ /* support for records */ \ do_klass(RecordComponent_klass, java_lang_reflect_RecordComponent ) \ \ + /* support for vectors*/ \ + do_klass(vector_VectorSupport_klass, jdk_internal_vm_vector_VectorSupport ) \ + do_klass(vector_VectorPayload_klass, jdk_internal_vm_vector_VectorPayload ) \ + do_klass(vector_Vector_klass, jdk_internal_vm_vector_Vector ) \ + do_klass(vector_VectorMask_klass, jdk_internal_vm_vector_VectorMask ) \ + do_klass(vector_VectorShuffle_klass, jdk_internal_vm_vector_VectorShuffle ) \ + \ /*end*/ class SystemDictionary : AllStatic { --- old/src/hotspot/share/classfile/vmSymbols.hpp 2020-08-07 00:44:10.000000000 +0300 +++ new/src/hotspot/share/classfile/vmSymbols.hpp 2020-08-07 00:44:10.000000000 +0300 @@ -80,6 +80,16 @@ template(java_lang_Integer_IntegerCache, "java/lang/Integer$IntegerCache") \ template(java_lang_Long, "java/lang/Long") \ template(java_lang_Long_LongCache, "java/lang/Long$LongCache") \ + \ + template(jdk_internal_vm_vector_VectorSupport, "jdk/internal/vm/vector/VectorSupport") \ + template(jdk_internal_vm_vector_VectorPayload, "jdk/internal/vm/vector/VectorSupport$VectorPayload") \ + template(jdk_internal_vm_vector_Vector, "jdk/internal/vm/vector/VectorSupport$Vector") \ + template(jdk_internal_vm_vector_VectorMask, "jdk/internal/vm/vector/VectorSupport$VectorMask") \ + template(jdk_internal_vm_vector_VectorShuffle, "jdk/internal/vm/vector/VectorSupport$VectorShuffle") \ + template(payload_name, "payload") \ + template(ETYPE_name, "ETYPE") \ + template(VLENGTH_name, "VLENGTH") \ + \ template(java_lang_Shutdown, "java/lang/Shutdown") \ template(java_lang_ref_Reference, "java/lang/ref/Reference") \ template(java_lang_ref_SoftReference, "java/lang/ref/SoftReference") \ @@ -1416,6 +1426,122 @@ do_intrinsic(_getAndSetReference, jdk_internal_misc_Unsafe, getAndSetReference_name, getAndSetReference_signature, F_R) \ do_name( getAndSetReference_name, "getAndSetReference") \ do_signature(getAndSetReference_signature, "(Ljava/lang/Object;JLjava/lang/Object;)Ljava/lang/Object;" ) \ + \ + /* Vector API intrinsification support */ \ + \ + do_intrinsic(_VectorUnaryOp, jdk_internal_vm_vector_VectorSupport, vector_unary_op_name, vector_unary_op_sig, F_S) \ + do_signature(vector_unary_op_sig, "(ILjava/lang/Class;Ljava/lang/Class;ILjava/lang/Object;Ljava/util/function/Function;)Ljava/lang/Object;") \ + do_name(vector_unary_op_name, "unaryOp") \ + \ + do_intrinsic(_VectorBinaryOp, jdk_internal_vm_vector_VectorSupport, vector_binary_op_name, vector_binary_op_sig, F_S) \ + do_signature(vector_binary_op_sig, "(ILjava/lang/Class;Ljava/lang/Class;ILjava/lang/Object;Ljava/lang/Object;" \ + "Ljava/util/function/BiFunction;)Ljava/lang/Object;") \ + do_name(vector_binary_op_name, "binaryOp") \ + \ + do_intrinsic(_VectorTernaryOp, jdk_internal_vm_vector_VectorSupport, vector_ternary_op_name, vector_ternary_op_sig, F_S) \ + do_signature(vector_ternary_op_sig, "(ILjava/lang/Class;Ljava/lang/Class;ILjava/lang/Object;Ljava/lang/Object;" \ + "Ljava/lang/Object;Ljdk/internal/vm/vector/VectorSupport$TernaryOperation;)Ljava/lang/Object;") \ + do_name(vector_ternary_op_name, "ternaryOp") \ + \ + do_intrinsic(_VectorBroadcastCoerced, jdk_internal_vm_vector_VectorSupport, vector_broadcast_coerced_name, vector_broadcast_coerced_sig, F_S)\ + do_signature(vector_broadcast_coerced_sig, "(Ljava/lang/Class;Ljava/lang/Class;IJLjdk/internal/vm/vector/VectorSupport$VectorSpecies;" \ + "Ljdk/internal/vm/vector/VectorSupport$BroadcastOperation;)Ljava/lang/Object;") \ + do_name(vector_broadcast_coerced_name, "broadcastCoerced") \ + \ + do_intrinsic(_VectorShuffleIota, jdk_internal_vm_vector_VectorSupport, vector_shuffle_step_iota_name, vector_shuffle_step_iota_sig, F_S) \ + do_signature(vector_shuffle_step_iota_sig, "(Ljava/lang/Class;Ljava/lang/Class;Ljdk/internal/vm/vector/VectorSupport$VectorSpecies;" \ + "IIIILjdk/internal/vm/vector/VectorSupport$ShuffleIotaOperation;)Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;") \ + do_name(vector_shuffle_step_iota_name, "shuffleIota") \ + \ + do_intrinsic(_VectorShuffleToVector, jdk_internal_vm_vector_VectorSupport, vector_shuffle_to_vector_name, vector_shuffle_to_vector_sig, F_S) \ + do_signature(vector_shuffle_to_vector_sig, "(Ljava/lang/Class;Ljava/lang/Class;Ljava/lang/Class;Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;" \ + "ILjdk/internal/vm/vector/VectorSupport$ShuffleToVectorOperation;)Ljava/lang/Object;") \ + do_name(vector_shuffle_to_vector_name, "shuffleToVector") \ + \ + do_intrinsic(_VectorLoadOp, jdk_internal_vm_vector_VectorSupport, vector_load_op_name, vector_load_op_sig, F_S) \ + do_signature(vector_load_op_sig, "(Ljava/lang/Class;Ljava/lang/Class;ILjava/lang/Object;JLjava/lang/Object;" \ + "ILjdk/internal/vm/vector/VectorSupport$VectorSpecies;Ljdk/internal/vm/vector/VectorSupport$LoadOperation;)Ljava/lang/Object;") \ + do_name(vector_load_op_name, "load") \ + \ + do_intrinsic(_VectorStoreOp, jdk_internal_vm_vector_VectorSupport, vector_store_op_name, vector_store_op_sig, F_S) \ + do_signature(vector_store_op_sig, "(Ljava/lang/Class;Ljava/lang/Class;ILjava/lang/Object;JLjdk/internal/vm/vector/VectorSupport$Vector;" \ + "Ljava/lang/Object;ILjdk/internal/vm/vector/VectorSupport$StoreVectorOperation;)V") \ + do_name(vector_store_op_name, "store") \ + \ + do_intrinsic(_VectorReductionCoerced, jdk_internal_vm_vector_VectorSupport, vector_reduction_coerced_name, vector_reduction_coerced_sig, F_S) \ + do_signature(vector_reduction_coerced_sig, "(ILjava/lang/Class;Ljava/lang/Class;ILjdk/internal/vm/vector/VectorSupport$Vector;Ljava/util/function/Function;)J") \ + do_name(vector_reduction_coerced_name, "reductionCoerced") \ + \ + do_intrinsic(_VectorTest, jdk_internal_vm_vector_VectorSupport, vector_test_name, vector_test_sig, F_S) \ + do_signature(vector_test_sig, "(ILjava/lang/Class;Ljava/lang/Class;ILjava/lang/Object;Ljava/lang/Object;Ljava/util/function/BiFunction;)Z") \ + do_name(vector_test_name, "test") \ + \ + do_intrinsic(_VectorBlend, jdk_internal_vm_vector_VectorSupport, vector_blend_name, vector_blend_sig, F_S) \ + do_signature(vector_blend_sig, "(Ljava/lang/Class;Ljava/lang/Class;Ljava/lang/Class;I" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;Ljdk/internal/vm/vector/VectorSupport$Vector;Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorBlendOp;)Ljdk/internal/vm/vector/VectorSupport$Vector;") \ + do_name(vector_blend_name, "blend") \ + \ + do_intrinsic(_VectorCompare, jdk_internal_vm_vector_VectorSupport, vector_compare_name, vector_compare_sig, F_S) \ + do_signature(vector_compare_sig, "(ILjava/lang/Class;Ljava/lang/Class;Ljava/lang/Class;I" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;" "Ljdk/internal/vm/vector/VectorSupport$Vector;" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorCompareOp;" ")" "Ljdk/internal/vm/vector/VectorSupport$VectorMask;") \ + do_name(vector_compare_name, "compare") \ + \ + do_intrinsic(_VectorRearrange, jdk_internal_vm_vector_VectorSupport, vector_rearrange_name, vector_rearrange_sig, F_S) \ + do_signature(vector_rearrange_sig, "(Ljava/lang/Class;Ljava/lang/Class;Ljava/lang/Class;I" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorRearrangeOp;)Ljdk/internal/vm/vector/VectorSupport$Vector;") \ + do_name(vector_rearrange_name, "rearrangeOp") \ + \ + do_intrinsic(_VectorExtract, jdk_internal_vm_vector_VectorSupport, vector_extract_name, vector_extract_sig, F_S) \ + do_signature(vector_extract_sig, "(Ljava/lang/Class;Ljava/lang/Class;I" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;I" \ + "Ljdk/internal/vm/vector/VectorSupport$VecExtractOp;)J") \ + do_name(vector_extract_name, "extract") \ + \ + do_intrinsic(_VectorInsert, jdk_internal_vm_vector_VectorSupport, vector_insert_name, vector_insert_sig, F_S) \ + do_signature(vector_insert_sig, "(Ljava/lang/Class;Ljava/lang/Class;I" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;IJ" \ + "Ljdk/internal/vm/vector/VectorSupport$VecInsertOp;)Ljdk/internal/vm/vector/VectorSupport$Vector;") \ + do_name(vector_insert_name, "insert") \ + \ + do_intrinsic(_VectorBroadcastInt, jdk_internal_vm_vector_VectorSupport, vector_broadcast_int_name, vector_broadcast_int_sig, F_S) \ + do_signature(vector_broadcast_int_sig, "(ILjava/lang/Class;Ljava/lang/Class;I" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;I" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorBroadcastIntOp;)Ljdk/internal/vm/vector/VectorSupport$Vector;") \ + do_name(vector_broadcast_int_name, "broadcastInt") \ + \ + do_intrinsic(_VectorConvert, jdk_internal_vm_vector_VectorSupport, vector_convert_name, vector_convert_sig, F_S) \ + do_signature(vector_convert_sig, "(ILjava/lang/Class;Ljava/lang/Class;I" \ + "Ljava/lang/Class;Ljava/lang/Class;I" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorPayload;" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorSpecies;" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorConvertOp;)Ljdk/internal/vm/vector/VectorSupport$VectorPayload;") \ + do_name(vector_convert_name, "convert") \ + \ + do_intrinsic(_VectorGatherOp, jdk_internal_vm_vector_VectorSupport, vector_gather_name, vector_gather_sig, F_S) \ + do_signature(vector_gather_sig, "(Ljava/lang/Class;Ljava/lang/Class;ILjava/lang/Class;" \ + "Ljava/lang/Object;J" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;" \ + "Ljava/lang/Object;I[II" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorSpecies;" \ + "Ljdk/internal/vm/vector/VectorSupport$LoadVectorOperationWithMap;)" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;") \ + do_name(vector_gather_name, "loadWithMap") \ + \ + do_intrinsic(_VectorScatterOp, jdk_internal_vm_vector_VectorSupport, vector_scatter_name, vector_scatter_sig, F_S) \ + do_signature(vector_scatter_sig, "(Ljava/lang/Class;Ljava/lang/Class;ILjava/lang/Class;" \ + "Ljava/lang/Object;J" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;Ljdk/internal/vm/vector/VectorSupport$Vector;" \ + "Ljava/lang/Object;I[II" \ + "Ljdk/internal/vm/vector/VectorSupport$StoreVectorOperationWithMap;)V") \ + do_name(vector_scatter_name, "storeWithMap") \ + \ + do_intrinsic(_VectorRebox, jdk_internal_vm_vector_VectorSupport, vector_rebox_name, vector_rebox_sig, F_S) \ + do_alias(vector_rebox_sig, object_object_signature) \ + do_name(vector_rebox_name, "maybeRebox") \ + \ \ /* (2) Bytecode intrinsics */ \ \ @@ -1600,7 +1726,7 @@ #undef VM_INTRINSIC_ENUM ID_LIMIT, - LAST_COMPILER_INLINE = _getAndSetReference, + LAST_COMPILER_INLINE = _VectorScatterOp, FIRST_MH_SIG_POLY = _invokeGeneric, FIRST_MH_STATIC = _linkToVirtual, LAST_MH_SIG_POLY = _linkToInterface, --- old/src/hotspot/share/code/debugInfo.hpp 2020-08-07 00:44:10.000000000 +0300 +++ new/src/hotspot/share/code/debugInfo.hpp 2020-08-07 00:44:10.000000000 +0300 @@ -42,6 +42,7 @@ // - ConstantValue describes a constant class ConstantOopReadValue; +class LocationValue; class ObjectValue; class ScopeValue: public ResourceObj { @@ -67,6 +68,11 @@ return (ObjectValue*)this; } + LocationValue* as_LocationValue() { + assert(is_location(), "must be"); + return (LocationValue*)this; + } + // Serialization of debugging information virtual void write_on(DebugInfoWriteStream* stream) = 0; static ScopeValue* read_from(DebugInfoReadStream* stream); --- old/src/hotspot/share/code/location.hpp 2020-08-07 00:44:11.000000000 +0300 +++ new/src/hotspot/share/code/location.hpp 2020-08-07 00:44:11.000000000 +0300 @@ -58,6 +58,7 @@ lng, // Long held in one register float_in_dbl, // Float held in double register dbl, // Double held in one register + vector, // Vector in one register addr, // JSR return address narrowoop // Narrow Oop (please GC me!) }; --- old/src/hotspot/share/opto/addnode.hpp 2020-08-07 00:44:11.000000000 +0300 +++ new/src/hotspot/share/opto/addnode.hpp 2020-08-07 00:44:11.000000000 +0300 @@ -279,6 +279,30 @@ virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); }; +//------------------------------MaxLNode--------------------------------------- +// MAXimum of 2 longs. +class MaxLNode : public MaxNode { +public: + MaxLNode(Node *in1, Node *in2) : MaxNode(in1, in2) {} + virtual int Opcode() const; + virtual const Type *add_ring(const Type*, const Type*) const { return TypeLong::LONG; } + virtual const Type *add_id() const { return TypeLong::make(min_jlong); } + virtual const Type *bottom_type() const { return TypeLong::LONG; } + virtual uint ideal_reg() const { return Op_RegL; } +}; + +//------------------------------MinLNode--------------------------------------- +// MINimum of 2 longs. +class MinLNode : public MaxNode { +public: + MinLNode(Node *in1, Node *in2) : MaxNode(in1, in2) {} + virtual int Opcode() const; + virtual const Type *add_ring(const Type*, const Type*) const { return TypeLong::LONG; } + virtual const Type *add_id() const { return TypeLong::make(max_jlong); } + virtual const Type *bottom_type() const { return TypeLong::LONG; } + virtual uint ideal_reg() const { return Op_RegL; } +}; + //------------------------------MaxFNode--------------------------------------- // Maximum of 2 floats. class MaxFNode : public MaxNode { --- old/src/hotspot/share/opto/c2_globals.hpp 2020-08-07 00:44:12.000000000 +0300 +++ new/src/hotspot/share/opto/c2_globals.hpp 2020-08-07 00:44:11.000000000 +0300 @@ -740,6 +740,15 @@ diagnostic(bool, UseMontgomerySquareIntrinsic, false, \ "Enables intrinsification of BigInteger.montgomerySquare()") \ \ + experimental(bool, EnableVectorSupport, false, \ + "Enables VectorSupport intrinsics") \ + \ + experimental(bool, EnableVectorReboxing, false, \ + "Enables reboxing of vectors") \ + \ + experimental(bool, EnableVectorAggressiveReboxing, false, \ + "Enables aggressive reboxing of vectors") \ + \ product(bool, UseTypeSpeculation, true, \ "Speculatively propagate types from profiles") \ \ --- old/src/hotspot/share/opto/c2compiler.cpp 2020-08-07 00:44:12.000000000 +0300 +++ new/src/hotspot/share/opto/c2compiler.cpp 2020-08-07 00:44:12.000000000 +0300 @@ -646,6 +646,28 @@ case vmIntrinsics::_isCompileConstant: case vmIntrinsics::_Preconditions_checkIndex: break; + + case vmIntrinsics::_VectorUnaryOp: + case vmIntrinsics::_VectorBinaryOp: + case vmIntrinsics::_VectorTernaryOp: + case vmIntrinsics::_VectorBroadcastCoerced: + case vmIntrinsics::_VectorShuffleIota: + case vmIntrinsics::_VectorShuffleToVector: + case vmIntrinsics::_VectorLoadOp: + case vmIntrinsics::_VectorStoreOp: + case vmIntrinsics::_VectorGatherOp: + case vmIntrinsics::_VectorScatterOp: + case vmIntrinsics::_VectorReductionCoerced: + case vmIntrinsics::_VectorTest: + case vmIntrinsics::_VectorBlend: + case vmIntrinsics::_VectorRearrange: + case vmIntrinsics::_VectorCompare: + case vmIntrinsics::_VectorBroadcastInt: + case vmIntrinsics::_VectorConvert: + case vmIntrinsics::_VectorInsert: + case vmIntrinsics::_VectorExtract: + return EnableVectorSupport; + default: return false; } --- old/src/hotspot/share/opto/callGenerator.cpp 2020-08-07 00:44:13.000000000 +0300 +++ new/src/hotspot/share/opto/callGenerator.cpp 2020-08-07 00:44:12.000000000 +0300 @@ -537,7 +537,7 @@ C->add_string_late_inline(this); - JVMState* new_jvms = DirectCallGenerator::generate(jvms); + JVMState* new_jvms = DirectCallGenerator::generate(jvms); return new_jvms; } @@ -561,7 +561,7 @@ C->add_boxing_late_inline(this); - JVMState* new_jvms = DirectCallGenerator::generate(jvms); + JVMState* new_jvms = DirectCallGenerator::generate(jvms); return new_jvms; } }; @@ -570,6 +570,28 @@ return new LateInlineBoxingCallGenerator(method, inline_cg); } +class LateInlineVectorReboxingCallGenerator : public LateInlineCallGenerator { + + public: + LateInlineVectorReboxingCallGenerator(ciMethod* method, CallGenerator* inline_cg) : + LateInlineCallGenerator(method, inline_cg, /*is_pure=*/true) {} + + virtual JVMState* generate(JVMState* jvms) { + Compile *C = Compile::current(); + + C->log_inline_id(this); + + C->add_vector_reboxing_late_inline(this); + + JVMState* new_jvms = DirectCallGenerator::generate(jvms); + return new_jvms; + } +}; + +// static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg); +CallGenerator* CallGenerator::for_vector_reboxing_late_inline(ciMethod* method, CallGenerator* inline_cg) { + return new LateInlineVectorReboxingCallGenerator(method, inline_cg); +} //---------------------------WarmCallGenerator-------------------------------- // Internal class which handles initial deferral of inlining decisions. class WarmCallGenerator : public CallGenerator { --- old/src/hotspot/share/opto/callGenerator.hpp 2020-08-07 00:44:13.000000000 +0300 +++ new/src/hotspot/share/opto/callGenerator.hpp 2020-08-07 00:44:13.000000000 +0300 @@ -132,6 +132,7 @@ static CallGenerator* for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const); static CallGenerator* for_string_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_boxing_late_inline(ciMethod* m, CallGenerator* inline_cg); + static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg); // How to make a call but defer the decision whether to inline or not. static CallGenerator* for_warm_call(WarmCallInfo* ci, --- old/src/hotspot/share/opto/castnode.cpp 2020-08-07 00:44:13.000000000 +0300 +++ new/src/hotspot/share/opto/castnode.cpp 2020-08-07 00:44:13.000000000 +0300 @@ -290,9 +290,17 @@ if (_carry_dependency) { return this; } - // Toned down to rescue meeting at a Phi 3 different oops all implementing - // the same interface. - return (phase->type(in(1)) == phase->type(this)) ? in(1) : this; + const Type* t = phase->type(in(1)); + if (EnableVectorReboxing && in(1)->Opcode() == Op_VectorBox) { + if (t->higher_equal_speculative(phase->type(this))) { + return in(1); + } + } else if (t == phase->type(this)) { + // Toned down to rescue meeting at a Phi 3 different oops all implementing + // the same interface. + return in(1); + } + return this; } //------------------------------Value------------------------------------------ --- old/src/hotspot/share/opto/cfgnode.cpp 2020-08-07 00:44:14.000000000 +0300 +++ new/src/hotspot/share/opto/cfgnode.cpp 2020-08-07 00:44:14.000000000 +0300 @@ -43,6 +43,7 @@ #include "opto/regmask.hpp" #include "opto/runtime.hpp" #include "opto/subnode.hpp" +#include "opto/vectornode.hpp" #include "utilities/vmError.hpp" // Portions of code courtesy of Clifford Click @@ -2376,6 +2377,47 @@ } #endif + // Phi (VB ... VB) => VB (Phi ...) (Phi ...) + if (EnableVectorReboxing && can_reshape && progress == NULL) { + PhaseIterGVN* igvn = phase->is_IterGVN(); + + bool all_inputs_are_equiv_vboxes = true; + for (uint i = 1; i < req(); ++i) { + Node* n = in(i); + if (in(i)->Opcode() != Op_VectorBox) { + all_inputs_are_equiv_vboxes = false; + break; + } + // Check that vector type of vboxes is equivalent + if (i != 1) { + if (Type::cmp(in(i-0)->in(VectorBoxNode::Value)->bottom_type(), + in(i-1)->in(VectorBoxNode::Value)->bottom_type()) != 0) { + all_inputs_are_equiv_vboxes = false; + break; + } + if (Type::cmp(in(i-0)->in(VectorBoxNode::Box)->bottom_type(), + in(i-1)->in(VectorBoxNode::Box)->bottom_type()) != 0) { + all_inputs_are_equiv_vboxes = false; + break; + } + } + } + + if (all_inputs_are_equiv_vboxes) { + VectorBoxNode* vbox = static_cast(in(1)); + PhiNode* new_vbox_phi = new PhiNode(r, vbox->box_type()); + PhiNode* new_vect_phi = new PhiNode(r, vbox->vec_type()); + for (uint i = 1; i < req(); ++i) { + VectorBoxNode* old_vbox = static_cast(in(i)); + new_vbox_phi->set_req(i, old_vbox->in(VectorBoxNode::Box)); + new_vect_phi->set_req(i, old_vbox->in(VectorBoxNode::Value)); + } + igvn->register_new_node_with_optimizer(new_vbox_phi, this); + igvn->register_new_node_with_optimizer(new_vect_phi, this); + progress = new VectorBoxNode(igvn->C, new_vbox_phi, new_vect_phi, vbox->box_type(), vbox->vec_type()); + } + } + return progress; // Return any progress } --- old/src/hotspot/share/opto/classes.hpp 2020-08-07 00:44:14.000000000 +0300 +++ new/src/hotspot/share/opto/classes.hpp 2020-08-07 00:44:14.000000000 +0300 @@ -198,9 +198,10 @@ macro(Mach) macro(MachProj) macro(MulAddS2I) +macro(MaxI) +macro(MaxL) macro(MaxD) macro(MaxF) -macro(MaxI) macro(MemBarAcquire) macro(LoadFence) macro(SetVectMaskI) @@ -212,9 +213,10 @@ macro(MemBarVolatile) macro(MemBarStoreStore) macro(MergeMem) -macro(MinD) -macro(MinF) macro(MinI) +macro(MinL) +macro(MinF) +macro(MinD) macro(ModD) macro(ModF) macro(ModI) @@ -229,6 +231,8 @@ macro(MulI) macro(MulL) macro(Multi) +macro(NegI) +macro(NegL) macro(NegD) macro(NegF) macro(NeverBranch) @@ -316,6 +320,8 @@ macro(MacroLogicV) macro(ThreadLocal) macro(Unlock) +macro(URShiftB) +macro(URShiftS) macro(URShiftI) macro(URShiftL) macro(XorI) @@ -358,6 +364,7 @@ macro(AbsVL) macro(AbsVF) macro(AbsVD) +macro(NegVI) macro(NegVF) macro(NegVD) macro(SqrtVD) @@ -387,7 +394,9 @@ macro(MinReductionV) macro(MaxReductionV) macro(LoadVector) +macro(LoadVectorGather) macro(StoreVector) +macro(StoreVectorScatter) macro(Pack) macro(PackB) macro(PackS) @@ -416,3 +425,24 @@ macro(LowerCase) macro(UpperCase) macro(Whitespace) +macro(VectorBox) +macro(VectorBoxAllocate) +macro(VectorUnbox) +macro(VectorMaskWrapper) +macro(VectorMaskCmp) +macro(VectorTest) +macro(VectorBlend) +macro(VectorRearrange) +macro(VectorLoadMask) +macro(VectorLoadShuffle) +macro(VectorLoadConst) +macro(VectorStoreMask) +macro(VectorReinterpret) +macro(VectorCast) +macro(VectorCastB2X) +macro(VectorCastS2X) +macro(VectorCastI2X) +macro(VectorCastL2X) +macro(VectorCastF2X) +macro(VectorCastD2X) +macro(VectorInsert) --- old/src/hotspot/share/opto/compile.cpp 2020-08-07 00:44:15.000000000 +0300 +++ new/src/hotspot/share/opto/compile.cpp 2020-08-07 00:44:14.000000000 +0300 @@ -68,6 +68,7 @@ #include "opto/runtime.hpp" #include "opto/stringopts.hpp" #include "opto/type.hpp" +#include "opto/vector.hpp" #include "opto/vectornode.hpp" #include "runtime/arguments.hpp" #include "runtime/sharedRuntime.hpp" @@ -411,6 +412,7 @@ remove_useless_late_inlines(&_string_late_inlines, useful); remove_useless_late_inlines(&_boxing_late_inlines, useful); remove_useless_late_inlines(&_late_inlines, useful); + remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful); debug_only(verify_graph_edges(true/*check for no_dead_code*/);) } @@ -543,6 +545,7 @@ _late_inlines(comp_arena(), 2, 0, NULL), _string_late_inlines(comp_arena(), 2, 0, NULL), _boxing_late_inlines(comp_arena(), 2, 0, NULL), + _vector_reboxing_late_inlines(comp_arena(), 2, 0, NULL), _late_inlines_pos(0), _number_of_mh_late_inlines(0), _print_inlining_stream(NULL), @@ -1946,6 +1949,8 @@ inline_incrementally_cleanup(igvn); + print_method(PHASE_INCREMENTAL_INLINE_STEP, 3); + if (failing()) return; } assert( igvn._worklist.size() == 0, "should be done with igvn" ); @@ -2080,6 +2085,16 @@ // so keep only the actual candidates for optimizations. cleanup_expensive_nodes(igvn); + assert(EnableVectorSupport || !has_vbox_nodes(), "sanity"); + if (EnableVectorSupport && has_vbox_nodes()) { + TracePhase tp("", &timers[_t_vector]); + PhaseVector pv(igvn); + pv.optimize_vector_boxes(); + + print_method(PHASE_ITER_GVN_AFTER_VECTOR, 2); + } + assert(!has_vbox_nodes(), "sanity"); + if (!failing() && RenumberLiveNodes && live_nodes() + NodeLimitFudgeFactor < unique()) { Compile::TracePhase tp("", &timers[_t_renumberLive]); initial_gvn()->replace_with(&igvn); @@ -2534,6 +2549,34 @@ (inputs.size() == 2 || inputs.size() == 3); } +void Compile::inline_vector_reboxing_calls() { + if (C->_vector_reboxing_late_inlines.length() > 0) { + PhaseGVN* gvn = C->initial_gvn(); + + _late_inlines_pos = C->_late_inlines.length(); + while (_vector_reboxing_late_inlines.length() > 0) { + CallGenerator* cg = _vector_reboxing_late_inlines.pop(); + cg->do_late_inline(); + if (failing()) return; + print_method(PHASE_INLINE_VECTOR_REBOX, cg->call_node()); + } + _vector_reboxing_late_inlines.trunc_to(0); + } +} + +bool Compile::has_vbox_nodes() { + if (C->_vector_reboxing_late_inlines.length() > 0) { + return true; + } + for (int macro_idx = C->macro_count() - 1; macro_idx >= 0; macro_idx--) { + Node * n = C->macro_node(macro_idx); + assert(n->is_macro(), "only macro nodes expected here"); + if (n->Opcode() == Op_VectorUnbox || n->Opcode() == Op_VectorBox || n->Opcode() == Op_VectorBoxAllocate) { + return true; + } + } + return false; +} void Compile::process_logic_cone_root(PhaseIterGVN &igvn, Node *n, VectorSet &visited) { assert(is_vector_bitwise_op(n), "not a root"); @@ -2602,8 +2645,8 @@ if (failing()) { return; } + print_method(PHASE_AFTER_MATCHING, 3); } - // In debug mode can dump m._nodes.dump() for mapping of ideal to machine // nodes. Mapping is only valid at the root of each matched subtree. NOT_PRODUCT( verify_graph_edges(); ) @@ -2791,7 +2834,8 @@ // Check for commutative opcode switch( nop ) { case Op_AddI: case Op_AddF: case Op_AddD: case Op_AddL: - case Op_MaxI: case Op_MinI: + case Op_MaxI: case Op_MaxL: case Op_MaxF: case Op_MaxD: + case Op_MinI: case Op_MinL: case Op_MinF: case Op_MinD: case Op_MulI: case Op_MulF: case Op_MulD: case Op_MulL: case Op_AndL: case Op_XorL: case Op_OrL: case Op_AndI: case Op_XorI: case Op_OrI: { @@ -3356,6 +3400,8 @@ case Op_LoadVector: case Op_StoreVector: + case Op_LoadVectorGather: + case Op_StoreVectorScatter: break; case Op_AddReductionVI: @@ -4571,26 +4617,43 @@ } } -void Compile::print_method(CompilerPhaseType cpt, int level, int idx) { +void Compile::print_method(CompilerPhaseType cpt, const char *name, int level, int idx) { EventCompilerPhase event; if (event.should_commit()) { CompilerEvent::PhaseEvent::post(event, C->_latest_stage_start_counter, cpt, C->_compile_id, level); } - #ifndef PRODUCT if (should_print(level)) { - char output[1024]; - if (idx != 0) { - jio_snprintf(output, sizeof(output), "%s:%d", CompilerPhaseTypeHelper::to_string(cpt), idx); - } else { - jio_snprintf(output, sizeof(output), "%s", CompilerPhaseTypeHelper::to_string(cpt)); - } - _printer->print_method(output, level); + _printer->print_method(name, level); } #endif C->_latest_stage_start_counter.stamp(); } +void Compile::print_method(CompilerPhaseType cpt, int level, int idx) { + char output[1024]; +#ifndef PRODUCT + if (idx != 0) { + jio_snprintf(output, sizeof(output), "%s:%d", CompilerPhaseTypeHelper::to_string(cpt), idx); + } else { + jio_snprintf(output, sizeof(output), "%s", CompilerPhaseTypeHelper::to_string(cpt)); + } +#endif + print_method(cpt, output, level, idx); +} + +void Compile::print_method(CompilerPhaseType cpt, Node* n, int level) { + ResourceMark rm; + stringStream ss; + ss.print_raw(CompilerPhaseTypeHelper::to_string(cpt)); + if (n != NULL) { + ss.print(": %d %s ", n->_idx, NodeClassNames[n->Opcode()]); + } else { + ss.print_raw(": NULL"); + } + C->print_method(cpt, ss.as_string(), level); +} + void Compile::end_method(int level) { EventCompilerPhase event; if (event.should_commit()) { --- old/src/hotspot/share/opto/compile.hpp 2020-08-07 00:44:15.000000000 +0300 +++ new/src/hotspot/share/opto/compile.hpp 2020-08-07 00:44:15.000000000 +0300 @@ -382,6 +382,8 @@ GrowableArray _boxing_late_inlines; // same but for boxing operations + GrowableArray _vector_reboxing_late_inlines; // same but for vector reboxing operations + int _late_inlines_pos; // Where in the queue should the next late inlining candidate go (emulate depth first inlining) uint _number_of_mh_late_inlines; // number of method handle late inlining still pending @@ -643,7 +645,9 @@ #endif } + void print_method(CompilerPhaseType cpt, const char *name, int level = 1, int idx = 0); void print_method(CompilerPhaseType cpt, int level = 1, int idx = 0); + void print_method(CompilerPhaseType cpt, Node* n, int level = 3); #ifndef PRODUCT void igv_print_method_to_file(const char* phase_name = "Debug", bool append = false); @@ -864,10 +868,13 @@ bool allow_intrinsics = true); bool should_delay_inlining(ciMethod* call_method, JVMState* jvms) { return should_delay_string_inlining(call_method, jvms) || - should_delay_boxing_inlining(call_method, jvms); + should_delay_boxing_inlining(call_method, jvms) || + should_delay_vector_inlining(call_method, jvms); } bool should_delay_string_inlining(ciMethod* call_method, JVMState* jvms); bool should_delay_boxing_inlining(ciMethod* call_method, JVMState* jvms); + bool should_delay_vector_inlining(ciMethod* call_method, JVMState* jvms); + bool should_delay_vector_reboxing_inlining(ciMethod* call_method, JVMState* jvms); // Helper functions to identify inlining potential at call-site ciMethod* optimize_virtual_call(ciMethod* caller, int bci, ciInstanceKlass* klass, @@ -939,6 +946,10 @@ _boxing_late_inlines.push(cg); } + void add_vector_reboxing_late_inline(CallGenerator* cg) { + _vector_reboxing_late_inlines.push(cg); + } + void remove_useless_late_inlines(GrowableArray* inlines, Unique_Node_List &useful); void process_print_inlining(); @@ -968,6 +979,9 @@ bool optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode); void remove_root_to_sfpts_edges(PhaseIterGVN& igvn); + void inline_vector_reboxing_calls(); + bool has_vbox_nodes(); + // Matching, CFG layout, allocation, code generation PhaseCFG* cfg() { return _cfg; } bool has_java_calls() const { return _java_calls > 0; } --- old/src/hotspot/share/opto/doCall.cpp 2020-08-07 00:44:16.000000000 +0300 +++ new/src/hotspot/share/opto/doCall.cpp 2020-08-07 00:44:15.000000000 +0300 @@ -134,6 +134,8 @@ if (cg->does_virtual_dispatch()) { cg_intrinsic = cg; cg = NULL; + } else if (should_delay_vector_inlining(callee, jvms)) { + return CallGenerator::for_late_inline(callee, cg); } else { return cg; } @@ -184,6 +186,8 @@ return CallGenerator::for_string_late_inline(callee, cg); } else if (should_delay_boxing_inlining(callee, jvms)) { return CallGenerator::for_boxing_late_inline(callee, cg); + } else if (should_delay_vector_reboxing_inlining(callee, jvms)) { + return CallGenerator::for_vector_reboxing_late_inline(callee, cg); } else if ((should_delay || AlwaysIncrementalInline)) { return CallGenerator::for_late_inline(callee, cg); } @@ -421,6 +425,14 @@ return false; } +bool Compile::should_delay_vector_inlining(ciMethod* call_method, JVMState* jvms) { + return EnableVectorSupport && call_method->is_vector_method(); +} + +bool Compile::should_delay_vector_reboxing_inlining(ciMethod* call_method, JVMState* jvms) { + return EnableVectorSupport && (call_method->intrinsic_id() == vmIntrinsics::_VectorRebox); +} + // uncommon-trap call-sites where callee is unloaded, uninitialized or will not link bool Parse::can_not_compile_call_site(ciMethod *dest_method, ciInstanceKlass* klass) { // Additional inputs to consider... --- old/src/hotspot/share/opto/lcm.cpp 2020-08-07 00:44:16.000000000 +0300 +++ new/src/hotspot/share/opto/lcm.cpp 2020-08-07 00:44:16.000000000 +0300 @@ -686,6 +686,7 @@ case Op_StoreP: case Op_StoreN: case Op_StoreVector: + case Op_StoreVectorScatter: case Op_StoreNKlass: for (uint k = 1; k < m->req(); k++) { Node *in = m->in(k); --- old/src/hotspot/share/opto/library_call.cpp 2020-08-07 00:44:16.000000000 +0300 +++ new/src/hotspot/share/opto/library_call.cpp 2020-08-07 00:44:16.000000000 +0300 @@ -37,15 +37,13 @@ #include "opto/addnode.hpp" #include "opto/arraycopynode.hpp" #include "opto/c2compiler.hpp" -#include "opto/callGenerator.hpp" #include "opto/castnode.hpp" #include "opto/cfgnode.hpp" #include "opto/convertnode.hpp" #include "opto/countbitsnode.hpp" -#include "opto/intrinsicnode.hpp" #include "opto/idealKit.hpp" +#include "opto/library_call.hpp" #include "opto/mathexactnode.hpp" -#include "opto/movenode.hpp" #include "opto/mulnode.hpp" #include "opto/narrowptrnode.hpp" #include "opto/opaquenode.hpp" @@ -60,291 +58,6 @@ #include "utilities/macros.hpp" #include "utilities/powerOfTwo.hpp" -class LibraryIntrinsic : public InlineCallGenerator { - // Extend the set of intrinsics known to the runtime: - public: - private: - bool _is_virtual; - bool _does_virtual_dispatch; - int8_t _predicates_count; // Intrinsic is predicated by several conditions - int8_t _last_predicate; // Last generated predicate - vmIntrinsics::ID _intrinsic_id; - - public: - LibraryIntrinsic(ciMethod* m, bool is_virtual, int predicates_count, bool does_virtual_dispatch, vmIntrinsics::ID id) - : InlineCallGenerator(m), - _is_virtual(is_virtual), - _does_virtual_dispatch(does_virtual_dispatch), - _predicates_count((int8_t)predicates_count), - _last_predicate((int8_t)-1), - _intrinsic_id(id) - { - } - virtual bool is_intrinsic() const { return true; } - virtual bool is_virtual() const { return _is_virtual; } - virtual bool is_predicated() const { return _predicates_count > 0; } - virtual int predicates_count() const { return _predicates_count; } - virtual bool does_virtual_dispatch() const { return _does_virtual_dispatch; } - virtual JVMState* generate(JVMState* jvms); - virtual Node* generate_predicate(JVMState* jvms, int predicate); - vmIntrinsics::ID intrinsic_id() const { return _intrinsic_id; } -}; - - -// Local helper class for LibraryIntrinsic: -class LibraryCallKit : public GraphKit { - private: - LibraryIntrinsic* _intrinsic; // the library intrinsic being called - Node* _result; // the result node, if any - int _reexecute_sp; // the stack pointer when bytecode needs to be reexecuted - - const TypeOopPtr* sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type); - - public: - LibraryCallKit(JVMState* jvms, LibraryIntrinsic* intrinsic) - : GraphKit(jvms), - _intrinsic(intrinsic), - _result(NULL) - { - // Check if this is a root compile. In that case we don't have a caller. - if (!jvms->has_method()) { - _reexecute_sp = sp(); - } else { - // Find out how many arguments the interpreter needs when deoptimizing - // and save the stack pointer value so it can used by uncommon_trap. - // We find the argument count by looking at the declared signature. - bool ignored_will_link; - ciSignature* declared_signature = NULL; - ciMethod* ignored_callee = caller()->get_method_at_bci(bci(), ignored_will_link, &declared_signature); - const int nargs = declared_signature->arg_size_for_bc(caller()->java_code_at_bci(bci())); - _reexecute_sp = sp() + nargs; // "push" arguments back on stack - } - } - - virtual LibraryCallKit* is_LibraryCallKit() const { return (LibraryCallKit*)this; } - - ciMethod* caller() const { return jvms()->method(); } - int bci() const { return jvms()->bci(); } - LibraryIntrinsic* intrinsic() const { return _intrinsic; } - vmIntrinsics::ID intrinsic_id() const { return _intrinsic->intrinsic_id(); } - ciMethod* callee() const { return _intrinsic->method(); } - - bool try_to_inline(int predicate); - Node* try_to_predicate(int predicate); - - void push_result() { - // Push the result onto the stack. - if (!stopped() && result() != NULL) { - BasicType bt = result()->bottom_type()->basic_type(); - push_node(bt, result()); - } - } - - private: - void fatal_unexpected_iid(vmIntrinsics::ID iid) { - fatal("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid)); - } - - void set_result(Node* n) { assert(_result == NULL, "only set once"); _result = n; } - void set_result(RegionNode* region, PhiNode* value); - Node* result() { return _result; } - - virtual int reexecute_sp() { return _reexecute_sp; } - - // Helper functions to inline natives - Node* generate_guard(Node* test, RegionNode* region, float true_prob); - Node* generate_slow_guard(Node* test, RegionNode* region); - Node* generate_fair_guard(Node* test, RegionNode* region); - Node* generate_negative_guard(Node* index, RegionNode* region, - // resulting CastII of index: - Node* *pos_index = NULL); - Node* generate_limit_guard(Node* offset, Node* subseq_length, - Node* array_length, - RegionNode* region); - void generate_string_range_check(Node* array, Node* offset, - Node* length, bool char_count); - Node* generate_current_thread(Node* &tls_output); - Node* load_mirror_from_klass(Node* klass); - Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null, - RegionNode* region, int null_path, - int offset); - Node* load_klass_from_mirror(Node* mirror, bool never_see_null, - RegionNode* region, int null_path) { - int offset = java_lang_Class::klass_offset(); - return load_klass_from_mirror_common(mirror, never_see_null, - region, null_path, - offset); - } - Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null, - RegionNode* region, int null_path) { - int offset = java_lang_Class::array_klass_offset(); - return load_klass_from_mirror_common(mirror, never_see_null, - region, null_path, - offset); - } - Node* generate_access_flags_guard(Node* kls, - int modifier_mask, int modifier_bits, - RegionNode* region); - Node* generate_interface_guard(Node* kls, RegionNode* region); - Node* generate_hidden_class_guard(Node* kls, RegionNode* region); - Node* generate_array_guard(Node* kls, RegionNode* region) { - return generate_array_guard_common(kls, region, false, false); - } - Node* generate_non_array_guard(Node* kls, RegionNode* region) { - return generate_array_guard_common(kls, region, false, true); - } - Node* generate_objArray_guard(Node* kls, RegionNode* region) { - return generate_array_guard_common(kls, region, true, false); - } - Node* generate_non_objArray_guard(Node* kls, RegionNode* region) { - return generate_array_guard_common(kls, region, true, true); - } - Node* generate_array_guard_common(Node* kls, RegionNode* region, - bool obj_array, bool not_array); - Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region); - CallJavaNode* generate_method_call(vmIntrinsics::ID method_id, - bool is_virtual = false, bool is_static = false); - CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) { - return generate_method_call(method_id, false, true); - } - CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) { - return generate_method_call(method_id, true, false); - } - Node * load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls); - Node * field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls); - - Node* make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae); - bool inline_string_compareTo(StrIntrinsicNode::ArgEnc ae); - bool inline_string_indexOf(StrIntrinsicNode::ArgEnc ae); - bool inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae); - Node* make_indexOf_node(Node* src_start, Node* src_count, Node* tgt_start, Node* tgt_count, - RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae); - bool inline_string_indexOfChar(); - bool inline_string_equals(StrIntrinsicNode::ArgEnc ae); - bool inline_string_toBytesU(); - bool inline_string_getCharsU(); - bool inline_string_copy(bool compress); - bool inline_string_char_access(bool is_store); - Node* round_double_node(Node* n); - bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName); - bool inline_math_native(vmIntrinsics::ID id); - bool inline_math(vmIntrinsics::ID id); - bool inline_double_math(vmIntrinsics::ID id); - template - bool inline_math_overflow(Node* arg1, Node* arg2); - void inline_math_mathExact(Node* math, Node* test); - bool inline_math_addExactI(bool is_increment); - bool inline_math_addExactL(bool is_increment); - bool inline_math_multiplyExactI(); - bool inline_math_multiplyExactL(); - bool inline_math_multiplyHigh(); - bool inline_math_negateExactI(); - bool inline_math_negateExactL(); - bool inline_math_subtractExactI(bool is_decrement); - bool inline_math_subtractExactL(bool is_decrement); - bool inline_min_max(vmIntrinsics::ID id); - bool inline_notify(vmIntrinsics::ID id); - Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y); - // This returns Type::AnyPtr, RawPtr, or OopPtr. - int classify_unsafe_addr(Node* &base, Node* &offset, BasicType type); - Node* make_unsafe_address(Node*& base, Node* offset, DecoratorSet decorators, BasicType type = T_ILLEGAL, bool can_cast = false); - - typedef enum { Relaxed, Opaque, Volatile, Acquire, Release } AccessKind; - DecoratorSet mo_decorator_for_access_kind(AccessKind kind); - bool inline_unsafe_access(bool is_store, BasicType type, AccessKind kind, bool is_unaligned); - static bool klass_needs_init_guard(Node* kls); - bool inline_unsafe_allocate(); - bool inline_unsafe_newArray(bool uninitialized); - bool inline_unsafe_writeback0(); - bool inline_unsafe_writebackSync0(bool is_pre); - bool inline_unsafe_copyMemory(); - bool inline_native_currentThread(); - - bool inline_native_time_funcs(address method, const char* funcName); -#ifdef JFR_HAVE_INTRINSICS - bool inline_native_classID(); - bool inline_native_getEventWriter(); -#endif - bool inline_native_Class_query(vmIntrinsics::ID id); - bool inline_native_subtype_check(); - bool inline_native_getLength(); - bool inline_array_copyOf(bool is_copyOfRange); - bool inline_array_equals(StrIntrinsicNode::ArgEnc ae); - bool inline_preconditions_checkIndex(); - void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array); - bool inline_native_clone(bool is_virtual); - bool inline_native_Reflection_getCallerClass(); - // Helper function for inlining native object hash method - bool inline_native_hashcode(bool is_virtual, bool is_static); - bool inline_native_getClass(); - - // Helper functions for inlining arraycopy - bool inline_arraycopy(); - AllocateArrayNode* tightly_coupled_allocation(Node* ptr, - RegionNode* slow_region); - JVMState* arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp); - void arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms, int saved_reexecute_sp, - uint new_idx); - - typedef enum { LS_get_add, LS_get_set, LS_cmp_swap, LS_cmp_swap_weak, LS_cmp_exchange } LoadStoreKind; - bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind, AccessKind access_kind); - bool inline_unsafe_fence(vmIntrinsics::ID id); - bool inline_onspinwait(); - bool inline_fp_conversions(vmIntrinsics::ID id); - bool inline_number_methods(vmIntrinsics::ID id); - bool inline_reference_get(); - bool inline_Class_cast(); - bool inline_aescrypt_Block(vmIntrinsics::ID id); - bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id); - bool inline_electronicCodeBook_AESCrypt(vmIntrinsics::ID id); - bool inline_counterMode_AESCrypt(vmIntrinsics::ID id); - Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting); - Node* inline_electronicCodeBook_AESCrypt_predicate(bool decrypting); - Node* inline_counterMode_AESCrypt_predicate(); - Node* get_key_start_from_aescrypt_object(Node* aescrypt_object); - Node* get_original_key_start_from_aescrypt_object(Node* aescrypt_object); - bool inline_ghash_processBlocks(); - bool inline_base64_encodeBlock(); - bool inline_digestBase_implCompress(vmIntrinsics::ID id); - bool inline_digestBase_implCompressMB(int predicate); - bool inline_digestBase_implCompressMB(Node* digestBaseObj, ciInstanceKlass* instklass, - bool long_state, address stubAddr, const char *stubName, - Node* src_start, Node* ofs, Node* limit); - Node* get_state_from_digest_object(Node *digestBase_object); - Node* get_long_state_from_digest_object(Node *digestBase_object); - Node* inline_digestBase_implCompressMB_predicate(int predicate); - bool inline_encodeISOArray(); - bool inline_updateCRC32(); - bool inline_updateBytesCRC32(); - bool inline_updateByteBufferCRC32(); - Node* get_table_from_crc32c_class(ciInstanceKlass *crc32c_class); - bool inline_updateBytesCRC32C(); - bool inline_updateDirectByteBufferCRC32C(); - bool inline_updateBytesAdler32(); - bool inline_updateByteBufferAdler32(); - bool inline_multiplyToLen(); - bool inline_hasNegatives(); - bool inline_squareToLen(); - bool inline_mulAdd(); - bool inline_montgomeryMultiply(); - bool inline_montgomerySquare(); - bool inline_bigIntegerShift(bool isRightShift); - bool inline_vectorizedMismatch(); - bool inline_fma(vmIntrinsics::ID id); - bool inline_character_compare(vmIntrinsics::ID id); - bool inline_fp_min_max(vmIntrinsics::ID id); - - bool inline_profileBoolean(); - bool inline_isCompileConstant(); - void clear_upper_avx() { -#ifdef X86 - if (UseAVX >= 2) { - C->set_clear_upper_avx(true); - } -#endif - } -}; - //---------------------------make_vm_intrinsic---------------------------- CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) { vmIntrinsics::ID id = m->intrinsic_id(); @@ -453,6 +166,7 @@ } C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed); C->print_inlining_update(this); + return NULL; } @@ -530,7 +244,6 @@ } assert(merged_memory(), ""); - switch (intrinsic_id()) { case vmIntrinsics::_hashCode: return inline_native_hashcode(intrinsic()->is_virtual(), !is_static); case vmIntrinsics::_identityHashCode: return inline_native_hashcode(/*!virtual*/ false, is_static); @@ -907,6 +620,45 @@ case vmIntrinsics::_minD: return inline_fp_min_max(intrinsic_id()); + case vmIntrinsics::_VectorUnaryOp: + return inline_vector_nary_operation(1); + case vmIntrinsics::_VectorBinaryOp: + return inline_vector_nary_operation(2); + case vmIntrinsics::_VectorTernaryOp: + return inline_vector_nary_operation(3); + case vmIntrinsics::_VectorBroadcastCoerced: + return inline_vector_broadcast_coerced(); + case vmIntrinsics::_VectorShuffleIota: + return inline_vector_shuffle_iota(); + case vmIntrinsics::_VectorShuffleToVector: + return inline_vector_shuffle_to_vector(); + case vmIntrinsics::_VectorLoadOp: + return inline_vector_mem_operation(/*is_store=*/false); + case vmIntrinsics::_VectorStoreOp: + return inline_vector_mem_operation(/*is_store=*/true); + case vmIntrinsics::_VectorGatherOp: + return inline_vector_gather_scatter(/*is_scatter*/ false); + case vmIntrinsics::_VectorScatterOp: + return inline_vector_gather_scatter(/*is_scatter*/ true); + case vmIntrinsics::_VectorReductionCoerced: + return inline_vector_reduction(); + case vmIntrinsics::_VectorTest: + return inline_vector_test(); + case vmIntrinsics::_VectorBlend: + return inline_vector_blend(); + case vmIntrinsics::_VectorRearrange: + return inline_vector_rearrange(); + case vmIntrinsics::_VectorCompare: + return inline_vector_compare(); + case vmIntrinsics::_VectorBroadcastInt: + return inline_vector_broadcast_int(); + case vmIntrinsics::_VectorConvert: + return inline_vector_convert(); + case vmIntrinsics::_VectorInsert: + return inline_vector_insert(); + case vmIntrinsics::_VectorExtract: + return inline_vector_extract(); + default: // If you get here, it may be that someone has added a new intrinsic // to the list in vmSymbols.hpp without implementing it here. @@ -2240,7 +1992,7 @@ } } -inline Node* LibraryCallKit::make_unsafe_address(Node*& base, Node* offset, DecoratorSet decorators, BasicType type, bool can_cast) { +Node* LibraryCallKit::make_unsafe_address(Node*& base, Node* offset, DecoratorSet decorators, BasicType type, bool can_cast) { Node* uncasted_base = base; int kind = classify_unsafe_addr(uncasted_base, offset, type); if (kind == Type::RawPtr) { --- old/src/hotspot/share/opto/matcher.cpp 2020-08-07 00:44:17.000000000 +0300 +++ new/src/hotspot/share/opto/matcher.cpp 2020-08-07 00:44:17.000000000 +0300 @@ -423,7 +423,7 @@ return rms; } -#define NOF_STACK_MASKS (3*6+5) +#define NOF_STACK_MASKS (3*11) // Create the initial stack mask used by values spilling to the stack. // Disallow any debug info in outgoing argument areas by setting the @@ -465,6 +465,18 @@ idealreg2spillmask [Op_VecY] = &rms[21]; idealreg2spillmask [Op_VecZ] = &rms[22]; + idealreg2debugmask [Op_VecS] = &rms[23]; + idealreg2debugmask [Op_VecD] = &rms[24]; + idealreg2debugmask [Op_VecX] = &rms[25]; + idealreg2debugmask [Op_VecY] = &rms[26]; + idealreg2debugmask [Op_VecZ] = &rms[27]; + + idealreg2mhdebugmask[Op_VecS] = &rms[28]; + idealreg2mhdebugmask[Op_VecD] = &rms[29]; + idealreg2mhdebugmask[Op_VecX] = &rms[30]; + idealreg2mhdebugmask[Op_VecY] = &rms[31]; + idealreg2mhdebugmask[Op_VecZ] = &rms[32]; + OptoReg::Name i; // At first, start with the empty mask @@ -511,13 +523,19 @@ if (Matcher::vector_size_supported(T_BYTE,4)) { *idealreg2spillmask[Op_VecS] = *idealreg2regmask[Op_VecS]; idealreg2spillmask[Op_VecS]->OR(C->FIRST_STACK_mask()); + } else { + *idealreg2spillmask[Op_VecS] = RegMask::Empty; } + if (Matcher::vector_size_supported(T_FLOAT,2)) { // For VecD we need dual alignment and 8 bytes (2 slots) for spills. // RA guarantees such alignment since it is needed for Double and Long values. *idealreg2spillmask[Op_VecD] = *idealreg2regmask[Op_VecD]; idealreg2spillmask[Op_VecD]->OR(aligned_stack_mask); + } else { + *idealreg2spillmask[Op_VecD] = RegMask::Empty; } + if (Matcher::vector_size_supported(T_FLOAT,4)) { // For VecX we need quadro alignment and 16 bytes (4 slots) for spills. // @@ -535,7 +553,10 @@ assert(aligned_stack_mask.is_AllStack(), "should be infinite stack"); *idealreg2spillmask[Op_VecX] = *idealreg2regmask[Op_VecX]; idealreg2spillmask[Op_VecX]->OR(aligned_stack_mask); + } else { + *idealreg2spillmask[Op_VecX] = RegMask::Empty; } + if (Matcher::vector_size_supported(T_FLOAT,8)) { // For VecY we need octo alignment and 32 bytes (8 slots) for spills. OptoReg::Name in = OptoReg::add(_in_arg_limit, -1); @@ -547,7 +568,10 @@ assert(aligned_stack_mask.is_AllStack(), "should be infinite stack"); *idealreg2spillmask[Op_VecY] = *idealreg2regmask[Op_VecY]; idealreg2spillmask[Op_VecY]->OR(aligned_stack_mask); + } else { + *idealreg2spillmask[Op_VecY] = RegMask::Empty; } + if (Matcher::vector_size_supported(T_FLOAT,16)) { // For VecZ we need enough alignment and 64 bytes (16 slots) for spills. OptoReg::Name in = OptoReg::add(_in_arg_limit, -1); @@ -559,7 +583,10 @@ assert(aligned_stack_mask.is_AllStack(), "should be infinite stack"); *idealreg2spillmask[Op_VecZ] = *idealreg2regmask[Op_VecZ]; idealreg2spillmask[Op_VecZ]->OR(aligned_stack_mask); + } else { + *idealreg2spillmask[Op_VecZ] = RegMask::Empty; } + if (UseFPUForSpilling) { // This mask logic assumes that the spill operations are // symmetric and that the registers involved are the same size. @@ -593,6 +620,12 @@ *idealreg2debugmask [Op_RegD]= *idealreg2spillmask[Op_RegD]; *idealreg2debugmask [Op_RegP]= *idealreg2spillmask[Op_RegP]; + *idealreg2debugmask [Op_VecS]= *idealreg2spillmask[Op_VecS]; + *idealreg2debugmask [Op_VecD]= *idealreg2spillmask[Op_VecD]; + *idealreg2debugmask [Op_VecX]= *idealreg2spillmask[Op_VecX]; + *idealreg2debugmask [Op_VecY]= *idealreg2spillmask[Op_VecY]; + *idealreg2debugmask [Op_VecZ]= *idealreg2spillmask[Op_VecZ]; + *idealreg2mhdebugmask[Op_RegN]= *idealreg2spillmask[Op_RegN]; *idealreg2mhdebugmask[Op_RegI]= *idealreg2spillmask[Op_RegI]; *idealreg2mhdebugmask[Op_RegL]= *idealreg2spillmask[Op_RegL]; @@ -600,6 +633,12 @@ *idealreg2mhdebugmask[Op_RegD]= *idealreg2spillmask[Op_RegD]; *idealreg2mhdebugmask[Op_RegP]= *idealreg2spillmask[Op_RegP]; + *idealreg2mhdebugmask[Op_VecS]= *idealreg2spillmask[Op_VecS]; + *idealreg2mhdebugmask[Op_VecD]= *idealreg2spillmask[Op_VecD]; + *idealreg2mhdebugmask[Op_VecX]= *idealreg2spillmask[Op_VecX]; + *idealreg2mhdebugmask[Op_VecY]= *idealreg2spillmask[Op_VecY]; + *idealreg2mhdebugmask[Op_VecZ]= *idealreg2spillmask[Op_VecZ]; + // Prevent stub compilations from attempting to reference // callee-saved registers from debug info bool exclude_soe = !Compile::current()->is_method_compilation(); @@ -615,6 +654,11 @@ idealreg2debugmask [Op_RegF]->Remove(i); // masks idealreg2debugmask [Op_RegD]->Remove(i); idealreg2debugmask [Op_RegP]->Remove(i); + idealreg2debugmask [Op_VecS]->Remove(i); + idealreg2debugmask [Op_VecD]->Remove(i); + idealreg2debugmask [Op_VecX]->Remove(i); + idealreg2debugmask [Op_VecY]->Remove(i); + idealreg2debugmask [Op_VecZ]->Remove(i); idealreg2mhdebugmask[Op_RegN]->Remove(i); idealreg2mhdebugmask[Op_RegI]->Remove(i); @@ -622,6 +666,11 @@ idealreg2mhdebugmask[Op_RegF]->Remove(i); idealreg2mhdebugmask[Op_RegD]->Remove(i); idealreg2mhdebugmask[Op_RegP]->Remove(i); + idealreg2mhdebugmask[Op_VecS]->Remove(i); + idealreg2mhdebugmask[Op_VecD]->Remove(i); + idealreg2mhdebugmask[Op_VecX]->Remove(i); + idealreg2mhdebugmask[Op_VecY]->Remove(i); + idealreg2mhdebugmask[Op_VecZ]->Remove(i); } } @@ -634,6 +683,11 @@ idealreg2mhdebugmask[Op_RegF]->SUBTRACT(save_mask); idealreg2mhdebugmask[Op_RegD]->SUBTRACT(save_mask); idealreg2mhdebugmask[Op_RegP]->SUBTRACT(save_mask); + idealreg2mhdebugmask[Op_VecS]->SUBTRACT(save_mask); + idealreg2mhdebugmask[Op_VecD]->SUBTRACT(save_mask); + idealreg2mhdebugmask[Op_VecX]->SUBTRACT(save_mask); + idealreg2mhdebugmask[Op_VecY]->SUBTRACT(save_mask); + idealreg2mhdebugmask[Op_VecZ]->SUBTRACT(save_mask); } //---------------------------is_save_on_entry---------------------------------- @@ -1926,7 +1980,6 @@ return false; } - bool Matcher::clone_node(Node* n, Node* m, Matcher::MStack& mstack) { // Must clone all producers of flags, or we will not match correctly. // Suppose a compare setting int-flags is shared (e.g., a switch-tree) @@ -2264,6 +2317,20 @@ n->del_req(3); break; } + case Op_VectorBlend: + case Op_VectorInsert: { + Node* pair = new BinaryNode(n->in(1), n->in(2)); + n->set_req(1, pair); + n->set_req(2, n->in(3)); + n->del_req(3); + break; + } + case Op_StoreVectorScatter: { + Node* pair = new BinaryNode(n->in(MemNode::ValueIn), n->in(MemNode::ValueIn+1)); + n->set_req(MemNode::ValueIn, pair); + n->del_req(MemNode::ValueIn+1); + break; + } case Op_MulAddS2I: { Node* pair1 = new BinaryNode(n->in(1), n->in(2)); Node* pair2 = new BinaryNode(n->in(3), n->in(4)); @@ -2273,6 +2340,12 @@ n->del_req(3); break; } + case Op_VectorMaskCmp: { + n->set_req(1, new BinaryNode(n->in(1), n->in(2))); + n->set_req(2, n->in(3)); + n->del_req(3); + break; + } default: break; } --- old/src/hotspot/share/opto/matcher.hpp 2020-08-07 00:44:17.000000000 +0300 +++ new/src/hotspot/share/opto/matcher.hpp 2020-08-07 00:44:17.000000000 +0300 @@ -337,6 +337,9 @@ // Vector ideal reg static const uint vector_ideal_reg(int len); + // Does the CPU supports vector variable shift instructions? + static bool supports_vector_variable_shifts(void); + // CPU supports misaligned vectors store/load. static const bool misaligned_vectors_ok(); --- old/src/hotspot/share/opto/memnode.cpp 2020-08-07 00:44:18.000000000 +0300 +++ new/src/hotspot/share/opto/memnode.cpp 2020-08-07 00:44:18.000000000 +0300 @@ -633,7 +633,8 @@ } if (st_offset != offset && st_offset != Type::OffsetBot) { - const int MAX_STORE = BytesPerLong; + const int MAX_STORE = MAX2(BytesPerLong, (int)MaxVectorSize); + assert(mem->as_Store()->memory_size() <= MAX_STORE, ""); if (st_offset >= offset + size_in_bytes || st_offset <= offset - MAX_STORE || st_offset <= offset - mem->as_Store()->memory_size()) { @@ -1101,7 +1102,12 @@ // (This is one of the few places where a generic PhaseTransform // can create new nodes. Think of it as lazily manifesting // virtually pre-existing constants.) - return phase->zerocon(memory_type()); + if (memory_type() != T_VOID) { + return phase->zerocon(memory_type()); + } else { + // TODO: materialize all-zero vector constant + assert(!isa_Load() || as_Load()->type()->isa_vect(), ""); + } } // A load from an initialization barrier can match a captured store. @@ -2546,6 +2552,8 @@ assert(Opcode() == st->Opcode() || st->Opcode() == Op_StoreVector || Opcode() == Op_StoreVector || + st->Opcode() == Op_StoreVectorScatter || + Opcode() == Op_StoreVectorScatter || phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw || (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy @@ -3729,7 +3737,7 @@ int InitializeNode::captured_store_insertion_point(intptr_t start, int size_in_bytes, PhaseTransform* phase) { - const int FAIL = 0, MAX_STORE = BytesPerLong; + const int FAIL = 0, MAX_STORE = MAX2(BytesPerLong, (int)MaxVectorSize); if (is_complete()) return FAIL; // arraycopy got here first; punt @@ -3759,6 +3767,7 @@ } return -(int)i; // not found; here is where to put it } else if (st_off < start) { + assert(st->as_Store()->memory_size() <= MAX_STORE, ""); if (size_in_bytes != 0 && start < st_off + MAX_STORE && start < st_off + st->as_Store()->memory_size()) { --- old/src/hotspot/share/opto/movenode.cpp 2020-08-07 00:44:18.000000000 +0300 +++ new/src/hotspot/share/opto/movenode.cpp 2020-08-07 00:44:18.000000000 +0300 @@ -363,6 +363,14 @@ return TypeD::make( v.get_jdouble() ); } +//------------------------------Identity---------------------------------------- +Node* MoveL2DNode::Identity(PhaseGVN* phase) { + if (in(1)->Opcode() == Op_MoveD2L) { + return in(1)->in(1); + } + return this; +} + //------------------------------Value------------------------------------------ const Type* MoveI2FNode::Value(PhaseGVN* phase) const { const Type *t = phase->type( in(1) ); @@ -374,6 +382,14 @@ return TypeF::make( v.get_jfloat() ); } +//------------------------------Identity---------------------------------------- +Node* MoveI2FNode::Identity(PhaseGVN* phase) { + if (in(1)->Opcode() == Op_MoveF2I) { + return in(1)->in(1); + } + return this; +} + //------------------------------Value------------------------------------------ const Type* MoveF2INode::Value(PhaseGVN* phase) const { const Type *t = phase->type( in(1) ); @@ -385,6 +401,14 @@ return TypeInt::make( v.get_jint() ); } +//------------------------------Identity---------------------------------------- +Node* MoveF2INode::Identity(PhaseGVN* phase) { + if (in(1)->Opcode() == Op_MoveI2F) { + return in(1)->in(1); + } + return this; +} + //------------------------------Value------------------------------------------ const Type* MoveD2LNode::Value(PhaseGVN* phase) const { const Type *t = phase->type( in(1) ); @@ -396,6 +420,14 @@ return TypeLong::make( v.get_jlong() ); } +//------------------------------Identity---------------------------------------- +Node* MoveD2LNode::Identity(PhaseGVN* phase) { + if (in(1)->Opcode() == Op_MoveL2D) { + return in(1)->in(1); + } + return this; +} + #ifndef PRODUCT //----------------------------BinaryNode--------------------------------------- // The set of related nodes for a BinaryNode is all data inputs and all outputs --- old/src/hotspot/share/opto/movenode.hpp 2020-08-07 00:44:19.000000000 +0300 +++ new/src/hotspot/share/opto/movenode.hpp 2020-08-07 00:44:19.000000000 +0300 @@ -105,6 +105,7 @@ virtual const Type *bottom_type() const { return Type::FLOAT; } virtual uint ideal_reg() const { return Op_RegF; } virtual const Type* Value(PhaseGVN* phase) const; + virtual Node* Identity(PhaseGVN* phase); }; class MoveL2DNode : public Node { @@ -114,6 +115,7 @@ virtual const Type *bottom_type() const { return Type::DOUBLE; } virtual uint ideal_reg() const { return Op_RegD; } virtual const Type* Value(PhaseGVN* phase) const; + virtual Node* Identity(PhaseGVN* phase); }; class MoveF2INode : public Node { @@ -123,6 +125,7 @@ virtual const Type *bottom_type() const { return TypeInt::INT; } virtual uint ideal_reg() const { return Op_RegI; } virtual const Type* Value(PhaseGVN* phase) const; + virtual Node* Identity(PhaseGVN* phase); }; class MoveD2LNode : public Node { @@ -132,6 +135,7 @@ virtual const Type *bottom_type() const { return TypeLong::LONG; } virtual uint ideal_reg() const { return Op_RegL; } virtual const Type* Value(PhaseGVN* phase) const; + virtual Node* Identity(PhaseGVN* phase); }; //------------------------------BinaryNode------------------------------------- --- old/src/hotspot/share/opto/mulnode.hpp 2020-08-07 00:44:19.000000000 +0300 +++ new/src/hotspot/share/opto/mulnode.hpp 2020-08-07 00:44:19.000000000 +0300 @@ -236,6 +236,25 @@ virtual uint ideal_reg() const { return Op_RegL; } }; +//------------------------------URShiftBNode----------------------------------- +// Logical shift right +class URShiftBNode : public Node { +public: + URShiftBNode( Node *in1, Node *in2 ) : Node(0,in1,in2) { + ShouldNotReachHere(); // only vector variant is used + } + virtual int Opcode() const; +}; + +//------------------------------URShiftSNode----------------------------------- +// Logical shift right +class URShiftSNode : public Node { +public: + URShiftSNode( Node *in1, Node *in2 ) : Node(0,in1,in2) { + ShouldNotReachHere(); // only vector variant is used + } + virtual int Opcode() const; +}; //------------------------------URShiftINode----------------------------------- // Logical shift right --- old/src/hotspot/share/opto/node.hpp 2020-08-07 00:44:20.000000000 +0300 +++ new/src/hotspot/share/opto/node.hpp 2020-08-07 00:44:19.000000000 +0300 @@ -152,7 +152,10 @@ class UnlockNode; class VectorNode; class LoadVectorNode; +class LoadVectorGatherNode; class StoreVectorNode; +class StoreVectorScatterNode; +class VectorMaskCmpNode; class VectorSet; typedef void (*NFunc)(Node&,void*); extern "C" { @@ -689,8 +692,10 @@ DEFINE_CLASS_ID(Mem, Node, 4) DEFINE_CLASS_ID(Load, Mem, 0) DEFINE_CLASS_ID(LoadVector, Load, 0) + DEFINE_CLASS_ID(LoadVectorGather, LoadVector, 0) DEFINE_CLASS_ID(Store, Mem, 1) DEFINE_CLASS_ID(StoreVector, Store, 0) + DEFINE_CLASS_ID(StoreVectorScatter, StoreVector, 0) DEFINE_CLASS_ID(LoadStore, Mem, 2) DEFINE_CLASS_ID(LoadStoreConditional, LoadStore, 0) DEFINE_CLASS_ID(CompareAndSwap, LoadStoreConditional, 0) @@ -715,6 +720,7 @@ DEFINE_CLASS_ID(Add, Node, 11) DEFINE_CLASS_ID(Mul, Node, 12) DEFINE_CLASS_ID(Vector, Node, 13) + DEFINE_CLASS_ID(VectorMaskCmp, Vector, 0) DEFINE_CLASS_ID(ClearArray, Node, 14) DEFINE_CLASS_ID(Halt, Node, 15) DEFINE_CLASS_ID(Opaque1, Node, 16) @@ -885,7 +891,10 @@ DEFINE_CLASS_QUERY(Type) DEFINE_CLASS_QUERY(Vector) DEFINE_CLASS_QUERY(LoadVector) + DEFINE_CLASS_QUERY(LoadVectorGather) DEFINE_CLASS_QUERY(StoreVector) + DEFINE_CLASS_QUERY(StoreVectorScatter) + DEFINE_CLASS_QUERY(VectorMaskCmp) DEFINE_CLASS_QUERY(Unlock) #undef DEFINE_CLASS_QUERY --- old/src/hotspot/share/opto/output.cpp 2020-08-07 00:44:20.000000000 +0300 +++ new/src/hotspot/share/opto/output.cpp 2020-08-07 00:44:20.000000000 +0300 @@ -826,6 +826,10 @@ ? Location::int_in_long : Location::normal )); } else if( t->base() == Type::NarrowOop ) { array->append(new_loc_value( C->regalloc(), regnum, Location::narrowoop )); + } else if ( t->base() == Type::VectorS || t->base() == Type::VectorD || + t->base() == Type::VectorX || t->base() == Type::VectorY || + t->base() == Type::VectorZ) { + array->append(new_loc_value( C->regalloc(), regnum, Location::vector )); } else { array->append(new_loc_value( C->regalloc(), regnum, C->regalloc()->is_oop(local) ? Location::oop : Location::normal )); } --- old/src/hotspot/share/opto/phase.cpp 2020-08-07 00:44:20.000000000 +0300 +++ new/src/hotspot/share/opto/phase.cpp 2020-08-07 00:44:20.000000000 +0300 @@ -78,6 +78,10 @@ } } tty->print_cr (" Renumber Live: %7.3f s", timers[_t_renumberLive].seconds()); + tty->print_cr (" Vector: %7.3f s", timers[_t_vector].seconds()); + tty->print_cr (" Box elimination: %7.3f s", timers[_t_vector_elimination].seconds()); + tty->print_cr (" IGVN: %7.3f s", timers[_t_vector_igvn].seconds()); + tty->print_cr (" Prune Useless: %7.3f s", timers[_t_vector_pru].seconds()); tty->print_cr (" IdealLoop: %7.3f s", timers[_t_idealLoop].seconds()); tty->print_cr (" IdealLoop Verify: %7.3f s", timers[_t_idealLoopVerify].seconds()); tty->print_cr (" Cond Const Prop: %7.3f s", timers[_t_ccp].seconds()); --- old/src/hotspot/share/opto/phase.hpp 2020-08-07 00:44:21.000000000 +0300 +++ new/src/hotspot/share/opto/phase.hpp 2020-08-07 00:44:21.000000000 +0300 @@ -59,6 +59,7 @@ Ideal_Loop, // Find idealized trip-counted loops Macro_Expand, // Expand macro nodes Peephole, // Apply peephole optimizations + Vector, Output, last_phase }; @@ -75,6 +76,10 @@ _t_incrInline_igvn, _t_incrInline_pru, _t_incrInline_inline, + _t_vector, + _t_vector_elimination, + _t_vector_igvn, + _t_vector_pru, _t_renumberLive, _t_idealLoop, _t_idealLoopVerify, --- old/src/hotspot/share/opto/phasetype.hpp 2020-08-07 00:44:21.000000000 +0300 +++ new/src/hotspot/share/opto/phasetype.hpp 2020-08-07 00:44:21.000000000 +0300 @@ -31,7 +31,14 @@ PHASE_BEFORE_REMOVEUSELESS, PHASE_AFTER_PARSING, PHASE_ITER_GVN1, + PHASE_EXPAND_VUNBOX, + PHASE_SCALARIZE_VBOX, + PHASE_INLINE_VECTOR_REBOX, + PHASE_EXPAND_VBOX, + PHASE_ELIMINATE_VBOX_ALLOC, PHASE_PHASEIDEAL_BEFORE_EA, + PHASE_ITER_GVN_AFTER_VECTOR, + PHASE_ITER_GVN_BEFORE_EA, PHASE_ITER_GVN_AFTER_EA, PHASE_ITER_GVN_AFTER_ELIMINATION, PHASE_PHASEIDEALLOOP1, @@ -41,6 +48,7 @@ PHASE_ITER_GVN2, PHASE_PHASEIDEALLOOP_ITERATIONS, PHASE_OPTIMIZE_FINISHED, + PHASE_AFTER_MATCHING, PHASE_GLOBAL_CODE_MOTION, PHASE_FINAL_CODE, PHASE_AFTER_EA, @@ -51,6 +59,7 @@ PHASE_BEFORE_MATCHING, PHASE_MATCHING, PHASE_INCREMENTAL_INLINE, + PHASE_INCREMENTAL_INLINE_STEP, PHASE_INCREMENTAL_BOXING_INLINE, PHASE_CALL_CATCH_CLEANUP, PHASE_INSERT_BARRIER, @@ -73,7 +82,14 @@ case PHASE_BEFORE_REMOVEUSELESS: return "Before RemoveUseless"; case PHASE_AFTER_PARSING: return "After Parsing"; case PHASE_ITER_GVN1: return "Iter GVN 1"; + case PHASE_EXPAND_VUNBOX: return "Expand VectorUnbox"; + case PHASE_SCALARIZE_VBOX: return "Scalarize VectorBox"; + case PHASE_INLINE_VECTOR_REBOX: return "Inline Vector Rebox Calls"; + case PHASE_EXPAND_VBOX: return "Expand VectorBox"; + case PHASE_ELIMINATE_VBOX_ALLOC: return "Eliminate VectorBoxAllocate"; case PHASE_PHASEIDEAL_BEFORE_EA: return "PhaseIdealLoop before EA"; + case PHASE_ITER_GVN_AFTER_VECTOR: return "Iter GVN after vector box elimination"; + case PHASE_ITER_GVN_BEFORE_EA: return "Iter GVN before EA"; case PHASE_ITER_GVN_AFTER_EA: return "Iter GVN after EA"; case PHASE_ITER_GVN_AFTER_ELIMINATION: return "Iter GVN after eliminating allocations and locks"; case PHASE_PHASEIDEALLOOP1: return "PhaseIdealLoop 1"; @@ -83,6 +99,7 @@ case PHASE_ITER_GVN2: return "Iter GVN 2"; case PHASE_PHASEIDEALLOOP_ITERATIONS: return "PhaseIdealLoop iterations"; case PHASE_OPTIMIZE_FINISHED: return "Optimize finished"; + case PHASE_AFTER_MATCHING: return "After Matching"; case PHASE_GLOBAL_CODE_MOTION: return "Global code motion"; case PHASE_FINAL_CODE: return "Final Code"; case PHASE_AFTER_EA: return "After Escape Analysis"; @@ -93,6 +110,7 @@ case PHASE_BEFORE_MATCHING: return "Before matching"; case PHASE_MATCHING: return "After matching"; case PHASE_INCREMENTAL_INLINE: return "Incremental Inline"; + case PHASE_INCREMENTAL_INLINE_STEP: return "Incremental Inline Step"; case PHASE_INCREMENTAL_BOXING_INLINE: return "Incremental Boxing Inline"; case PHASE_CALL_CATCH_CLEANUP: return "Call catch cleanup"; case PHASE_INSERT_BARRIER: return "Insert barrier"; --- old/src/hotspot/share/opto/subnode.hpp 2020-08-07 00:44:22.000000000 +0300 +++ new/src/hotspot/share/opto/subnode.hpp 2020-08-07 00:44:21.000000000 +0300 @@ -404,6 +404,28 @@ NegNode( Node *in1 ) : Node(0,in1) {} }; +//------------------------------NegINode--------------------------------------- +// Negate value an int. For int values, negation is the same as subtraction +// from zero +class NegINode : public NegNode { +public: + NegINode(Node *in1) : NegNode(in1) {} + virtual int Opcode() const; + const Type *bottom_type() const { return TypeInt::INT; } + virtual uint ideal_reg() const { return Op_RegI; } +}; + +//------------------------------NegLNode--------------------------------------- +// Negate value an int. For int values, negation is the same as subtraction +// from zero +class NegLNode : public NegNode { +public: + NegLNode(Node *in1) : NegNode(in1) {} + virtual int Opcode() const; + const Type *bottom_type() const { return TypeLong::LONG; } + virtual uint ideal_reg() const { return Op_RegL; } +}; + //------------------------------NegFNode--------------------------------------- // Negate value a float. Negating 0.0 returns -0.0, but subtracting from // zero returns +0.0 (per JVM spec on 'fneg' bytecode). As subtraction --- old/src/hotspot/share/opto/superword.cpp 2020-08-07 00:44:22.000000000 +0300 +++ new/src/hotspot/share/opto/superword.cpp 2020-08-07 00:44:22.000000000 +0300 @@ -2745,7 +2745,7 @@ } } // Move shift count into vector register. - cnt = VectorNode::shift_count(p0, cnt, vlen, velt_basic_type(p0)); + cnt = VectorNode::shift_count(p0->Opcode(), cnt, vlen, velt_basic_type(p0)); _igvn.register_new_node_with_optimizer(cnt); _phase->set_ctrl(cnt, _phase->get_ctrl(opd)); return cnt; --- old/src/hotspot/share/opto/type.cpp 2020-08-07 00:44:22.000000000 +0300 +++ new/src/hotspot/share/opto/type.cpp 2020-08-07 00:44:22.000000000 +0300 @@ -436,16 +436,22 @@ BOTTOM = make(Bottom); // Everything HALF = make(Half); // Placeholder half of doublewide type + TypeF::MAX = TypeF::make(max_jfloat); // Float MAX + TypeF::MIN = TypeF::make(min_jfloat); // Float MIN TypeF::ZERO = TypeF::make(0.0); // Float 0 (positive zero) TypeF::ONE = TypeF::make(1.0); // Float 1 TypeF::POS_INF = TypeF::make(jfloat_cast(POSITIVE_INFINITE_F)); TypeF::NEG_INF = TypeF::make(-jfloat_cast(POSITIVE_INFINITE_F)); + TypeD::MAX = TypeD::make(max_jdouble); // Double MAX + TypeD::MIN = TypeD::make(min_jdouble); // Double MIN TypeD::ZERO = TypeD::make(0.0); // Double 0 (positive zero) TypeD::ONE = TypeD::make(1.0); // Double 1 TypeD::POS_INF = TypeD::make(jdouble_cast(POSITIVE_INFINITE_D)); TypeD::NEG_INF = TypeD::make(-jdouble_cast(POSITIVE_INFINITE_D)); + TypeInt::MAX = TypeInt::make(max_jint); // Int MAX + TypeInt::MIN = TypeInt::make(min_jint); // Int MIN TypeInt::MINUS_1 = TypeInt::make(-1); // -1 TypeInt::ZERO = TypeInt::make( 0); // 0 TypeInt::ONE = TypeInt::make( 1); // 1 @@ -474,6 +480,8 @@ assert( TypeInt::CC_GE == TypeInt::BOOL, "types must match for CmpL to work" ); assert( (juint)(TypeInt::CC->_hi - TypeInt::CC->_lo) <= SMALLINT, "CC is truly small"); + TypeLong::MAX = TypeLong::make(max_jlong); // Long MAX + TypeLong::MIN = TypeLong::make(min_jlong); // Long MIN TypeLong::MINUS_1 = TypeLong::make(-1); // -1 TypeLong::ZERO = TypeLong::make( 0); // 0 TypeLong::ONE = TypeLong::make( 1); // 1 @@ -1109,6 +1117,8 @@ //============================================================================= // Convenience common pre-built types. +const TypeF *TypeF::MAX; // Floating point max +const TypeF *TypeF::MIN; // Floating point min const TypeF *TypeF::ZERO; // Floating point zero const TypeF *TypeF::ONE; // Floating point one const TypeF *TypeF::POS_INF; // Floating point positive infinity @@ -1219,6 +1229,8 @@ //============================================================================= // Convenience common pre-built types. +const TypeD *TypeD::MAX; // Floating point max +const TypeD *TypeD::MIN; // Floating point min const TypeD *TypeD::ZERO; // Floating point zero const TypeD *TypeD::ONE; // Floating point one const TypeD *TypeD::POS_INF; // Floating point positive infinity @@ -1325,6 +1337,8 @@ //============================================================================= // Convience common pre-built types. +const TypeInt *TypeInt::MAX; // INT_MAX +const TypeInt *TypeInt::MIN; // INT_MIN const TypeInt *TypeInt::MINUS_1;// -1 const TypeInt *TypeInt::ZERO; // 0 const TypeInt *TypeInt::ONE; // 1 @@ -1594,6 +1608,8 @@ //============================================================================= // Convenience common pre-built types. +const TypeLong *TypeLong::MAX; +const TypeLong *TypeLong::MIN; const TypeLong *TypeLong::MINUS_1;// -1 const TypeLong *TypeLong::ZERO; // 0 const TypeLong *TypeLong::ONE; // 1 --- old/src/hotspot/share/opto/type.hpp 2020-08-07 00:44:23.000000000 +0300 +++ new/src/hotspot/share/opto/type.hpp 2020-08-07 00:44:23.000000000 +0300 @@ -481,6 +481,8 @@ virtual const Type *xmeet( const Type *t ) const; virtual const Type *xdual() const; // Compute dual right now. // Convenience common pre-built types. + static const TypeF *MAX; + static const TypeF *MIN; static const TypeF *ZERO; // positive zero only static const TypeF *ONE; static const TypeF *POS_INF; @@ -510,6 +512,8 @@ virtual const Type *xmeet( const Type *t ) const; virtual const Type *xdual() const; // Compute dual right now. // Convenience common pre-built types. + static const TypeD *MAX; + static const TypeD *MIN; static const TypeD *ZERO; // positive zero only static const TypeD *ONE; static const TypeD *POS_INF; @@ -553,6 +557,8 @@ virtual const Type *narrow( const Type *t ) const; // Do not kill _widen bits. // Convenience common pre-built types. + static const TypeInt *MAX; + static const TypeInt *MIN; static const TypeInt *MINUS_1; static const TypeInt *ZERO; static const TypeInt *ONE; @@ -618,6 +624,8 @@ virtual const Type *widen( const Type *t, const Type* limit_type ) const; virtual const Type *narrow( const Type *t ) const; // Convenience common pre-built types. + static const TypeLong *MAX; + static const TypeLong *MIN; static const TypeLong *MINUS_1; static const TypeLong *ZERO; static const TypeLong *ONE; --- old/src/hotspot/share/opto/vectornode.cpp 2020-08-07 00:44:23.000000000 +0300 +++ new/src/hotspot/share/opto/vectornode.cpp 2020-08-07 00:44:23.000000000 +0300 @@ -24,6 +24,7 @@ #include "precompiled.hpp" #include "memory/allocation.inline.hpp" #include "opto/connode.hpp" +#include "opto/subnode.hpp" #include "opto/vectornode.hpp" #include "utilities/powerOfTwo.hpp" @@ -117,12 +118,51 @@ case Op_AbsL: assert(bt == T_LONG, "must be"); return Op_AbsVL; + case Op_MinI: + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: return Op_MinV; + default: ShouldNotReachHere(); return 0; + } + case Op_MinL: + assert(bt == T_LONG, "must be"); + return Op_MinV; + case Op_MinF: + assert(bt == T_FLOAT, "must be"); + return Op_MinV; + case Op_MinD: + assert(bt == T_DOUBLE, "must be"); + return Op_MinV; + case Op_MaxI: + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: return Op_MaxV; + default: ShouldNotReachHere(); return 0; + } + case Op_MaxL: + assert(bt == T_LONG, "must be"); + return Op_MaxV; + case Op_MaxF: + assert(bt == T_FLOAT, "must be"); + return Op_MaxV; + case Op_MaxD: + assert(bt == T_DOUBLE, "must be"); + return Op_MaxV; case Op_AbsF: assert(bt == T_FLOAT, "must be"); return Op_AbsVF; case Op_AbsD: assert(bt == T_DOUBLE, "must be"); return Op_AbsVD; + case Op_NegI: + assert(bt == T_INT, "must be"); + return Op_NegVI; case Op_NegF: assert(bt == T_FLOAT, "must be"); return Op_NegVF; @@ -169,6 +209,12 @@ case Op_RShiftL: assert(bt == T_LONG, "must be"); return Op_RShiftVL; + case Op_URShiftB: + assert(bt == T_BYTE, "must be"); + return Op_URShiftVB; + case Op_URShiftS: + assert(bt == T_SHORT, "must be"); + return Op_URShiftVS; case Op_URShiftI: switch (bt) { case T_BOOLEAN:return Op_URShiftVB; @@ -194,18 +240,6 @@ case Op_XorI: case Op_XorL: return Op_XorV; - case Op_MinF: - assert(bt == T_FLOAT, "must be"); - return Op_MinV; - case Op_MinD: - assert(bt == T_DOUBLE, "must be"); - return Op_MinV; - case Op_MaxF: - assert(bt == T_FLOAT, "must be"); - return Op_MaxV; - case Op_MaxD: - assert(bt == T_DOUBLE, "must be"); - return Op_MaxV; case Op_LoadB: case Op_LoadUB: @@ -232,6 +266,28 @@ } } +int VectorNode::replicate_opcode(BasicType bt) { + switch(bt) { + case T_BOOLEAN: + case T_BYTE: + return Op_ReplicateB; + case T_SHORT: + case T_CHAR: + return Op_ReplicateS; + case T_INT: + return Op_ReplicateI; + case T_LONG: + return Op_ReplicateL; + case T_FLOAT: + return Op_ReplicateF; + case T_DOUBLE: + return Op_ReplicateD; + default: + assert(false, "wrong type: %s", type2name(bt)); + return 0; + } +} + // Also used to check if the code generator // supports the vector operation. bool VectorNode::implemented(int opc, uint vlen, BasicType bt) { @@ -284,6 +340,16 @@ } } +bool VectorNode::is_vshift_cnt(Node* n) { + switch (n->Opcode()) { + case Op_LShiftCntV: + case Op_RShiftCntV: + return true; + default: + return false; + } +} + // Check if input is loop invariant vector. bool VectorNode::is_invariant_vector(Node* n) { // Only Replicate vector nodes are loop invariant for now. @@ -350,12 +416,10 @@ } } -// Return the vector version of a scalar operation node. -VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt) { - const TypeVect* vt = TypeVect::make(bt, vlen); - int vopc = VectorNode::opcode(opc, bt); +// Make a vector node for binary operation +VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, const TypeVect* vt) { // This method should not be called for unimplemented vectors. - guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]); + guarantee(vopc > 0, "vopc must be > 0"); switch (vopc) { case Op_AddVB: return new AddVBNode(n1, n2, vt); case Op_AddVS: return new AddVSNode(n1, n2, vt); @@ -381,13 +445,17 @@ case Op_DivVF: return new DivVFNode(n1, n2, vt); case Op_DivVD: return new DivVDNode(n1, n2, vt); + case Op_MinV: return new MinVNode(n1, n2, vt); + case Op_MaxV: return new MaxVNode(n1, n2, vt); + + case Op_AbsVF: return new AbsVFNode(n1, vt); + case Op_AbsVD: return new AbsVDNode(n1, vt); case Op_AbsVB: return new AbsVBNode(n1, vt); case Op_AbsVS: return new AbsVSNode(n1, vt); case Op_AbsVI: return new AbsVINode(n1, vt); case Op_AbsVL: return new AbsVLNode(n1, vt); - case Op_AbsVF: return new AbsVFNode(n1, vt); - case Op_AbsVD: return new AbsVDNode(n1, vt); + case Op_NegVI: return new NegVINode(n1, vt); case Op_NegVF: return new NegVFNode(n1, vt); case Op_NegVD: return new NegVDNode(n1, vt); @@ -415,9 +483,6 @@ case Op_OrV: return new OrVNode (n1, n2, vt); case Op_XorV: return new XorVNode(n1, n2, vt); - case Op_MinV: return new MinVNode(n1, n2, vt); - case Op_MaxV: return new MaxVNode(n1, n2, vt); - case Op_RoundDoubleModeV: return new RoundDoubleModeVNode(n1, n2, vt); case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt); @@ -427,11 +492,19 @@ } } -VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) { +// Return the vector version of a scalar binary operation node. +VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt) { const TypeVect* vt = TypeVect::make(bt, vlen); int vopc = VectorNode::opcode(opc, bt); // This method should not be called for unimplemented vectors. guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]); + return make(vopc, n1, n2, vt); +} + +// Make a vector node for ternary operation +VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, Node* n3, const TypeVect* vt) { + // This method should not be called for unimplemented vectors. + guarantee(vopc > 0, "vopc must be > 0"); switch (vopc) { case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt); case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt); @@ -441,6 +514,15 @@ } } +// Return the vector version of a scalar ternary operation node. +VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) { + const TypeVect* vt = TypeVect::make(bt, vlen); + int vopc = VectorNode::opcode(opc, bt); + // This method should not be called for unimplemented vectors. + guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]); + return make(vopc, n1, n2, n3, vt); +} + // Scalar promotion VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t) { BasicType bt = opd_t->array_element_basic_type(); @@ -467,21 +549,22 @@ } } -VectorNode* VectorNode::shift_count(Node* shift, Node* cnt, uint vlen, BasicType bt) { - assert(VectorNode::is_shift(shift), "sanity"); +VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt) { // Match shift count type with shift vector type. const TypeVect* vt = TypeVect::make(bt, vlen); - switch (shift->Opcode()) { + switch (opc) { case Op_LShiftI: case Op_LShiftL: return new LShiftCntVNode(cnt, vt); case Op_RShiftI: case Op_RShiftL: + case Op_URShiftB: + case Op_URShiftS: case Op_URShiftI: case Op_URShiftL: return new RShiftCntVNode(cnt, vt); default: - fatal("Missed vector creation for '%s'", NodeClassNames[shift->Opcode()]); + fatal("Missed vector creation for '%s'", NodeClassNames[opc]); return NULL; } } @@ -628,29 +711,37 @@ return new StoreVectorNode(ctl, mem, adr, atyp, val); } +int ExtractNode::opcode(BasicType bt) { + switch (bt) { + case T_BOOLEAN: return Op_ExtractUB; + case T_BYTE: return Op_ExtractB; + case T_CHAR: return Op_ExtractC; + case T_SHORT: return Op_ExtractS; + case T_INT: return Op_ExtractI; + case T_LONG: return Op_ExtractL; + case T_FLOAT: return Op_ExtractF; + case T_DOUBLE: return Op_ExtractD; + default: + assert(false, "wrong type: %s", type2name(bt)); + return 0; + } +} + // Extract a scalar element of vector. Node* ExtractNode::make(Node* v, uint position, BasicType bt) { assert((int)position < Matcher::max_vector_size(bt), "pos in range"); ConINode* pos = ConINode::make((int)position); switch (bt) { - case T_BOOLEAN: - return new ExtractUBNode(v, pos); - case T_BYTE: - return new ExtractBNode(v, pos); - case T_CHAR: - return new ExtractCNode(v, pos); - case T_SHORT: - return new ExtractSNode(v, pos); - case T_INT: - return new ExtractINode(v, pos); - case T_LONG: - return new ExtractLNode(v, pos); - case T_FLOAT: - return new ExtractFNode(v, pos); - case T_DOUBLE: - return new ExtractDNode(v, pos); + case T_BOOLEAN: return new ExtractUBNode(v, pos); + case T_BYTE: return new ExtractBNode(v, pos); + case T_CHAR: return new ExtractCNode(v, pos); + case T_SHORT: return new ExtractSNode(v, pos); + case T_INT: return new ExtractINode(v, pos); + case T_LONG: return new ExtractLNode(v, pos); + case T_FLOAT: return new ExtractFNode(v, pos); + case T_DOUBLE: return new ExtractDNode(v, pos); default: - fatal("Type '%s' is not supported for vectors", type2name(bt)); + assert(false, "wrong type: %s", type2name(bt)); return NULL; } } @@ -659,8 +750,16 @@ int vopc = opc; switch (opc) { case Op_AddI: - assert(bt == T_INT, "must be"); - vopc = Op_AddReductionVI; + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_AddReductionVI; + break; + default: ShouldNotReachHere(); return 0; + } break; case Op_AddL: assert(bt == T_LONG, "must be"); @@ -675,8 +774,16 @@ vopc = Op_AddReductionVD; break; case Op_MulI: - assert(bt == T_INT, "must be"); - vopc = Op_MulReductionVI; + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_MulReductionVI; + break; + default: ShouldNotReachHere(); return 0; + } break; case Op_MulL: assert(bt == T_LONG, "must be"); @@ -690,6 +797,22 @@ assert(bt == T_DOUBLE, "must be"); vopc = Op_MulReductionVD; break; + case Op_MinI: + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_MinReductionV; + break; + default: ShouldNotReachHere(); return 0; + } + break; + case Op_MinL: + assert(bt == T_LONG, "must be"); + vopc = Op_MinReductionV; + break; case Op_MinF: assert(bt == T_FLOAT, "must be"); vopc = Op_MinReductionV; @@ -698,6 +821,22 @@ assert(bt == T_DOUBLE, "must be"); vopc = Op_MinReductionV; break; + case Op_MaxI: + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_MaxReductionV; + break; + default: ShouldNotReachHere(); return 0; + } + break; + case Op_MaxL: + assert(bt == T_LONG, "must be"); + vopc = Op_MaxReductionV; + break; case Op_MaxF: assert(bt == T_FLOAT, "must be"); vopc = Op_MaxReductionV; @@ -707,24 +846,48 @@ vopc = Op_MaxReductionV; break; case Op_AndI: - assert(bt == T_INT, "must be"); - vopc = Op_AndReductionV; + switch (bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_AndReductionV; + break; + default: ShouldNotReachHere(); return 0; + } break; case Op_AndL: assert(bt == T_LONG, "must be"); vopc = Op_AndReductionV; break; case Op_OrI: - assert(bt == T_INT, "must be"); - vopc = Op_OrReductionV; + switch(bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_OrReductionV; + break; + default: ShouldNotReachHere(); return 0; + } break; case Op_OrL: assert(bt == T_LONG, "must be"); vopc = Op_OrReductionV; break; case Op_XorI: - assert(bt == T_INT, "must be"); - vopc = Op_XorReductionV; + switch(bt) { + case T_BOOLEAN: + case T_CHAR: return 0; + case T_BYTE: + case T_SHORT: + case T_INT: + vopc = Op_XorReductionV; + break; + default: ShouldNotReachHere(); return 0; + } break; case Op_XorL: assert(bt == T_LONG, "must be"); @@ -759,11 +922,116 @@ case Op_OrReductionV: return new OrReductionVNode(ctrl, n1, n2); case Op_XorReductionV: return new XorReductionVNode(ctrl, n1, n2); default: - fatal("Missed vector creation for '%s'", NodeClassNames[vopc]); + assert(false, "unknown node: %s", NodeClassNames[vopc]); return NULL; } } +VectorStoreMaskNode* VectorStoreMaskNode::make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem) { + assert(in->bottom_type()->isa_vect(), "sanity"); + const TypeVect* vt = TypeVect::make(T_BOOLEAN, num_elem); + int elem_size = type2aelembytes(in_type); + return new VectorStoreMaskNode(in, gvn.intcon(elem_size), vt); +} + +VectorCastNode* VectorCastNode::make(int vopc, Node* n1, BasicType bt, uint vlen) { + const TypeVect* vt = TypeVect::make(bt, vlen); + switch (vopc) { + case Op_VectorCastB2X: return new VectorCastB2XNode(n1, vt); + case Op_VectorCastS2X: return new VectorCastS2XNode(n1, vt); + case Op_VectorCastI2X: return new VectorCastI2XNode(n1, vt); + case Op_VectorCastL2X: return new VectorCastL2XNode(n1, vt); + case Op_VectorCastF2X: return new VectorCastF2XNode(n1, vt); + case Op_VectorCastD2X: return new VectorCastD2XNode(n1, vt); + default: + assert(false, "unknown node: %s", NodeClassNames[vopc]); + return NULL; + } +} + +int VectorCastNode::opcode(BasicType bt) { + switch (bt) { + case T_BYTE: return Op_VectorCastB2X; + case T_SHORT: return Op_VectorCastS2X; + case T_INT: return Op_VectorCastI2X; + case T_LONG: return Op_VectorCastL2X; + case T_FLOAT: return Op_VectorCastF2X; + case T_DOUBLE: return Op_VectorCastD2X; + default: + assert(false, "unknown type: %s", type2name(bt)); + return 0; + } +} + +Node* ReductionNode::make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt) { + int vopc = opcode(opc, bt); + guarantee(vopc != opc, "Vector reduction for '%s' is not implemented", NodeClassNames[opc]); + + switch (vopc) { + case Op_AndReductionV: + switch (bt) { + case T_BYTE: + case T_SHORT: + case T_INT: + return gvn.makecon(TypeInt::MINUS_1); + case T_LONG: + return gvn.makecon(TypeLong::MINUS_1); + default: + fatal("Missed vector creation for '%s' as the basic type is not correct.", NodeClassNames[vopc]); + return NULL; + } + break; + case Op_AddReductionVI: // fallthrough + case Op_AddReductionVL: // fallthrough + case Op_AddReductionVF: // fallthrough + case Op_AddReductionVD: + case Op_OrReductionV: + case Op_XorReductionV: + return gvn.zerocon(bt); + case Op_MulReductionVI: + return gvn.makecon(TypeInt::ONE); + case Op_MulReductionVL: + return gvn.makecon(TypeLong::ONE); + case Op_MulReductionVF: + return gvn.makecon(TypeF::ONE); + case Op_MulReductionVD: + return gvn.makecon(TypeD::ONE); + case Op_MinReductionV: + switch (bt) { + case T_BYTE: + case T_SHORT: + case T_INT: + return gvn.makecon(TypeInt::MAX); + case T_LONG: + return gvn.makecon(TypeLong::MAX); + case T_FLOAT: + return gvn.makecon(TypeF::POS_INF); + case T_DOUBLE: + return gvn.makecon(TypeD::POS_INF); + default: Unimplemented(); return NULL; + } + break; + case Op_MaxReductionV: + switch (bt) { + case T_BYTE: + case T_SHORT: + case T_INT: + return gvn.makecon(TypeInt::MIN); + case T_LONG: + return gvn.makecon(TypeLong::MIN); + case T_FLOAT: + return gvn.makecon(TypeF::NEG_INF); + case T_DOUBLE: + return gvn.makecon(TypeD::NEG_INF); + default: Unimplemented(); return NULL; + } + break; + default: + fatal("Missed vector creation for '%s'", NodeClassNames[vopc]); + return NULL; + } +} + bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) { if (is_java_primitive(bt) && (vlen > 1) && is_power_of_2(vlen) && @@ -775,7 +1043,7 @@ } MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node* in3, - uint truth_table, const TypeVect* vt) { + uint truth_table, const TypeVect* vt) { assert(truth_table <= 0xFF, "invalid"); assert(in1->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch"); assert(in2->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch"); @@ -784,3 +1052,51 @@ return new MacroLogicVNode(in1, in2, in3, fn, vt); } +#ifndef PRODUCT +void VectorMaskCmpNode::dump_spec(outputStream *st) const { + st->print(" %d #", _predicate); _type->dump_on(st); +} +#endif // PRODUCT + +Node* VectorReinterpretNode::Identity(PhaseGVN *phase) { + Node* n = in(1); + if (n->Opcode() == Op_VectorReinterpret) { + if (Type::cmp(bottom_type(), n->in(1)->bottom_type()) == 0) { + return n->in(1); + } + } + return this; +} + +Node* VectorInsertNode::make(Node* vec, Node* new_val, int position) { + assert(position < (int)vec->bottom_type()->is_vect()->length(), "pos in range"); + ConINode* pos = ConINode::make(position); + return new VectorInsertNode(vec, new_val, pos, vec->bottom_type()->is_vect()); +} + +Node* VectorUnboxNode::Identity(PhaseGVN *phase) { + Node* n = obj()->uncast(); + if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) { + if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) { + return n->in(VectorBoxNode::Value); + } + } + return this; +} + +const TypeFunc* VectorBoxNode::vec_box_type(const TypeInstPtr* box_type) { + const Type** fields = TypeTuple::fields(0); + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields); + + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = box_type; + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields); + + return TypeFunc::make(domain, range); +} + +#ifndef PRODUCT +void VectorBoxAllocateNode::dump_spec(outputStream *st) const { + CallStaticJavaNode::dump_spec(st); +} +#endif // !PRODUCT --- old/src/hotspot/share/opto/vectornode.hpp 2020-08-07 00:44:24.000000000 +0300 +++ new/src/hotspot/share/opto/vectornode.hpp 2020-08-07 00:44:24.000000000 +0300 @@ -24,6 +24,7 @@ #ifndef SHARE_OPTO_VECTORNODE_HPP #define SHARE_OPTO_VECTORNODE_HPP +#include "opto/callnode.hpp" #include "opto/matcher.hpp" #include "opto/memnode.hpp" #include "opto/node.hpp" @@ -68,13 +69,17 @@ virtual uint ideal_reg() const { return Matcher::vector_ideal_reg(vect_type()->length_in_bytes()); } static VectorNode* scalar2vector(Node* s, uint vlen, const Type* opd_t); - static VectorNode* shift_count(Node* shift, Node* cnt, uint vlen, BasicType bt); + static VectorNode* shift_count(int opc, Node* cnt, uint vlen, BasicType bt); static VectorNode* make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt); + static VectorNode* make(int vopc, Node* n1, Node* n2, const TypeVect* vt); static VectorNode* make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt); + static VectorNode* make(int vopc, Node* n1, Node* n2, Node* n3, const TypeVect* vt); static int opcode(int opc, BasicType bt); + static int replicate_opcode(BasicType bt); static bool implemented(int opc, uint vlen, BasicType bt); static bool is_shift(Node* n); + static bool is_vshift_cnt(Node* n); static bool is_type_transition_short_to_int(Node* n); static bool is_type_transition_to_int(Node* n); static bool is_muladds2i(Node* n); @@ -156,9 +161,10 @@ static ReductionNode* make(int opc, Node *ctrl, Node* in1, Node* in2, BasicType bt); static int opcode(int opc, BasicType bt); static bool implemented(int opc, uint vlen, BasicType bt); + static Node* make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt); virtual const Type* bottom_type() const { - BasicType vbt = in(2)->bottom_type()->is_vect()->element_basic_type(); + BasicType vbt = in(1)->bottom_type()->basic_type(); return Type::get_const_basic_type(vbt); } @@ -168,13 +174,11 @@ }; //------------------------------AddReductionVINode-------------------------------------- -// Vector add int as a reduction +// Vector add byte, short and int as a reduction class AddReductionVINode : public ReductionNode { public: AddReductionVINode(Node * ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} virtual int Opcode() const; - virtual const Type* bottom_type() const { return TypeInt::INT; } - virtual uint ideal_reg() const { return Op_RegI; } }; //------------------------------AddReductionVLNode-------------------------------------- @@ -183,8 +187,6 @@ public: AddReductionVLNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} virtual int Opcode() const; - virtual const Type* bottom_type() const { return TypeLong::LONG; } - virtual uint ideal_reg() const { return Op_RegL; } }; //------------------------------AddReductionVFNode-------------------------------------- @@ -193,8 +195,6 @@ public: AddReductionVFNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} virtual int Opcode() const; - virtual const Type* bottom_type() const { return Type::FLOAT; } - virtual uint ideal_reg() const { return Op_RegF; } }; //------------------------------AddReductionVDNode-------------------------------------- @@ -203,8 +203,6 @@ public: AddReductionVDNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} virtual int Opcode() const; - virtual const Type* bottom_type() const { return Type::DOUBLE; } - virtual uint ideal_reg() const { return Op_RegD; } }; //------------------------------SubVBNode-------------------------------------- @@ -344,13 +342,11 @@ }; //------------------------------MulReductionVINode-------------------------------------- -// Vector multiply int as a reduction +// Vector multiply byte, short and int as a reduction class MulReductionVINode : public ReductionNode { public: MulReductionVINode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} virtual int Opcode() const; - virtual const Type* bottom_type() const { return TypeInt::INT; } - virtual uint ideal_reg() const { return Op_RegI; } }; //------------------------------MulReductionVLNode-------------------------------------- @@ -359,8 +355,6 @@ public: MulReductionVLNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} virtual int Opcode() const; - virtual const Type* bottom_type() const { return TypeLong::LONG; } - virtual uint ideal_reg() const { return Op_RegI; } }; //------------------------------MulReductionVFNode-------------------------------------- @@ -369,8 +363,6 @@ public: MulReductionVFNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} virtual int Opcode() const; - virtual const Type* bottom_type() const { return Type::FLOAT; } - virtual uint ideal_reg() const { return Op_RegF; } }; //------------------------------MulReductionVDNode-------------------------------------- @@ -379,8 +371,6 @@ public: MulReductionVDNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} virtual int Opcode() const; - virtual const Type* bottom_type() const { return Type::DOUBLE; } - virtual uint ideal_reg() const { return Op_RegD; } }; //------------------------------DivVFNode-------------------------------------- @@ -415,10 +405,26 @@ virtual int Opcode() const; }; +//------------------------------MinVNode-------------------------------------- +// Vector Min +class MinVNode : public VectorNode { +public: + MinVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {} + virtual int Opcode() const; +}; + +//------------------------------MaxVNode-------------------------------------- +// Vector Max +class MaxVNode : public VectorNode { + public: + MaxVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {} + virtual int Opcode() const; +}; + //------------------------------AbsVINode-------------------------------------- // Vector Abs int class AbsVINode : public VectorNode { -public: + public: AbsVINode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {} virtual int Opcode() const; }; @@ -447,6 +453,14 @@ virtual int Opcode() const; }; +//------------------------------NegVINode-------------------------------------- +// Vector Neg int +class NegVINode : public VectorNode { + public: + NegVINode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {} + virtual int Opcode() const; +}; + //------------------------------NegVFNode-------------------------------------- // Vector Neg float class NegVFNode : public VectorNode { @@ -614,64 +628,48 @@ virtual int Opcode() const; }; -//------------------------------OrVNode--------------------------------------- -// Vector or integer -class OrVNode : public VectorNode { +//------------------------------AndReductionVNode-------------------------------------- +// Vector and byte, short, int, long as a reduction +class AndReductionVNode : public ReductionNode { public: - OrVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {} + AndReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} virtual int Opcode() const; }; -//------------------------------XorVNode--------------------------------------- -// Vector xor integer -class XorVNode : public VectorNode { +//------------------------------OrVNode--------------------------------------- +// Vector or byte, short, int, long as a reduction +class OrVNode : public VectorNode { public: - XorVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {} - virtual int Opcode() const; -}; - -//------------------------------AndReductionVNode-------------------------------------- -// Vector and int, long as a reduction -class AndReductionVNode : public ReductionNode { -public: - AndReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} + OrVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {} virtual int Opcode() const; }; //------------------------------OrReductionVNode-------------------------------------- -// Vector or int, long as a reduction +// Vector xor byte, short, int, long as a reduction class OrReductionVNode : public ReductionNode { -public: + public: OrReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} virtual int Opcode() const; }; //------------------------------XorReductionVNode-------------------------------------- -// Vector xor int, long as a reduction +// Vector and int, long as a reduction class XorReductionVNode : public ReductionNode { -public: + public: XorReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} virtual int Opcode() const; }; -//------------------------------MinVNode-------------------------------------- -// Vector min -class MinVNode : public VectorNode { -public: - MinVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {} - virtual int Opcode() const; -}; - -//------------------------------MaxVNode-------------------------------------- -// Vector max -class MaxVNode : public VectorNode { -public: - MaxVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {} +//------------------------------XorVNode--------------------------------------- +// Vector xor integer +class XorVNode : public VectorNode { + public: + XorVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {} virtual int Opcode() const; }; //------------------------------MinReductionVNode-------------------------------------- -// Vector min as a reduction +// Vector min byte, short, int, long, float, double as a reduction class MinReductionVNode : public ReductionNode { public: MinReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} @@ -679,7 +677,7 @@ }; //------------------------------MaxReductionVNode-------------------------------------- -// Vector max as a reduction +// Vector min byte, short, int, long, float, double as a reduction class MaxReductionVNode : public ReductionNode { public: MaxReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {} @@ -716,13 +714,28 @@ uint element_size(void) { return type2aelembytes(vect_type()->element_basic_type()); } }; +//------------------------------LoadVectorGatherNode------------------------------ +// Load Vector from memory via index map +class LoadVectorGatherNode : public LoadVectorNode { + public: + LoadVectorGatherNode(Node* c, Node* mem, Node* adr, const TypePtr* at, const TypeVect* vt, Node* indices) + : LoadVectorNode(c, mem, adr, at, vt) { + init_class_id(Class_LoadVectorGather); + assert(indices->bottom_type()->is_vect(), "indices must be in vector"); + add_req(indices); + assert(req() == MemNode::ValueIn + 1, "match_edge expects that last input is in MemNode::ValueIn"); + } + + virtual int Opcode() const; + virtual uint match_edge(uint idx) const { return idx == MemNode::Address || idx == MemNode::ValueIn; } +}; + //------------------------------StoreVectorNode-------------------------------- // Store Vector to memory class StoreVectorNode : public StoreNode { public: StoreVectorNode(Node* c, Node* mem, Node* adr, const TypePtr* at, Node* val) : StoreNode(c, mem, adr, at, val, MemNode::unordered) { - assert(val->is_Vector() || val->is_LoadVector(), "sanity"); init_class_id(Class_StoreVector); set_mismatched_access(); } @@ -743,6 +756,23 @@ uint element_size(void) { return type2aelembytes(vect_type()->element_basic_type()); } }; +//------------------------------StoreVectorScatterNode------------------------------ +// Store Vector into memory via index map + + class StoreVectorScatterNode : public StoreVectorNode { + public: + StoreVectorScatterNode(Node* c, Node* mem, Node* adr, const TypePtr* at, Node* val, Node* indices) + : StoreVectorNode(c, mem, adr, at, val) { + init_class_id(Class_StoreVectorScatter); + assert(indices->bottom_type()->is_vect(), "indices must be in vector"); + add_req(indices); + assert(req() == MemNode::ValueIn + 2, "match_edge expects that last input is in MemNode::ValueIn+1"); + } + virtual int Opcode() const; + virtual uint match_edge(uint idx) const { return idx == MemNode::Address || + idx == MemNode::ValueIn || + idx == MemNode::ValueIn + 1; } +}; //=========================Promote_Scalar_to_Vector============================ @@ -884,6 +914,12 @@ }; +class VectorLoadConstNode : public VectorNode { + public: + VectorLoadConstNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {} + virtual int Opcode() const; +}; + //========================Extract_Scalar_from_Vector=========================== //------------------------------ExtractNode------------------------------------ @@ -897,6 +933,7 @@ uint pos() const { return in(2)->get_int(); } static Node* make(Node* v, uint position, BasicType bt); + static int opcode(BasicType bt); }; //------------------------------ExtractBNode----------------------------------- @@ -925,7 +962,7 @@ public: ExtractCNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {} virtual int Opcode() const; - virtual const Type *bottom_type() const { return TypeInt::INT; } + virtual const Type *bottom_type() const { return TypeInt::CHAR; } virtual uint ideal_reg() const { return Op_RegI; } }; @@ -935,7 +972,7 @@ public: ExtractSNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {} virtual int Opcode() const; - virtual const Type *bottom_type() const { return TypeInt::INT; } + virtual const Type *bottom_type() const { return TypeInt::SHORT; } virtual uint ideal_reg() const { return Op_RegI; } }; @@ -1003,4 +1040,284 @@ static MacroLogicVNode* make(PhaseGVN& igvn, Node* in1, Node* in2, Node* in3, uint truth_table, const TypeVect* vt); }; +class VectorMaskCmpNode : public VectorNode { + private: + BoolTest::mask _predicate; + + protected: + uint size_of() const { return sizeof(*this); } + + public: + VectorMaskCmpNode(BoolTest::mask predicate, Node* in1, Node* in2, ConINode* predicate_node, const TypeVect* vt) : + VectorNode(in1, in2, predicate_node, vt), + _predicate(predicate) { + assert(in1->bottom_type()->is_vect()->element_basic_type() == in2->bottom_type()->is_vect()->element_basic_type(), + "VectorMaskCmp inputs must have same type for elements"); + assert(in1->bottom_type()->is_vect()->length() == in2->bottom_type()->is_vect()->length(), + "VectorMaskCmp inputs must have same number of elements"); + init_class_id(Class_VectorMaskCmp); + } + + virtual int Opcode() const; + virtual uint hash() const { return VectorNode::hash() + _predicate; } + virtual bool cmp( const Node &n ) const { + return VectorNode::cmp(n) && _predicate == ((VectorMaskCmpNode&)n)._predicate; + } + BoolTest::mask get_predicate() { return _predicate; } +#ifndef PRODUCT + virtual void dump_spec(outputStream *st) const; +#endif // !PRODUCT +}; + +// Used to wrap other vector nodes in order to add masking functionality. +class VectorMaskWrapperNode : public VectorNode { + public: + VectorMaskWrapperNode(Node* vector, Node* mask) + : VectorNode(vector, mask, vector->bottom_type()->is_vect()) { + assert(mask->is_VectorMaskCmp(), "VectorMaskWrapper requires that second argument be a mask"); + } + + virtual int Opcode() const; + Node* vector_val() const { return in(1); } + Node* vector_mask() const { return in(2); } +}; + +class VectorTestNode : public Node { + private: + BoolTest::mask _predicate; + + protected: + uint size_of() const { return sizeof(*this); } + + public: + VectorTestNode( Node *in1, Node *in2, BoolTest::mask predicate) : Node(NULL, in1, in2), _predicate(predicate) { + assert(in1->is_Vector() || in1->is_LoadVector(), "must be vector"); + assert(in2->is_Vector() || in2->is_LoadVector(), "must be vector"); + assert(in1->bottom_type()->is_vect()->element_basic_type() == in2->bottom_type()->is_vect()->element_basic_type(), + "same type elements are needed"); + assert(in1->bottom_type()->is_vect()->length() == in2->bottom_type()->is_vect()->length(), + "same number of elements is needed"); + } + virtual int Opcode() const; + virtual uint hash() const { return Node::hash() + _predicate; } + virtual bool cmp( const Node &n ) const { + return Node::cmp(n) && _predicate == ((VectorTestNode&)n)._predicate; + } + virtual const Type *bottom_type() const { return TypeInt::BOOL; } + virtual uint ideal_reg() const { return Op_RegI; } // TODO Should be RegFlags but due to missing comparison flags for BoolTest + // in middle-end, we make it boolean result directly. + BoolTest::mask get_predicate() const { return _predicate; } +}; + +class VectorBlendNode : public VectorNode { + public: + VectorBlendNode(Node* vec1, Node* vec2, Node* mask) + : VectorNode(vec1, vec2, mask, vec1->bottom_type()->is_vect()) { + // assert(mask->is_VectorMask(), "VectorBlendNode requires that third argument be a mask"); + } + + virtual int Opcode() const; + Node* vec1() const { return in(1); } + Node* vec2() const { return in(2); } + Node* vec_mask() const { return in(3); } +}; + +class VectorRearrangeNode : public VectorNode { + public: + VectorRearrangeNode(Node* vec1, Node* shuffle) + : VectorNode(vec1, shuffle, vec1->bottom_type()->is_vect()) { + // assert(mask->is_VectorMask(), "VectorBlendNode requires that third argument be a mask"); + } + + virtual int Opcode() const; + Node* vec1() const { return in(1); } + Node* vec_shuffle() const { return in(2); } +}; + + +class VectorLoadMaskNode : public VectorNode { + public: + VectorLoadMaskNode(Node* in, const TypeVect* vt) + : VectorNode(in, vt) { + assert(in->is_LoadVector(), "expected load vector"); + assert(in->as_LoadVector()->vect_type()->element_basic_type() == T_BOOLEAN, "must be boolean"); + } + + virtual int Opcode() const; +}; + +class VectorLoadShuffleNode : public VectorNode { + public: + VectorLoadShuffleNode(Node* in, const TypeVect* vt) + : VectorNode(in, vt) { + assert(in->is_LoadVector(), "expected load vector"); + assert(in->as_LoadVector()->vect_type()->element_basic_type() == T_BYTE, "must be BYTE"); + } + + int GetOutShuffleSize() const { return type2aelembytes(vect_type()->element_basic_type()); } + virtual int Opcode() const; +}; + +class VectorStoreMaskNode : public VectorNode { + protected: + VectorStoreMaskNode(Node* in1, ConINode* in2, const TypeVect* vt) + : VectorNode(in1, in2, vt) { } + + public: + virtual int Opcode() const; + + static VectorStoreMaskNode* make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem); +}; + +// This is intended for use as a simple reinterpret node that has no cast. +class VectorReinterpretNode : public VectorNode { + private: + const TypeVect* _src_vt; + protected: + uint size_of() const { return sizeof(*this); } + public: + VectorReinterpretNode(Node* in, const TypeVect* src_vt, const TypeVect* dst_vt) + : VectorNode(in, dst_vt), _src_vt(src_vt) { } + + virtual uint hash() const { return VectorNode::hash() + _src_vt->hash(); } + virtual bool cmp( const Node &n ) const { + return VectorNode::cmp(n) && !Type::cmp(_src_vt,((VectorReinterpretNode&)n)._src_vt); + } + virtual Node *Identity(PhaseGVN *phase); + + virtual int Opcode() const; +}; + +class VectorCastNode : public VectorNode { + public: + VectorCastNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {} + virtual int Opcode() const; + + static VectorCastNode* make(int vopc, Node* n1, BasicType bt, uint vlen); + static int opcode(BasicType bt); + static bool implemented(BasicType bt, uint vlen); +}; + +class VectorCastB2XNode : public VectorCastNode { + public: + VectorCastB2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) { + assert(in->bottom_type()->is_vect()->element_basic_type() == T_BYTE, "must be byte"); + } + virtual int Opcode() const; +}; + +class VectorCastS2XNode : public VectorCastNode { + public: + VectorCastS2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) { + assert(in->bottom_type()->is_vect()->element_basic_type() == T_SHORT, "must be short"); + } + virtual int Opcode() const; +}; + +class VectorCastI2XNode : public VectorCastNode { + public: + VectorCastI2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) { + assert(in->bottom_type()->is_vect()->element_basic_type() == T_INT, "must be int"); + } + virtual int Opcode() const; +}; + +class VectorCastL2XNode : public VectorCastNode { + public: + VectorCastL2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) { + assert(in->bottom_type()->is_vect()->element_basic_type() == T_LONG, "must be long"); + } + virtual int Opcode() const; +}; + +class VectorCastF2XNode : public VectorCastNode { + public: + VectorCastF2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) { + assert(in->bottom_type()->is_vect()->element_basic_type() == T_FLOAT, "must be float"); + } + virtual int Opcode() const; +}; + +class VectorCastD2XNode : public VectorCastNode { + public: + VectorCastD2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) { + assert(in->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE, "must be double"); + } + virtual int Opcode() const; +}; + +class VectorInsertNode : public VectorNode { + public: + VectorInsertNode(Node* vsrc, Node* new_val, ConINode* pos, const TypeVect* vt) : VectorNode(vsrc, new_val, (Node*)pos, vt) { + assert(pos->get_int() >= 0, "positive constants"); + assert(pos->get_int() < (int)vt->length(), "index must be less than vector length"); + assert(Type::cmp(vt, vsrc->bottom_type()) == 0, "input and output must be same type"); + } + virtual int Opcode() const; + uint pos() const { return in(3)->get_int(); } + + static Node* make(Node* vec, Node* new_val, int position); +}; + +class VectorBoxNode : public Node { + private: + const TypeInstPtr* const _box_type; + const TypeVect* const _vec_type; + public: + enum { + Box = 1, + Value = 2 + }; + VectorBoxNode(Compile* C, Node* box, Node* val, + const TypeInstPtr* box_type, const TypeVect* vt) + : Node(NULL, box, val), _box_type(box_type), _vec_type(vt) { + init_flags(Flag_is_macro); + C->add_macro_node(this); + } + + const TypeInstPtr* box_type() const { assert(_box_type != NULL, ""); return _box_type; }; + const TypeVect* vec_type() const { assert(_vec_type != NULL, ""); return _vec_type; }; + + virtual int Opcode() const; + virtual const Type* bottom_type() const { return _box_type; } + virtual uint ideal_reg() const { return box_type()->ideal_reg(); } + virtual uint size_of() const { return sizeof(*this); } + + static const TypeFunc* vec_box_type(const TypeInstPtr* box_type); +}; + +class VectorBoxAllocateNode : public CallStaticJavaNode { + public: + VectorBoxAllocateNode(Compile* C, const TypeInstPtr* vbox_type) + : CallStaticJavaNode(C, VectorBoxNode::vec_box_type(vbox_type), NULL, NULL, -1) { + init_flags(Flag_is_macro); + C->add_macro_node(this); + } + + virtual int Opcode() const; +#ifndef PRODUCT + virtual void dump_spec(outputStream *st) const; +#endif // !PRODUCT +}; + +class VectorUnboxNode : public VectorNode { + private: + bool _shuffle_to_vector; + protected: + uint size_of() const { return sizeof(*this); } + public: + VectorUnboxNode(Compile* C, const TypeVect* vec_type, Node* obj, Node* mem, bool shuffle_to_vector) + : VectorNode(mem, obj, vec_type) { + _shuffle_to_vector = shuffle_to_vector; + init_flags(Flag_is_macro); + C->add_macro_node(this); + } + + virtual int Opcode() const; + Node* obj() const { return in(2); } + Node* mem() const { return in(1); } + virtual Node *Identity(PhaseGVN *phase); + bool is_shuffle_to_vector() { return _shuffle_to_vector; } +}; + #endif // SHARE_OPTO_VECTORNODE_HPP --- old/src/hotspot/share/prims/nativeLookup.cpp 2020-08-07 00:44:24.000000000 +0300 +++ new/src/hotspot/share/prims/nativeLookup.cpp 2020-08-07 00:44:24.000000000 +0300 @@ -118,6 +118,7 @@ void JNICALL JVM_RegisterMethodHandleMethods(JNIEnv *env, jclass unsafecls); void JNICALL JVM_RegisterPerfMethods(JNIEnv *env, jclass perfclass); void JNICALL JVM_RegisterWhiteBoxMethods(JNIEnv *env, jclass wbclass); + void JNICALL JVM_RegisterVectorSupportMethods(JNIEnv *env, jclass vsclass); #if INCLUDE_JVMCI jobject JNICALL JVM_GetJVMCIRuntime(JNIEnv *env, jclass c); void JNICALL JVM_RegisterJVMCINatives(JNIEnv *env, jclass compilerToVMClass); @@ -132,6 +133,7 @@ { CC"Java_java_lang_invoke_MethodHandleNatives_registerNatives", NULL, FN_PTR(JVM_RegisterMethodHandleMethods) }, { CC"Java_jdk_internal_perf_Perf_registerNatives", NULL, FN_PTR(JVM_RegisterPerfMethods) }, { CC"Java_sun_hotspot_WhiteBox_registerNatives", NULL, FN_PTR(JVM_RegisterWhiteBoxMethods) }, + { CC"Java_jdk_internal_vm_vector_VectorSupport_registerNatives", NULL, FN_PTR(JVM_RegisterVectorSupportMethods)}, #if INCLUDE_JVMCI { CC"Java_jdk_vm_ci_runtime_JVMCI_initializeRuntime", NULL, FN_PTR(JVM_GetJVMCIRuntime) }, { CC"Java_jdk_vm_ci_hotspot_CompilerToVM_registerNatives", NULL, FN_PTR(JVM_RegisterJVMCINatives) }, --- old/src/hotspot/share/runtime/arguments.cpp 2020-08-07 00:44:25.000000000 +0300 +++ new/src/hotspot/share/runtime/arguments.cpp 2020-08-07 00:44:24.000000000 +0300 @@ -4190,8 +4190,23 @@ if (!UseBiasedLocking) { UseOptoBiasInlining = false; } -#endif + if (!EnableVectorSupport) { + if (!FLAG_IS_DEFAULT(EnableVectorReboxing) && EnableVectorReboxing) { + warning("Disabling EnableVectorReboxing since EnableVectorSupport is turned off."); + } + FLAG_SET_DEFAULT(EnableVectorReboxing, false); + + if (!FLAG_IS_DEFAULT(EnableVectorAggressiveReboxing) && EnableVectorAggressiveReboxing) { + if (!EnableVectorReboxing) { + warning("Disabling EnableVectorAggressiveReboxing since EnableVectorReboxing is turned off."); + } else { + warning("Disabling EnableVectorAggressiveReboxing since EnableVectorSupport is turned off."); + } + } + FLAG_SET_DEFAULT(EnableVectorAggressiveReboxing, false); + } +#endif // COMPILER2 return JNI_OK; } --- old/src/hotspot/share/runtime/deoptimization.cpp 2020-08-07 00:44:25.000000000 +0300 +++ new/src/hotspot/share/runtime/deoptimization.cpp 2020-08-07 00:44:25.000000000 +0300 @@ -49,6 +49,7 @@ #include "oops/typeArrayOop.inline.hpp" #include "oops/verifyOopClosure.hpp" #include "prims/jvmtiThreadState.hpp" +#include "prims/vectorSupport.hpp" #include "runtime/atomic.hpp" #include "runtime/biasedLocking.hpp" #include "runtime/deoptimization.hpp" @@ -1002,7 +1003,15 @@ #endif // INCLUDE_JVMCI || INCLUDE_AOT InstanceKlass* ik = InstanceKlass::cast(k); if (obj == NULL) { +#ifdef COMPILER2 + if (EnableVectorSupport && VectorSupport::is_vector(ik)) { + obj = VectorSupport::allocate_vector(ik, fr, reg_map, sv, THREAD); + } else { + obj = ik->allocate_instance(THREAD); + } +#else obj = ik->allocate_instance(THREAD); +#endif // COMPILER2 } } else if (k->is_typeArray_klass()) { TypeArrayKlass* ak = TypeArrayKlass::cast(k); @@ -1339,6 +1348,11 @@ continue; } #endif // INCLUDE_JVMCI || INCLUDE_AOT +#ifdef COMPILER2 + if (EnableVectorSupport && VectorSupport::is_vector(k)) { + continue; // skip field reassignment for vectors + } +#endif if (k->is_instance_klass()) { InstanceKlass* ik = InstanceKlass::cast(k); reassign_fields_by_klass(ik, fr, reg_map, sv, 0, obj(), skip_internal); --- old/src/hotspot/share/runtime/stackValue.cpp 2020-08-07 00:44:25.000000000 +0300 +++ new/src/hotspot/share/runtime/stackValue.cpp 2020-08-07 00:44:25.000000000 +0300 @@ -146,8 +146,12 @@ value.ji = *(jint*)value_addr; return new StackValue(value.p); } - case Location::invalid: + case Location::invalid: { return new StackValue(); + } + case Location::vector: { + ShouldNotReachHere(); // should be handled by Deoptimization::realloc_objects() + } default: ShouldNotReachHere(); } @@ -218,7 +222,7 @@ st->print("NULL"); } st->print(" <" INTPTR_FORMAT ">", p2i(_handle_value())); - break; + break; case T_CONFLICT: st->print("conflict"); --- old/src/hotspot/share/runtime/vmStructs.cpp 2020-08-07 00:44:26.000000000 +0300 +++ new/src/hotspot/share/runtime/vmStructs.cpp 2020-08-07 00:44:26.000000000 +0300 @@ -1501,6 +1501,8 @@ declare_c2_type(MaxNode, AddNode) \ declare_c2_type(MaxINode, MaxNode) \ declare_c2_type(MinINode, MaxNode) \ + declare_c2_type(MaxLNode, MaxNode) \ + declare_c2_type(MinLNode, MaxNode) \ declare_c2_type(MaxFNode, MaxNode) \ declare_c2_type(MinFNode, MaxNode) \ declare_c2_type(MaxDNode, MaxNode) \ @@ -1735,6 +1737,8 @@ declare_c2_type(AbsDNode, AbsNode) \ declare_c2_type(CmpLTMaskNode, Node) \ declare_c2_type(NegNode, Node) \ + declare_c2_type(NegINode, NegNode) \ + declare_c2_type(NegLNode, NegNode) \ declare_c2_type(NegFNode, NegNode) \ declare_c2_type(NegDNode, NegNode) \ declare_c2_type(AtanDNode, Node) \ @@ -1744,10 +1748,12 @@ declare_c2_type(ReverseBytesLNode, Node) \ declare_c2_type(ReductionNode, Node) \ declare_c2_type(VectorNode, Node) \ - declare_c2_type(AbsVBNode, VectorNode) \ - declare_c2_type(AbsVSNode, VectorNode) \ - declare_c2_type(AbsVINode, VectorNode) \ - declare_c2_type(AbsVLNode, VectorNode) \ + declare_c2_type(AbsVFNode, VectorNode) \ + declare_c2_type(AbsVDNode, VectorNode) \ + declare_c2_type(AbsVBNode, VectorNode) \ + declare_c2_type(AbsVSNode, VectorNode) \ + declare_c2_type(AbsVINode, VectorNode) \ + declare_c2_type(AbsVLNode, VectorNode) \ declare_c2_type(AddVBNode, VectorNode) \ declare_c2_type(AddVSNode, VectorNode) \ declare_c2_type(AddVINode, VectorNode) \ @@ -1773,6 +1779,7 @@ declare_c2_type(MulVFNode, VectorNode) \ declare_c2_type(MulReductionVFNode, ReductionNode) \ declare_c2_type(MulVDNode, VectorNode) \ + declare_c2_type(NegVINode, VectorNode) \ declare_c2_type(NegVFNode, VectorNode) \ declare_c2_type(NegVDNode, VectorNode) \ declare_c2_type(FmaVDNode, VectorNode) \ @@ -1795,6 +1802,8 @@ declare_c2_type(URShiftVSNode, VectorNode) \ declare_c2_type(URShiftVINode, VectorNode) \ declare_c2_type(URShiftVLNode, VectorNode) \ + declare_c2_type(MinReductionVNode, ReductionNode) \ + declare_c2_type(MaxReductionVNode, ReductionNode) \ declare_c2_type(AndVNode, VectorNode) \ declare_c2_type(AndReductionVNode, ReductionNode) \ declare_c2_type(OrVNode, VectorNode) \ @@ -1803,8 +1812,6 @@ declare_c2_type(XorReductionVNode, ReductionNode) \ declare_c2_type(MaxVNode, VectorNode) \ declare_c2_type(MinVNode, VectorNode) \ - declare_c2_type(MaxReductionVNode, ReductionNode) \ - declare_c2_type(MinReductionVNode, ReductionNode) \ declare_c2_type(LoadVectorNode, LoadNode) \ declare_c2_type(StoreVectorNode, StoreNode) \ declare_c2_type(ReplicateBNode, VectorNode) \ @@ -1842,6 +1849,27 @@ declare_c2_type(OverflowMulLNode, OverflowLNode) \ declare_c2_type(FmaDNode, Node) \ declare_c2_type(FmaFNode, Node) \ + declare_c2_type(LoadVectorGatherNode, LoadVectorNode) \ + declare_c2_type(StoreVectorScatterNode, StoreVectorNode) \ + declare_c2_type(VectorLoadMaskNode, VectorNode) \ + declare_c2_type(VectorLoadShuffleNode, VectorNode) \ + declare_c2_type(VectorStoreMaskNode, VectorNode) \ + declare_c2_type(VectorBlendNode, VectorNode) \ + declare_c2_type(VectorRearrangeNode, VectorNode) \ + declare_c2_type(VectorMaskWrapperNode, VectorNode) \ + declare_c2_type(VectorMaskCmpNode, VectorNode) \ + declare_c2_type(VectorCastB2XNode, VectorNode) \ + declare_c2_type(VectorCastS2XNode, VectorNode) \ + declare_c2_type(VectorCastI2XNode, VectorNode) \ + declare_c2_type(VectorCastL2XNode, VectorNode) \ + declare_c2_type(VectorCastF2XNode, VectorNode) \ + declare_c2_type(VectorCastD2XNode, VectorNode) \ + declare_c2_type(VectorInsertNode, VectorNode) \ + declare_c2_type(VectorUnboxNode, VectorNode) \ + declare_c2_type(VectorReinterpretNode, VectorNode) \ + declare_c2_type(VectorBoxNode, Node) \ + declare_c2_type(VectorBoxAllocateNode, CallStaticJavaNode) \ + declare_c2_type(VectorTestNode, Node) \ \ /*********************/ \ /* Adapter Blob Entries */ \ --- old/src/hotspot/share/utilities/globalDefinitions.hpp 2020-08-07 00:44:26.000000000 +0300 +++ new/src/hotspot/share/utilities/globalDefinitions.hpp 2020-08-07 00:44:26.000000000 +0300 @@ -233,6 +233,9 @@ return (byte_size + (HeapWordSize-1)) >> LogHeapWordSize; } +inline jfloat jfloat_cast(jint x); +inline jdouble jdouble_cast(jlong x); + //------------------------------------------- // Constant for jlong (standardized by C++11) @@ -243,6 +246,13 @@ const jlong min_jlong = CONST64(0x8000000000000000); const jlong max_jlong = CONST64(0x7fffffffffffffff); +//------------------------------------------- +// Constant for jdouble +const jlong min_jlongDouble = CONST64(0x0000000000000001); +const jdouble min_jdouble = jdouble_cast(min_jlongDouble); +const jlong max_jlongDouble = CONST64(0x7fefffffffffffff); +const jdouble max_jdouble = jdouble_cast(max_jlongDouble); + const size_t K = 1024; const size_t M = K*K; const size_t G = M*K; @@ -465,6 +475,11 @@ const jint min_jint = (jint)1 << (sizeof(jint)*BitsPerByte-1); // 0x80000000 == smallest jint const jint max_jint = (juint)min_jint - 1; // 0x7FFFFFFF == largest jint +const jint min_jintFloat = (jint)(0x00000001); +const jfloat min_jfloat = jfloat_cast(min_jintFloat); +const jint max_jintFloat = (jint)(0x7f7fffff); +const jfloat max_jfloat = jfloat_cast(max_jintFloat); + //---------------------------------------------------------------------------------------------------- // JVM spec restrictions @@ -669,6 +684,14 @@ return (t == T_OBJECT || t == T_ARRAY); } +inline bool is_integral_type(BasicType t) { + return is_subword_type(t) || t == T_INT || t == T_LONG; +} + +inline bool is_floating_point_type(BasicType t) { + return (t == T_FLOAT || t == T_DOUBLE); +} + extern char type2char_tab[T_CONFLICT+1]; // Map a BasicType to a jchar inline char type2char(BasicType t) { return (uint)t < T_CONFLICT+1 ? type2char_tab[t] : 0; } extern int type2size[T_CONFLICT+1]; // Map BasicType to result stack elements --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java 2020-08-07 00:44:27.000000000 +0300 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java 2020-08-07 00:44:27.000000000 +0300 @@ -495,6 +495,31 @@ } } + if (isJDK16OrHigher()) { + // The following are intrinsics required for the Vector API + add(toBeInvestigated, + "jdk/internal/vm/vector/VectorSupport.binaryOp(ILjava/lang/Class;Ljava/lang/Class;ILjava/lang/Object;Ljava/lang/Object;Ljava/util/function/BiFunction;)Ljava/lang/Object;", + "jdk/internal/vm/vector/VectorSupport.blend(Ljava/lang/Class;Ljava/lang/Class;Ljava/lang/Class;ILjdk/internal/vm/vector/VectorSupport$Vector;Ljdk/internal/vm/vector/VectorSupport$Vector;Ljdk/internal/vm/vector/VectorSupport$VectorMask;Ljdk/internal/vm/vector/VectorSupport$VectorBlendOp;)Ljdk/internal/vm/vector/VectorSupport$Vector;", + "jdk/internal/vm/vector/VectorSupport.broadcastCoerced(Ljava/lang/Class;Ljava/lang/Class;IJLjdk/internal/vm/vector/VectorSupport$VectorSpecies;Ljdk/internal/vm/vector/VectorSupport$BroadcastOperation;)Ljava/lang/Object;", + "jdk/internal/vm/vector/VectorSupport.broadcastInt(ILjava/lang/Class;Ljava/lang/Class;ILjdk/internal/vm/vector/VectorSupport$Vector;ILjdk/internal/vm/vector/VectorSupport$VectorBroadcastIntOp;)Ljdk/internal/vm/vector/VectorSupport$Vector;", + "jdk/internal/vm/vector/VectorSupport.compare(ILjava/lang/Class;Ljava/lang/Class;Ljava/lang/Class;ILjdk/internal/vm/vector/VectorSupport$Vector;Ljdk/internal/vm/vector/VectorSupport$Vector;Ljdk/internal/vm/vector/VectorSupport$VectorCompareOp;)Ljdk/internal/vm/vector/VectorSupport$VectorMask;", + "jdk/internal/vm/vector/VectorSupport.convert(ILjava/lang/Class;Ljava/lang/Class;ILjava/lang/Class;Ljava/lang/Class;ILjdk/internal/vm/vector/VectorSupport$VectorPayload;Ljdk/internal/vm/vector/VectorSupport$VectorSpecies;Ljdk/internal/vm/vector/VectorSupport$VectorConvertOp;)Ljdk/internal/vm/vector/VectorSupport$VectorPayload;", + "jdk/internal/vm/vector/VectorSupport.extract(Ljava/lang/Class;Ljava/lang/Class;ILjdk/internal/vm/vector/VectorSupport$Vector;ILjdk/internal/vm/vector/VectorSupport$VecExtractOp;)J", + "jdk/internal/vm/vector/VectorSupport.insert(Ljava/lang/Class;Ljava/lang/Class;ILjdk/internal/vm/vector/VectorSupport$Vector;IJLjdk/internal/vm/vector/VectorSupport$VecInsertOp;)Ljdk/internal/vm/vector/VectorSupport$Vector;", + "jdk/internal/vm/vector/VectorSupport.load(Ljava/lang/Class;Ljava/lang/Class;ILjava/lang/Object;JLjava/lang/Object;ILjdk/internal/vm/vector/VectorSupport$VectorSpecies;Ljdk/internal/vm/vector/VectorSupport$LoadOperation;)Ljava/lang/Object;", + "jdk/internal/vm/vector/VectorSupport.loadWithMap(Ljava/lang/Class;Ljava/lang/Class;ILjava/lang/Class;Ljava/lang/Object;JLjdk/internal/vm/vector/VectorSupport$Vector;Ljava/lang/Object;I[IILjdk/internal/vm/vector/VectorSupport$VectorSpecies;Ljdk/internal/vm/vector/VectorSupport$LoadVectorOperationWithMap;)Ljdk/internal/vm/vector/VectorSupport$Vector;", + "jdk/internal/vm/vector/VectorSupport.maybeRebox(Ljava/lang/Object;)Ljava/lang/Object;", + "jdk/internal/vm/vector/VectorSupport.rearrangeOp(Ljava/lang/Class;Ljava/lang/Class;Ljava/lang/Class;ILjdk/internal/vm/vector/VectorSupport$Vector;Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;Ljdk/internal/vm/vector/VectorSupport$VectorRearrangeOp;)Ljdk/internal/vm/vector/VectorSupport$Vector;", + "jdk/internal/vm/vector/VectorSupport.reductionCoerced(ILjava/lang/Class;Ljava/lang/Class;ILjdk/internal/vm/vector/VectorSupport$Vector;Ljava/util/function/Function;)J", + "jdk/internal/vm/vector/VectorSupport.shuffleIota(Ljava/lang/Class;Ljava/lang/Class;Ljdk/internal/vm/vector/VectorSupport$VectorSpecies;IIIILjdk/internal/vm/vector/VectorSupport$ShuffleIotaOperation;)Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;", + "jdk/internal/vm/vector/VectorSupport.shuffleToVector(Ljava/lang/Class;Ljava/lang/Class;Ljava/lang/Class;Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;ILjdk/internal/vm/vector/VectorSupport$ShuffleToVectorOperation;)Ljava/lang/Object;", + "jdk/internal/vm/vector/VectorSupport.store(Ljava/lang/Class;Ljava/lang/Class;ILjava/lang/Object;JLjdk/internal/vm/vector/VectorSupport$Vector;Ljava/lang/Object;ILjdk/internal/vm/vector/VectorSupport$StoreVectorOperation;)V", + "jdk/internal/vm/vector/VectorSupport.storeWithMap(Ljava/lang/Class;Ljava/lang/Class;ILjava/lang/Class;Ljava/lang/Object;JLjdk/internal/vm/vector/VectorSupport$Vector;Ljdk/internal/vm/vector/VectorSupport$Vector;Ljava/lang/Object;I[IILjdk/internal/vm/vector/VectorSupport$StoreVectorOperationWithMap;)V", + "jdk/internal/vm/vector/VectorSupport.ternaryOp(ILjava/lang/Class;Ljava/lang/Class;ILjava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljdk/internal/vm/vector/VectorSupport$TernaryOperation;)Ljava/lang/Object;", + "jdk/internal/vm/vector/VectorSupport.test(ILjava/lang/Class;Ljava/lang/Class;ILjava/lang/Object;Ljava/lang/Object;Ljava/util/function/BiFunction;)Z", + "jdk/internal/vm/vector/VectorSupport.unaryOp(ILjava/lang/Class;Ljava/lang/Class;ILjava/lang/Object;Ljava/util/function/Function;)Ljava/lang/Object;"); + } + /* * The intrinsics down here are known to be implemented but they are not always enabled on * the HotSpot side (e.g., because they require certain CPU features). So, we are ignoring @@ -606,6 +631,10 @@ return JavaVersionUtil.JAVA_SPEC >= 16; } + private static boolean isJDK16OrHigher() { + return JavaVersionUtil.JAVA_SPEC >= 16; + } + public interface Refiner { void refine(CheckGraalIntrinsics checker); } --- /dev/null 2020-08-07 00:44:27.000000000 +0300 +++ new/src/hotspot/share/opto/library_call.hpp 2020-08-07 00:44:27.000000000 +0300 @@ -0,0 +1,348 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "ci/ciMethod.hpp" +#include "classfile/javaClasses.hpp" +#include "opto/callGenerator.hpp" +#include "opto/graphKit.hpp" +#include "opto/castnode.hpp" +#include "opto/convertnode.hpp" +#include "opto/intrinsicnode.hpp" +#include "opto/movenode.hpp" + +class LibraryIntrinsic : public InlineCallGenerator { + // Extend the set of intrinsics known to the runtime: + public: + private: + bool _is_virtual; + bool _does_virtual_dispatch; + int8_t _predicates_count; // Intrinsic is predicated by several conditions + int8_t _last_predicate; // Last generated predicate + vmIntrinsics::ID _intrinsic_id; + + public: + LibraryIntrinsic(ciMethod* m, bool is_virtual, int predicates_count, bool does_virtual_dispatch, vmIntrinsics::ID id) + : InlineCallGenerator(m), + _is_virtual(is_virtual), + _does_virtual_dispatch(does_virtual_dispatch), + _predicates_count((int8_t)predicates_count), + _last_predicate((int8_t)-1), + _intrinsic_id(id) + { + } + virtual bool is_intrinsic() const { return true; } + virtual bool is_virtual() const { return _is_virtual; } + virtual bool is_predicated() const { return _predicates_count > 0; } + virtual int predicates_count() const { return _predicates_count; } + virtual bool does_virtual_dispatch() const { return _does_virtual_dispatch; } + virtual JVMState* generate(JVMState* jvms); + virtual Node* generate_predicate(JVMState* jvms, int predicate); + vmIntrinsics::ID intrinsic_id() const { return _intrinsic_id; } +}; + + +// Local helper class for LibraryIntrinsic: +class LibraryCallKit : public GraphKit { + private: + LibraryIntrinsic* _intrinsic; // the library intrinsic being called + Node* _result; // the result node, if any + int _reexecute_sp; // the stack pointer when bytecode needs to be reexecuted + + const TypeOopPtr* sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type); + + public: + LibraryCallKit(JVMState* jvms, LibraryIntrinsic* intrinsic) + : GraphKit(jvms), + _intrinsic(intrinsic), + _result(NULL) + { + // Check if this is a root compile. In that case we don't have a caller. + if (!jvms->has_method()) { + _reexecute_sp = sp(); + } else { + // Find out how many arguments the interpreter needs when deoptimizing + // and save the stack pointer value so it can used by uncommon_trap. + // We find the argument count by looking at the declared signature. + bool ignored_will_link; + ciSignature* declared_signature = NULL; + ciMethod* ignored_callee = caller()->get_method_at_bci(bci(), ignored_will_link, &declared_signature); + const int nargs = declared_signature->arg_size_for_bc(caller()->java_code_at_bci(bci())); + _reexecute_sp = sp() + nargs; // "push" arguments back on stack + } + } + + virtual LibraryCallKit* is_LibraryCallKit() const { return (LibraryCallKit*)this; } + + ciMethod* caller() const { return jvms()->method(); } + int bci() const { return jvms()->bci(); } + LibraryIntrinsic* intrinsic() const { return _intrinsic; } + vmIntrinsics::ID intrinsic_id() const { return _intrinsic->intrinsic_id(); } + ciMethod* callee() const { return _intrinsic->method(); } + + bool try_to_inline(int predicate); + Node* try_to_predicate(int predicate); + + void push_result() { + // Push the result onto the stack. + if (!stopped() && result() != NULL) { + BasicType bt = result()->bottom_type()->basic_type(); + push_node(bt, result()); + } + } + + private: + void fatal_unexpected_iid(vmIntrinsics::ID iid) { + fatal("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid)); + } + + void set_result(Node* n) { assert(_result == NULL, "only set once"); _result = n; } + void set_result(RegionNode* region, PhiNode* value); + Node* result() { return _result; } + + virtual int reexecute_sp() { return _reexecute_sp; } + + // Helper functions to inline natives + Node* generate_guard(Node* test, RegionNode* region, float true_prob); + Node* generate_slow_guard(Node* test, RegionNode* region); + Node* generate_fair_guard(Node* test, RegionNode* region); + Node* generate_negative_guard(Node* index, RegionNode* region, + // resulting CastII of index: + Node* *pos_index = NULL); + Node* generate_limit_guard(Node* offset, Node* subseq_length, + Node* array_length, + RegionNode* region); + void generate_string_range_check(Node* array, Node* offset, + Node* length, bool char_count); + Node* generate_current_thread(Node* &tls_output); + Node* load_mirror_from_klass(Node* klass); + Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null, + RegionNode* region, int null_path, + int offset); + Node* load_klass_from_mirror(Node* mirror, bool never_see_null, + RegionNode* region, int null_path) { + int offset = java_lang_Class::klass_offset(); + return load_klass_from_mirror_common(mirror, never_see_null, + region, null_path, + offset); + } + Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null, + RegionNode* region, int null_path) { + int offset = java_lang_Class::array_klass_offset(); + return load_klass_from_mirror_common(mirror, never_see_null, + region, null_path, + offset); + } + Node* generate_access_flags_guard(Node* kls, + int modifier_mask, int modifier_bits, + RegionNode* region); + Node* generate_interface_guard(Node* kls, RegionNode* region); + Node* generate_hidden_class_guard(Node* kls, RegionNode* region); + Node* generate_array_guard(Node* kls, RegionNode* region) { + return generate_array_guard_common(kls, region, false, false); + } + Node* generate_non_array_guard(Node* kls, RegionNode* region) { + return generate_array_guard_common(kls, region, false, true); + } + Node* generate_objArray_guard(Node* kls, RegionNode* region) { + return generate_array_guard_common(kls, region, true, false); + } + Node* generate_non_objArray_guard(Node* kls, RegionNode* region) { + return generate_array_guard_common(kls, region, true, true); + } + Node* generate_array_guard_common(Node* kls, RegionNode* region, + bool obj_array, bool not_array); + Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region); + CallJavaNode* generate_method_call(vmIntrinsics::ID method_id, + bool is_virtual = false, bool is_static = false); + CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) { + return generate_method_call(method_id, false, true); + } + CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) { + return generate_method_call(method_id, true, false); + } + Node * load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls); + Node * field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls); + + Node* make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae); + bool inline_string_compareTo(StrIntrinsicNode::ArgEnc ae); + bool inline_string_indexOf(StrIntrinsicNode::ArgEnc ae); + bool inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae); + Node* make_indexOf_node(Node* src_start, Node* src_count, Node* tgt_start, Node* tgt_count, + RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae); + bool inline_string_indexOfChar(); + bool inline_string_equals(StrIntrinsicNode::ArgEnc ae); + bool inline_string_toBytesU(); + bool inline_string_getCharsU(); + bool inline_string_copy(bool compress); + bool inline_string_char_access(bool is_store); + Node* round_double_node(Node* n); + bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName); + bool inline_math_native(vmIntrinsics::ID id); + bool inline_math(vmIntrinsics::ID id); + bool inline_double_math(vmIntrinsics::ID id); + template + bool inline_math_overflow(Node* arg1, Node* arg2); + void inline_math_mathExact(Node* math, Node* test); + bool inline_math_addExactI(bool is_increment); + bool inline_math_addExactL(bool is_increment); + bool inline_math_multiplyExactI(); + bool inline_math_multiplyExactL(); + bool inline_math_multiplyHigh(); + bool inline_math_negateExactI(); + bool inline_math_negateExactL(); + bool inline_math_subtractExactI(bool is_decrement); + bool inline_math_subtractExactL(bool is_decrement); + bool inline_min_max(vmIntrinsics::ID id); + bool inline_notify(vmIntrinsics::ID id); + Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y); + // This returns Type::AnyPtr, RawPtr, or OopPtr. + int classify_unsafe_addr(Node* &base, Node* &offset, BasicType type); + Node* make_unsafe_address(Node*& base, Node* offset, DecoratorSet decorators, BasicType type = T_ILLEGAL, bool can_cast = false); + + typedef enum { Relaxed, Opaque, Volatile, Acquire, Release } AccessKind; + DecoratorSet mo_decorator_for_access_kind(AccessKind kind); + bool inline_unsafe_access(bool is_store, BasicType type, AccessKind kind, bool is_unaligned); + static bool klass_needs_init_guard(Node* kls); + bool inline_unsafe_allocate(); + bool inline_unsafe_newArray(bool uninitialized); + bool inline_unsafe_writeback0(); + bool inline_unsafe_writebackSync0(bool is_pre); + bool inline_unsafe_copyMemory(); + bool inline_native_currentThread(); + + bool inline_native_time_funcs(address method, const char* funcName); +#ifdef JFR_HAVE_INTRINSICS + bool inline_native_classID(); + bool inline_native_getEventWriter(); +#endif + bool inline_native_Class_query(vmIntrinsics::ID id); + bool inline_native_subtype_check(); + bool inline_native_getLength(); + bool inline_array_copyOf(bool is_copyOfRange); + bool inline_array_equals(StrIntrinsicNode::ArgEnc ae); + bool inline_preconditions_checkIndex(); + void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array); + bool inline_native_clone(bool is_virtual); + bool inline_native_Reflection_getCallerClass(); + // Helper function for inlining native object hash method + bool inline_native_hashcode(bool is_virtual, bool is_static); + bool inline_native_getClass(); + + // Helper functions for inlining arraycopy + bool inline_arraycopy(); + AllocateArrayNode* tightly_coupled_allocation(Node* ptr, + RegionNode* slow_region); + JVMState* arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp); + void arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms, int saved_reexecute_sp, + uint new_idx); + + typedef enum { LS_get_add, LS_get_set, LS_cmp_swap, LS_cmp_swap_weak, LS_cmp_exchange } LoadStoreKind; + bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind, AccessKind access_kind); + bool inline_unsafe_fence(vmIntrinsics::ID id); + bool inline_onspinwait(); + bool inline_fp_conversions(vmIntrinsics::ID id); + bool inline_number_methods(vmIntrinsics::ID id); + bool inline_reference_get(); + bool inline_Class_cast(); + bool inline_aescrypt_Block(vmIntrinsics::ID id); + bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id); + bool inline_electronicCodeBook_AESCrypt(vmIntrinsics::ID id); + bool inline_counterMode_AESCrypt(vmIntrinsics::ID id); + Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting); + Node* inline_electronicCodeBook_AESCrypt_predicate(bool decrypting); + Node* inline_counterMode_AESCrypt_predicate(); + Node* get_key_start_from_aescrypt_object(Node* aescrypt_object); + Node* get_original_key_start_from_aescrypt_object(Node* aescrypt_object); + bool inline_ghash_processBlocks(); + bool inline_base64_encodeBlock(); + bool inline_digestBase_implCompress(vmIntrinsics::ID id); + bool inline_digestBase_implCompressMB(int predicate); + bool inline_digestBase_implCompressMB(Node* digestBaseObj, ciInstanceKlass* instklass, + bool long_state, address stubAddr, const char *stubName, + Node* src_start, Node* ofs, Node* limit); + Node* get_state_from_digest_object(Node *digestBase_object); + Node* get_long_state_from_digest_object(Node *digestBase_object); + Node* inline_digestBase_implCompressMB_predicate(int predicate); + bool inline_encodeISOArray(); + bool inline_updateCRC32(); + bool inline_updateBytesCRC32(); + bool inline_updateByteBufferCRC32(); + Node* get_table_from_crc32c_class(ciInstanceKlass *crc32c_class); + bool inline_updateBytesCRC32C(); + bool inline_updateDirectByteBufferCRC32C(); + bool inline_updateBytesAdler32(); + bool inline_updateByteBufferAdler32(); + bool inline_multiplyToLen(); + bool inline_hasNegatives(); + bool inline_squareToLen(); + bool inline_mulAdd(); + bool inline_montgomeryMultiply(); + bool inline_montgomerySquare(); + bool inline_bigIntegerShift(bool isRightShift); + bool inline_vectorizedMismatch(); + bool inline_fma(vmIntrinsics::ID id); + bool inline_character_compare(vmIntrinsics::ID id); + bool inline_fp_min_max(vmIntrinsics::ID id); + + bool inline_profileBoolean(); + bool inline_isCompileConstant(); + + // Vector API support + bool inline_vector_nary_operation(int n); + bool inline_vector_broadcast_coerced(); + bool inline_vector_shuffle_to_vector(); + bool inline_vector_shuffle_iota(); + bool inline_vector_mem_operation(bool is_store); + bool inline_vector_gather_scatter(bool is_scatter); + bool inline_vector_reduction(); + bool inline_vector_test(); + bool inline_vector_blend(); + bool inline_vector_rearrange(); + bool inline_vector_compare(); + bool inline_vector_broadcast_int(); + bool inline_vector_convert(); + bool inline_vector_extract(); + bool inline_vector_insert(); + Node* box_vector(Node* in, const TypeInstPtr* vbox_type, BasicType bt, int num_elem); + Node* unbox_vector(Node* in, const TypeInstPtr* vbox_type, BasicType bt, int num_elem, bool shuffle_to_vector = false); + Node* shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem); + + enum VectorMaskUseType { + VecMaskUseLoad, + VecMaskUseStore, + VecMaskUseAll, + VecMaskNotUsed + }; + + bool arch_supports_vector(int op, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args = false); + + void clear_upper_avx() { +#ifdef X86 + if (UseAVX >= 2) { + C->set_clear_upper_avx(true); + } +#endif + } +}; + --- /dev/null 2020-08-07 00:44:28.000000000 +0300 +++ new/src/hotspot/share/opto/vector.cpp 2020-08-07 00:44:27.000000000 +0300 @@ -0,0 +1,466 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "opto/castnode.hpp" +#include "opto/graphKit.hpp" +#include "opto/phaseX.hpp" +#include "opto/rootnode.hpp" +#include "opto/vector.hpp" +#include "utilities/macros.hpp" + +void PhaseVector::optimize_vector_boxes() { + Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); + + // Signal GraphKit it's post-parse phase. + assert(C->inlining_incrementally() == false, "sanity"); + C->set_inlining_incrementally(true); + + C->for_igvn()->clear(); + C->initial_gvn()->replace_with(&_igvn); + + expand_vunbox_nodes(); + scalarize_vbox_nodes(); + + C->inline_vector_reboxing_calls(); + + expand_vbox_nodes(); + eliminate_vbox_alloc_nodes(); + + C->set_inlining_incrementally(false); + + do_cleanup(); +} + +void PhaseVector::do_cleanup() { + if (C->failing()) return; + { + Compile::TracePhase tp("vector_pru", &timers[_t_vector_pru]); + ResourceMark rm; + PhaseRemoveUseless pru(C->initial_gvn(), C->for_igvn()); + if (C->failing()) return; + } + { + Compile::TracePhase tp("incrementalInline_igvn", &timers[_t_vector_igvn]); + _igvn = PhaseIterGVN(C->initial_gvn()); + _igvn.optimize(); + if (C->failing()) return; + } + C->print_method(PHASE_ITER_GVN_BEFORE_EA, 3); +} + +void PhaseVector::scalarize_vbox_nodes() { + if (C->failing()) return; + + if (!EnableVectorReboxing) { + return; // don't scalarize vector boxes + } + + int macro_idx = C->macro_count() - 1; + while (macro_idx >= 0) { + Node * n = C->macro_node(macro_idx); + assert(n->is_macro(), "only macro nodes expected here"); + if (n->Opcode() == Op_VectorBox) { + VectorBoxNode* vbox = static_cast(n); + scalarize_vbox_node(vbox); + if (C->failing()) return; + C->print_method(PHASE_SCALARIZE_VBOX, vbox, 3); + } + if (C->failing()) return; + macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); + } +} + +void PhaseVector::expand_vbox_nodes() { + if (C->failing()) return; + + int macro_idx = C->macro_count() - 1; + while (macro_idx >= 0) { + Node * n = C->macro_node(macro_idx); + assert(n->is_macro(), "only macro nodes expected here"); + if (n->Opcode() == Op_VectorBox) { + VectorBoxNode* vbox = static_cast(n); + expand_vbox_node(vbox); + if (C->failing()) return; + } + if (C->failing()) return; + macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); + } +} + +void PhaseVector::expand_vunbox_nodes() { + if (C->failing()) return; + + int macro_idx = C->macro_count() - 1; + while (macro_idx >= 0) { + Node * n = C->macro_node(macro_idx); + assert(n->is_macro(), "only macro nodes expected here"); + if (n->Opcode() == Op_VectorUnbox) { + VectorUnboxNode* vec_unbox = static_cast(n); + expand_vunbox_node(vec_unbox); + if (C->failing()) return; + C->print_method(PHASE_EXPAND_VUNBOX, vec_unbox, 3); + } + if (C->failing()) return; + macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); + } +} + +void PhaseVector::eliminate_vbox_alloc_nodes() { + if (C->failing()) return; + + int macro_idx = C->macro_count() - 1; + while (macro_idx >= 0) { + Node * n = C->macro_node(macro_idx); + assert(n->is_macro(), "only macro nodes expected here"); + if (n->Opcode() == Op_VectorBoxAllocate) { + VectorBoxAllocateNode* vbox_alloc = static_cast(n); + eliminate_vbox_alloc_node(vbox_alloc); + if (C->failing()) return; + C->print_method(PHASE_ELIMINATE_VBOX_ALLOC, vbox_alloc, 3); + } + if (C->failing()) return; + macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); + } +} + +static JVMState* clone_jvms(Compile* C, SafePointNode* sfpt) { + JVMState* new_jvms = sfpt->jvms()->clone_shallow(C); + uint size = sfpt->req(); + SafePointNode* map = new SafePointNode(size, new_jvms); + for (uint i = 0; i < size; i++) { + map->init_req(i, sfpt->in(i)); + } + new_jvms->set_map(map); + return new_jvms; +} + +void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) { + Node* vec_value = vec_box->in(VectorBoxNode::Value); + PhaseGVN& gvn = *C->initial_gvn(); + + // Process merged VBAs + + if (EnableVectorAggressiveReboxing) { + Unique_Node_List calls(C->comp_arena()); + for (DUIterator_Fast imax, i = vec_box->fast_outs(imax); i < imax; i++) { + Node* use = vec_box->fast_out(i); + if (use->is_CallJava()) { + CallJavaNode* call = use->as_CallJava(); + if (call->has_non_debug_use(vec_box) && vec_box->in(VectorBoxNode::Box)->is_Phi()) { + calls.push(call); + } + } + } + + while (calls.size() > 0) { + CallJavaNode* call = calls.pop()->as_CallJava(); + // Attach new VBA to the call and use it instead of Phi (VBA ... VBA). + + JVMState* jvms = clone_jvms(C, call); + GraphKit kit(jvms); + PhaseGVN& gvn = kit.gvn(); + + // Adjust JVMS from post-call to pre-call state: put args on stack + uint nargs = call->method()->arg_size(); + kit.ensure_stack(kit.sp() + nargs); + for (uint i = TypeFunc::Parms; i < call->tf()->domain()->cnt(); i++) { + kit.push(call->in(i)); + } + jvms = kit.sync_jvms(); + + Node* new_vbox = NULL; + { + PreserveReexecuteState prs(&kit); + + kit.jvms()->set_should_reexecute(true); + + const TypeInstPtr* vbox_type = vec_box->box_type(); + const TypeVect* vect_type = vec_box->vec_type(); + Node* vect = vec_box->in(VectorBoxNode::Value); + + VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type); + kit.set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true); + kit.make_slow_call_ex(alloc, C->env()->Throwable_klass(), /*separate_io_proj=*/true, /*deoptimize=*/true); + kit.set_i_o(gvn.transform( new ProjNode(alloc, TypeFunc::I_O) )); + kit.set_all_memory(gvn.transform( new ProjNode(alloc, TypeFunc::Memory) )); + Node* ret = gvn.transform(new ProjNode(alloc, TypeFunc::Parms)); + + new_vbox = gvn.transform(new VectorBoxNode(C, ret, vect, vbox_type, vect_type)); + + kit.replace_in_map(vec_box, new_vbox); + } + + kit.dec_sp(nargs); + jvms = kit.sync_jvms(); + + call->set_req(TypeFunc::Control , kit.control()); + call->set_req(TypeFunc::I_O , kit.i_o()); + call->set_req(TypeFunc::Memory , kit.reset_memory()); + call->set_req(TypeFunc::FramePtr, kit.frameptr()); + call->replace_edge(vec_box, new_vbox); + + C->record_for_igvn(call); + } + } + + // Process debug uses at safepoints + Unique_Node_List safepoints(C->comp_arena()); + + for (DUIterator_Fast imax, i = vec_box->fast_outs(imax); i < imax; i++) { + Node* use = vec_box->fast_out(i); + if (use->is_SafePoint()) { + SafePointNode* sfpt = use->as_SafePoint(); + if (!sfpt->is_Call() || !sfpt->as_Call()->has_non_debug_use(vec_box)) { + safepoints.push(sfpt); + } + } + } + + while (safepoints.size() > 0) { + SafePointNode* sfpt = safepoints.pop()->as_SafePoint(); + + uint first_ind = (sfpt->req() - sfpt->jvms()->scloff()); + Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(), +#ifdef ASSERT + NULL, +#endif // ASSERT + first_ind, /*n_fields=*/1); + sobj->init_req(0, C->root()); + sfpt->add_req(vec_value); + + sobj = gvn.transform(sobj); + + JVMState *jvms = sfpt->jvms(); + + jvms->set_endoff(sfpt->req()); + // Now make a pass over the debug information replacing any references + // to the allocated object with "sobj" + int start = jvms->debug_start(); + int end = jvms->debug_end(); + sfpt->replace_edges_in_range(vec_box, sobj, start, end); + + C->record_for_igvn(sfpt); + } +} + +void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) { + if (vec_box->outcnt() > 0) { + Node* vbox = vec_box->in(VectorBoxNode::Box); + Node* vect = vec_box->in(VectorBoxNode::Value); + Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(), vec_box->vec_type()); + C->gvn_replace_by(vec_box, result); + C->print_method(PHASE_EXPAND_VBOX, vec_box, 3); + } + C->remove_macro_node(vec_box); +} + +Node* PhaseVector::expand_vbox_node_helper(Node* vbox, + Node* vect, + const TypeInstPtr* box_type, + const TypeVect* vect_type) { + if (vbox->is_Phi() && vect->is_Phi()) { + assert(vbox->as_Phi()->region() == vect->as_Phi()->region(), ""); + Node* new_phi = new PhiNode(vbox->as_Phi()->region(), box_type); + for (uint i = 1; i < vbox->req(); i++) { + Node* new_box = expand_vbox_node_helper(vbox->in(i), vect->in(i), box_type, vect_type); + new_phi->set_req(i, new_box); + } + new_phi = C->initial_gvn()->transform(new_phi); + return new_phi; + } else if (vbox->is_Proj() && vbox->in(0)->Opcode() == Op_VectorBoxAllocate) { + VectorBoxAllocateNode* vbox_alloc = static_cast(vbox->in(0)); + return expand_vbox_alloc_node(vbox_alloc, vect, box_type, vect_type); + } else { + assert(!vbox->is_Phi(), ""); + // TODO: assert that expanded vbox is initialized with the same value (vect). + return vbox; // already expanded + } +} + +static bool is_vector_mask(ciKlass* klass) { + return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); +} + +static bool is_vector_shuffle(ciKlass* klass) { + return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); +} + +Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc, + Node* value, + const TypeInstPtr* box_type, + const TypeVect* vect_type) { + JVMState* jvms = clone_jvms(C, vbox_alloc); + GraphKit kit(jvms); + PhaseGVN& gvn = kit.gvn(); + + ciInstanceKlass* box_klass = box_type->klass()->as_instance_klass(); + BasicType bt = vect_type->element_basic_type(); + int num_elem = vect_type->length(); + + bool is_mask = is_vector_mask(box_klass); + if (is_mask && bt != T_BOOLEAN) { + value = gvn.transform(VectorStoreMaskNode::make(gvn, value, bt, num_elem)); + // Although type of mask depends on its definition, in terms of storage everything is stored in boolean array. + bt = T_BOOLEAN; + assert(value->as_Vector()->bottom_type()->is_vect()->element_basic_type() == bt, + "must be consistent with mask representation"); + } + + // Generate array allocation for the field which holds the values. + const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(bt)); + Node* arr = kit.new_array(kit.makecon(array_klass), kit.intcon(num_elem), 1); + + // Store the vector value into the array. + // (The store should be captured by InitializeNode and turned into initialized store later.) + Node* arr_adr = kit.array_element_address(arr, kit.intcon(0), bt); + const TypePtr* arr_adr_type = arr_adr->bottom_type()->is_ptr(); + Node* arr_mem = kit.memory(arr_adr); + Node* vstore = gvn.transform(StoreVectorNode::make(0, + kit.control(), + arr_mem, + arr_adr, + arr_adr_type, + value, + num_elem)); + kit.set_memory(vstore, arr_adr_type); + + C->set_max_vector_size(MAX2(C->max_vector_size(), vect_type->length_in_bytes())); + + // Generate the allocate for the Vector object. + const TypeKlassPtr* klass_type = box_type->as_klass_type(); + Node* klass_node = kit.makecon(klass_type); + Node* vec_obj = kit.new_instance(klass_node); + + // Store the allocated array into object. + ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbol::payload_name(), + ciSymbol::object_signature(), + false); + assert(field != NULL, ""); + Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes()); + const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr(); + + // The store should be captured by InitializeNode and turned into initialized store later. + Node* field_store = gvn.transform(kit.access_store_at(vec_obj, + vec_field, + vec_adr_type, + arr, + TypeOopPtr::make_from_klass(field->type()->as_klass()), + T_OBJECT, + IN_HEAP)); + kit.set_memory(field_store, vec_adr_type); + + kit.replace_call(vbox_alloc, vec_obj, true); + C->remove_macro_node(vbox_alloc); + + return vec_obj; +} + +void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) { + if (vec_unbox->outcnt() > 0) { + GraphKit kit; + PhaseGVN& gvn = kit.gvn(); + + Node* obj = vec_unbox->obj(); + const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr(); + ciInstanceKlass* from_kls = tinst->klass()->as_instance_klass(); + BasicType bt = vec_unbox->vect_type()->element_basic_type(); + BasicType masktype = bt; + BasicType elem_bt; + + if (is_vector_mask(from_kls)) { + bt = T_BOOLEAN; + } else if (is_vector_shuffle(from_kls)) { + if (vec_unbox->is_shuffle_to_vector() == true) { + elem_bt = bt; + } + bt = T_BYTE; + } + + ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbol::payload_name(), + ciSymbol::object_signature(), + false); + assert(field != NULL, ""); + int offset = field->offset_in_bytes(); + Node* vec_adr = kit.basic_plus_adr(obj, offset); + + Node* mem = vec_unbox->mem(); + Node* ctrl = vec_unbox->in(0); + Node* vec_field_ld = LoadNode::make(gvn, + ctrl, + mem, + vec_adr, + vec_adr->bottom_type()->is_ptr(), + TypeOopPtr::make_from_klass(field->type()->as_klass()), + T_OBJECT, + MemNode::unordered); + vec_field_ld = gvn.transform(vec_field_ld); + + // For proper aliasing, attach concrete payload type. + ciKlass* payload_klass = ciTypeArrayKlass::make(bt); + const Type* payload_type = TypeAryPtr::make_from_klass(payload_klass)->cast_to_ptr_type(TypePtr::NotNull); + vec_field_ld = gvn.transform(new CastPPNode(vec_field_ld, payload_type)); + + Node* adr = kit.array_element_address(vec_field_ld, gvn.intcon(0), bt); + const TypePtr* adr_type = adr->bottom_type()->is_ptr(); + const TypeVect* vt = vec_unbox->bottom_type()->is_vect(); + int num_elem = vt->length(); + Node* vec_val_load = LoadVectorNode::make(0, + ctrl, + mem, + adr, + adr_type, + num_elem, + bt); + vec_val_load = gvn.transform(vec_val_load); + + C->set_max_vector_size(MAX2(C->max_vector_size(), vt->length_in_bytes())); + + if (is_vector_mask(from_kls) && masktype != T_BOOLEAN) { + assert(vec_unbox->bottom_type()->is_vect()->element_basic_type() == masktype, "expect mask type consistency"); + vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::make(masktype, num_elem))); + } else if (is_vector_shuffle(from_kls)) { + if (vec_unbox->is_shuffle_to_vector() == false) { + assert(vec_unbox->bottom_type()->is_vect()->element_basic_type() == masktype, "expect shuffle type consistency"); + vec_val_load = gvn.transform(new VectorLoadShuffleNode(vec_val_load, TypeVect::make(masktype, num_elem))); + } else if (elem_bt != T_BYTE) { + vec_val_load = gvn.transform(VectorCastNode::make(Op_VectorCastB2X, vec_val_load, elem_bt, num_elem)); + } + } + + gvn.hash_delete(vec_unbox); + vec_unbox->disconnect_inputs(NULL, C); + C->gvn_replace_by(vec_unbox, vec_val_load); + } + C->remove_macro_node(vec_unbox); +} + +void PhaseVector::eliminate_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc) { + JVMState* jvms = clone_jvms(C, vbox_alloc); + GraphKit kit(jvms); + // Remove VBA, but leave a safepoint behind. + // Otherwise, it may end up with a loop without any safepoint polls. + kit.replace_call(vbox_alloc, kit.map(), true); + C->remove_macro_node(vbox_alloc); +} --- /dev/null 2020-08-07 00:44:28.000000000 +0300 +++ new/src/hotspot/share/opto/vector.hpp 2020-08-07 00:44:28.000000000 +0300 @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OPTO_VECTOR_HPP +#define SHARE_OPTO_VECTOR_HPP + +#include "opto/node.hpp" +#include "opto/phaseX.hpp" +#include "opto/type.hpp" +#include "opto/vectornode.hpp" + +class PhaseVector : public Phase { + private: + PhaseIterGVN& _igvn; + + void expand_vbox_nodes(); + void expand_vbox_node(VectorBoxNode* vec_box); + Node* expand_vbox_node_helper(Node* vbox, + Node* vect, + const TypeInstPtr* box_type, + const TypeVect* vect_type); + Node* expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc, + Node* value, + const TypeInstPtr* box_type, + const TypeVect* vect_type); + void scalarize_vbox_nodes(); + void scalarize_vbox_node(VectorBoxNode* vec_box); + void expand_vunbox_nodes(); + void expand_vunbox_node(VectorUnboxNode* vec_box); + void eliminate_vbox_alloc_nodes(); + void eliminate_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc); + void do_cleanup(); + void scalarize_vector_boxes(); + void expand_vector_boxes(); + + public: + PhaseVector(PhaseIterGVN& igvn) : Phase(Vector), _igvn(igvn) {} + void optimize_vector_boxes(); +}; + +#endif // SHARE_OPTO_VECTOR_HPP --- /dev/null 2020-08-07 00:44:28.000000000 +0300 +++ new/src/hotspot/share/opto/vectorIntrinsics.cpp 2020-08-07 00:44:28.000000000 +0300 @@ -0,0 +1,1594 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/vmSymbols.hpp" +#include "opto/library_call.hpp" +#include "opto/runtime.hpp" +#include "opto/vectornode.hpp" +#include "prims/vectorSupport.hpp" + +bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) { + // Check that the operation is valid. + if (sopc <= 0) { +#ifndef PRODUCT + if (C->print_intrinsics()) { + tty->print_cr(" ** Rejected intrinsification because no valid vector op could be extracted"); + } +#endif + return false; + } + + // Check that architecture supports this op-size-type combination. + if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) { +#ifndef PRODUCT + if (C->print_intrinsics()) { + tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support it", + NodeClassNames[sopc], type2name(type), num_elem); + } +#endif + return false; + } else { + assert(Matcher::match_rule_supported(sopc), "must be supported"); + } + + if (!has_scalar_args && VectorNode::is_vector_shift(sopc) && + Matcher::supports_vector_variable_shifts() == false) { + if (C->print_intrinsics()) { + tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts", + NodeClassNames[sopc], type2name(type), num_elem); + } + return false; + } + + // Check whether mask unboxing is supported. + if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseLoad) { + if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) { + #ifndef PRODUCT + if (C->print_intrinsics()) { + tty->print_cr(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it", + NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem); + } + #endif + return false; + } + } + + // Check whether mask boxing is supported. + if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseStore) { + if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) { + #ifndef PRODUCT + if (C->print_intrinsics()) { + tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it", + NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem); + } + #endif + return false; + } + } + + return true; +} + +static bool is_vector_mask(ciKlass* klass) { + return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); +} + +static bool is_vector_shuffle(ciKlass* klass) { + return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); +} + +static bool is_klass_initialized(const TypeInstPtr* vec_klass) { + assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass(), "klass instance expected"); + ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass(); + return klass->is_initialized(); +} + +#ifdef ASSERT +static bool is_vector(ciKlass* klass) { + return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass()); +} + +static bool check_vbox(const TypeInstPtr* vbox_type) { + assert(vbox_type->klass_is_exact(), ""); + + ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass(); + assert(is_vector(ik), "not a vector"); + + ciField* fd1 = ik->get_field_by_name(ciSymbol::ETYPE_name(), ciSymbol::class_signature(), /* is_static */ true); + assert(fd1 != NULL, "element type info is missing"); + + ciConstant val1 = fd1->constant_value(); + BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type(); + assert(is_java_primitive(elem_bt), "element type info is missing"); + + ciField* fd2 = ik->get_field_by_name(ciSymbol::VLENGTH_name(), ciSymbol::int_signature(), /* is_static */ true); + assert(fd2 != NULL, "vector length info is missing"); + + ciConstant val2 = fd2->constant_value(); + assert(val2.as_int() > 0, "vector length info is missing"); + + return true; +} +#endif + +Node* LibraryCallKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, + BasicType elem_bt, int num_elem) { + assert(EnableVectorSupport, ""); + const TypeVect* vec_type = TypeVect::make(elem_bt, num_elem); + + VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type); + set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true); + make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true); + set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) )); + set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) )); + Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms)); + + assert(check_vbox(vbox_type), ""); + VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vec_type); + return gvn().transform(vbox); +} + +Node* LibraryCallKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) { + assert(EnableVectorSupport, ""); + const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr(); + if (vbox_type->klass() != vbox_type_v->klass()) { + return NULL; // arguments don't agree on vector shapes + } + if (vbox_type_v->maybe_null()) { + return NULL; // no nulls are allowed + } + assert(check_vbox(vbox_type), ""); + const TypeVect* vec_type = TypeVect::make(elem_bt, num_elem); + Node* unbox = gvn().transform(new VectorUnboxNode(C, vec_type, v, merged_memory(), shuffle_to_vector)); + return unbox; +} + +// public static +// +// VM unaryOp(int oprId, Class vmClass, Class elementType, int length, +// VM vm, +// Function defaultImpl) { +// +// public static +// +// VM binaryOp(int oprId, Class vmClass, Class elementType, int length, +// VM vm1, VM vm2, +// BiFunction defaultImpl) { +// +// public static +// +// VM ternaryOp(int oprId, Class vmClass, Class elementType, int length, +// VM vm1, VM vm2, VM vm3, +// TernaryOperation defaultImpl) { +// +bool LibraryCallKit::inline_vector_nary_operation(int n) { + const TypeInt* opr = gvn().type(argument(0))->is_int(); + const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr(); + const TypeInt* vlen = gvn().type(argument(3))->is_int(); + + if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(1)->Opcode()], + NodeClassNames[argument(2)->Opcode()], + NodeClassNames[argument(3)->Opcode()]); + } + return false; // not enough info for intrinsification + } + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); + } + return false; // should be primitive type + } + if (!is_klass_initialized(vector_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + BasicType elem_bt = elem_type->basic_type(); + int num_elem = vlen->get_con(); + int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt); + int sopc = VectorNode::opcode(opc, elem_bt); + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad. + if (!arch_supports_vector(sopc, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseAll : VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d", + n, sopc, num_elem, type2name(elem_bt), + is_vector_mask(vbox_klass) ? 1 : 0); + } + return false; // not supported + } + + Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL; + switch (n) { + case 3: { + opd3 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem); + if (opd3 == NULL) { + if (C->print_intrinsics()) { + tty->print_cr(" ** unbox failed v3=%s", + NodeClassNames[argument(6)->Opcode()]); + } + return false; + } + // fall-through + } + case 2: { + opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); + if (opd2 == NULL) { + if (C->print_intrinsics()) { + tty->print_cr(" ** unbox failed v2=%s", + NodeClassNames[argument(5)->Opcode()]); + } + return false; + } + // fall-through + } + case 1: { + opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); + if (opd1 == NULL) { + if (C->print_intrinsics()) { + tty->print_cr(" ** unbox failed v1=%s", + NodeClassNames[argument(4)->Opcode()]); + } + return false; + } + break; + } + default: fatal("unsupported arity: %d", n); + } + + Node* operation = NULL; + const TypeVect* vt = TypeVect::make(elem_bt, num_elem); + switch (n) { + case 1: + case 2: { + operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, vt)); + break; + } + case 3: { + operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, opd3, vt)); + break; + } + default: fatal("unsupported arity: %d", n); + } + // Wrap it up in VectorBox to keep object type information. + Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem); + set_result(vbox); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + +// , E> +// Sh ShuffleIota(Class E, Class ShuffleClass, Vector.Species s, int length, +// int start, int step, int wrap, ShuffleIotaOperation defaultImpl) +bool LibraryCallKit::inline_vector_shuffle_iota() { + const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->is_instptr(); + const TypeInt* vlen = gvn().type(argument(3))->is_int(); + Node* start = argument(4); + const TypeInt* start_val = gvn().type(start)->is_int(); + Node* step = argument(5); + const TypeInt* step_val = gvn().type(step)->is_int(); + const TypeInt* wrap = gvn().type(argument(6))->is_int(); + + if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) || + shuffle_klass->const_oop() == NULL || !wrap->is_con()) { + return false; // not enough info for intrinsification + } + if (!is_klass_initialized(shuffle_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + + int do_wrap = wrap->get_con(); + int num_elem = vlen->get_con(); + BasicType elem_bt = T_BYTE; + + if (num_elem < 4) + return false; + + if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) { + return false; + } + if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) { + return false; + } + if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) { + return false; + } + if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) { + return false; + } + if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) { + return false; + } + + const Type * type_bt = Type::get_const_basic_type(elem_bt); + const TypeVect * vt = TypeVect::make(type_bt, num_elem); + + Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt)); + + if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) { + Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt)); + res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt)); + } else if (step_val->get_con() > 1) { + Node* cnt = gvn().makecon(TypeInt::make(log2_int(step_val->get_con()))); + res = gvn().transform(VectorNode::make(Op_LShiftVB, res, cnt, vt)); + } + + if (!start_val->is_con() || start_val->get_con() != 0) { + Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt)); + res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt)); + } + + Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1)); + Node * bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt)); + if(do_wrap) { + // Wrap the indices greater than lane count. + res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt)); + } else { + ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(1)); + Node * lane_cnt = gvn().makecon(TypeInt::make(num_elem)); + Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt)); + Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vt)); + + // Make the indices greater than lane count as -ve values. This matches the java side implementation. + res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt)); + Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt)); + res = gvn().transform(new VectorBlendNode(biased_val, res, mask)); + } + + ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass); + + // Wrap it up in VectorBox to keep object type information. + res = box_vector(res, shuffle_box_type, elem_bt, num_elem); + set_result(res); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + +// , E> +// VM shuffleToVector(Class VecClass, ClassE , Class ShuffleClass, Sh s, int length, +// ShuffleToVectorOperation defaultImpl) +bool LibraryCallKit::inline_vector_shuffle_to_vector() { + const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); + const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->is_instptr(); + Node* shuffle = argument(3); + const TypeInt* vlen = gvn().type(argument(4))->is_int(); + + if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) { + return false; // not enough info for intrinsification + } + if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + + int num_elem = vlen->get_con(); + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + BasicType elem_bt = elem_type->basic_type(); + + if (num_elem < 4) { + return false; + } + + int cast_vopc = VectorCastNode::opcode(T_BYTE); // from shuffle of type T_BYTE + // Make sure that cast is implemented to particular type/size combination. + if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s", + cast_vopc, num_elem, type2name(elem_bt)); + } + return false; + } + + ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass); + + // Unbox shuffle with true flag to indicate its load shuffle to vector + Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, elem_bt, num_elem, true); + + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + // Box vector + Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem); + set_result(res); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + +// > +// V broadcastCoerced(Class vectorClass, Class elementType, int vlen, +// long bits, +// LongFunction defaultImpl) +bool LibraryCallKit::inline_vector_broadcast_coerced() { + const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); + const TypeInt* vlen = gvn().type(argument(2))->is_int(); + + if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(1)->Opcode()], + NodeClassNames[argument(2)->Opcode()]); + } + return false; // not enough info for intrinsification + } + + if (!is_klass_initialized(vector_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); + } + return false; // should be primitive type + } + BasicType elem_bt = elem_type->basic_type(); + int num_elem = vlen->get_con(); + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad. + if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, + (is_vector_mask(vbox_klass) ? VecMaskUseStore : VecMaskNotUsed), true /*has_scalar_args*/)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d", + num_elem, type2name(elem_bt), + is_vector_mask(vbox_klass) ? 1 : 0); + } + return false; // not supported + } + + Node* bits = argument(3); // long + + Node* elem = NULL; + switch (elem_bt) { + case T_BOOLEAN: // fall-through + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_CHAR: // fall-through + case T_INT: { + elem = gvn().transform(new ConvL2INode(bits)); + break; + } + case T_DOUBLE: { + elem = gvn().transform(new MoveL2DNode(bits)); + break; + } + case T_FLOAT: { + bits = gvn().transform(new ConvL2INode(bits)); + elem = gvn().transform(new MoveI2FNode(bits)); + break; + } + case T_LONG: { + elem = bits; // no conversion needed + break; + } + default: fatal("%s", type2name(elem_bt)); + } + + Node* broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt)); + broadcast = gvn().transform(broadcast); + + Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem); + set_result(box); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + +// > +// V load(Class vectorClass, Class elementType, int vlen, +// Object base, long offset, +// /* Vector.Mask m*/ +// Object container, int index, +// LoadOperation defaultImpl) { +// +// > +// void store(Class vectorClass, Class elementType, int vlen, +// Object base, long offset, +// V v, /*Vector.Mask m*/ +// Object container, int index, +// StoreVectorOperation defaultImpl) { + +bool LibraryCallKit::inline_vector_mem_operation(bool is_store) { + const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); + const TypeInt* vlen = gvn().type(argument(2))->is_int(); + + if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(1)->Opcode()], + NodeClassNames[argument(2)->Opcode()]); + } + return false; // not enough info for intrinsification + } + if (!is_klass_initialized(vector_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); + } + return false; // should be primitive type + } + BasicType elem_bt = elem_type->basic_type(); + int num_elem = vlen->get_con(); + + // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad. + if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no", + is_store, is_store ? "store" : "load", + num_elem, type2name(elem_bt)); + } + return false; // not supported + } + + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + bool is_mask = is_vector_mask(vbox_klass); + + Node* base = argument(3); + Node* offset = ConvL2X(argument(4)); + DecoratorSet decorators = C2_UNSAFE_ACCESS; + Node* addr = make_unsafe_address(base, offset, decorators, (is_mask ? T_BOOLEAN : elem_bt), true); + + // Can base be NULL? Otherwise, always on-heap access. + bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base)); + + const TypePtr *addr_type = gvn().type(addr)->isa_ptr(); + const TypeAryPtr* arr_type = addr_type->isa_aryptr(); + + // Now handle special case where load/store happens from/to byte array but element type is not byte. + bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE; + // Handle loading masks. + // If there is no consistency between array and vector element types, it must be special byte array case or loading masks + if (arr_type != NULL && !using_byte_array && elem_bt != arr_type->elem()->array_element_basic_type() && !is_mask) { + return false; + } + // Since we are using byte array, we need to double check that the byte operations are supported by backend. + if (using_byte_array) { + int byte_num_elem = num_elem * type2aelembytes(elem_bt); + if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no", + is_store, is_store ? "store" : "load", + byte_num_elem, type2name(elem_bt)); + } + return false; // not supported + } + } + if (is_mask) { + if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=%d op=%s/mask vlen=%d etype=bit ismask=no", + is_store, is_store ? "store" : "load", + num_elem); + } + return false; // not supported + } + if (!is_store) { + if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) { + return false; // not supported + } + } else { + if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) { + return false; // not supported + } + } + } + + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + if (can_access_non_heap) { + insert_mem_bar(Op_MemBarCPUOrder); + } + + if (is_store) { + Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem); + if (val == NULL) { + return false; // operand unboxing failed + } + set_all_memory(reset_memory()); + + // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector. + int store_num_elem = num_elem; + if (using_byte_array) { + store_num_elem = num_elem * type2aelembytes(elem_bt); + const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem); + val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type)); + } + + Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem)); + set_memory(vstore, addr_type); + } else { + // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load. + Node* vload = NULL; + if (using_byte_array) { + int load_num_elem = num_elem * type2aelembytes(elem_bt); + vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE)); + const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem); + vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type)); + } else { + // Special handle for masks + if (is_mask) { + vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN)); + const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem); + vload = gvn().transform(new VectorLoadMaskNode(vload, to_vect_type)); + } else { + vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt)); + } + } + Node* box = box_vector(vload, vbox_type, elem_bt, num_elem); + set_result(box); + } + + if (can_access_non_heap) { + insert_mem_bar(Op_MemBarCPUOrder); + } + + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + +// , W extends IntVector, E, S extends VectorSpecies> +// void loadWithMap(Class vectorClass, Class E, int length, Class vectorIndexClass, +// Object base, long offset, // Unsafe addressing +// W index_vector, +// C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation +// LoadVectorOperationWithMap defaultImpl) +// +// , W extends IntVector> +// void storeWithMap(Class vectorClass, Class elementType, int length, Class vectorIndexClass, +// Object base, long offset, // Unsafe addressing +// W index_vector, V v, +// C container, int index, int[] indexMap, int indexM, // Arguments for default implementation +// StoreVectorOperationWithMap defaultImpl) { +// +bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) { + const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); + const TypeInt* vlen = gvn().type(argument(2))->is_int(); + const TypeInstPtr* vector_idx_klass = gvn().type(argument(3))->is_instptr(); + + if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(1)->Opcode()], + NodeClassNames[argument(2)->Opcode()], + NodeClassNames[argument(3)->Opcode()]); + } + return false; // not enough info for intrinsification + } + + if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); + } + return false; // should be primitive type + } + BasicType elem_bt = elem_type->basic_type(); + int num_elem = vlen->get_con(); + + if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no", + is_scatter, is_scatter ? "scatter" : "gather", + num_elem, type2name(elem_bt)); + } + return false; // not supported + } + + // Check that the vector holding indices is supported by architecture + if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int ismask=no", + is_scatter, is_scatter ? "scatter" : "gather", + num_elem); + } + return false; // not supported + } + + Node* base = argument(4); + Node* offset = ConvL2X(argument(5)); + Node* addr = make_unsafe_address(base, offset, C2_UNSAFE_ACCESS, elem_bt, true); + + const TypePtr *addr_type = gvn().type(addr)->isa_ptr(); + const TypeAryPtr* arr_type = addr_type->isa_aryptr(); + + // The array must be consistent with vector type + if (arr_type == NULL || (arr_type != NULL && elem_bt != arr_type->elem()->array_element_basic_type())) { + return false; + } + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass(); + + if (vbox_idx_klass == NULL) { + return false; + } + + const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass); + + Node* index_vect = unbox_vector(argument(7), vbox_idx_type, T_INT, num_elem); + if (index_vect == NULL) { + return false; + } + const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem); + if (is_scatter) { + Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem); + if (val == NULL) { + return false; // operand unboxing failed + } + set_all_memory(reset_memory()); + + Node* vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect)); + set_memory(vstore, addr_type); + } else { + Node* vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect)); + + Node* box = box_vector(vload, vbox_type, elem_bt, num_elem); + set_result(box); + } + + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + +// > +// long reductionCoerced(int oprId, Class vectorClass, Class elementType, int vlen, +// V v, +// Function defaultImpl) + +bool LibraryCallKit::inline_vector_reduction() { + const TypeInt* opr = gvn().type(argument(0))->is_int(); + const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr(); + const TypeInt* vlen = gvn().type(argument(3))->is_int(); + + if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(1)->Opcode()], + NodeClassNames[argument(2)->Opcode()], + NodeClassNames[argument(3)->Opcode()]); + } + return false; // not enough info for intrinsification + } + if (!is_klass_initialized(vector_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); + } + return false; // should be primitive type + } + BasicType elem_bt = elem_type->basic_type(); + int num_elem = vlen->get_con(); + + int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt); + int sopc = ReductionNode::opcode(opc, elem_bt); + + // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad. + if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s ismask=no", + sopc, num_elem, type2name(elem_bt)); + } + return false; + } + + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + Node* opd = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); + if (opd == NULL) { + return false; // operand unboxing failed + } + + Node* init = ReductionNode::make_reduction_input(gvn(), opc, elem_bt); + Node* rn = gvn().transform(ReductionNode::make(opc, NULL, init, opd, elem_bt)); + + Node* bits = NULL; + switch (elem_bt) { + case T_BYTE: + case T_SHORT: + case T_INT: { + bits = gvn().transform(new ConvI2LNode(rn)); + break; + } + case T_FLOAT: { + rn = gvn().transform(new MoveF2INode(rn)); + bits = gvn().transform(new ConvI2LNode(rn)); + break; + } + case T_DOUBLE: { + bits = gvn().transform(new MoveD2LNode(rn)); + break; + } + case T_LONG: { + bits = rn; // no conversion needed + break; + } + default: fatal("%s", type2name(elem_bt)); + } + set_result(bits); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + +// public static boolean test(int cond, Class vectorClass, Class elementType, int vlen, +// V v1, V v2, +// BiFunction defaultImpl) { +// +bool LibraryCallKit::inline_vector_test() { + const TypeInt* cond = gvn().type(argument(0))->is_int(); + const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr(); + const TypeInt* vlen = gvn().type(argument(3))->is_int(); + + if (!cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(1)->Opcode()], + NodeClassNames[argument(2)->Opcode()], + NodeClassNames[argument(3)->Opcode()]); + } + return false; // not enough info for intrinsification + } + if (!is_klass_initialized(vector_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); + } + return false; // should be primitive type + } + BasicType elem_bt = elem_type->basic_type(); + int num_elem = vlen->get_con(); + BoolTest::mask booltest = (BoolTest::mask)cond->get_con(); + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d", + cond->get_con(), num_elem, type2name(elem_bt), + is_vector_mask(vbox_klass)); + } + return false; + } + + Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); + Node* opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); + if (opd1 == NULL || opd2 == NULL) { + return false; // operand unboxing failed + } + Node* test = new VectorTestNode(opd1, opd2, booltest); + test = gvn().transform(test); + + set_result(test); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + +// public static +// +// V blend(Class vectorClass, Class maskClass, Class elementType, int vlen, +// V v1, V v2, M m, +// VectorBlendOp defaultImpl) { ... +// +bool LibraryCallKit::inline_vector_blend() { + const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); + const TypeInstPtr* mask_klass = gvn().type(argument(1))->is_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr(); + const TypeInt* vlen = gvn().type(argument(3))->is_int(); + + if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL || + elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(1)->Opcode()], + NodeClassNames[argument(2)->Opcode()], + NodeClassNames[argument(3)->Opcode()]); + } + return false; // not enough info for intrinsification + } + if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); + } + return false; // should be primitive type + } + BasicType elem_bt = elem_type->basic_type(); + BasicType mask_bt = elem_bt; + int num_elem = vlen->get_con(); + + if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload", + num_elem, type2name(elem_bt)); + } + return false; // not supported + } + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass); + + Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); + Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); + Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem); + + if (v1 == NULL || v2 == NULL || mask == NULL) { + return false; // operand unboxing failed + } + + Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask)); + + Node* box = box_vector(blend, vbox_type, elem_bt, num_elem); + set_result(box); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + +// public static , +// M extends Vector.Mask, +// S extends Vector.Shape, E> +// M compare(int cond, Class vectorClass, Class maskClass, Class elementType, int vlen, +// V v1, V v2, +// VectorCompareOp defaultImpl) { ... +// +bool LibraryCallKit::inline_vector_compare() { + const TypeInt* cond = gvn().type(argument(0))->is_int(); + const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr(); + const TypeInstPtr* mask_klass = gvn().type(argument(2))->is_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(3))->is_instptr(); + const TypeInt* vlen = gvn().type(argument(4))->is_int(); + + if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL || + elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(1)->Opcode()], + NodeClassNames[argument(2)->Opcode()], + NodeClassNames[argument(3)->Opcode()], + NodeClassNames[argument(4)->Opcode()]); + } + return false; // not enough info for intrinsification + } + if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); + } + return false; // should be primitive type + } + + int num_elem = vlen->get_con(); + BasicType elem_bt = elem_type->basic_type(); + BasicType mask_bt = elem_bt; + + if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore", + cond->get_con(), num_elem, type2name(elem_bt)); + } + return false; + } + + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass); + + Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); + Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem); + + if (v1 == NULL || v2 == NULL) { + return false; // operand unboxing failed + } + BoolTest::mask pred = (BoolTest::mask)cond->get_con(); + ConINode* pred_node = (ConINode*)gvn().makecon(cond); + + const TypeVect* vt = TypeVect::make(mask_bt, num_elem); + Node* operation = gvn().transform(new VectorMaskCmpNode(pred, v1, v2, pred_node, vt)); + + Node* box = box_vector(operation, mbox_type, mask_bt, num_elem); + set_result(box); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + +// public static +// +// V rearrangeOp(Class vectorClass, Class shuffleClass, Class< ? > elementType, int vlen, +// V v1, Sh sh, +// VectorSwizzleOp defaultImpl) { ... + +bool LibraryCallKit::inline_vector_rearrange() { + const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); + const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->is_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr(); + const TypeInt* vlen = gvn().type(argument(3))->is_int(); + + if (shuffle_klass->const_oop() == NULL || vector_klass->const_oop() == NULL || + elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(1)->Opcode()], + NodeClassNames[argument(2)->Opcode()], + NodeClassNames[argument(3)->Opcode()]); + } + return false; // not enough info for intrinsification + } + if (!is_klass_initialized(vector_klass) || !is_klass_initialized(shuffle_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); + } + return false; // should be primitive type + } + BasicType elem_bt = elem_type->basic_type(); + BasicType shuffle_bt = elem_bt; + int num_elem = vlen->get_con(); + + if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no", + num_elem, type2name(elem_bt)); + } + return false; // not supported + } + if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no", + num_elem, type2name(elem_bt)); + } + return false; // not supported + } + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass); + + Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); + Node* shuffle = unbox_vector(argument(5), shbox_type, shuffle_bt, num_elem); + + if (v1 == NULL || shuffle == NULL) { + return false; // operand unboxing failed + } + + Node* rearrange = gvn().transform(new VectorRearrangeNode(v1, shuffle)); + + Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem); + set_result(box); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + +Node* LibraryCallKit::shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) { + assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported"); + juint mask = (type2aelembytes(bt) * BitsPerByte - 1); + Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask))); + Node* mcnt = gvn().transform(new AndINode(cnt, nmask)); + return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt)); +} + +// public static +// > +// V broadcastInt(int opr, Class vectorClass, Class elementType, int vlen, +// V v, int i, +// VectorBroadcastIntOp defaultImpl) { +// +bool LibraryCallKit::inline_vector_broadcast_int() { + const TypeInt* opr = gvn().type(argument(0))->is_int(); + const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr(); + const TypeInt* vlen = gvn().type(argument(3))->is_int(); + + if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(1)->Opcode()], + NodeClassNames[argument(2)->Opcode()], + NodeClassNames[argument(3)->Opcode()]); + } + return false; // not enough info for intrinsification + } + if (!is_klass_initialized(vector_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); + } + return false; // should be primitive type + } + BasicType elem_bt = elem_type->basic_type(); + int num_elem = vlen->get_con(); + int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt); + int sopc = VectorNode::opcode(opc, elem_bt); + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, true /*has_scalar_args*/)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s ismask=no", + sopc, num_elem, type2name(elem_bt)); + } + return false; // not supported + } + Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); + Node* opd2 = shift_count(argument(5), opc, elem_bt, num_elem); + if (opd1 == NULL || opd2 == NULL) { + return false; + } + Node* operation = gvn().transform(VectorNode::make(opc, opd1, opd2, num_elem, elem_bt)); + + Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem); + set_result(vbox); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + +// public static +// VOUT convert(int oprId, +// Class fromVectorClass, Class fromElementType, int fromVLen, +// Class toVectorClass, Class toElementType, int toVLen, +// VIN v, S s, +// VectorConvertOp defaultImpl) { +// +bool LibraryCallKit::inline_vector_convert() { + const TypeInt* opr = gvn().type(argument(0))->is_int(); + + const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->is_instptr(); + const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->is_instptr(); + const TypeInt* vlen_from = gvn().type(argument(3))->is_int(); + + const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->is_instptr(); + const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->is_instptr(); + const TypeInt* vlen_to = gvn().type(argument(6))->is_int(); + + if (!opr->is_con() || + vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() || + vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(1)->Opcode()], + NodeClassNames[argument(2)->Opcode()], + NodeClassNames[argument(3)->Opcode()], + NodeClassNames[argument(4)->Opcode()], + NodeClassNames[argument(5)->Opcode()], + NodeClassNames[argument(6)->Opcode()]); + } + return false; // not enough info for intrinsification + } + if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + + assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST || + opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode"); + bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST); + + ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass(); + ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass(); + if (is_vector_shuffle(vbox_klass_from) || is_vector_shuffle(vbox_klass_to)) { + return false; // vector shuffles aren't supported + } + bool is_mask = is_vector_mask(vbox_klass_from); + + ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type_from->is_primitive_type()) { + return false; // should be primitive type + } + BasicType elem_bt_from = elem_type_from->basic_type(); + ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type_to->is_primitive_type()) { + return false; // should be primitive type + } + BasicType elem_bt_to = elem_type_to->basic_type(); + if (is_mask && elem_bt_from != elem_bt_to) { + return false; // type mismatch + } + int num_elem_from = vlen_from->get_con(); + int num_elem_to = vlen_to->get_con(); + + // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed + // since we may need to change size. + if (!arch_supports_vector(Op_VectorReinterpret, + num_elem_from, + elem_bt_from, + is_mask ? VecMaskUseAll : VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d", + is_cast ? "cast" : "reinterpret", + num_elem_from, type2name(elem_bt_from), is_mask); + } + return false; + } + + // Check whether we can support resizing/reinterpreting to the new size. + if (!arch_supports_vector(Op_VectorReinterpret, + num_elem_to, + elem_bt_to, + is_mask ? VecMaskUseAll : VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d", + is_cast ? "cast" : "reinterpret", + num_elem_to, type2name(elem_bt_to), is_mask); + } + return false; + } + + // At this point, we know that both input and output vector registers are supported + // by the architecture. Next check if the casted type is simply to same type - which means + // that it is actually a resize and not a cast. + if (is_cast && elem_bt_from == elem_bt_to) { + is_cast = false; + } + + const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from); + + Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from); + if (opd1 == NULL) { + return false; + } + + const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from); + const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to); + + Node* op = opd1; + if (is_cast) { + assert(!is_mask, "masks cannot be casted"); + int cast_vopc = VectorCastNode::opcode(elem_bt_from); + // Make sure that cast is implemented to particular type/size combination. + if (!arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d", + cast_vopc, + num_elem_to, type2name(elem_bt_to), is_mask); + } + return false; + } + + if (num_elem_from < num_elem_to) { + // Since input and output number of elements are not consistent, we need to make sure we + // properly size. Thus, first make a cast that retains the number of elements from source. + // In case the size exceeds the arch size, we do the minimum. + int num_elem_for_cast = MIN2(num_elem_from, Matcher::max_vector_size(elem_bt_to)); + + // It is possible that arch does not support this intermediate vector size + // TODO More complex logic required here to handle this corner case for the sizes. + if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d", + cast_vopc, + num_elem_for_cast, type2name(elem_bt_to), is_mask); + } + return false; + } + + op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast)); + // Now ensure that the destination gets properly resized to needed size. + op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type)); + } else if (num_elem_from > num_elem_to) { + // Since number elements from input is larger than output, simply reduce size of input (we are supposed to + // drop top elements anyway). + int num_elem_for_resize = MAX2(num_elem_to, Matcher::min_vector_size(elem_bt_to)); + + // It is possible that arch does not support this intermediate vector size + // TODO More complex logic required here to handle this corner case for the sizes. + if (!arch_supports_vector(Op_VectorReinterpret, + num_elem_for_resize, + elem_bt_from, + VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d", + num_elem_for_resize, type2name(elem_bt_from), is_mask); + } + return false; + } + + op = gvn().transform(new VectorReinterpretNode(op, + src_type, + TypeVect::make(elem_bt_from, + num_elem_for_resize))); + op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to)); + } else { + // Since input and output number of elements match, and since we know this vector size is + // supported, simply do a cast with no resize needed. + op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to)); + } + } else if (Type::cmp(src_type, dst_type) != 0) { + assert(!is_cast, "must be reinterpret"); + op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type)); + } + + const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to); + Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to); + set_result(vbox); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to)))); + return true; +} + +// public static +// > +// V insert(Class vectorClass, Class elementType, int vlen, +// V vec, int ix, long val, +// VecInsertOp defaultImpl) { +// +bool LibraryCallKit::inline_vector_insert() { + const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); + const TypeInt* vlen = gvn().type(argument(2))->is_int(); + const TypeInt* idx = gvn().type(argument(4))->is_int(); + + if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(1)->Opcode()], + NodeClassNames[argument(2)->Opcode()], + NodeClassNames[argument(4)->Opcode()]); + } + return false; // not enough info for intrinsification + } + if (!is_klass_initialized(vector_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); + } + return false; // should be primitive type + } + BasicType elem_bt = elem_type->basic_type(); + int num_elem = vlen->get_con(); + if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no", + num_elem, type2name(elem_bt)); + } + return false; // not supported + } + + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem); + if (opd == NULL) { + return false; + } + + Node* insert_val = argument(5); + assert(gvn().type(insert_val)->isa_long() != NULL, "expected to be long"); + + // Convert insert value back to its appropriate type. + switch (elem_bt) { + case T_BYTE: + insert_val = gvn().transform(new ConvL2INode(insert_val)); + insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE)); + break; + case T_SHORT: + insert_val = gvn().transform(new ConvL2INode(insert_val)); + insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT)); + break; + case T_INT: + insert_val = gvn().transform(new ConvL2INode(insert_val)); + break; + case T_FLOAT: + insert_val = gvn().transform(new ConvL2INode(insert_val)); + insert_val = gvn().transform(new MoveI2FNode(insert_val)); + break; + case T_DOUBLE: + insert_val = gvn().transform(new MoveL2DNode(insert_val)); + break; + case T_LONG: + // no conversion needed + break; + default: fatal("%s", type2name(elem_bt)); break; + } + + Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con())); + + Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem); + set_result(vbox); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + +// public static +// > +// long extract(Class vectorClass, Class elementType, int vlen, +// V vec, int ix, +// VecExtractOp defaultImpl) { +// +bool LibraryCallKit::inline_vector_extract() { + const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); + const TypeInt* vlen = gvn().type(argument(2))->is_int(); + const TypeInt* idx = gvn().type(argument(4))->is_int(); + + if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(1)->Opcode()], + NodeClassNames[argument(2)->Opcode()], + NodeClassNames[argument(4)->Opcode()]); + } + return false; // not enough info for intrinsification + } + if (!is_klass_initialized(vector_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); + } + return false; // should be primitive type + } + BasicType elem_bt = elem_type->basic_type(); + int num_elem = vlen->get_con(); + int vopc = ExtractNode::opcode(elem_bt); + if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no", + num_elem, type2name(elem_bt)); + } + return false; // not supported + } + + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem); + if (opd == NULL) { + return false; + } + + Node* operation = gvn().transform(ExtractNode::make(opd, idx->get_con(), elem_bt)); + + Node* bits = NULL; + switch (elem_bt) { + case T_BYTE: + case T_SHORT: + case T_INT: { + bits = gvn().transform(new ConvI2LNode(operation)); + break; + } + case T_FLOAT: { + bits = gvn().transform(new MoveF2INode(operation)); + bits = gvn().transform(new ConvI2LNode(bits)); + break; + } + case T_DOUBLE: { + bits = gvn().transform(new MoveD2LNode(operation)); + break; + } + case T_LONG: { + bits = operation; // no conversion needed + break; + } + default: fatal("%s", type2name(elem_bt)); + } + + set_result(bits); + return true; +} + --- /dev/null 2020-08-07 00:44:29.000000000 +0300 +++ new/src/hotspot/share/prims/vectorSupport.cpp 2020-08-07 00:44:28.000000000 +0300 @@ -0,0 +1,429 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "jni.h" +#include "jvm.h" +#include "classfile/javaClasses.inline.hpp" +#include "code/location.hpp" +#include "prims/vectorSupport.hpp" +#include "runtime/fieldDescriptor.inline.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/interfaceSupport.inline.hpp" +#include "runtime/jniHandles.inline.hpp" +#include "runtime/stackValue.hpp" + +#ifdef COMPILER2 +#include "opto/matcher.hpp" // Matcher::max_vector_size(BasicType) +#endif // COMPILER2 + +bool VectorSupport::is_vector(Klass* klass) { + return klass->is_subclass_of(SystemDictionary::vector_VectorPayload_klass()); +} + +bool VectorSupport::is_vector_mask(Klass* klass) { + return klass->is_subclass_of(SystemDictionary::vector_VectorMask_klass()); +} + +bool VectorSupport::is_vector_shuffle(Klass* klass) { + return klass->is_subclass_of(SystemDictionary::vector_VectorShuffle_klass()); +} + +BasicType VectorSupport::klass2bt(InstanceKlass* ik) { + assert(ik->is_subclass_of(SystemDictionary::vector_VectorPayload_klass()), "%s not a VectorPayload", ik->name()->as_C_string()); + fieldDescriptor fd; // find_field initializes fd if found + // static final Class ETYPE; + Klass* holder = ik->find_field(vmSymbols::ETYPE_name(), vmSymbols::class_signature(), &fd); + + assert(holder != NULL, "sanity"); + assert(fd.is_static(), ""); + assert(fd.offset() > 0, ""); + + if (is_vector_shuffle(ik)) { + return T_BYTE; + } else { // vector and mask + oop value = ik->java_mirror()->obj_field(fd.offset()); + BasicType elem_bt = java_lang_Class::as_BasicType(value); + return elem_bt; + } +} + +jint VectorSupport::klass2length(InstanceKlass* ik) { + fieldDescriptor fd; // find_field initializes fd if found + // static final int VLENGTH; + Klass* holder = ik->find_field(vmSymbols::VLENGTH_name(), vmSymbols::int_signature(), &fd); + + assert(holder != NULL, "sanity"); + assert(fd.is_static(), ""); + assert(fd.offset() > 0, ""); + + jint vlen = ik->java_mirror()->int_field(fd.offset()); + assert(vlen > 0, ""); + return vlen; +} + +void VectorSupport::init_vector_array(typeArrayOop arr, BasicType elem_bt, int num_elem, address value_addr) { + int elem_size = type2aelembytes(elem_bt); + for (int i = 0; i < num_elem; i++) { + switch (elem_bt) { + case T_BYTE: { + jbyte elem_value = *(jbyte*) (value_addr + i * elem_size); + arr->byte_at_put(i, elem_value); + break; + } + case T_SHORT: { + jshort elem_value = *(jshort*) (value_addr + i * elem_size); + arr->short_at_put(i, elem_value); + break; + } + case T_INT: { + jint elem_value = *(jint*) (value_addr + i * elem_size); + arr->int_at_put(i, elem_value); + break; + } + case T_LONG: { + jlong elem_value = *(jlong*) (value_addr + i * elem_size); + arr->long_at_put(i, elem_value); + break; + } + case T_FLOAT: { + jfloat elem_value = *(jfloat*) (value_addr + i * elem_size); + arr->float_at_put(i, elem_value); + break; + } + case T_DOUBLE: { + jdouble elem_value = *(jdouble*) (value_addr + i * elem_size); + arr->double_at_put(i, elem_value); + break; + } + default: + fatal("unsupported: %s", type2name(elem_bt)); + } + } +} + +void VectorSupport::init_mask_array(typeArrayOop arr, BasicType elem_bt, int num_elem, address value_addr) { + int elem_size = type2aelembytes(elem_bt); + + for (int i = 0; i < num_elem; i++) { + switch (elem_bt) { + case T_BYTE: { + jbyte elem_value = *(jbyte*) (value_addr + i * elem_size); + arr->bool_at_put(i, elem_value != 0); + break; + } + case T_SHORT: { + jshort elem_value = *(jshort*) (value_addr + i * elem_size); + arr->bool_at_put(i, elem_value != 0); + break; + } + case T_INT: // fall-through + case T_FLOAT: { + jint elem_value = *(jint*) (value_addr + i * elem_size); + arr->bool_at_put(i, elem_value != 0); + break; + } + case T_LONG: // fall-through + case T_DOUBLE: { + jlong elem_value = *(jlong*) (value_addr + i * elem_size); + arr->bool_at_put(i, elem_value != 0); + break; + } + default: + fatal("unsupported: %s", type2name(elem_bt)); + } + } +} + +oop VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, BasicType elem_bt, int num_elem, address value_addr, TRAPS) { + + bool is_mask = is_vector_mask(ik); + + // On-heap vector values are represented as primitive arrays. + TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(is_mask ? T_BOOLEAN : elem_bt)); + + typeArrayOop arr = tak->allocate(num_elem, CHECK_NULL); // safepoint + + if (is_mask) { + init_mask_array(arr, elem_bt, num_elem, value_addr); + } else { + init_vector_array(arr, elem_bt, num_elem, value_addr); + } + return arr; +} + +oop VectorSupport::allocate_vector(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ObjectValue* ov, TRAPS) { + assert(is_vector(ik), "%s not a vector", ik->name()->as_C_string()); + assert(ov->field_size() == 1, "%s not a vector", ik->name()->as_C_string()); + + // Vector value in an aligned adjacent tuple (1, 2, 4, 8, or 16 slots). + LocationValue* loc_value = ov->field_at(0)->as_LocationValue(); + + BasicType elem_bt = klass2bt(ik); + int num_elem = klass2length(ik); + + Handle vbox = ik->allocate_instance_handle(CHECK_NULL); + + Location loc = loc_value->location(); + + oop payload = NULL; + if (loc.type() == Location::vector) { + address value_addr = loc.is_register() + // Value was in a callee-save register + ? reg_map->location(VMRegImpl::as_VMReg(loc.register_number())) + // Else value was directly saved on the stack. The frame's original stack pointer, + // before any extension by its callee (due to Compiler1 linkage on SPARC), must be used. + : ((address)fr->unextended_sp()) + loc.stack_offset(); + payload = allocate_vector_payload_helper(ik, elem_bt, num_elem, value_addr, CHECK_NULL); // safepoint + } else { + // assert(false, "interesting"); + StackValue* value = StackValue::create_stack_value(fr, reg_map, loc_value); + payload = value->get_obj()(); + } + vector_VectorPayload::set_payload(vbox(), payload); + return vbox(); +} + +#ifdef COMPILER2 +int VectorSupport::vop2ideal(jint id, BasicType bt) { + VectorOperation vop = (VectorOperation)id; + switch (vop) { + case VECTOR_OP_ADD: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: return Op_AddI; + case T_LONG: return Op_AddL; + case T_FLOAT: return Op_AddF; + case T_DOUBLE: return Op_AddD; + default: fatal("ADD: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_SUB: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: return Op_SubI; + case T_LONG: return Op_SubL; + case T_FLOAT: return Op_SubF; + case T_DOUBLE: return Op_SubD; + default: fatal("SUB: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_MUL: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: return Op_MulI; + case T_LONG: return Op_MulL; + case T_FLOAT: return Op_MulF; + case T_DOUBLE: return Op_MulD; + default: fatal("MUL: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_DIV: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: return Op_DivI; + case T_LONG: return Op_DivL; + case T_FLOAT: return Op_DivF; + case T_DOUBLE: return Op_DivD; + default: fatal("DIV: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_MIN: { + switch (bt) { + case T_BYTE: + case T_SHORT: + case T_INT: return Op_MinI; + case T_LONG: return Op_MinL; + case T_FLOAT: return Op_MinF; + case T_DOUBLE: return Op_MinD; + default: fatal("MIN: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_MAX: { + switch (bt) { + case T_BYTE: + case T_SHORT: + case T_INT: return Op_MaxI; + case T_LONG: return Op_MaxL; + case T_FLOAT: return Op_MaxF; + case T_DOUBLE: return Op_MaxD; + default: fatal("MAX: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_ABS: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: return Op_AbsI; + case T_LONG: return Op_AbsL; + case T_FLOAT: return Op_AbsF; + case T_DOUBLE: return Op_AbsD; + default: fatal("ABS: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_NEG: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: return Op_NegI; + case T_FLOAT: return Op_NegF; + case T_DOUBLE: return Op_NegD; + default: fatal("NEG: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_AND: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: return Op_AndI; + case T_LONG: return Op_AndL; + default: fatal("AND: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_OR: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: return Op_OrI; + case T_LONG: return Op_OrL; + default: fatal("OR: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_XOR: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: return Op_XorI; + case T_LONG: return Op_XorL; + default: fatal("XOR: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_SQRT: { + switch (bt) { + case T_FLOAT: return Op_SqrtF; + case T_DOUBLE: return Op_SqrtD; + default: fatal("SQRT: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_FMA: { + switch (bt) { + case T_FLOAT: return Op_FmaF; + case T_DOUBLE: return Op_FmaD; + default: fatal("FMA: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_LSHIFT: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: return Op_LShiftI; + case T_LONG: return Op_LShiftL; + default: fatal("LSHIFT: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_RSHIFT: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: return Op_RShiftI; + case T_LONG: return Op_RShiftL; + default: fatal("RSHIFT: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_URSHIFT: { + switch (bt) { + case T_BYTE: return Op_URShiftB; + case T_SHORT: return Op_URShiftS; + case T_INT: return Op_URShiftI; + case T_LONG: return Op_URShiftL; + default: fatal("URSHIFT: %s", type2name(bt)); + } + break; + } + default: fatal("unknown op: %d", vop); + } + return 0; // Unimplemented +} +#endif // COMPILER2 + +/** + * Implementation of the jdk.internal.vm.vector.VectorSupport class + */ + +JVM_ENTRY(jint, VectorSupport_GetMaxLaneCount(JNIEnv *env, jclass vsclazz, jobject clazz)) { +#ifdef COMPILER2 + oop mirror = JNIHandles::resolve_non_null(clazz); + if (java_lang_Class::is_primitive(mirror)) { + BasicType bt = java_lang_Class::primitive_type(mirror); + return Matcher::max_vector_size(bt); + } +#endif // COMPILER2 + return -1; +} JVM_END + +// JVM_RegisterVectorSupportMethods + +#define LANG "Ljava/lang/" +#define CLS LANG "Class;" + +#define CC (char*) /*cast a literal from (const char*)*/ +#define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f) + +static JNINativeMethod jdk_internal_vm_vector_VectorSupport_methods[] = { + {CC "getMaxLaneCount", CC "(" CLS ")I", FN_PTR(VectorSupport_GetMaxLaneCount)} +}; + +#undef CC +#undef FN_PTR + +#undef LANG +#undef CLS + +// This function is exported, used by NativeLookup. + +JVM_ENTRY(void, JVM_RegisterVectorSupportMethods(JNIEnv* env, jclass vsclass)) { + ThreadToNativeFromVM ttnfv(thread); + + int ok = env->RegisterNatives(vsclass, jdk_internal_vm_vector_VectorSupport_methods, sizeof(jdk_internal_vm_vector_VectorSupport_methods)/sizeof(JNINativeMethod)); + guarantee(ok == 0, "register jdk.internal.vm.vector.VectorSupport natives"); +} JVM_END --- /dev/null 2020-08-07 00:44:29.000000000 +0300 +++ new/src/hotspot/share/prims/vectorSupport.hpp 2020-08-07 00:44:29.000000000 +0300 @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_PRIMS_VECTORSUPPORT_HPP +#define SHARE_PRIMS_VECTORSUPPORT_HPP + +#include "jni.h" +#include "code/debugInfo.hpp" +#include "memory/allocation.hpp" +#include "oops/typeArrayOop.inline.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" +#include "utilities/exceptions.hpp" + +extern "C" { + void JNICALL JVM_RegisterVectorSupportMethods(JNIEnv* env, jclass vsclass); +} + +class VectorSupport : AllStatic { + private: + static void init_mask_array(typeArrayOop arr, BasicType elem_bt, int num_elem, address value_addr); + static void init_vector_array(typeArrayOop arr, BasicType elem_bt, int num_elem, address value_addr); + static oop allocate_vector_payload_helper(InstanceKlass* ik, BasicType elem_bt, int num_elem, address value_addr, TRAPS); + + static BasicType klass2bt(InstanceKlass* ik); + static jint klass2length(InstanceKlass* ik); + + public: + + // Should be aligned with constants in jdk.internal.vm.vector.VectorSupport + enum VectorOperation { + // Unary + VECTOR_OP_ABS = 0, + VECTOR_OP_NEG = 1, + VECTOR_OP_SQRT = 2, + + // Binary + VECTOR_OP_ADD = 4, + VECTOR_OP_SUB = 5, + VECTOR_OP_MUL = 6, + VECTOR_OP_DIV = 7, + VECTOR_OP_MIN = 8, + VECTOR_OP_MAX = 9, + VECTOR_OP_AND = 10, + VECTOR_OP_OR = 11, + VECTOR_OP_XOR = 12, + + // Ternary + VECTOR_OP_FMA = 13, + + // Broadcast int + VECTOR_OP_LSHIFT = 14, + VECTOR_OP_RSHIFT = 15, + VECTOR_OP_URSHIFT = 16, + + // Convert + VECTOR_OP_CAST = 17, + VECTOR_OP_REINTERPRET = 18 + }; + + static int vop2ideal(jint vop, BasicType bt); + + static oop allocate_vector(InstanceKlass* holder, frame* fr, RegisterMap* reg_map, ObjectValue* sv, TRAPS); + + static bool is_vector(Klass* klass); + static bool is_vector_mask(Klass* klass); + static bool is_vector_shuffle(Klass* klass); +}; +#endif // SHARE_PRIMS_VECTORSUPPORT_HPP --- /dev/null 2020-08-07 00:44:29.000000000 +0300 +++ new/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java 2020-08-07 00:44:29.000000000 +0300 @@ -0,0 +1,468 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package jdk.internal.vm.vector; + +import jdk.internal.HotSpotIntrinsicCandidate; +import jdk.internal.misc.Unsafe; +import jdk.internal.vm.annotation.ForceInline; + +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.util.Objects; +import java.util.function.*; + +public class VectorSupport { + static { + registerNatives(); + } + + private static final Unsafe U = Unsafe.getUnsafe(); + + // Unary + public static final int VECTOR_OP_ABS = 0; + public static final int VECTOR_OP_NEG = 1; + public static final int VECTOR_OP_SQRT = 2; + + // Binary + public static final int VECTOR_OP_ADD = 4; + public static final int VECTOR_OP_SUB = 5; + public static final int VECTOR_OP_MUL = 6; + public static final int VECTOR_OP_DIV = 7; + public static final int VECTOR_OP_MIN = 8; + public static final int VECTOR_OP_MAX = 9; + + public static final int VECTOR_OP_AND = 10; + public static final int VECTOR_OP_OR = 11; + public static final int VECTOR_OP_XOR = 12; + + // Ternary + public static final int VECTOR_OP_FMA = 13; + + // Broadcast int + public static final int VECTOR_OP_LSHIFT = 14; + public static final int VECTOR_OP_RSHIFT = 15; + public static final int VECTOR_OP_URSHIFT = 16; + + public static final int VECTOR_OP_CAST = 17; + public static final int VECTOR_OP_REINTERPRET = 18; + + // enum BoolTest + public static final int BT_eq = 0; + public static final int BT_ne = 4; + public static final int BT_le = 5; + public static final int BT_ge = 7; + public static final int BT_lt = 3; + public static final int BT_gt = 1; + public static final int BT_overflow = 2; + public static final int BT_no_overflow = 6; + + // BasicType codes, for primitives only: + public static final int + T_FLOAT = 6, + T_DOUBLE = 7, + T_BYTE = 8, + T_SHORT = 9, + T_INT = 10, + T_LONG = 11; + + /* ============================================================================ */ + + public static class VectorSpecies {} + + public static class VectorPayload { + private final Object payload; // array of primitives + + public VectorPayload(Object payload) { + this.payload = payload; + } + + protected final Object getPayload() { + return VectorSupport.maybeRebox(this).payload; + } + } + + public static class Vector extends VectorPayload { + public Vector(Object payload) { + super(payload); + } + } + + public static class VectorShuffle extends VectorPayload { + public VectorShuffle(Object payload) { + super(payload); + } + } + public static class VectorMask extends VectorPayload { + public VectorMask(Object payload) { + super(payload); + } + } + + /* ============================================================================ */ + public interface BroadcastOperation> { + VM broadcast(long l, S s); + } + + @HotSpotIntrinsicCandidate + public static + > + VM broadcastCoerced(Class vmClass, Class E, int length, + long bits, S s, + BroadcastOperation defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.broadcast(bits, s); + } + + /* ============================================================================ */ + public interface ShuffleIotaOperation> { + VectorShuffle apply(int length, int start, int step, S s); + } + + @HotSpotIntrinsicCandidate + public static + > + VectorShuffle shuffleIota(Class E, Class ShuffleClass, S s, int length, + int start, int step, int wrap, ShuffleIotaOperation defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(length, start, step, s); + } + + public interface ShuffleToVectorOperation { + VM apply(Sh s); + } + + @HotSpotIntrinsicCandidate + public static + , E> + VM shuffleToVector(Class VM, ClassE , Class ShuffleClass, Sh s, int length, + ShuffleToVectorOperation defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(s); + } + + /* ============================================================================ */ + public interface IndexOperation, E, S extends VectorSpecies> { + V index(V v, int step, S s); + } + + //FIXME @HotSpotIntrinsicCandidate + public static + , E, S extends VectorSpecies> + V indexVector(Class vClass, Class E, int length, + V v, int step, S s, + IndexOperation defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.index(v, step, s); + } + + /* ============================================================================ */ + + @HotSpotIntrinsicCandidate + public static + > + long reductionCoerced(int oprId, Class vectorClass, Class elementType, int length, + V v, + Function defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(v); + } + + /* ============================================================================ */ + + public interface VecExtractOp { + long apply(V v1, int idx); + } + + @HotSpotIntrinsicCandidate + public static + > + long extract(Class vectorClass, Class elementType, int vlen, + V vec, int ix, + VecExtractOp defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(vec, ix); + } + + /* ============================================================================ */ + + public interface VecInsertOp { + V apply(V v1, int idx, long val); + } + + @HotSpotIntrinsicCandidate + public static + > + V insert(Class vectorClass, Class elementType, int vlen, + V vec, int ix, long val, + VecInsertOp defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(vec, ix, val); + } + + /* ============================================================================ */ + + @HotSpotIntrinsicCandidate + public static + + VM unaryOp(int oprId, Class vmClass, Class elementType, int length, + VM vm, + Function defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(vm); + } + + /* ============================================================================ */ + + @HotSpotIntrinsicCandidate + public static + + VM binaryOp(int oprId, Class vmClass, Class elementType, int length, + VM vm1, VM vm2, + BiFunction defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(vm1, vm2); + } + + /* ============================================================================ */ + + public interface TernaryOperation { + V apply(V v1, V v2, V v3); + } + + @HotSpotIntrinsicCandidate + public static + + VM ternaryOp(int oprId, Class vmClass, Class elementType, int length, + VM vm1, VM vm2, VM vm3, + TernaryOperation defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(vm1, vm2, vm3); + } + + /* ============================================================================ */ + + // Memory operations + + public interface LoadOperation> { + V load(C container, int index, S s); + } + + @HotSpotIntrinsicCandidate + public static + > + VM load(Class vmClass, Class E, int length, + Object base, long offset, // Unsafe addressing + C container, int index, S s, // Arguments for default implementation + LoadOperation defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.load(container, index, s); + } + + /* ============================================================================ */ + + public interface LoadVectorOperationWithMap, E, S extends VectorSpecies> { + V loadWithMap(C container, int index, int[] indexMap, int indexM, S s); + } + + @HotSpotIntrinsicCandidate + public static + , W extends Vector, E, S extends VectorSpecies> + V loadWithMap(Class vectorClass, Class E, int length, Class vectorIndexClass, + Object base, long offset, // Unsafe addressing + W index_vector, + C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation + LoadVectorOperationWithMap defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.loadWithMap(container, index, indexMap, indexM, s); + } + + /* ============================================================================ */ + + public interface StoreVectorOperation> { + void store(C container, int index, V v); + } + + @HotSpotIntrinsicCandidate + public static + > + void store(Class vectorClass, Class elementType, int length, + Object base, long offset, // Unsafe addressing + V v, + C container, int index, // Arguments for default implementation + StoreVectorOperation defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + defaultImpl.store(container, index, v); + } + + /* ============================================================================ */ + + public interface StoreVectorOperationWithMap> { + void storeWithMap(C container, int index, V v, int[] indexMap, int indexM); + } + + @HotSpotIntrinsicCandidate + public static + , W extends Vector> + void storeWithMap(Class vectorClass, Class elementType, int length, Class vectorIndexClass, + Object base, long offset, // Unsafe addressing + W index_vector, V v, + C container, int index, int[] indexMap, int indexM, // Arguments for default implementation + StoreVectorOperationWithMap defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + defaultImpl.storeWithMap(container, index, v, indexMap, indexM); + } + + /* ============================================================================ */ + + @HotSpotIntrinsicCandidate + public static + + boolean test(int cond, Class vmClass, Class elementType, int length, + VM vm1, VM vm2, + BiFunction defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(vm1, vm2); + } + + /* ============================================================================ */ + + public interface VectorCompareOp { + M apply(int cond, V v1, V v2); + } + + @HotSpotIntrinsicCandidate + public static , + M extends VectorMask, + E> + M compare(int cond, Class vectorClass, Class maskClass, Class elementType, int length, + V v1, V v2, + VectorCompareOp defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(cond, v1, v2); + } + + /* ============================================================================ */ + + public interface VectorRearrangeOp, + Sh extends VectorShuffle, + E> { + V apply(V v1, Sh shuffle); + } + + @HotSpotIntrinsicCandidate + public static + , + Sh extends VectorShuffle, + E> + V rearrangeOp(Class vectorClass, Class shuffleClass, Class elementType, int vlen, + V v1, Sh sh, + VectorRearrangeOp defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(v1, sh); + } + + /* ============================================================================ */ + + public interface VectorBlendOp, + M extends VectorMask, + E> { + V apply(V v1, V v2, M mask); + } + + @HotSpotIntrinsicCandidate + public static + , + M extends VectorMask, + E> + V blend(Class vectorClass, Class maskClass, Class elementType, int length, + V v1, V v2, M m, + VectorBlendOp defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(v1, v2, m); + } + + /* ============================================================================ */ + + public interface VectorBroadcastIntOp> { + V apply(V v, int n); + } + + @HotSpotIntrinsicCandidate + public static + > + V broadcastInt(int opr, Class vectorClass, Class elementType, int length, + V v, int n, + VectorBroadcastIntOp defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(v, n); + } + + /* ============================================================================ */ + + public interface VectorConvertOp { + VOUT apply(VIN v, S species); + } + + // Users of this intrinsic assume that it respects + // REGISTER_ENDIAN, which is currently ByteOrder.LITTLE_ENDIAN. + // See javadoc for REGISTER_ENDIAN. + + @HotSpotIntrinsicCandidate + public static > + VOUT convert(int oprId, + Class fromVectorClass, Class fromElementType, int fromVLen, + Class toVectorClass, Class toElementType, int toVLen, + VIN v, S s, + VectorConvertOp defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(v, s); + } + + /* ============================================================================ */ + + @HotSpotIntrinsicCandidate + public static V maybeRebox(V v) { + // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses. + // TODO: move the fence generation into C2. Generate only when reboxing is taking place. + U.loadFence(); + return v; + } + + /* ============================================================================ */ + + // query the JVM's supported vector sizes and types + public static native int getMaxLaneCount(Class etype); + + /* ============================================================================ */ + + public static boolean isNonCapturingLambda(Object o) { + return o.getClass().getDeclaredFields().length == 0; + } + + /* ============================================================================ */ + + private static native int registerNatives(); +}