317 bool inline_updateDirectByteBufferCRC32C(); 318 bool inline_updateBytesAdler32(); 319 bool inline_updateByteBufferAdler32(); 320 bool inline_multiplyToLen(); 321 bool inline_hasNegatives(); 322 bool inline_squareToLen(); 323 bool inline_mulAdd(); 324 bool inline_montgomeryMultiply(); 325 bool inline_montgomerySquare(); 326 bool inline_vectorizedMismatch(); 327 bool inline_fma(vmIntrinsics::ID id); 328 bool inline_character_compare(vmIntrinsics::ID id); 329 bool inline_fp_min_max(vmIntrinsics::ID id); 330 331 bool inline_profileBoolean(); 332 bool inline_isCompileConstant(); 333 334 // Vector API support 335 bool inline_vector_nary_operation(int n); 336 bool inline_vector_broadcast_coerced(); 337 bool inline_vector_mem_operation(bool is_store); 338 bool inline_vector_gather_scatter(bool is_scatter); 339 bool inline_vector_reduction(); 340 bool inline_vector_test(); 341 bool inline_vector_blend(); 342 bool inline_vector_rearrange(); 343 bool inline_vector_compare(); 344 bool inline_vector_broadcast_int(); 345 bool inline_vector_cast_reinterpret(bool is_cast); 346 bool inline_vector_extract(); 347 bool inline_vector_insert(); 348 Node* box_vector(Node* in, const TypeInstPtr* vbox_type, BasicType bt, int num_elem); 349 Node* unbox_vector(Node* in, const TypeInstPtr* vbox_type, BasicType bt, int num_elem); 350 Node* shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem); 351 Node* gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2); 352 void set_vector_result(Node* result, bool set_res = true); 353 354 void clear_upper_avx() { 355 #ifdef X86 356 if (UseAVX >= 2) { 893 case vmIntrinsics::_isDigit: 894 case vmIntrinsics::_isLowerCase: 895 case vmIntrinsics::_isUpperCase: 896 case vmIntrinsics::_isWhitespace: 897 return inline_character_compare(intrinsic_id()); 898 899 case vmIntrinsics::_maxF: 900 case vmIntrinsics::_minF: 901 case vmIntrinsics::_maxD: 902 case vmIntrinsics::_minD: 903 return inline_fp_min_max(intrinsic_id()); 904 905 case vmIntrinsics::_VectorUnaryOp: 906 return inline_vector_nary_operation(1); 907 case vmIntrinsics::_VectorBinaryOp: 908 return inline_vector_nary_operation(2); 909 case vmIntrinsics::_VectorTernaryOp: 910 return inline_vector_nary_operation(3); 911 case vmIntrinsics::_VectorBroadcastCoerced: 912 return inline_vector_broadcast_coerced(); 913 case vmIntrinsics::_VectorLoadOp: 914 return inline_vector_mem_operation(/*is_store=*/false); 915 case vmIntrinsics::_VectorStoreOp: 916 return inline_vector_mem_operation(/*is_store=*/true); 917 case vmIntrinsics::_VectorGatherOp: 918 return inline_vector_gather_scatter(/*is_scatter*/ false); 919 case vmIntrinsics::_VectorScatterOp: 920 return inline_vector_gather_scatter(/*is_scatter*/ true); 921 case vmIntrinsics::_VectorReductionCoerced: 922 return inline_vector_reduction(); 923 case vmIntrinsics::_VectorTest: 924 return inline_vector_test(); 925 case vmIntrinsics::_VectorBlend: 926 return inline_vector_blend(); 927 case vmIntrinsics::_VectorRearrange: 928 return inline_vector_rearrange(); 929 case vmIntrinsics::_VectorCompare: 930 return inline_vector_compare(); 931 case vmIntrinsics::_VectorBroadcastInt: 932 return inline_vector_broadcast_int(); 6968 return false; 6969 } 6970 } else { 6971 switch (n) { 6972 case 1: 6973 case 2: { 6974 operation = _gvn.transform(VectorNode::make(opc, opd1, opd2, num_elem, elem_bt)); 6975 break; 6976 } 6977 case 3: { 6978 operation = _gvn.transform(VectorNode::make(opc, opd1, opd2, opd3, num_elem, elem_bt)); 6979 break; 6980 } 6981 default: fatal("unsupported arity: %d", n); 6982 } 6983 } 6984 // Wrap it up in VectorBox to keep object type information. 6985 operation = box_vector(operation, vbox_type, elem_bt, num_elem); 6986 set_vector_result(operation); 6987 6988 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 6989 return true; 6990 } 6991 6992 // <V extends Vector<?,?>> 6993 // V broadcastCoerced(Class<?> vectorClass, Class<?> elementType, int vlen, 6994 // long bits, 6995 // LongFunction<V> defaultImpl) 6996 bool LibraryCallKit::inline_vector_broadcast_coerced() { 6997 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); 6998 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); 6999 const TypeInt* vlen = gvn().type(argument(2))->is_int(); 7000 7001 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { 7002 return false; // not enough info for intrinsification 7003 } 7004 7005 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 7006 if (!elem_type->is_primitive_type()) { 7007 return false; // should be primitive type | 317 bool inline_updateDirectByteBufferCRC32C(); 318 bool inline_updateBytesAdler32(); 319 bool inline_updateByteBufferAdler32(); 320 bool inline_multiplyToLen(); 321 bool inline_hasNegatives(); 322 bool inline_squareToLen(); 323 bool inline_mulAdd(); 324 bool inline_montgomeryMultiply(); 325 bool inline_montgomerySquare(); 326 bool inline_vectorizedMismatch(); 327 bool inline_fma(vmIntrinsics::ID id); 328 bool inline_character_compare(vmIntrinsics::ID id); 329 bool inline_fp_min_max(vmIntrinsics::ID id); 330 331 bool inline_profileBoolean(); 332 bool inline_isCompileConstant(); 333 334 // Vector API support 335 bool inline_vector_nary_operation(int n); 336 bool inline_vector_broadcast_coerced(); 337 bool inline_vector_shuffle_to_vector(); 338 bool inline_vector_shuffle_iota(); 339 bool inline_vector_mem_operation(bool is_store); 340 bool inline_vector_gather_scatter(bool is_scatter); 341 bool inline_vector_reduction(); 342 bool inline_vector_test(); 343 bool inline_vector_blend(); 344 bool inline_vector_rearrange(); 345 bool inline_vector_compare(); 346 bool inline_vector_broadcast_int(); 347 bool inline_vector_cast_reinterpret(bool is_cast); 348 bool inline_vector_extract(); 349 bool inline_vector_insert(); 350 Node* box_vector(Node* in, const TypeInstPtr* vbox_type, BasicType bt, int num_elem); 351 Node* unbox_vector(Node* in, const TypeInstPtr* vbox_type, BasicType bt, int num_elem); 352 Node* shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem); 353 Node* gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2); 354 void set_vector_result(Node* result, bool set_res = true); 355 356 void clear_upper_avx() { 357 #ifdef X86 358 if (UseAVX >= 2) { 895 case vmIntrinsics::_isDigit: 896 case vmIntrinsics::_isLowerCase: 897 case vmIntrinsics::_isUpperCase: 898 case vmIntrinsics::_isWhitespace: 899 return inline_character_compare(intrinsic_id()); 900 901 case vmIntrinsics::_maxF: 902 case vmIntrinsics::_minF: 903 case vmIntrinsics::_maxD: 904 case vmIntrinsics::_minD: 905 return inline_fp_min_max(intrinsic_id()); 906 907 case vmIntrinsics::_VectorUnaryOp: 908 return inline_vector_nary_operation(1); 909 case vmIntrinsics::_VectorBinaryOp: 910 return inline_vector_nary_operation(2); 911 case vmIntrinsics::_VectorTernaryOp: 912 return inline_vector_nary_operation(3); 913 case vmIntrinsics::_VectorBroadcastCoerced: 914 return inline_vector_broadcast_coerced(); 915 case vmIntrinsics::_VectorShuffleIota: 916 return inline_vector_shuffle_iota(); 917 case vmIntrinsics::_VectorShuffleToVector: 918 return inline_vector_shuffle_to_vector(); 919 case vmIntrinsics::_VectorLoadOp: 920 return inline_vector_mem_operation(/*is_store=*/false); 921 case vmIntrinsics::_VectorStoreOp: 922 return inline_vector_mem_operation(/*is_store=*/true); 923 case vmIntrinsics::_VectorGatherOp: 924 return inline_vector_gather_scatter(/*is_scatter*/ false); 925 case vmIntrinsics::_VectorScatterOp: 926 return inline_vector_gather_scatter(/*is_scatter*/ true); 927 case vmIntrinsics::_VectorReductionCoerced: 928 return inline_vector_reduction(); 929 case vmIntrinsics::_VectorTest: 930 return inline_vector_test(); 931 case vmIntrinsics::_VectorBlend: 932 return inline_vector_blend(); 933 case vmIntrinsics::_VectorRearrange: 934 return inline_vector_rearrange(); 935 case vmIntrinsics::_VectorCompare: 936 return inline_vector_compare(); 937 case vmIntrinsics::_VectorBroadcastInt: 938 return inline_vector_broadcast_int(); 6974 return false; 6975 } 6976 } else { 6977 switch (n) { 6978 case 1: 6979 case 2: { 6980 operation = _gvn.transform(VectorNode::make(opc, opd1, opd2, num_elem, elem_bt)); 6981 break; 6982 } 6983 case 3: { 6984 operation = _gvn.transform(VectorNode::make(opc, opd1, opd2, opd3, num_elem, elem_bt)); 6985 break; 6986 } 6987 default: fatal("unsupported arity: %d", n); 6988 } 6989 } 6990 // Wrap it up in VectorBox to keep object type information. 6991 operation = box_vector(operation, vbox_type, elem_bt, num_elem); 6992 set_vector_result(operation); 6993 6994 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 6995 return true; 6996 } 6997 6998 // <Sh extends VectorShuffle<E>, E> 6999 // Sh ShuffleIota(Class<?> E, Class<?> ShuffleClass, Vector.Species<E> s, int length, 7000 // int step, ShuffleIotaOperation<Sh, E> defaultImpl) 7001 bool LibraryCallKit::inline_vector_shuffle_iota() { 7002 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->is_instptr(); 7003 const TypeInt* vlen = gvn().type(argument(3))->is_int(); 7004 Node* step = argument(4); 7005 7006 if (!vlen->is_con() || shuffle_klass->const_oop() == NULL) { 7007 return false; // not enough info for intrinsification 7008 } 7009 7010 int num_elem = vlen->get_con(); 7011 BasicType elem_bt = T_BYTE; 7012 7013 if (num_elem < 4) 7014 return false; 7015 7016 if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) { 7017 return false; 7018 } 7019 if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) { 7020 return false; 7021 } 7022 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) { 7023 return false; 7024 } 7025 7026 const TypeVect * vt = TypeVect::make(Type::get_const_basic_type(elem_bt), num_elem); 7027 7028 Node* iota = _gvn.transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt)); 7029 Node* bcast_step = _gvn.transform(VectorNode::scalar2vector(step, num_elem, Type::get_const_basic_type(elem_bt))); 7030 7031 Node* bcast_mod = _gvn.transform(VectorNode::scalar2vector(gvn().makecon(TypeInt::make(num_elem-1)), 7032 num_elem, Type::get_const_basic_type(elem_bt))); 7033 Node* add = _gvn.transform(VectorNode::make(Op_AddI, iota, bcast_step, num_elem, elem_bt)); 7034 Node* res = _gvn.transform(VectorNode::make(Op_AndI, add, bcast_mod, num_elem, elem_bt)); 7035 7036 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass(); 7037 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass); 7038 7039 // Wrap it up in VectorBox to keep object type information. 7040 res = box_vector(res, shuffle_box_type, elem_bt, num_elem); 7041 7042 set_vector_result(res); 7043 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 7044 return true; 7045 } 7046 7047 // <VM ,Sh extends VectorShuffle<E>, E> 7048 // VM shuffleToVector(Class<VM> VecClass, Class<?>E , Class<?> ShuffleClass, Sh s, int length, 7049 // ShuffleToVectorOperation<VM,Sh,E> defaultImpl) 7050 bool LibraryCallKit::inline_vector_shuffle_to_vector() { 7051 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); 7052 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); 7053 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->is_instptr(); 7054 Node* shuffle = argument(3); 7055 const TypeInt* vlen = gvn().type(argument(4))->is_int(); 7056 7057 if (!vlen->is_con() || shuffle_klass->const_oop() == NULL) { 7058 return false; // not enough info for intrinsification 7059 } 7060 7061 int num_elem = vlen->get_con(); 7062 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 7063 BasicType elem_bt = elem_type->basic_type(); 7064 7065 if (num_elem < 4) { 7066 return false; 7067 } 7068 7069 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) { 7070 return false; // not supported 7071 } 7072 7073 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass(); 7074 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass); 7075 7076 // Unbox shuffle 7077 Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, elem_bt, num_elem); 7078 7079 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 7080 const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 7081 7082 // Box vector 7083 Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem); 7084 set_vector_result(res); 7085 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 7086 return true; 7087 } 7088 7089 // <V extends Vector<?,?>> 7090 // V broadcastCoerced(Class<?> vectorClass, Class<?> elementType, int vlen, 7091 // long bits, 7092 // LongFunction<V> defaultImpl) 7093 bool LibraryCallKit::inline_vector_broadcast_coerced() { 7094 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); 7095 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); 7096 const TypeInt* vlen = gvn().type(argument(2))->is_int(); 7097 7098 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { 7099 return false; // not enough info for intrinsification 7100 } 7101 7102 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 7103 if (!elem_type->is_primitive_type()) { 7104 return false; // should be primitive type |