12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "opto/library_call.hpp"
28 #include "opto/runtime.hpp"
29 #include "opto/vectornode.hpp"
30 #include "prims/vectorSupport.hpp"
31
32 bool LibraryCallKit::arch_supports_vector(int op, int num_elem, BasicType type, VectorMaskUseType mask_use_type) {
33 // Check that the operation is valid.
34 if (op <= 0) {
35 #ifndef PRODUCT
36 if (C->print_intrinsics()) {
37 tty->print_cr(" ** Rejected intrinsification because no valid vector op could be extracted");
38 }
39 #endif
40 return false;
41 }
42
43 // Check that architecture supports this op-size-type combination.
44 if (!Matcher::match_rule_supported_vector(op, num_elem, type)) {
45 #ifndef PRODUCT
46 if (C->print_intrinsics()) {
47 tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
48 NodeClassNames[op], type2name(type), num_elem);
49 }
50 #endif
51 return false;
52 } else {
53 assert(Matcher::match_rule_supported(op), "must be supported");
54 }
55
56 // Check whether mask unboxing is supported.
57 if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseLoad) {
58 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) {
59 #ifndef PRODUCT
60 if (C->print_intrinsics()) {
61 tty->print_cr(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
62 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
63 }
64 #endif
65 return false;
66 }
67 }
68
69 // Check whether mask boxing is supported.
70 if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseStore) {
71 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) {
72 #ifndef PRODUCT
73 if (C->print_intrinsics()) {
74 tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
75 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
76 }
77 #endif
78 return false;
79 }
80 }
81
82 return true;
83 }
84
85 static int get_sopc(int opc, BasicType elem_bt, int arity) {
86 #ifdef X86
87 // Variable shift handling
88 if (arity == 2) {
89 switch (opc) {
90 case Op_LShiftI:
91 case Op_LShiftL:
92 return Op_VLShiftV;
93 case Op_RShiftI:
94 case Op_RShiftL:
95 return Op_VRShiftV;
96 case Op_URShiftB:
97 case Op_URShiftS:
98 case Op_URShiftI:
99 case Op_URShiftL:
100 return Op_VURShiftV;
101
102 default: break;
103 }
104 }
105 #endif
106 return VectorNode::opcode(opc, elem_bt);
107 }
108
109 static bool is_vector_mask(ciKlass* klass) {
110 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
111 }
112
113 static bool is_vector_shuffle(ciKlass* klass) {
114 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
115 }
116
117 #ifdef ASSERT
118 static bool is_vector(ciKlass* klass) {
119 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
120 }
121
122 static bool check_vbox(const TypeInstPtr* vbox_type) {
123 assert(vbox_type->klass_is_exact(), "");
124
125 ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass();
126 assert(is_vector(ik), "not a vector");
127
128 ciField* fd1 = ik->get_field_by_name(ciSymbol::ETYPE_name(), ciSymbol::class_signature(), /* is_static */ true);
129 assert(fd1 != NULL, "element type info is missing");
130
131 ciConstant val1 = fd1->constant_value();
132 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
133 assert(is_java_primitive(elem_bt), "element type info is missing");
134
135 ciField* fd2 = ik->get_field_by_name(ciSymbol::VLENGTH_name(), ciSymbol::int_signature(), /* is_static */ true);
136 assert(fd2 != NULL, "vector length info is missing");
198 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
199 const TypeInt* vlen = gvn().type(argument(3))->is_int();
200
201 if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
202 if (C->print_intrinsics()) {
203 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
204 NodeClassNames[argument(0)->Opcode()],
205 NodeClassNames[argument(1)->Opcode()],
206 NodeClassNames[argument(2)->Opcode()],
207 NodeClassNames[argument(3)->Opcode()]);
208 }
209 return false; // not enough info for intrinsification
210 }
211 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
212 if (!elem_type->is_primitive_type()) {
213 if (C->print_intrinsics()) {
214 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
215 }
216 return false; // should be primitive type
217 }
218 BasicType elem_bt = elem_type->basic_type();
219 int num_elem = vlen->get_con();
220 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
221 int sopc = get_sopc(opc, elem_bt, n);
222 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
223 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
224
225 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
226 if (!arch_supports_vector(sopc, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseAll : VecMaskNotUsed)) {
227 if (C->print_intrinsics()) {
228 tty->print_cr(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d",
229 n, sopc, num_elem, type2name(elem_bt),
230 is_vector_mask(vbox_klass) ? 1 : 0);
231 }
232 return false; // not supported
233 }
234
235 Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL;
236 switch (n) {
237 case 3: {
238 opd3 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
239 if (opd3 == NULL) {
240 if (C->print_intrinsics()) {
241 tty->print_cr(" ** unbox failed v3=%s",
290 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
291 return true;
292 }
293
294 // <Sh extends VectorShuffle<E>, E>
295 // Sh ShuffleIota(Class<?> E, Class<?> ShuffleClass, Vector.Species<E> s, int length,
296 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
297 bool LibraryCallKit::inline_vector_shuffle_iota() {
298 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->is_instptr();
299 const TypeInt* vlen = gvn().type(argument(3))->is_int();
300 Node* start = argument(4);
301 const TypeInt* start_val = gvn().type(start)->is_int();
302 Node* step = argument(5);
303 const TypeInt* step_val = gvn().type(step)->is_int();
304 const TypeInt* wrap = gvn().type(argument(6))->is_int();
305
306 if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) ||
307 shuffle_klass->const_oop() == NULL || !wrap->is_con()) {
308 return false; // not enough info for intrinsification
309 }
310
311 int do_wrap = wrap->get_con();
312 int num_elem = vlen->get_con();
313 BasicType elem_bt = T_BYTE;
314
315 if (num_elem < 4)
316 return false;
317
318 if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) {
319 return false;
320 }
321 if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) {
322 return false;
323 }
324 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
325 return false;
326 }
327 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
328 return false;
329 }
369 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
370 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
371
372 // Wrap it up in VectorBox to keep object type information.
373 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
374 set_result(res);
375 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
376 return true;
377 }
378
379 // <VM ,Sh extends VectorShuffle<E>, E>
380 // VM shuffleToVector(Class<VM> VecClass, Class<?>E , Class<?> ShuffleClass, Sh s, int length,
381 // ShuffleToVectorOperation<VM,Sh,E> defaultImpl)
382 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
383 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
384 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr();
385 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->is_instptr();
386 Node* shuffle = argument(3);
387 const TypeInt* vlen = gvn().type(argument(4))->is_int();
388
389 if (!vlen->is_con() || shuffle_klass->const_oop() == NULL) {
390 return false; // not enough info for intrinsification
391 }
392
393 int num_elem = vlen->get_con();
394 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
395 BasicType elem_bt = elem_type->basic_type();
396
397 if (num_elem < 4) {
398 return false;
399 }
400
401 int cast_vopc = VectorCastNode::opcode(T_BYTE); // from shuffle of type T_BYTE
402 // Make sure that cast is implemented to particular type/size combination.
403 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
404 if (C->print_intrinsics()) {
405 tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
406 cast_vopc, num_elem, type2name(elem_bt));
407 }
408 return false;
409 }
410
411 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
426
427 // <V extends Vector<?,?>>
428 // V broadcastCoerced(Class<?> vectorClass, Class<?> elementType, int vlen,
429 // long bits,
430 // LongFunction<V> defaultImpl)
431 bool LibraryCallKit::inline_vector_broadcast_coerced() {
432 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
433 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr();
434 const TypeInt* vlen = gvn().type(argument(2))->is_int();
435
436 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
437 if (C->print_intrinsics()) {
438 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s",
439 NodeClassNames[argument(0)->Opcode()],
440 NodeClassNames[argument(1)->Opcode()],
441 NodeClassNames[argument(2)->Opcode()]);
442 }
443 return false; // not enough info for intrinsification
444 }
445
446 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
447 if (!elem_type->is_primitive_type()) {
448 if (C->print_intrinsics()) {
449 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
450 }
451 return false; // should be primitive type
452 }
453 BasicType elem_bt = elem_type->basic_type();
454 int num_elem = vlen->get_con();
455 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
456 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
457
458 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
459 if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt,
460 is_vector_mask(vbox_klass) ? VecMaskUseStore : VecMaskNotUsed)) {
461 if (C->print_intrinsics()) {
462 tty->print_cr(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d",
463 num_elem, type2name(elem_bt),
464 is_vector_mask(vbox_klass) ? 1 : 0);
465 }
466 return false; // not supported
467 }
468
469 Node* bits = argument(3); // long
470
471 Node* elem = NULL;
472 switch (elem_bt) {
473 case T_BOOLEAN: // fall-through
474 case T_BYTE: // fall-through
475 case T_SHORT: // fall-through
476 case T_CHAR: // fall-through
477 case T_INT: {
478 elem = gvn().transform(new ConvL2INode(bits));
479 break;
480 }
514 // void store(Class<?> vectorClass, Class<?> elementType, int vlen,
515 // Object base, long offset,
516 // V v, /*Vector.Mask<E,S> m*/
517 // Object container, int index,
518 // StoreVectorOperation<C, V> defaultImpl) {
519
520 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
521 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
522 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr();
523 const TypeInt* vlen = gvn().type(argument(2))->is_int();
524
525 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
526 if (C->print_intrinsics()) {
527 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s",
528 NodeClassNames[argument(0)->Opcode()],
529 NodeClassNames[argument(1)->Opcode()],
530 NodeClassNames[argument(2)->Opcode()]);
531 }
532 return false; // not enough info for intrinsification
533 }
534
535 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
536 if (!elem_type->is_primitive_type()) {
537 if (C->print_intrinsics()) {
538 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
539 }
540 return false; // should be primitive type
541 }
542 BasicType elem_bt = elem_type->basic_type();
543 int num_elem = vlen->get_con();
544
545 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
546 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
547 if (C->print_intrinsics()) {
548 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
549 is_store, is_store ? "store" : "load",
550 num_elem, type2name(elem_bt));
551 }
552 return false; // not supported
553 }
672 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
673 // StoreVectorOperationWithMap<C, V> defaultImpl) {
674 //
675 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
676 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
677 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr();
678 const TypeInt* vlen = gvn().type(argument(2))->is_int();
679 const TypeInstPtr* vector_idx_klass = gvn().type(argument(3))->is_instptr();
680
681 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) {
682 if (C->print_intrinsics()) {
683 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
684 NodeClassNames[argument(0)->Opcode()],
685 NodeClassNames[argument(1)->Opcode()],
686 NodeClassNames[argument(2)->Opcode()],
687 NodeClassNames[argument(3)->Opcode()]);
688 }
689 return false; // not enough info for intrinsification
690 }
691
692 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
693 if (!elem_type->is_primitive_type()) {
694 if (C->print_intrinsics()) {
695 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
696 }
697 return false; // should be primitive type
698 }
699 BasicType elem_bt = elem_type->basic_type();
700 int num_elem = vlen->get_con();
701
702 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
703 if (C->print_intrinsics()) {
704 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
705 is_scatter, is_scatter ? "scatter" : "gather",
706 num_elem, type2name(elem_bt));
707 }
708 return false; // not supported
709 }
710
711 // Check that the vector holding indices is supported by architecture
769 // long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int vlen,
770 // V v,
771 // Function<V,Long> defaultImpl)
772
773 bool LibraryCallKit::inline_vector_reduction() {
774 const TypeInt* opr = gvn().type(argument(0))->is_int();
775 const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr();
776 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
777 const TypeInt* vlen = gvn().type(argument(3))->is_int();
778
779 if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
780 if (C->print_intrinsics()) {
781 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
782 NodeClassNames[argument(0)->Opcode()],
783 NodeClassNames[argument(1)->Opcode()],
784 NodeClassNames[argument(2)->Opcode()],
785 NodeClassNames[argument(3)->Opcode()]);
786 }
787 return false; // not enough info for intrinsification
788 }
789 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
790 if (!elem_type->is_primitive_type()) {
791 if (C->print_intrinsics()) {
792 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
793 }
794 return false; // should be primitive type
795 }
796 BasicType elem_bt = elem_type->basic_type();
797 int num_elem = vlen->get_con();
798
799 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
800 int sopc = ReductionNode::opcode(opc, elem_bt);
801
802 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
803 if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed)) {
804 if (C->print_intrinsics()) {
805 tty->print_cr(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s ismask=no",
806 sopc, num_elem, type2name(elem_bt));
807 }
808 return false;
850 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
851 // V v1, V v2,
852 // BiFunction<V, V, Boolean> defaultImpl) {
853 //
854 bool LibraryCallKit::inline_vector_test() {
855 const TypeInt* cond = gvn().type(argument(0))->is_int();
856 const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr();
857 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
858 const TypeInt* vlen = gvn().type(argument(3))->is_int();
859
860 if (!cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
861 if (C->print_intrinsics()) {
862 tty->print_cr(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
863 NodeClassNames[argument(0)->Opcode()],
864 NodeClassNames[argument(1)->Opcode()],
865 NodeClassNames[argument(2)->Opcode()],
866 NodeClassNames[argument(3)->Opcode()]);
867 }
868 return false; // not enough info for intrinsification
869 }
870 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
871 if (!elem_type->is_primitive_type()) {
872 if (C->print_intrinsics()) {
873 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
874 }
875 return false; // should be primitive type
876 }
877 BasicType elem_bt = elem_type->basic_type();
878 int num_elem = vlen->get_con();
879 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
880 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
881 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
882
883 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
884 if (C->print_intrinsics()) {
885 tty->print_cr(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
886 cond->get_con(), num_elem, type2name(elem_bt),
887 is_vector_mask(vbox_klass));
888 }
889 return false;
908 // V v1, V v2, M m,
909 // VectorBlendOp<V,M> defaultImpl) { ...
910 //
911 bool LibraryCallKit::inline_vector_blend() {
912 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
913 const TypeInstPtr* mask_klass = gvn().type(argument(1))->is_instptr();
914 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
915 const TypeInt* vlen = gvn().type(argument(3))->is_int();
916
917 if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
918 elem_klass->const_oop() == NULL || !vlen->is_con()) {
919 if (C->print_intrinsics()) {
920 tty->print_cr(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
921 NodeClassNames[argument(0)->Opcode()],
922 NodeClassNames[argument(1)->Opcode()],
923 NodeClassNames[argument(2)->Opcode()],
924 NodeClassNames[argument(3)->Opcode()]);
925 }
926 return false; // not enough info for intrinsification
927 }
928 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
929 if (!elem_type->is_primitive_type()) {
930 if (C->print_intrinsics()) {
931 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
932 }
933 return false; // should be primitive type
934 }
935 BasicType elem_bt = elem_type->basic_type();
936 BasicType mask_bt = elem_bt;
937 int num_elem = vlen->get_con();
938
939 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
940 if (C->print_intrinsics()) {
941 tty->print_cr(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
942 num_elem, type2name(elem_bt));
943 }
944 return false; // not supported
945 }
946 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
947 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
974 //
975 bool LibraryCallKit::inline_vector_compare() {
976 const TypeInt* cond = gvn().type(argument(0))->is_int();
977 const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr();
978 const TypeInstPtr* mask_klass = gvn().type(argument(2))->is_instptr();
979 const TypeInstPtr* elem_klass = gvn().type(argument(3))->is_instptr();
980 const TypeInt* vlen = gvn().type(argument(4))->is_int();
981
982 if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
983 elem_klass->const_oop() == NULL || !vlen->is_con()) {
984 if (C->print_intrinsics()) {
985 tty->print_cr(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
986 NodeClassNames[argument(0)->Opcode()],
987 NodeClassNames[argument(1)->Opcode()],
988 NodeClassNames[argument(2)->Opcode()],
989 NodeClassNames[argument(3)->Opcode()],
990 NodeClassNames[argument(4)->Opcode()]);
991 }
992 return false; // not enough info for intrinsification
993 }
994 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
995 if (!elem_type->is_primitive_type()) {
996 if (C->print_intrinsics()) {
997 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
998 }
999 return false; // should be primitive type
1000 }
1001
1002 int num_elem = vlen->get_con();
1003 BasicType elem_bt = elem_type->basic_type();
1004 BasicType mask_bt = elem_bt;
1005
1006 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1007 if (C->print_intrinsics()) {
1008 tty->print_cr(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1009 cond->get_con(), num_elem, type2name(elem_bt));
1010 }
1011 return false;
1012 }
1013
1032 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1033 set_result(box);
1034 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1035 return true;
1036 }
1037
1038 // public static
1039 // <V extends Vector, Sh extends Shuffle>
1040 // V rearrangeOp(Class<V> vectorClass, Class<Sh> shuffleClass, Class< ? > elementType, int vlen,
1041 // V v1, Sh sh,
1042 // VectorSwizzleOp<V, Sh, S, E> defaultImpl) { ...
1043
1044 bool LibraryCallKit::inline_vector_rearrange() {
1045 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
1046 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->is_instptr();
1047 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
1048 const TypeInt* vlen = gvn().type(argument(3))->is_int();
1049
1050 if (shuffle_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1051 elem_klass->const_oop() == NULL || !vlen->is_con()) {
1052 return false; // not enough info for intrinsification
1053 if (C->print_intrinsics()) {
1054 tty->print_cr(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1055 NodeClassNames[argument(0)->Opcode()],
1056 NodeClassNames[argument(1)->Opcode()],
1057 NodeClassNames[argument(2)->Opcode()],
1058 NodeClassNames[argument(3)->Opcode()]);
1059 }
1060 }
1061 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1062 if (!elem_type->is_primitive_type()) {
1063 if (C->print_intrinsics()) {
1064 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1065 }
1066 return false; // should be primitive type
1067 }
1068 BasicType elem_bt = elem_type->basic_type();
1069 BasicType shuffle_bt = elem_bt;
1070 int num_elem = vlen->get_con();
1071
1072 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1073 if (C->print_intrinsics()) {
1074 tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1075 num_elem, type2name(elem_bt));
1076 }
1077 return false; // not supported
1078 }
1079 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed)) {
1117 // V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int vlen,
1118 // V v, int i,
1119 // VectorBroadcastIntOp<V> defaultImpl) {
1120 //
1121 bool LibraryCallKit::inline_vector_broadcast_int() {
1122 const TypeInt* opr = gvn().type(argument(0))->is_int();
1123 const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr();
1124 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
1125 const TypeInt* vlen = gvn().type(argument(3))->is_int();
1126
1127 if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1128 if (C->print_intrinsics()) {
1129 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1130 NodeClassNames[argument(0)->Opcode()],
1131 NodeClassNames[argument(1)->Opcode()],
1132 NodeClassNames[argument(2)->Opcode()],
1133 NodeClassNames[argument(3)->Opcode()]);
1134 }
1135 return false; // not enough info for intrinsification
1136 }
1137 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1138 if (!elem_type->is_primitive_type()) {
1139 if (C->print_intrinsics()) {
1140 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1141 }
1142 return false; // should be primitive type
1143 }
1144 BasicType elem_bt = elem_type->basic_type();
1145 int num_elem = vlen->get_con();
1146 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1147 int sopc = get_sopc(opc, elem_bt, 1); // get_node_id(opr->get_con(), elem_bt);
1148 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1149 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1150
1151 if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed)) {
1152 if (C->print_intrinsics()) {
1153 tty->print_cr(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s ismask=no",
1154 sopc, num_elem, type2name(elem_bt));
1155 }
1156 return false; // not supported
1157 }
1158 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1159 Node* opd2 = shift_count(argument(5), opc, elem_bt, num_elem);
1160 if (opd1 == NULL || opd2 == NULL) {
1161 return false;
1162 }
1163 Node* operation = gvn().transform(VectorNode::make(opc, opd1, opd2, num_elem, elem_bt));
1164
1165 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
1166 set_result(vbox);
1167 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1168 return true;
1169 }
1170
1171 // public static <VOUT extends VectorPayload,
1186
1187 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->is_instptr();
1188 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->is_instptr();
1189 const TypeInt* vlen_to = gvn().type(argument(6))->is_int();
1190
1191 if (!opr->is_con() ||
1192 vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() ||
1193 vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) {
1194 if (C->print_intrinsics()) {
1195 tty->print_cr(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
1196 NodeClassNames[argument(0)->Opcode()],
1197 NodeClassNames[argument(1)->Opcode()],
1198 NodeClassNames[argument(2)->Opcode()],
1199 NodeClassNames[argument(3)->Opcode()],
1200 NodeClassNames[argument(4)->Opcode()],
1201 NodeClassNames[argument(5)->Opcode()],
1202 NodeClassNames[argument(6)->Opcode()]);
1203 }
1204 return false; // not enough info for intrinsification
1205 }
1206
1207 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
1208 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
1209 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST);
1210
1211 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
1212 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
1213 if (is_vector_shuffle(vbox_klass_from) || is_vector_shuffle(vbox_klass_to)) {
1214 return false; // vector shuffles aren't supported
1215 }
1216 bool is_mask = is_vector_mask(vbox_klass_from);
1217
1218 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
1219 if (!elem_type_from->is_primitive_type()) {
1220 return false; // should be primitive type
1221 }
1222 BasicType elem_bt_from = elem_type_from->basic_type();
1223 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
1224 if (!elem_type_to->is_primitive_type()) {
1225 return false; // should be primitive type
1354 // V insert(Class<? extends V> vectorClass, Class<?> elementType, int vlen,
1355 // V vec, int ix, long val,
1356 // VecInsertOp<V> defaultImpl) {
1357 //
1358 bool LibraryCallKit::inline_vector_insert() {
1359 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
1360 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr();
1361 const TypeInt* vlen = gvn().type(argument(2))->is_int();
1362 const TypeInt* idx = gvn().type(argument(4))->is_int();
1363
1364 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
1365 if (C->print_intrinsics()) {
1366 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
1367 NodeClassNames[argument(0)->Opcode()],
1368 NodeClassNames[argument(1)->Opcode()],
1369 NodeClassNames[argument(2)->Opcode()],
1370 NodeClassNames[argument(4)->Opcode()]);
1371 }
1372 return false; // not enough info for intrinsification
1373 }
1374 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1375 if (!elem_type->is_primitive_type()) {
1376 if (C->print_intrinsics()) {
1377 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1378 }
1379 return false; // should be primitive type
1380 }
1381 BasicType elem_bt = elem_type->basic_type();
1382 int num_elem = vlen->get_con();
1383 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
1384 if (C->print_intrinsics()) {
1385 tty->print_cr(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
1386 num_elem, type2name(elem_bt));
1387 }
1388 return false; // not supported
1389 }
1390
1391 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1392 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1393
1438 // long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
1439 // V vec, int ix,
1440 // VecExtractOp<V> defaultImpl) {
1441 //
1442 bool LibraryCallKit::inline_vector_extract() {
1443 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
1444 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr();
1445 const TypeInt* vlen = gvn().type(argument(2))->is_int();
1446 const TypeInt* idx = gvn().type(argument(4))->is_int();
1447
1448 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
1449 if (C->print_intrinsics()) {
1450 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
1451 NodeClassNames[argument(0)->Opcode()],
1452 NodeClassNames[argument(1)->Opcode()],
1453 NodeClassNames[argument(2)->Opcode()],
1454 NodeClassNames[argument(4)->Opcode()]);
1455 }
1456 return false; // not enough info for intrinsification
1457 }
1458 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1459 if (!elem_type->is_primitive_type()) {
1460 if (C->print_intrinsics()) {
1461 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1462 }
1463 return false; // should be primitive type
1464 }
1465 BasicType elem_bt = elem_type->basic_type();
1466 int num_elem = vlen->get_con();
1467 int vopc = ExtractNode::opcode(elem_bt);
1468 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
1469 if (C->print_intrinsics()) {
1470 tty->print_cr(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
1471 num_elem, type2name(elem_bt));
1472 }
1473 return false; // not supported
1474 }
1475
1476 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1477 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
|
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "opto/library_call.hpp"
28 #include "opto/runtime.hpp"
29 #include "opto/vectornode.hpp"
30 #include "prims/vectorSupport.hpp"
31
32 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
33 // Check that the operation is valid.
34 if (sopc <= 0) {
35 #ifndef PRODUCT
36 if (C->print_intrinsics()) {
37 tty->print_cr(" ** Rejected intrinsification because no valid vector op could be extracted");
38 }
39 #endif
40 return false;
41 }
42
43 // Check that architecture supports this op-size-type combination.
44 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
45 #ifndef PRODUCT
46 if (C->print_intrinsics()) {
47 tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
48 NodeClassNames[sopc], type2name(type), num_elem);
49 }
50 #endif
51 return false;
52 } else {
53 assert(Matcher::match_rule_supported(sopc), "must be supported");
54 }
55
56 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
57 Matcher::supports_vector_variable_shifts() == false) {
58 if (C->print_intrinsics()) {
59 tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
60 NodeClassNames[sopc], type2name(type), num_elem);
61 }
62 return false;
63 }
64
65 // Check whether mask unboxing is supported.
66 if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseLoad) {
67 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) {
68 #ifndef PRODUCT
69 if (C->print_intrinsics()) {
70 tty->print_cr(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
71 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
72 }
73 #endif
74 return false;
75 }
76 }
77
78 // Check whether mask boxing is supported.
79 if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseStore) {
80 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) {
81 #ifndef PRODUCT
82 if (C->print_intrinsics()) {
83 tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
84 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
85 }
86 #endif
87 return false;
88 }
89 }
90
91 return true;
92 }
93
94 static bool is_vector_mask(ciKlass* klass) {
95 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
96 }
97
98 static bool is_vector_shuffle(ciKlass* klass) {
99 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
100 }
101
102 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
103 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass(), "klass instance expected");
104 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
105 return klass->is_initialized();
106 }
107
108 #ifdef ASSERT
109 static bool is_vector(ciKlass* klass) {
110 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
111 }
112
113 static bool check_vbox(const TypeInstPtr* vbox_type) {
114 assert(vbox_type->klass_is_exact(), "");
115
116 ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass();
117 assert(is_vector(ik), "not a vector");
118
119 ciField* fd1 = ik->get_field_by_name(ciSymbol::ETYPE_name(), ciSymbol::class_signature(), /* is_static */ true);
120 assert(fd1 != NULL, "element type info is missing");
121
122 ciConstant val1 = fd1->constant_value();
123 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
124 assert(is_java_primitive(elem_bt), "element type info is missing");
125
126 ciField* fd2 = ik->get_field_by_name(ciSymbol::VLENGTH_name(), ciSymbol::int_signature(), /* is_static */ true);
127 assert(fd2 != NULL, "vector length info is missing");
189 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
190 const TypeInt* vlen = gvn().type(argument(3))->is_int();
191
192 if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
193 if (C->print_intrinsics()) {
194 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
195 NodeClassNames[argument(0)->Opcode()],
196 NodeClassNames[argument(1)->Opcode()],
197 NodeClassNames[argument(2)->Opcode()],
198 NodeClassNames[argument(3)->Opcode()]);
199 }
200 return false; // not enough info for intrinsification
201 }
202 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
203 if (!elem_type->is_primitive_type()) {
204 if (C->print_intrinsics()) {
205 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
206 }
207 return false; // should be primitive type
208 }
209 if (!is_klass_initialized(vector_klass)) {
210 if (C->print_intrinsics()) {
211 tty->print_cr(" ** klass argument not initialized");
212 }
213 return false;
214 }
215 BasicType elem_bt = elem_type->basic_type();
216 int num_elem = vlen->get_con();
217 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
218 int sopc = VectorNode::opcode(opc, elem_bt);
219 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
220 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
221
222 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
223 if (!arch_supports_vector(sopc, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseAll : VecMaskNotUsed)) {
224 if (C->print_intrinsics()) {
225 tty->print_cr(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d",
226 n, sopc, num_elem, type2name(elem_bt),
227 is_vector_mask(vbox_klass) ? 1 : 0);
228 }
229 return false; // not supported
230 }
231
232 Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL;
233 switch (n) {
234 case 3: {
235 opd3 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
236 if (opd3 == NULL) {
237 if (C->print_intrinsics()) {
238 tty->print_cr(" ** unbox failed v3=%s",
287 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
288 return true;
289 }
290
291 // <Sh extends VectorShuffle<E>, E>
292 // Sh ShuffleIota(Class<?> E, Class<?> ShuffleClass, Vector.Species<E> s, int length,
293 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
294 bool LibraryCallKit::inline_vector_shuffle_iota() {
295 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->is_instptr();
296 const TypeInt* vlen = gvn().type(argument(3))->is_int();
297 Node* start = argument(4);
298 const TypeInt* start_val = gvn().type(start)->is_int();
299 Node* step = argument(5);
300 const TypeInt* step_val = gvn().type(step)->is_int();
301 const TypeInt* wrap = gvn().type(argument(6))->is_int();
302
303 if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) ||
304 shuffle_klass->const_oop() == NULL || !wrap->is_con()) {
305 return false; // not enough info for intrinsification
306 }
307 if (!is_klass_initialized(shuffle_klass)) {
308 if (C->print_intrinsics()) {
309 tty->print_cr(" ** klass argument not initialized");
310 }
311 return false;
312 }
313
314 int do_wrap = wrap->get_con();
315 int num_elem = vlen->get_con();
316 BasicType elem_bt = T_BYTE;
317
318 if (num_elem < 4)
319 return false;
320
321 if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) {
322 return false;
323 }
324 if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) {
325 return false;
326 }
327 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
328 return false;
329 }
330 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
331 return false;
332 }
372 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
373 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
374
375 // Wrap it up in VectorBox to keep object type information.
376 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
377 set_result(res);
378 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
379 return true;
380 }
381
382 // <VM ,Sh extends VectorShuffle<E>, E>
383 // VM shuffleToVector(Class<VM> VecClass, Class<?>E , Class<?> ShuffleClass, Sh s, int length,
384 // ShuffleToVectorOperation<VM,Sh,E> defaultImpl)
385 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
386 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
387 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr();
388 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->is_instptr();
389 Node* shuffle = argument(3);
390 const TypeInt* vlen = gvn().type(argument(4))->is_int();
391
392 if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) {
393 return false; // not enough info for intrinsification
394 }
395 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
396 if (C->print_intrinsics()) {
397 tty->print_cr(" ** klass argument not initialized");
398 }
399 return false;
400 }
401
402 int num_elem = vlen->get_con();
403 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
404 BasicType elem_bt = elem_type->basic_type();
405
406 if (num_elem < 4) {
407 return false;
408 }
409
410 int cast_vopc = VectorCastNode::opcode(T_BYTE); // from shuffle of type T_BYTE
411 // Make sure that cast is implemented to particular type/size combination.
412 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
413 if (C->print_intrinsics()) {
414 tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
415 cast_vopc, num_elem, type2name(elem_bt));
416 }
417 return false;
418 }
419
420 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
435
436 // <V extends Vector<?,?>>
437 // V broadcastCoerced(Class<?> vectorClass, Class<?> elementType, int vlen,
438 // long bits,
439 // LongFunction<V> defaultImpl)
440 bool LibraryCallKit::inline_vector_broadcast_coerced() {
441 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
442 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr();
443 const TypeInt* vlen = gvn().type(argument(2))->is_int();
444
445 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
446 if (C->print_intrinsics()) {
447 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s",
448 NodeClassNames[argument(0)->Opcode()],
449 NodeClassNames[argument(1)->Opcode()],
450 NodeClassNames[argument(2)->Opcode()]);
451 }
452 return false; // not enough info for intrinsification
453 }
454
455 if (!is_klass_initialized(vector_klass)) {
456 if (C->print_intrinsics()) {
457 tty->print_cr(" ** klass argument not initialized");
458 }
459 return false;
460 }
461 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
462 if (!elem_type->is_primitive_type()) {
463 if (C->print_intrinsics()) {
464 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
465 }
466 return false; // should be primitive type
467 }
468 BasicType elem_bt = elem_type->basic_type();
469 int num_elem = vlen->get_con();
470 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
471 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
472
473 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
474 if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt,
475 (is_vector_mask(vbox_klass) ? VecMaskUseStore : VecMaskNotUsed), true /*has_scalar_args*/)) {
476 if (C->print_intrinsics()) {
477 tty->print_cr(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d",
478 num_elem, type2name(elem_bt),
479 is_vector_mask(vbox_klass) ? 1 : 0);
480 }
481 return false; // not supported
482 }
483
484 Node* bits = argument(3); // long
485
486 Node* elem = NULL;
487 switch (elem_bt) {
488 case T_BOOLEAN: // fall-through
489 case T_BYTE: // fall-through
490 case T_SHORT: // fall-through
491 case T_CHAR: // fall-through
492 case T_INT: {
493 elem = gvn().transform(new ConvL2INode(bits));
494 break;
495 }
529 // void store(Class<?> vectorClass, Class<?> elementType, int vlen,
530 // Object base, long offset,
531 // V v, /*Vector.Mask<E,S> m*/
532 // Object container, int index,
533 // StoreVectorOperation<C, V> defaultImpl) {
534
535 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
536 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
537 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr();
538 const TypeInt* vlen = gvn().type(argument(2))->is_int();
539
540 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
541 if (C->print_intrinsics()) {
542 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s",
543 NodeClassNames[argument(0)->Opcode()],
544 NodeClassNames[argument(1)->Opcode()],
545 NodeClassNames[argument(2)->Opcode()]);
546 }
547 return false; // not enough info for intrinsification
548 }
549 if (!is_klass_initialized(vector_klass)) {
550 if (C->print_intrinsics()) {
551 tty->print_cr(" ** klass argument not initialized");
552 }
553 return false;
554 }
555
556 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
557 if (!elem_type->is_primitive_type()) {
558 if (C->print_intrinsics()) {
559 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
560 }
561 return false; // should be primitive type
562 }
563 BasicType elem_bt = elem_type->basic_type();
564 int num_elem = vlen->get_con();
565
566 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
567 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
568 if (C->print_intrinsics()) {
569 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
570 is_store, is_store ? "store" : "load",
571 num_elem, type2name(elem_bt));
572 }
573 return false; // not supported
574 }
693 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
694 // StoreVectorOperationWithMap<C, V> defaultImpl) {
695 //
696 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
697 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
698 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr();
699 const TypeInt* vlen = gvn().type(argument(2))->is_int();
700 const TypeInstPtr* vector_idx_klass = gvn().type(argument(3))->is_instptr();
701
702 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) {
703 if (C->print_intrinsics()) {
704 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
705 NodeClassNames[argument(0)->Opcode()],
706 NodeClassNames[argument(1)->Opcode()],
707 NodeClassNames[argument(2)->Opcode()],
708 NodeClassNames[argument(3)->Opcode()]);
709 }
710 return false; // not enough info for intrinsification
711 }
712
713 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
714 if (C->print_intrinsics()) {
715 tty->print_cr(" ** klass argument not initialized");
716 }
717 return false;
718 }
719 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
720 if (!elem_type->is_primitive_type()) {
721 if (C->print_intrinsics()) {
722 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
723 }
724 return false; // should be primitive type
725 }
726 BasicType elem_bt = elem_type->basic_type();
727 int num_elem = vlen->get_con();
728
729 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
730 if (C->print_intrinsics()) {
731 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
732 is_scatter, is_scatter ? "scatter" : "gather",
733 num_elem, type2name(elem_bt));
734 }
735 return false; // not supported
736 }
737
738 // Check that the vector holding indices is supported by architecture
796 // long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int vlen,
797 // V v,
798 // Function<V,Long> defaultImpl)
799
800 bool LibraryCallKit::inline_vector_reduction() {
801 const TypeInt* opr = gvn().type(argument(0))->is_int();
802 const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr();
803 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
804 const TypeInt* vlen = gvn().type(argument(3))->is_int();
805
806 if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
807 if (C->print_intrinsics()) {
808 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
809 NodeClassNames[argument(0)->Opcode()],
810 NodeClassNames[argument(1)->Opcode()],
811 NodeClassNames[argument(2)->Opcode()],
812 NodeClassNames[argument(3)->Opcode()]);
813 }
814 return false; // not enough info for intrinsification
815 }
816 if (!is_klass_initialized(vector_klass)) {
817 if (C->print_intrinsics()) {
818 tty->print_cr(" ** klass argument not initialized");
819 }
820 return false;
821 }
822 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
823 if (!elem_type->is_primitive_type()) {
824 if (C->print_intrinsics()) {
825 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
826 }
827 return false; // should be primitive type
828 }
829 BasicType elem_bt = elem_type->basic_type();
830 int num_elem = vlen->get_con();
831
832 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
833 int sopc = ReductionNode::opcode(opc, elem_bt);
834
835 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
836 if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed)) {
837 if (C->print_intrinsics()) {
838 tty->print_cr(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s ismask=no",
839 sopc, num_elem, type2name(elem_bt));
840 }
841 return false;
883 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
884 // V v1, V v2,
885 // BiFunction<V, V, Boolean> defaultImpl) {
886 //
887 bool LibraryCallKit::inline_vector_test() {
888 const TypeInt* cond = gvn().type(argument(0))->is_int();
889 const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr();
890 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
891 const TypeInt* vlen = gvn().type(argument(3))->is_int();
892
893 if (!cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
894 if (C->print_intrinsics()) {
895 tty->print_cr(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
896 NodeClassNames[argument(0)->Opcode()],
897 NodeClassNames[argument(1)->Opcode()],
898 NodeClassNames[argument(2)->Opcode()],
899 NodeClassNames[argument(3)->Opcode()]);
900 }
901 return false; // not enough info for intrinsification
902 }
903 if (!is_klass_initialized(vector_klass)) {
904 if (C->print_intrinsics()) {
905 tty->print_cr(" ** klass argument not initialized");
906 }
907 return false;
908 }
909 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
910 if (!elem_type->is_primitive_type()) {
911 if (C->print_intrinsics()) {
912 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
913 }
914 return false; // should be primitive type
915 }
916 BasicType elem_bt = elem_type->basic_type();
917 int num_elem = vlen->get_con();
918 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
919 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
920 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
921
922 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
923 if (C->print_intrinsics()) {
924 tty->print_cr(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
925 cond->get_con(), num_elem, type2name(elem_bt),
926 is_vector_mask(vbox_klass));
927 }
928 return false;
947 // V v1, V v2, M m,
948 // VectorBlendOp<V,M> defaultImpl) { ...
949 //
950 bool LibraryCallKit::inline_vector_blend() {
951 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
952 const TypeInstPtr* mask_klass = gvn().type(argument(1))->is_instptr();
953 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
954 const TypeInt* vlen = gvn().type(argument(3))->is_int();
955
956 if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
957 elem_klass->const_oop() == NULL || !vlen->is_con()) {
958 if (C->print_intrinsics()) {
959 tty->print_cr(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
960 NodeClassNames[argument(0)->Opcode()],
961 NodeClassNames[argument(1)->Opcode()],
962 NodeClassNames[argument(2)->Opcode()],
963 NodeClassNames[argument(3)->Opcode()]);
964 }
965 return false; // not enough info for intrinsification
966 }
967 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
968 if (C->print_intrinsics()) {
969 tty->print_cr(" ** klass argument not initialized");
970 }
971 return false;
972 }
973 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
974 if (!elem_type->is_primitive_type()) {
975 if (C->print_intrinsics()) {
976 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
977 }
978 return false; // should be primitive type
979 }
980 BasicType elem_bt = elem_type->basic_type();
981 BasicType mask_bt = elem_bt;
982 int num_elem = vlen->get_con();
983
984 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
985 if (C->print_intrinsics()) {
986 tty->print_cr(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
987 num_elem, type2name(elem_bt));
988 }
989 return false; // not supported
990 }
991 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
992 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1019 //
1020 bool LibraryCallKit::inline_vector_compare() {
1021 const TypeInt* cond = gvn().type(argument(0))->is_int();
1022 const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr();
1023 const TypeInstPtr* mask_klass = gvn().type(argument(2))->is_instptr();
1024 const TypeInstPtr* elem_klass = gvn().type(argument(3))->is_instptr();
1025 const TypeInt* vlen = gvn().type(argument(4))->is_int();
1026
1027 if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1028 elem_klass->const_oop() == NULL || !vlen->is_con()) {
1029 if (C->print_intrinsics()) {
1030 tty->print_cr(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1031 NodeClassNames[argument(0)->Opcode()],
1032 NodeClassNames[argument(1)->Opcode()],
1033 NodeClassNames[argument(2)->Opcode()],
1034 NodeClassNames[argument(3)->Opcode()],
1035 NodeClassNames[argument(4)->Opcode()]);
1036 }
1037 return false; // not enough info for intrinsification
1038 }
1039 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1040 if (C->print_intrinsics()) {
1041 tty->print_cr(" ** klass argument not initialized");
1042 }
1043 return false;
1044 }
1045 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1046 if (!elem_type->is_primitive_type()) {
1047 if (C->print_intrinsics()) {
1048 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1049 }
1050 return false; // should be primitive type
1051 }
1052
1053 int num_elem = vlen->get_con();
1054 BasicType elem_bt = elem_type->basic_type();
1055 BasicType mask_bt = elem_bt;
1056
1057 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1058 if (C->print_intrinsics()) {
1059 tty->print_cr(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1060 cond->get_con(), num_elem, type2name(elem_bt));
1061 }
1062 return false;
1063 }
1064
1083 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1084 set_result(box);
1085 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1086 return true;
1087 }
1088
1089 // public static
1090 // <V extends Vector, Sh extends Shuffle>
1091 // V rearrangeOp(Class<V> vectorClass, Class<Sh> shuffleClass, Class< ? > elementType, int vlen,
1092 // V v1, Sh sh,
1093 // VectorSwizzleOp<V, Sh, S, E> defaultImpl) { ...
1094
1095 bool LibraryCallKit::inline_vector_rearrange() {
1096 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
1097 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->is_instptr();
1098 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
1099 const TypeInt* vlen = gvn().type(argument(3))->is_int();
1100
1101 if (shuffle_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1102 elem_klass->const_oop() == NULL || !vlen->is_con()) {
1103 if (C->print_intrinsics()) {
1104 tty->print_cr(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1105 NodeClassNames[argument(0)->Opcode()],
1106 NodeClassNames[argument(1)->Opcode()],
1107 NodeClassNames[argument(2)->Opcode()],
1108 NodeClassNames[argument(3)->Opcode()]);
1109 }
1110 return false; // not enough info for intrinsification
1111 }
1112 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(shuffle_klass)) {
1113 if (C->print_intrinsics()) {
1114 tty->print_cr(" ** klass argument not initialized");
1115 }
1116 return false;
1117 }
1118 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1119 if (!elem_type->is_primitive_type()) {
1120 if (C->print_intrinsics()) {
1121 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1122 }
1123 return false; // should be primitive type
1124 }
1125 BasicType elem_bt = elem_type->basic_type();
1126 BasicType shuffle_bt = elem_bt;
1127 int num_elem = vlen->get_con();
1128
1129 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1130 if (C->print_intrinsics()) {
1131 tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1132 num_elem, type2name(elem_bt));
1133 }
1134 return false; // not supported
1135 }
1136 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed)) {
1174 // V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int vlen,
1175 // V v, int i,
1176 // VectorBroadcastIntOp<V> defaultImpl) {
1177 //
1178 bool LibraryCallKit::inline_vector_broadcast_int() {
1179 const TypeInt* opr = gvn().type(argument(0))->is_int();
1180 const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr();
1181 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
1182 const TypeInt* vlen = gvn().type(argument(3))->is_int();
1183
1184 if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1185 if (C->print_intrinsics()) {
1186 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1187 NodeClassNames[argument(0)->Opcode()],
1188 NodeClassNames[argument(1)->Opcode()],
1189 NodeClassNames[argument(2)->Opcode()],
1190 NodeClassNames[argument(3)->Opcode()]);
1191 }
1192 return false; // not enough info for intrinsification
1193 }
1194 if (!is_klass_initialized(vector_klass)) {
1195 if (C->print_intrinsics()) {
1196 tty->print_cr(" ** klass argument not initialized");
1197 }
1198 return false;
1199 }
1200 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1201 if (!elem_type->is_primitive_type()) {
1202 if (C->print_intrinsics()) {
1203 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1204 }
1205 return false; // should be primitive type
1206 }
1207 BasicType elem_bt = elem_type->basic_type();
1208 int num_elem = vlen->get_con();
1209 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1210 int sopc = VectorNode::opcode(opc, elem_bt);
1211 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1212 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1213
1214 if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, true /*has_scalar_args*/)) {
1215 if (C->print_intrinsics()) {
1216 tty->print_cr(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s ismask=no",
1217 sopc, num_elem, type2name(elem_bt));
1218 }
1219 return false; // not supported
1220 }
1221 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1222 Node* opd2 = shift_count(argument(5), opc, elem_bt, num_elem);
1223 if (opd1 == NULL || opd2 == NULL) {
1224 return false;
1225 }
1226 Node* operation = gvn().transform(VectorNode::make(opc, opd1, opd2, num_elem, elem_bt));
1227
1228 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
1229 set_result(vbox);
1230 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1231 return true;
1232 }
1233
1234 // public static <VOUT extends VectorPayload,
1249
1250 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->is_instptr();
1251 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->is_instptr();
1252 const TypeInt* vlen_to = gvn().type(argument(6))->is_int();
1253
1254 if (!opr->is_con() ||
1255 vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() ||
1256 vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) {
1257 if (C->print_intrinsics()) {
1258 tty->print_cr(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
1259 NodeClassNames[argument(0)->Opcode()],
1260 NodeClassNames[argument(1)->Opcode()],
1261 NodeClassNames[argument(2)->Opcode()],
1262 NodeClassNames[argument(3)->Opcode()],
1263 NodeClassNames[argument(4)->Opcode()],
1264 NodeClassNames[argument(5)->Opcode()],
1265 NodeClassNames[argument(6)->Opcode()]);
1266 }
1267 return false; // not enough info for intrinsification
1268 }
1269 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
1270 if (C->print_intrinsics()) {
1271 tty->print_cr(" ** klass argument not initialized");
1272 }
1273 return false;
1274 }
1275
1276 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
1277 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
1278 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST);
1279
1280 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
1281 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
1282 if (is_vector_shuffle(vbox_klass_from) || is_vector_shuffle(vbox_klass_to)) {
1283 return false; // vector shuffles aren't supported
1284 }
1285 bool is_mask = is_vector_mask(vbox_klass_from);
1286
1287 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
1288 if (!elem_type_from->is_primitive_type()) {
1289 return false; // should be primitive type
1290 }
1291 BasicType elem_bt_from = elem_type_from->basic_type();
1292 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
1293 if (!elem_type_to->is_primitive_type()) {
1294 return false; // should be primitive type
1423 // V insert(Class<? extends V> vectorClass, Class<?> elementType, int vlen,
1424 // V vec, int ix, long val,
1425 // VecInsertOp<V> defaultImpl) {
1426 //
1427 bool LibraryCallKit::inline_vector_insert() {
1428 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
1429 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr();
1430 const TypeInt* vlen = gvn().type(argument(2))->is_int();
1431 const TypeInt* idx = gvn().type(argument(4))->is_int();
1432
1433 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
1434 if (C->print_intrinsics()) {
1435 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
1436 NodeClassNames[argument(0)->Opcode()],
1437 NodeClassNames[argument(1)->Opcode()],
1438 NodeClassNames[argument(2)->Opcode()],
1439 NodeClassNames[argument(4)->Opcode()]);
1440 }
1441 return false; // not enough info for intrinsification
1442 }
1443 if (!is_klass_initialized(vector_klass)) {
1444 if (C->print_intrinsics()) {
1445 tty->print_cr(" ** klass argument not initialized");
1446 }
1447 return false;
1448 }
1449 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1450 if (!elem_type->is_primitive_type()) {
1451 if (C->print_intrinsics()) {
1452 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1453 }
1454 return false; // should be primitive type
1455 }
1456 BasicType elem_bt = elem_type->basic_type();
1457 int num_elem = vlen->get_con();
1458 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
1459 if (C->print_intrinsics()) {
1460 tty->print_cr(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
1461 num_elem, type2name(elem_bt));
1462 }
1463 return false; // not supported
1464 }
1465
1466 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1467 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1468
1513 // long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
1514 // V vec, int ix,
1515 // VecExtractOp<V> defaultImpl) {
1516 //
1517 bool LibraryCallKit::inline_vector_extract() {
1518 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
1519 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr();
1520 const TypeInt* vlen = gvn().type(argument(2))->is_int();
1521 const TypeInt* idx = gvn().type(argument(4))->is_int();
1522
1523 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
1524 if (C->print_intrinsics()) {
1525 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
1526 NodeClassNames[argument(0)->Opcode()],
1527 NodeClassNames[argument(1)->Opcode()],
1528 NodeClassNames[argument(2)->Opcode()],
1529 NodeClassNames[argument(4)->Opcode()]);
1530 }
1531 return false; // not enough info for intrinsification
1532 }
1533 if (!is_klass_initialized(vector_klass)) {
1534 if (C->print_intrinsics()) {
1535 tty->print_cr(" ** klass argument not initialized");
1536 }
1537 return false;
1538 }
1539 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1540 if (!elem_type->is_primitive_type()) {
1541 if (C->print_intrinsics()) {
1542 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1543 }
1544 return false; // should be primitive type
1545 }
1546 BasicType elem_bt = elem_type->basic_type();
1547 int num_elem = vlen->get_con();
1548 int vopc = ExtractNode::opcode(elem_bt);
1549 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
1550 if (C->print_intrinsics()) {
1551 tty->print_cr(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
1552 num_elem, type2name(elem_bt));
1553 }
1554 return false; // not supported
1555 }
1556
1557 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1558 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
|