1 /* 2 * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "classfile/vmSymbols.hpp" 27 #include "opto/library_call.hpp" 28 #include "opto/runtime.hpp" 29 #include "opto/vectornode.hpp" 30 #include "prims/vectorSupport.hpp" 31 32 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) { 33 // Check that the operation is valid. 34 if (sopc <= 0) { 35 #ifndef PRODUCT 36 if (C->print_intrinsics()) { 37 tty->print_cr(" ** Rejected intrinsification because no valid vector op could be extracted"); 38 } 39 #endif 40 return false; 41 } 42 43 // Check that architecture supports this op-size-type combination. 44 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) { 45 #ifndef PRODUCT 46 if (C->print_intrinsics()) { 47 tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support it", 48 NodeClassNames[sopc], type2name(type), num_elem); 49 } 50 #endif 51 return false; 52 } else { 53 assert(Matcher::match_rule_supported(sopc), "must be supported"); 54 } 55 56 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) && 57 Matcher::supports_vector_variable_shifts() == false) { 58 if (C->print_intrinsics()) { 59 tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts", 60 NodeClassNames[sopc], type2name(type), num_elem); 61 } 62 return false; 63 } 64 65 // Check whether mask unboxing is supported. 66 if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseLoad) { 67 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) { 68 #ifndef PRODUCT 69 if (C->print_intrinsics()) { 70 tty->print_cr(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it", 71 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem); 72 } 73 #endif 74 return false; 75 } 76 } 77 78 // Check whether mask boxing is supported. 79 if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseStore) { 80 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) { 81 #ifndef PRODUCT 82 if (C->print_intrinsics()) { 83 tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it", 84 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem); 85 } 86 #endif 87 return false; 88 } 89 } 90 91 return true; 92 } 93 94 static bool is_vector_mask(ciKlass* klass) { 95 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); 96 } 97 98 static bool is_vector_shuffle(ciKlass* klass) { 99 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); 100 } 101 102 static bool is_klass_initialized(const TypeInstPtr* vec_klass) { 103 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass(), "klass instance expected"); 104 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass(); 105 return klass->is_initialized(); 106 } 107 108 #ifdef ASSERT 109 static bool is_vector(ciKlass* klass) { 110 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass()); 111 } 112 113 static bool check_vbox(const TypeInstPtr* vbox_type) { 114 assert(vbox_type->klass_is_exact(), ""); 115 116 ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass(); 117 assert(is_vector(ik), "not a vector"); 118 119 ciField* fd1 = ik->get_field_by_name(ciSymbol::ETYPE_name(), ciSymbol::class_signature(), /* is_static */ true); 120 assert(fd1 != NULL, "element type info is missing"); 121 122 ciConstant val1 = fd1->constant_value(); 123 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type(); 124 assert(is_java_primitive(elem_bt), "element type info is missing"); 125 126 ciField* fd2 = ik->get_field_by_name(ciSymbol::VLENGTH_name(), ciSymbol::int_signature(), /* is_static */ true); 127 assert(fd2 != NULL, "vector length info is missing"); 128 129 ciConstant val2 = fd2->constant_value(); 130 assert(val2.as_int() > 0, "vector length info is missing"); 131 132 return true; 133 } 134 #endif 135 136 Node* LibraryCallKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, 137 BasicType elem_bt, int num_elem) { 138 assert(EnableVectorSupport, ""); 139 const TypeVect* vec_type = TypeVect::make(elem_bt, num_elem); 140 141 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type); 142 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true); 143 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true); 144 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) )); 145 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) )); 146 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms)); 147 148 assert(check_vbox(vbox_type), ""); 149 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vec_type); 150 return gvn().transform(vbox); 151 } 152 153 Node* LibraryCallKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) { 154 assert(EnableVectorSupport, ""); 155 const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr(); 156 if (vbox_type->klass() != vbox_type_v->klass()) { 157 return NULL; // arguments don't agree on vector shapes 158 } 159 if (vbox_type_v->maybe_null()) { 160 return NULL; // no nulls are allowed 161 } 162 assert(check_vbox(vbox_type), ""); 163 const TypeVect* vec_type = TypeVect::make(elem_bt, num_elem); 164 Node* unbox = gvn().transform(new VectorUnboxNode(C, vec_type, v, merged_memory(), shuffle_to_vector)); 165 return unbox; 166 } 167 168 // public static 169 // <VM> 170 // VM unaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length, 171 // VM vm, 172 // Function<VM, VM> defaultImpl) { 173 // 174 // public static 175 // <VM> 176 // VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length, 177 // VM vm1, VM vm2, 178 // BiFunction<VM, VM, VM> defaultImpl) { 179 // 180 // public static 181 // <VM> 182 // VM ternaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length, 183 // VM vm1, VM vm2, VM vm3, 184 // TernaryOperation<VM> defaultImpl) { 185 // 186 bool LibraryCallKit::inline_vector_nary_operation(int n) { 187 const TypeInt* opr = gvn().type(argument(0))->is_int(); 188 const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr(); 189 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr(); 190 const TypeInt* vlen = gvn().type(argument(3))->is_int(); 191 192 if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { 193 if (C->print_intrinsics()) { 194 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s", 195 NodeClassNames[argument(0)->Opcode()], 196 NodeClassNames[argument(1)->Opcode()], 197 NodeClassNames[argument(2)->Opcode()], 198 NodeClassNames[argument(3)->Opcode()]); 199 } 200 return false; // not enough info for intrinsification 201 } 202 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 203 if (!elem_type->is_primitive_type()) { 204 if (C->print_intrinsics()) { 205 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); 206 } 207 return false; // should be primitive type 208 } 209 if (!is_klass_initialized(vector_klass)) { 210 if (C->print_intrinsics()) { 211 tty->print_cr(" ** klass argument not initialized"); 212 } 213 return false; 214 } 215 BasicType elem_bt = elem_type->basic_type(); 216 int num_elem = vlen->get_con(); 217 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt); 218 int sopc = VectorNode::opcode(opc, elem_bt); 219 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 220 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 221 222 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad. 223 if (!arch_supports_vector(sopc, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseAll : VecMaskNotUsed)) { 224 if (C->print_intrinsics()) { 225 tty->print_cr(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d", 226 n, sopc, num_elem, type2name(elem_bt), 227 is_vector_mask(vbox_klass) ? 1 : 0); 228 } 229 return false; // not supported 230 } 231 232 Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL; 233 switch (n) { 234 case 3: { 235 opd3 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem); 236 if (opd3 == NULL) { 237 if (C->print_intrinsics()) { 238 tty->print_cr(" ** unbox failed v3=%s", 239 NodeClassNames[argument(6)->Opcode()]); 240 } 241 return false; 242 } 243 // fall-through 244 } 245 case 2: { 246 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); 247 if (opd2 == NULL) { 248 if (C->print_intrinsics()) { 249 tty->print_cr(" ** unbox failed v2=%s", 250 NodeClassNames[argument(5)->Opcode()]); 251 } 252 return false; 253 } 254 // fall-through 255 } 256 case 1: { 257 opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); 258 if (opd1 == NULL) { 259 if (C->print_intrinsics()) { 260 tty->print_cr(" ** unbox failed v1=%s", 261 NodeClassNames[argument(4)->Opcode()]); 262 } 263 return false; 264 } 265 break; 266 } 267 default: fatal("unsupported arity: %d", n); 268 } 269 270 Node* operation = NULL; 271 const TypeVect* vt = TypeVect::make(elem_bt, num_elem); 272 switch (n) { 273 case 1: 274 case 2: { 275 operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, vt)); 276 break; 277 } 278 case 3: { 279 operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, opd3, vt)); 280 break; 281 } 282 default: fatal("unsupported arity: %d", n); 283 } 284 // Wrap it up in VectorBox to keep object type information. 285 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem); 286 set_result(vbox); 287 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 288 return true; 289 } 290 291 // <Sh extends VectorShuffle<E>, E> 292 // Sh ShuffleIota(Class<?> E, Class<?> ShuffleClass, Vector.Species<E> s, int length, 293 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl) 294 bool LibraryCallKit::inline_vector_shuffle_iota() { 295 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->is_instptr(); 296 const TypeInt* vlen = gvn().type(argument(3))->is_int(); 297 Node* start = argument(4); 298 const TypeInt* start_val = gvn().type(start)->is_int(); 299 Node* step = argument(5); 300 const TypeInt* step_val = gvn().type(step)->is_int(); 301 const TypeInt* wrap = gvn().type(argument(6))->is_int(); 302 303 if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) || 304 shuffle_klass->const_oop() == NULL || !wrap->is_con()) { 305 return false; // not enough info for intrinsification 306 } 307 if (!is_klass_initialized(shuffle_klass)) { 308 if (C->print_intrinsics()) { 309 tty->print_cr(" ** klass argument not initialized"); 310 } 311 return false; 312 } 313 314 int do_wrap = wrap->get_con(); 315 int num_elem = vlen->get_con(); 316 BasicType elem_bt = T_BYTE; 317 318 if (num_elem < 4) 319 return false; 320 321 if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) { 322 return false; 323 } 324 if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) { 325 return false; 326 } 327 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) { 328 return false; 329 } 330 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) { 331 return false; 332 } 333 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) { 334 return false; 335 } 336 337 const Type * type_bt = Type::get_const_basic_type(elem_bt); 338 const TypeVect * vt = TypeVect::make(type_bt, num_elem); 339 340 Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt)); 341 342 if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) { 343 Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt)); 344 res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt)); 345 } else if (step_val->get_con() > 1) { 346 Node* cnt = gvn().makecon(TypeInt::make(log2_int(step_val->get_con()))); 347 res = gvn().transform(VectorNode::make(Op_LShiftVB, res, cnt, vt)); 348 } 349 350 if (!start_val->is_con() || start_val->get_con() != 0) { 351 Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt)); 352 res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt)); 353 } 354 355 Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1)); 356 Node * bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt)); 357 if(do_wrap) { 358 // Wrap the indices greater than lane count. 359 res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt)); 360 } else { 361 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(1)); 362 Node * lane_cnt = gvn().makecon(TypeInt::make(num_elem)); 363 Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt)); 364 Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vt)); 365 366 // Make the indices greater than lane count as -ve values. This matches the java side implementation. 367 res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt)); 368 Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt)); 369 res = gvn().transform(new VectorBlendNode(biased_val, res, mask)); 370 } 371 372 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass(); 373 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass); 374 375 // Wrap it up in VectorBox to keep object type information. 376 res = box_vector(res, shuffle_box_type, elem_bt, num_elem); 377 set_result(res); 378 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 379 return true; 380 } 381 382 // <VM ,Sh extends VectorShuffle<E>, E> 383 // VM shuffleToVector(Class<VM> VecClass, Class<?>E , Class<?> ShuffleClass, Sh s, int length, 384 // ShuffleToVectorOperation<VM,Sh,E> defaultImpl) 385 bool LibraryCallKit::inline_vector_shuffle_to_vector() { 386 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); 387 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); 388 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->is_instptr(); 389 Node* shuffle = argument(3); 390 const TypeInt* vlen = gvn().type(argument(4))->is_int(); 391 392 if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) { 393 return false; // not enough info for intrinsification 394 } 395 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) { 396 if (C->print_intrinsics()) { 397 tty->print_cr(" ** klass argument not initialized"); 398 } 399 return false; 400 } 401 402 int num_elem = vlen->get_con(); 403 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 404 BasicType elem_bt = elem_type->basic_type(); 405 406 if (num_elem < 4) { 407 return false; 408 } 409 410 int cast_vopc = VectorCastNode::opcode(T_BYTE); // from shuffle of type T_BYTE 411 // Make sure that cast is implemented to particular type/size combination. 412 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) { 413 if (C->print_intrinsics()) { 414 tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s", 415 cast_vopc, num_elem, type2name(elem_bt)); 416 } 417 return false; 418 } 419 420 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass(); 421 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass); 422 423 // Unbox shuffle with true flag to indicate its load shuffle to vector 424 Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, elem_bt, num_elem, true); 425 426 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 427 const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 428 429 // Box vector 430 Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem); 431 set_result(res); 432 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 433 return true; 434 } 435 436 // <V extends Vector<?,?>> 437 // V broadcastCoerced(Class<?> vectorClass, Class<?> elementType, int vlen, 438 // long bits, 439 // LongFunction<V> defaultImpl) 440 bool LibraryCallKit::inline_vector_broadcast_coerced() { 441 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); 442 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); 443 const TypeInt* vlen = gvn().type(argument(2))->is_int(); 444 445 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { 446 if (C->print_intrinsics()) { 447 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s", 448 NodeClassNames[argument(0)->Opcode()], 449 NodeClassNames[argument(1)->Opcode()], 450 NodeClassNames[argument(2)->Opcode()]); 451 } 452 return false; // not enough info for intrinsification 453 } 454 455 if (!is_klass_initialized(vector_klass)) { 456 if (C->print_intrinsics()) { 457 tty->print_cr(" ** klass argument not initialized"); 458 } 459 return false; 460 } 461 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 462 if (!elem_type->is_primitive_type()) { 463 if (C->print_intrinsics()) { 464 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); 465 } 466 return false; // should be primitive type 467 } 468 BasicType elem_bt = elem_type->basic_type(); 469 int num_elem = vlen->get_con(); 470 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 471 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 472 473 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad. 474 if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, 475 (is_vector_mask(vbox_klass) ? VecMaskUseStore : VecMaskNotUsed), true /*has_scalar_args*/)) { 476 if (C->print_intrinsics()) { 477 tty->print_cr(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d", 478 num_elem, type2name(elem_bt), 479 is_vector_mask(vbox_klass) ? 1 : 0); 480 } 481 return false; // not supported 482 } 483 484 Node* bits = argument(3); // long 485 486 Node* elem = NULL; 487 switch (elem_bt) { 488 case T_BOOLEAN: // fall-through 489 case T_BYTE: // fall-through 490 case T_SHORT: // fall-through 491 case T_CHAR: // fall-through 492 case T_INT: { 493 elem = gvn().transform(new ConvL2INode(bits)); 494 break; 495 } 496 case T_DOUBLE: { 497 elem = gvn().transform(new MoveL2DNode(bits)); 498 break; 499 } 500 case T_FLOAT: { 501 bits = gvn().transform(new ConvL2INode(bits)); 502 elem = gvn().transform(new MoveI2FNode(bits)); 503 break; 504 } 505 case T_LONG: { 506 elem = bits; // no conversion needed 507 break; 508 } 509 default: fatal("%s", type2name(elem_bt)); 510 } 511 512 Node* broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt)); 513 broadcast = gvn().transform(broadcast); 514 515 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem); 516 set_result(box); 517 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 518 return true; 519 } 520 521 // <C, V extends Vector<?,?>> 522 // V load(Class<?> vectorClass, Class<?> elementType, int vlen, 523 // Object base, long offset, 524 // /* Vector.Mask<E,S> m*/ 525 // Object container, int index, 526 // LoadOperation<C, VM> defaultImpl) { 527 // 528 // <C, V extends Vector<?,?>> 529 // void store(Class<?> vectorClass, Class<?> elementType, int vlen, 530 // Object base, long offset, 531 // V v, /*Vector.Mask<E,S> m*/ 532 // Object container, int index, 533 // StoreVectorOperation<C, V> defaultImpl) { 534 535 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) { 536 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); 537 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); 538 const TypeInt* vlen = gvn().type(argument(2))->is_int(); 539 540 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { 541 if (C->print_intrinsics()) { 542 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s", 543 NodeClassNames[argument(0)->Opcode()], 544 NodeClassNames[argument(1)->Opcode()], 545 NodeClassNames[argument(2)->Opcode()]); 546 } 547 return false; // not enough info for intrinsification 548 } 549 if (!is_klass_initialized(vector_klass)) { 550 if (C->print_intrinsics()) { 551 tty->print_cr(" ** klass argument not initialized"); 552 } 553 return false; 554 } 555 556 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 557 if (!elem_type->is_primitive_type()) { 558 if (C->print_intrinsics()) { 559 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); 560 } 561 return false; // should be primitive type 562 } 563 BasicType elem_bt = elem_type->basic_type(); 564 int num_elem = vlen->get_con(); 565 566 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad. 567 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) { 568 if (C->print_intrinsics()) { 569 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no", 570 is_store, is_store ? "store" : "load", 571 num_elem, type2name(elem_bt)); 572 } 573 return false; // not supported 574 } 575 576 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 577 bool is_mask = is_vector_mask(vbox_klass); 578 579 Node* base = argument(3); 580 Node* offset = ConvL2X(argument(4)); 581 DecoratorSet decorators = C2_UNSAFE_ACCESS; 582 Node* addr = make_unsafe_address(base, offset, decorators, (is_mask ? T_BOOLEAN : elem_bt), true); 583 584 // Can base be NULL? Otherwise, always on-heap access. 585 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base)); 586 587 const TypePtr *addr_type = gvn().type(addr)->isa_ptr(); 588 const TypeAryPtr* arr_type = addr_type->isa_aryptr(); 589 590 // Now handle special case where load/store happens from/to byte array but element type is not byte. 591 bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE; 592 // Handle loading masks. 593 // If there is no consistency between array and vector element types, it must be special byte array case or loading masks 594 if (arr_type != NULL && !using_byte_array && elem_bt != arr_type->elem()->array_element_basic_type() && !is_mask) { 595 return false; 596 } 597 // Since we are using byte array, we need to double check that the byte operations are supported by backend. 598 if (using_byte_array) { 599 int byte_num_elem = num_elem * type2aelembytes(elem_bt); 600 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)) { 601 if (C->print_intrinsics()) { 602 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no", 603 is_store, is_store ? "store" : "load", 604 byte_num_elem, type2name(elem_bt)); 605 } 606 return false; // not supported 607 } 608 } 609 if (is_mask) { 610 if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) { 611 if (C->print_intrinsics()) { 612 tty->print_cr(" ** not supported: arity=%d op=%s/mask vlen=%d etype=bit ismask=no", 613 is_store, is_store ? "store" : "load", 614 num_elem); 615 } 616 return false; // not supported 617 } 618 if (!is_store) { 619 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) { 620 return false; // not supported 621 } 622 } else { 623 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) { 624 return false; // not supported 625 } 626 } 627 } 628 629 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 630 631 if (can_access_non_heap) { 632 insert_mem_bar(Op_MemBarCPUOrder); 633 } 634 635 if (is_store) { 636 Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem); 637 if (val == NULL) { 638 return false; // operand unboxing failed 639 } 640 set_all_memory(reset_memory()); 641 642 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector. 643 int store_num_elem = num_elem; 644 if (using_byte_array) { 645 store_num_elem = num_elem * type2aelembytes(elem_bt); 646 const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem); 647 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type)); 648 } 649 650 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem)); 651 set_memory(vstore, addr_type); 652 } else { 653 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load. 654 Node* vload = NULL; 655 if (using_byte_array) { 656 int load_num_elem = num_elem * type2aelembytes(elem_bt); 657 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE)); 658 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem); 659 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type)); 660 } else { 661 // Special handle for masks 662 if (is_mask) { 663 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN)); 664 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem); 665 vload = gvn().transform(new VectorLoadMaskNode(vload, to_vect_type)); 666 } else { 667 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt)); 668 } 669 } 670 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem); 671 set_result(box); 672 } 673 674 if (can_access_non_heap) { 675 insert_mem_bar(Op_MemBarCPUOrder); 676 } 677 678 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 679 return true; 680 } 681 682 // <C, V extends Vector<?>, W extends IntVector, E, S extends VectorSpecies<E>> 683 // void loadWithMap(Class<?> vectorClass, Class<E> E, int length, Class<?> vectorIndexClass, 684 // Object base, long offset, // Unsafe addressing 685 // W index_vector, 686 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation 687 // LoadVectorOperationWithMap<C, V, E, S> defaultImpl) 688 // 689 // <C, V extends Vector<?>, W extends IntVector> 690 // void storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass, 691 // Object base, long offset, // Unsafe addressing 692 // W index_vector, V v, 693 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation 694 // StoreVectorOperationWithMap<C, V> defaultImpl) { 695 // 696 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) { 697 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); 698 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); 699 const TypeInt* vlen = gvn().type(argument(2))->is_int(); 700 const TypeInstPtr* vector_idx_klass = gvn().type(argument(3))->is_instptr(); 701 702 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) { 703 if (C->print_intrinsics()) { 704 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s", 705 NodeClassNames[argument(0)->Opcode()], 706 NodeClassNames[argument(1)->Opcode()], 707 NodeClassNames[argument(2)->Opcode()], 708 NodeClassNames[argument(3)->Opcode()]); 709 } 710 return false; // not enough info for intrinsification 711 } 712 713 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) { 714 if (C->print_intrinsics()) { 715 tty->print_cr(" ** klass argument not initialized"); 716 } 717 return false; 718 } 719 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 720 if (!elem_type->is_primitive_type()) { 721 if (C->print_intrinsics()) { 722 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); 723 } 724 return false; // should be primitive type 725 } 726 BasicType elem_bt = elem_type->basic_type(); 727 int num_elem = vlen->get_con(); 728 729 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) { 730 if (C->print_intrinsics()) { 731 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no", 732 is_scatter, is_scatter ? "scatter" : "gather", 733 num_elem, type2name(elem_bt)); 734 } 735 return false; // not supported 736 } 737 738 // Check that the vector holding indices is supported by architecture 739 if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) { 740 if (C->print_intrinsics()) { 741 tty->print_cr(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int ismask=no", 742 is_scatter, is_scatter ? "scatter" : "gather", 743 num_elem); 744 } 745 return false; // not supported 746 } 747 748 Node* base = argument(4); 749 Node* offset = ConvL2X(argument(5)); 750 Node* addr = make_unsafe_address(base, offset, C2_UNSAFE_ACCESS, elem_bt, true); 751 752 const TypePtr *addr_type = gvn().type(addr)->isa_ptr(); 753 const TypeAryPtr* arr_type = addr_type->isa_aryptr(); 754 755 // The array must be consistent with vector type 756 if (arr_type == NULL || (arr_type != NULL && elem_bt != arr_type->elem()->array_element_basic_type())) { 757 return false; 758 } 759 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 760 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 761 762 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass(); 763 764 if (vbox_idx_klass == NULL) { 765 return false; 766 } 767 768 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass); 769 770 Node* index_vect = unbox_vector(argument(7), vbox_idx_type, T_INT, num_elem); 771 if (index_vect == NULL) { 772 return false; 773 } 774 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem); 775 if (is_scatter) { 776 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem); 777 if (val == NULL) { 778 return false; // operand unboxing failed 779 } 780 set_all_memory(reset_memory()); 781 782 Node* vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect)); 783 set_memory(vstore, addr_type); 784 } else { 785 Node* vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect)); 786 787 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem); 788 set_result(box); 789 } 790 791 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 792 return true; 793 } 794 795 // <V extends Vector<?,?>> 796 // long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int vlen, 797 // V v, 798 // Function<V,Long> defaultImpl) 799 800 bool LibraryCallKit::inline_vector_reduction() { 801 const TypeInt* opr = gvn().type(argument(0))->is_int(); 802 const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr(); 803 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr(); 804 const TypeInt* vlen = gvn().type(argument(3))->is_int(); 805 806 if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { 807 if (C->print_intrinsics()) { 808 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s", 809 NodeClassNames[argument(0)->Opcode()], 810 NodeClassNames[argument(1)->Opcode()], 811 NodeClassNames[argument(2)->Opcode()], 812 NodeClassNames[argument(3)->Opcode()]); 813 } 814 return false; // not enough info for intrinsification 815 } 816 if (!is_klass_initialized(vector_klass)) { 817 if (C->print_intrinsics()) { 818 tty->print_cr(" ** klass argument not initialized"); 819 } 820 return false; 821 } 822 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 823 if (!elem_type->is_primitive_type()) { 824 if (C->print_intrinsics()) { 825 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); 826 } 827 return false; // should be primitive type 828 } 829 BasicType elem_bt = elem_type->basic_type(); 830 int num_elem = vlen->get_con(); 831 832 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt); 833 int sopc = ReductionNode::opcode(opc, elem_bt); 834 835 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad. 836 if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed)) { 837 if (C->print_intrinsics()) { 838 tty->print_cr(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s ismask=no", 839 sopc, num_elem, type2name(elem_bt)); 840 } 841 return false; 842 } 843 844 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 845 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 846 847 Node* opd = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); 848 if (opd == NULL) { 849 return false; // operand unboxing failed 850 } 851 852 Node* init = ReductionNode::make_reduction_input(gvn(), opc, elem_bt); 853 Node* rn = gvn().transform(ReductionNode::make(opc, NULL, init, opd, elem_bt)); 854 855 Node* bits = NULL; 856 switch (elem_bt) { 857 case T_BYTE: 858 case T_SHORT: 859 case T_INT: { 860 bits = gvn().transform(new ConvI2LNode(rn)); 861 break; 862 } 863 case T_FLOAT: { 864 rn = gvn().transform(new MoveF2INode(rn)); 865 bits = gvn().transform(new ConvI2LNode(rn)); 866 break; 867 } 868 case T_DOUBLE: { 869 bits = gvn().transform(new MoveD2LNode(rn)); 870 break; 871 } 872 case T_LONG: { 873 bits = rn; // no conversion needed 874 break; 875 } 876 default: fatal("%s", type2name(elem_bt)); 877 } 878 set_result(bits); 879 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 880 return true; 881 } 882 883 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen, 884 // V v1, V v2, 885 // BiFunction<V, V, Boolean> defaultImpl) { 886 // 887 bool LibraryCallKit::inline_vector_test() { 888 const TypeInt* cond = gvn().type(argument(0))->is_int(); 889 const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr(); 890 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr(); 891 const TypeInt* vlen = gvn().type(argument(3))->is_int(); 892 893 if (!cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { 894 if (C->print_intrinsics()) { 895 tty->print_cr(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s", 896 NodeClassNames[argument(0)->Opcode()], 897 NodeClassNames[argument(1)->Opcode()], 898 NodeClassNames[argument(2)->Opcode()], 899 NodeClassNames[argument(3)->Opcode()]); 900 } 901 return false; // not enough info for intrinsification 902 } 903 if (!is_klass_initialized(vector_klass)) { 904 if (C->print_intrinsics()) { 905 tty->print_cr(" ** klass argument not initialized"); 906 } 907 return false; 908 } 909 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 910 if (!elem_type->is_primitive_type()) { 911 if (C->print_intrinsics()) { 912 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); 913 } 914 return false; // should be primitive type 915 } 916 BasicType elem_bt = elem_type->basic_type(); 917 int num_elem = vlen->get_con(); 918 BoolTest::mask booltest = (BoolTest::mask)cond->get_con(); 919 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 920 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 921 922 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) { 923 if (C->print_intrinsics()) { 924 tty->print_cr(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d", 925 cond->get_con(), num_elem, type2name(elem_bt), 926 is_vector_mask(vbox_klass)); 927 } 928 return false; 929 } 930 931 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); 932 Node* opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); 933 if (opd1 == NULL || opd2 == NULL) { 934 return false; // operand unboxing failed 935 } 936 Node* test = new VectorTestNode(opd1, opd2, booltest); 937 test = gvn().transform(test); 938 939 set_result(test); 940 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 941 return true; 942 } 943 944 // public static 945 // <V extends Vector, M extends Mask> 946 // V blend(Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int vlen, 947 // V v1, V v2, M m, 948 // VectorBlendOp<V,M> defaultImpl) { ... 949 // 950 bool LibraryCallKit::inline_vector_blend() { 951 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); 952 const TypeInstPtr* mask_klass = gvn().type(argument(1))->is_instptr(); 953 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr(); 954 const TypeInt* vlen = gvn().type(argument(3))->is_int(); 955 956 if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL || 957 elem_klass->const_oop() == NULL || !vlen->is_con()) { 958 if (C->print_intrinsics()) { 959 tty->print_cr(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s", 960 NodeClassNames[argument(0)->Opcode()], 961 NodeClassNames[argument(1)->Opcode()], 962 NodeClassNames[argument(2)->Opcode()], 963 NodeClassNames[argument(3)->Opcode()]); 964 } 965 return false; // not enough info for intrinsification 966 } 967 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) { 968 if (C->print_intrinsics()) { 969 tty->print_cr(" ** klass argument not initialized"); 970 } 971 return false; 972 } 973 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 974 if (!elem_type->is_primitive_type()) { 975 if (C->print_intrinsics()) { 976 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); 977 } 978 return false; // should be primitive type 979 } 980 BasicType elem_bt = elem_type->basic_type(); 981 BasicType mask_bt = elem_bt; 982 int num_elem = vlen->get_con(); 983 984 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) { 985 if (C->print_intrinsics()) { 986 tty->print_cr(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload", 987 num_elem, type2name(elem_bt)); 988 } 989 return false; // not supported 990 } 991 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 992 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 993 994 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass(); 995 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass); 996 997 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); 998 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); 999 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem); 1000 1001 if (v1 == NULL || v2 == NULL || mask == NULL) { 1002 return false; // operand unboxing failed 1003 } 1004 1005 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask)); 1006 1007 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem); 1008 set_result(box); 1009 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 1010 return true; 1011 } 1012 1013 // public static <V extends Vector<E,S>, 1014 // M extends Vector.Mask<E,S>, 1015 // S extends Vector.Shape, E> 1016 // M compare(int cond, Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int vlen, 1017 // V v1, V v2, 1018 // VectorCompareOp<V,M> defaultImpl) { ... 1019 // 1020 bool LibraryCallKit::inline_vector_compare() { 1021 const TypeInt* cond = gvn().type(argument(0))->is_int(); 1022 const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr(); 1023 const TypeInstPtr* mask_klass = gvn().type(argument(2))->is_instptr(); 1024 const TypeInstPtr* elem_klass = gvn().type(argument(3))->is_instptr(); 1025 const TypeInt* vlen = gvn().type(argument(4))->is_int(); 1026 1027 if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL || 1028 elem_klass->const_oop() == NULL || !vlen->is_con()) { 1029 if (C->print_intrinsics()) { 1030 tty->print_cr(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s", 1031 NodeClassNames[argument(0)->Opcode()], 1032 NodeClassNames[argument(1)->Opcode()], 1033 NodeClassNames[argument(2)->Opcode()], 1034 NodeClassNames[argument(3)->Opcode()], 1035 NodeClassNames[argument(4)->Opcode()]); 1036 } 1037 return false; // not enough info for intrinsification 1038 } 1039 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) { 1040 if (C->print_intrinsics()) { 1041 tty->print_cr(" ** klass argument not initialized"); 1042 } 1043 return false; 1044 } 1045 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 1046 if (!elem_type->is_primitive_type()) { 1047 if (C->print_intrinsics()) { 1048 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); 1049 } 1050 return false; // should be primitive type 1051 } 1052 1053 int num_elem = vlen->get_con(); 1054 BasicType elem_bt = elem_type->basic_type(); 1055 BasicType mask_bt = elem_bt; 1056 1057 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) { 1058 if (C->print_intrinsics()) { 1059 tty->print_cr(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore", 1060 cond->get_con(), num_elem, type2name(elem_bt)); 1061 } 1062 return false; 1063 } 1064 1065 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 1066 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 1067 1068 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass(); 1069 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass); 1070 1071 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); 1072 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem); 1073 1074 if (v1 == NULL || v2 == NULL) { 1075 return false; // operand unboxing failed 1076 } 1077 BoolTest::mask pred = (BoolTest::mask)cond->get_con(); 1078 ConINode* pred_node = (ConINode*)gvn().makecon(cond); 1079 1080 const TypeVect* vt = TypeVect::make(mask_bt, num_elem); 1081 Node* operation = gvn().transform(new VectorMaskCmpNode(pred, v1, v2, pred_node, vt)); 1082 1083 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem); 1084 set_result(box); 1085 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 1086 return true; 1087 } 1088 1089 // public static 1090 // <V extends Vector, Sh extends Shuffle> 1091 // V rearrangeOp(Class<V> vectorClass, Class<Sh> shuffleClass, Class< ? > elementType, int vlen, 1092 // V v1, Sh sh, 1093 // VectorSwizzleOp<V, Sh, S, E> defaultImpl) { ... 1094 1095 bool LibraryCallKit::inline_vector_rearrange() { 1096 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); 1097 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->is_instptr(); 1098 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr(); 1099 const TypeInt* vlen = gvn().type(argument(3))->is_int(); 1100 1101 if (shuffle_klass->const_oop() == NULL || vector_klass->const_oop() == NULL || 1102 elem_klass->const_oop() == NULL || !vlen->is_con()) { 1103 if (C->print_intrinsics()) { 1104 tty->print_cr(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s", 1105 NodeClassNames[argument(0)->Opcode()], 1106 NodeClassNames[argument(1)->Opcode()], 1107 NodeClassNames[argument(2)->Opcode()], 1108 NodeClassNames[argument(3)->Opcode()]); 1109 } 1110 return false; // not enough info for intrinsification 1111 } 1112 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(shuffle_klass)) { 1113 if (C->print_intrinsics()) { 1114 tty->print_cr(" ** klass argument not initialized"); 1115 } 1116 return false; 1117 } 1118 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 1119 if (!elem_type->is_primitive_type()) { 1120 if (C->print_intrinsics()) { 1121 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); 1122 } 1123 return false; // should be primitive type 1124 } 1125 BasicType elem_bt = elem_type->basic_type(); 1126 BasicType shuffle_bt = elem_bt; 1127 int num_elem = vlen->get_con(); 1128 1129 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) { 1130 if (C->print_intrinsics()) { 1131 tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no", 1132 num_elem, type2name(elem_bt)); 1133 } 1134 return false; // not supported 1135 } 1136 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed)) { 1137 if (C->print_intrinsics()) { 1138 tty->print_cr(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no", 1139 num_elem, type2name(elem_bt)); 1140 } 1141 return false; // not supported 1142 } 1143 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 1144 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 1145 1146 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass(); 1147 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass); 1148 1149 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); 1150 Node* shuffle = unbox_vector(argument(5), shbox_type, shuffle_bt, num_elem); 1151 1152 if (v1 == NULL || shuffle == NULL) { 1153 return false; // operand unboxing failed 1154 } 1155 1156 Node* rearrange = gvn().transform(new VectorRearrangeNode(v1, shuffle)); 1157 1158 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem); 1159 set_result(box); 1160 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 1161 return true; 1162 } 1163 1164 Node* LibraryCallKit::shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) { 1165 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported"); 1166 juint mask = (type2aelembytes(bt) * BitsPerByte - 1); 1167 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask))); 1168 Node* mcnt = gvn().transform(new AndINode(cnt, nmask)); 1169 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt)); 1170 } 1171 1172 // public static 1173 // <V extends Vector<?,?>> 1174 // V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int vlen, 1175 // V v, int i, 1176 // VectorBroadcastIntOp<V> defaultImpl) { 1177 // 1178 bool LibraryCallKit::inline_vector_broadcast_int() { 1179 const TypeInt* opr = gvn().type(argument(0))->is_int(); 1180 const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr(); 1181 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr(); 1182 const TypeInt* vlen = gvn().type(argument(3))->is_int(); 1183 1184 if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { 1185 if (C->print_intrinsics()) { 1186 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s", 1187 NodeClassNames[argument(0)->Opcode()], 1188 NodeClassNames[argument(1)->Opcode()], 1189 NodeClassNames[argument(2)->Opcode()], 1190 NodeClassNames[argument(3)->Opcode()]); 1191 } 1192 return false; // not enough info for intrinsification 1193 } 1194 if (!is_klass_initialized(vector_klass)) { 1195 if (C->print_intrinsics()) { 1196 tty->print_cr(" ** klass argument not initialized"); 1197 } 1198 return false; 1199 } 1200 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 1201 if (!elem_type->is_primitive_type()) { 1202 if (C->print_intrinsics()) { 1203 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); 1204 } 1205 return false; // should be primitive type 1206 } 1207 BasicType elem_bt = elem_type->basic_type(); 1208 int num_elem = vlen->get_con(); 1209 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt); 1210 int sopc = VectorNode::opcode(opc, elem_bt); 1211 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 1212 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 1213 1214 if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, true /*has_scalar_args*/)) { 1215 if (C->print_intrinsics()) { 1216 tty->print_cr(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s ismask=no", 1217 sopc, num_elem, type2name(elem_bt)); 1218 } 1219 return false; // not supported 1220 } 1221 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); 1222 Node* opd2 = shift_count(argument(5), opc, elem_bt, num_elem); 1223 if (opd1 == NULL || opd2 == NULL) { 1224 return false; 1225 } 1226 Node* operation = gvn().transform(VectorNode::make(opc, opd1, opd2, num_elem, elem_bt)); 1227 1228 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem); 1229 set_result(vbox); 1230 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 1231 return true; 1232 } 1233 1234 // public static <VOUT extends VectorPayload, 1235 // VIN extends VectorPayload, 1236 // S extends VectorSpecies> 1237 // VOUT convert(int oprId, 1238 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen, 1239 // Class<?> toVectorClass, Class<?> toElementType, int toVLen, 1240 // VIN v, S s, 1241 // VectorConvertOp<VOUT, VIN, S> defaultImpl) { 1242 // 1243 bool LibraryCallKit::inline_vector_convert() { 1244 const TypeInt* opr = gvn().type(argument(0))->is_int(); 1245 1246 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->is_instptr(); 1247 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->is_instptr(); 1248 const TypeInt* vlen_from = gvn().type(argument(3))->is_int(); 1249 1250 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->is_instptr(); 1251 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->is_instptr(); 1252 const TypeInt* vlen_to = gvn().type(argument(6))->is_int(); 1253 1254 if (!opr->is_con() || 1255 vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() || 1256 vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) { 1257 if (C->print_intrinsics()) { 1258 tty->print_cr(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s", 1259 NodeClassNames[argument(0)->Opcode()], 1260 NodeClassNames[argument(1)->Opcode()], 1261 NodeClassNames[argument(2)->Opcode()], 1262 NodeClassNames[argument(3)->Opcode()], 1263 NodeClassNames[argument(4)->Opcode()], 1264 NodeClassNames[argument(5)->Opcode()], 1265 NodeClassNames[argument(6)->Opcode()]); 1266 } 1267 return false; // not enough info for intrinsification 1268 } 1269 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) { 1270 if (C->print_intrinsics()) { 1271 tty->print_cr(" ** klass argument not initialized"); 1272 } 1273 return false; 1274 } 1275 1276 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST || 1277 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode"); 1278 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST); 1279 1280 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass(); 1281 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass(); 1282 if (is_vector_shuffle(vbox_klass_from) || is_vector_shuffle(vbox_klass_to)) { 1283 return false; // vector shuffles aren't supported 1284 } 1285 bool is_mask = is_vector_mask(vbox_klass_from); 1286 1287 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type(); 1288 if (!elem_type_from->is_primitive_type()) { 1289 return false; // should be primitive type 1290 } 1291 BasicType elem_bt_from = elem_type_from->basic_type(); 1292 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type(); 1293 if (!elem_type_to->is_primitive_type()) { 1294 return false; // should be primitive type 1295 } 1296 BasicType elem_bt_to = elem_type_to->basic_type(); 1297 if (is_mask && elem_bt_from != elem_bt_to) { 1298 return false; // type mismatch 1299 } 1300 int num_elem_from = vlen_from->get_con(); 1301 int num_elem_to = vlen_to->get_con(); 1302 1303 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed 1304 // since we may need to change size. 1305 if (!arch_supports_vector(Op_VectorReinterpret, 1306 num_elem_from, 1307 elem_bt_from, 1308 is_mask ? VecMaskUseAll : VecMaskNotUsed)) { 1309 if (C->print_intrinsics()) { 1310 tty->print_cr(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d", 1311 is_cast ? "cast" : "reinterpret", 1312 num_elem_from, type2name(elem_bt_from), is_mask); 1313 } 1314 return false; 1315 } 1316 1317 // Check whether we can support resizing/reinterpreting to the new size. 1318 if (!arch_supports_vector(Op_VectorReinterpret, 1319 num_elem_to, 1320 elem_bt_to, 1321 is_mask ? VecMaskUseAll : VecMaskNotUsed)) { 1322 if (C->print_intrinsics()) { 1323 tty->print_cr(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d", 1324 is_cast ? "cast" : "reinterpret", 1325 num_elem_to, type2name(elem_bt_to), is_mask); 1326 } 1327 return false; 1328 } 1329 1330 // At this point, we know that both input and output vector registers are supported 1331 // by the architecture. Next check if the casted type is simply to same type - which means 1332 // that it is actually a resize and not a cast. 1333 if (is_cast && elem_bt_from == elem_bt_to) { 1334 is_cast = false; 1335 } 1336 1337 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from); 1338 1339 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from); 1340 if (opd1 == NULL) { 1341 return false; 1342 } 1343 1344 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from); 1345 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to); 1346 1347 Node* op = opd1; 1348 if (is_cast) { 1349 assert(!is_mask, "masks cannot be casted"); 1350 int cast_vopc = VectorCastNode::opcode(elem_bt_from); 1351 // Make sure that cast is implemented to particular type/size combination. 1352 if (!arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) { 1353 if (C->print_intrinsics()) { 1354 tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d", 1355 cast_vopc, 1356 num_elem_to, type2name(elem_bt_to), is_mask); 1357 } 1358 return false; 1359 } 1360 1361 if (num_elem_from < num_elem_to) { 1362 // Since input and output number of elements are not consistent, we need to make sure we 1363 // properly size. Thus, first make a cast that retains the number of elements from source. 1364 // In case the size exceeds the arch size, we do the minimum. 1365 int num_elem_for_cast = MIN2(num_elem_from, Matcher::max_vector_size(elem_bt_to)); 1366 1367 // It is possible that arch does not support this intermediate vector size 1368 // TODO More complex logic required here to handle this corner case for the sizes. 1369 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) { 1370 if (C->print_intrinsics()) { 1371 tty->print_cr(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d", 1372 cast_vopc, 1373 num_elem_for_cast, type2name(elem_bt_to), is_mask); 1374 } 1375 return false; 1376 } 1377 1378 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast)); 1379 // Now ensure that the destination gets properly resized to needed size. 1380 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type)); 1381 } else if (num_elem_from > num_elem_to) { 1382 // Since number elements from input is larger than output, simply reduce size of input (we are supposed to 1383 // drop top elements anyway). 1384 int num_elem_for_resize = MAX2(num_elem_to, Matcher::min_vector_size(elem_bt_to)); 1385 1386 // It is possible that arch does not support this intermediate vector size 1387 // TODO More complex logic required here to handle this corner case for the sizes. 1388 if (!arch_supports_vector(Op_VectorReinterpret, 1389 num_elem_for_resize, 1390 elem_bt_from, 1391 VecMaskNotUsed)) { 1392 if (C->print_intrinsics()) { 1393 tty->print_cr(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d", 1394 num_elem_for_resize, type2name(elem_bt_from), is_mask); 1395 } 1396 return false; 1397 } 1398 1399 op = gvn().transform(new VectorReinterpretNode(op, 1400 src_type, 1401 TypeVect::make(elem_bt_from, 1402 num_elem_for_resize))); 1403 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to)); 1404 } else { 1405 // Since input and output number of elements match, and since we know this vector size is 1406 // supported, simply do a cast with no resize needed. 1407 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to)); 1408 } 1409 } else if (Type::cmp(src_type, dst_type) != 0) { 1410 assert(!is_cast, "must be reinterpret"); 1411 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type)); 1412 } 1413 1414 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to); 1415 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to); 1416 set_result(vbox); 1417 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to)))); 1418 return true; 1419 } 1420 1421 // public static 1422 // <V extends Vector<?>> 1423 // V insert(Class<? extends V> vectorClass, Class<?> elementType, int vlen, 1424 // V vec, int ix, long val, 1425 // VecInsertOp<V> defaultImpl) { 1426 // 1427 bool LibraryCallKit::inline_vector_insert() { 1428 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); 1429 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); 1430 const TypeInt* vlen = gvn().type(argument(2))->is_int(); 1431 const TypeInt* idx = gvn().type(argument(4))->is_int(); 1432 1433 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) { 1434 if (C->print_intrinsics()) { 1435 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s", 1436 NodeClassNames[argument(0)->Opcode()], 1437 NodeClassNames[argument(1)->Opcode()], 1438 NodeClassNames[argument(2)->Opcode()], 1439 NodeClassNames[argument(4)->Opcode()]); 1440 } 1441 return false; // not enough info for intrinsification 1442 } 1443 if (!is_klass_initialized(vector_klass)) { 1444 if (C->print_intrinsics()) { 1445 tty->print_cr(" ** klass argument not initialized"); 1446 } 1447 return false; 1448 } 1449 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 1450 if (!elem_type->is_primitive_type()) { 1451 if (C->print_intrinsics()) { 1452 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); 1453 } 1454 return false; // should be primitive type 1455 } 1456 BasicType elem_bt = elem_type->basic_type(); 1457 int num_elem = vlen->get_con(); 1458 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) { 1459 if (C->print_intrinsics()) { 1460 tty->print_cr(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no", 1461 num_elem, type2name(elem_bt)); 1462 } 1463 return false; // not supported 1464 } 1465 1466 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 1467 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 1468 1469 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem); 1470 if (opd == NULL) { 1471 return false; 1472 } 1473 1474 Node* insert_val = argument(5); 1475 assert(gvn().type(insert_val)->isa_long() != NULL, "expected to be long"); 1476 1477 // Convert insert value back to its appropriate type. 1478 switch (elem_bt) { 1479 case T_BYTE: 1480 insert_val = gvn().transform(new ConvL2INode(insert_val)); 1481 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE)); 1482 break; 1483 case T_SHORT: 1484 insert_val = gvn().transform(new ConvL2INode(insert_val)); 1485 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT)); 1486 break; 1487 case T_INT: 1488 insert_val = gvn().transform(new ConvL2INode(insert_val)); 1489 break; 1490 case T_FLOAT: 1491 insert_val = gvn().transform(new ConvL2INode(insert_val)); 1492 insert_val = gvn().transform(new MoveI2FNode(insert_val)); 1493 break; 1494 case T_DOUBLE: 1495 insert_val = gvn().transform(new MoveL2DNode(insert_val)); 1496 break; 1497 case T_LONG: 1498 // no conversion needed 1499 break; 1500 default: fatal("%s", type2name(elem_bt)); break; 1501 } 1502 1503 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con())); 1504 1505 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem); 1506 set_result(vbox); 1507 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); 1508 return true; 1509 } 1510 1511 // public static 1512 // <V extends Vector<?>> 1513 // long extract(Class<?> vectorClass, Class<?> elementType, int vlen, 1514 // V vec, int ix, 1515 // VecExtractOp<V> defaultImpl) { 1516 // 1517 bool LibraryCallKit::inline_vector_extract() { 1518 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr(); 1519 const TypeInstPtr* elem_klass = gvn().type(argument(1))->is_instptr(); 1520 const TypeInt* vlen = gvn().type(argument(2))->is_int(); 1521 const TypeInt* idx = gvn().type(argument(4))->is_int(); 1522 1523 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) { 1524 if (C->print_intrinsics()) { 1525 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s", 1526 NodeClassNames[argument(0)->Opcode()], 1527 NodeClassNames[argument(1)->Opcode()], 1528 NodeClassNames[argument(2)->Opcode()], 1529 NodeClassNames[argument(4)->Opcode()]); 1530 } 1531 return false; // not enough info for intrinsification 1532 } 1533 if (!is_klass_initialized(vector_klass)) { 1534 if (C->print_intrinsics()) { 1535 tty->print_cr(" ** klass argument not initialized"); 1536 } 1537 return false; 1538 } 1539 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); 1540 if (!elem_type->is_primitive_type()) { 1541 if (C->print_intrinsics()) { 1542 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type()); 1543 } 1544 return false; // should be primitive type 1545 } 1546 BasicType elem_bt = elem_type->basic_type(); 1547 int num_elem = vlen->get_con(); 1548 int vopc = ExtractNode::opcode(elem_bt); 1549 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) { 1550 if (C->print_intrinsics()) { 1551 tty->print_cr(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no", 1552 num_elem, type2name(elem_bt)); 1553 } 1554 return false; // not supported 1555 } 1556 1557 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); 1558 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); 1559 1560 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem); 1561 if (opd == NULL) { 1562 return false; 1563 } 1564 1565 Node* operation = gvn().transform(ExtractNode::make(opd, idx->get_con(), elem_bt)); 1566 1567 Node* bits = NULL; 1568 switch (elem_bt) { 1569 case T_BYTE: 1570 case T_SHORT: 1571 case T_INT: { 1572 bits = gvn().transform(new ConvI2LNode(operation)); 1573 break; 1574 } 1575 case T_FLOAT: { 1576 bits = gvn().transform(new MoveF2INode(operation)); 1577 bits = gvn().transform(new ConvI2LNode(bits)); 1578 break; 1579 } 1580 case T_DOUBLE: { 1581 bits = gvn().transform(new MoveD2LNode(operation)); 1582 break; 1583 } 1584 case T_LONG: { 1585 bits = operation; // no conversion needed 1586 break; 1587 } 1588 default: fatal("%s", type2name(elem_bt)); 1589 } 1590 1591 set_result(bits); 1592 return true; 1593 } 1594