1 /*
   2  * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/vmSymbols.hpp"
  27 #include "opto/library_call.hpp"
  28 #include "opto/runtime.hpp"
  29 #include "opto/vectornode.hpp"
  30 #include "prims/vectorSupport.hpp"
  31 
  32 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
  33   // Check that the operation is valid.
  34   if (sopc <= 0) {
  35 #ifndef PRODUCT
  36     if (C->print_intrinsics()) {
  37       tty->print_cr("  ** Rejected intrinsification because no valid vector op could be extracted");
  38     }
  39 #endif
  40     return false;
  41   }
  42 
  43   // Check that architecture supports this op-size-type combination.
  44   if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
  45 #ifndef PRODUCT
  46     if (C->print_intrinsics()) {
  47       tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support it",
  48                     NodeClassNames[sopc], type2name(type), num_elem);
  49     }
  50 #endif
  51     return false;
  52   } else {
  53     assert(Matcher::match_rule_supported(sopc), "must be supported");
  54   }
  55 
  56   if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
  57       Matcher::supports_vector_variable_shifts() == false) {
  58     if (C->print_intrinsics()) {
  59       tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
  60                     NodeClassNames[sopc], type2name(type), num_elem);
  61     }
  62     return false;
  63   }
  64 
  65   // Check whether mask unboxing is supported.
  66   if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseLoad) {
  67     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) {
  68     #ifndef PRODUCT
  69       if (C->print_intrinsics()) {
  70         tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
  71                       NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
  72       }
  73     #endif
  74       return false;
  75     }
  76   }
  77 
  78   // Check whether mask boxing is supported.
  79   if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseStore) {
  80     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) {
  81     #ifndef PRODUCT
  82       if (C->print_intrinsics()) {
  83         tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
  84                       NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
  85       }
  86     #endif
  87       return false;
  88     }
  89   }
  90 
  91   return true;
  92 }
  93 
  94 static bool is_vector_mask(ciKlass* klass) {
  95   return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
  96 }
  97 
  98 static bool is_vector_shuffle(ciKlass* klass) {
  99   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
 100 }
 101 
 102 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
 103   assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass(), "klass instance expected");
 104   ciInstanceKlass* klass =  vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
 105   return klass->is_initialized();
 106 }
 107 
 108 #ifdef ASSERT
 109 static bool is_vector(ciKlass* klass) {
 110   return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
 111 }
 112 
 113 static bool check_vbox(const TypeInstPtr* vbox_type) {
 114   assert(vbox_type->klass_is_exact(), "");
 115 
 116   ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass();
 117   assert(is_vector(ik), "not a vector");
 118 
 119   ciField* fd1 = ik->get_field_by_name(ciSymbol::ETYPE_name(), ciSymbol::class_signature(), /* is_static */ true);
 120   assert(fd1 != NULL, "element type info is missing");
 121 
 122   ciConstant val1 = fd1->constant_value();
 123   BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
 124   assert(is_java_primitive(elem_bt), "element type info is missing");
 125 
 126   ciField* fd2 = ik->get_field_by_name(ciSymbol::VLENGTH_name(), ciSymbol::int_signature(), /* is_static */ true);
 127   assert(fd2 != NULL, "vector length info is missing");
 128 
 129   ciConstant val2 = fd2->constant_value();
 130   assert(val2.as_int() > 0, "vector length info is missing");
 131 
 132   return true;
 133 }
 134 #endif
 135 
 136 Node* LibraryCallKit::box_vector(Node* vector, const TypeInstPtr* vbox_type,
 137                                  BasicType elem_bt, int num_elem) {
 138   assert(EnableVectorSupport, "");
 139   const TypeVect* vec_type = TypeVect::make(elem_bt, num_elem);
 140 
 141   VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
 142   set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
 143   make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true);
 144   set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
 145   set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
 146   Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
 147 
 148   assert(check_vbox(vbox_type), "");
 149   VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vec_type);
 150   return gvn().transform(vbox);
 151 }
 152 
 153 Node* LibraryCallKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
 154   assert(EnableVectorSupport, "");
 155   const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr();
 156   if (vbox_type->klass() != vbox_type_v->klass()) {
 157     return NULL; // arguments don't agree on vector shapes
 158   }
 159   if (vbox_type_v->maybe_null()) {
 160     return NULL; // no nulls are allowed
 161   }
 162   assert(check_vbox(vbox_type), "");
 163   const TypeVect* vec_type = TypeVect::make(elem_bt, num_elem);
 164   Node* unbox = gvn().transform(new VectorUnboxNode(C, vec_type, v, merged_memory(), shuffle_to_vector));
 165   return unbox;
 166 }
 167 
 168 // public static
 169 // <VM>
 170 // VM unaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
 171 //            VM vm,
 172 //            Function<VM, VM> defaultImpl) {
 173 //
 174 // public static
 175 // <VM>
 176 // VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
 177 //             VM vm1, VM vm2,
 178 //             BiFunction<VM, VM, VM> defaultImpl) {
 179 //
 180 // public static
 181 // <VM>
 182 // VM ternaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
 183 //              VM vm1, VM vm2, VM vm3,
 184 //              TernaryOperation<VM> defaultImpl) {
 185 //
 186 bool LibraryCallKit::inline_vector_nary_operation(int n) {
 187   const TypeInt* opr              = gvn().type(argument(0))->is_int();
 188   const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr();
 189   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->is_instptr();
 190   const TypeInt* vlen             = gvn().type(argument(3))->is_int();
 191 
 192   if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 193     if (C->print_intrinsics()) {
 194       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
 195                     NodeClassNames[argument(0)->Opcode()],
 196                     NodeClassNames[argument(1)->Opcode()],
 197                     NodeClassNames[argument(2)->Opcode()],
 198                     NodeClassNames[argument(3)->Opcode()]);
 199     }
 200     return false; // not enough info for intrinsification
 201   }
 202   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 203   if (!elem_type->is_primitive_type()) {
 204     if (C->print_intrinsics()) {
 205       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 206     }
 207     return false; // should be primitive type
 208   }
 209   if (!is_klass_initialized(vector_klass)) {
 210     if (C->print_intrinsics()) {
 211       tty->print_cr("  ** klass argument not initialized");
 212     }
 213     return false;
 214   }
 215   BasicType elem_bt = elem_type->basic_type();
 216   int num_elem = vlen->get_con();
 217   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
 218   int sopc = VectorNode::opcode(opc, elem_bt);
 219   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 220   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 221 
 222   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 223   if (!arch_supports_vector(sopc, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseAll : VecMaskNotUsed)) {
 224     if (C->print_intrinsics()) {
 225       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d",
 226                     n, sopc, num_elem, type2name(elem_bt),
 227                     is_vector_mask(vbox_klass) ? 1 : 0);
 228     }
 229     return false; // not supported
 230   }
 231 
 232   Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL;
 233   switch (n) {
 234     case 3: {
 235       opd3 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
 236       if (opd3 == NULL) {
 237         if (C->print_intrinsics()) {
 238           tty->print_cr("  ** unbox failed v3=%s",
 239                         NodeClassNames[argument(6)->Opcode()]);
 240         }
 241         return false;
 242       }
 243       // fall-through
 244     }
 245     case 2: {
 246       opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
 247       if (opd2 == NULL) {
 248         if (C->print_intrinsics()) {
 249           tty->print_cr("  ** unbox failed v2=%s",
 250                         NodeClassNames[argument(5)->Opcode()]);
 251         }
 252         return false;
 253       }
 254       // fall-through
 255     }
 256     case 1: {
 257       opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
 258       if (opd1 == NULL) {
 259         if (C->print_intrinsics()) {
 260           tty->print_cr("  ** unbox failed v1=%s",
 261                         NodeClassNames[argument(4)->Opcode()]);
 262         }
 263         return false;
 264       }
 265       break;
 266     }
 267     default: fatal("unsupported arity: %d", n);
 268   }
 269 
 270   Node* operation = NULL;
 271   const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
 272   switch (n) {
 273     case 1:
 274     case 2: {
 275       operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, vt));
 276       break;
 277     }
 278     case 3: {
 279       operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, opd3, vt));
 280       break;
 281     }
 282     default: fatal("unsupported arity: %d", n);
 283   }
 284   // Wrap it up in VectorBox to keep object type information.
 285   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 286   set_result(vbox);
 287   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 288   return true;
 289 }
 290 
 291 // <Sh extends VectorShuffle<E>,  E>
 292 //  Sh ShuffleIota(Class<?> E, Class<?> ShuffleClass, Vector.Species<E> s, int length,
 293 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 294 bool LibraryCallKit::inline_vector_shuffle_iota() {
 295   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->is_instptr();
 296   const TypeInt* vlen             = gvn().type(argument(3))->is_int();
 297   Node* start                     = argument(4);
 298   const TypeInt* start_val        = gvn().type(start)->is_int();
 299   Node* step                      = argument(5);
 300   const TypeInt* step_val         = gvn().type(step)->is_int();
 301   const TypeInt* wrap             = gvn().type(argument(6))->is_int();
 302 
 303   if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) ||
 304       shuffle_klass->const_oop() == NULL || !wrap->is_con()) {
 305     return false; // not enough info for intrinsification
 306   }
 307   if (!is_klass_initialized(shuffle_klass)) {
 308     if (C->print_intrinsics()) {
 309       tty->print_cr("  ** klass argument not initialized");
 310     }
 311     return false;
 312   }
 313 
 314   int do_wrap = wrap->get_con();
 315   int num_elem = vlen->get_con();
 316   BasicType elem_bt = T_BYTE;
 317 
 318   if (num_elem < 4)
 319     return false;
 320 
 321   if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) {
 322     return false;
 323   }
 324   if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) {
 325     return false;
 326   }
 327   if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
 328     return false;
 329   }
 330   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 331     return false;
 332   }
 333   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
 334     return false;
 335   }
 336 
 337   const Type * type_bt = Type::get_const_basic_type(elem_bt);
 338   const TypeVect * vt  = TypeVect::make(type_bt, num_elem);
 339 
 340   Node* res =  gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
 341 
 342   if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) {
 343     Node* bcast_step     = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
 344     res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt));
 345   } else if (step_val->get_con() > 1) {
 346     Node* cnt = gvn().makecon(TypeInt::make(log2_int(step_val->get_con())));
 347     res = gvn().transform(VectorNode::make(Op_LShiftVB, res, cnt, vt));
 348   }
 349 
 350   if (!start_val->is_con() || start_val->get_con() != 0) {
 351     Node* bcast_start    = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
 352     res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt));
 353   }
 354 
 355   Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
 356   Node * bcast_mod  = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
 357   if(do_wrap)  {
 358     // Wrap the indices greater than lane count.
 359     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 360   } else {
 361     ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(1));
 362     Node * lane_cnt  = gvn().makecon(TypeInt::make(num_elem));
 363     Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
 364     Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vt));
 365 
 366     // Make the indices greater than lane count as -ve values. This matches the java side implementation.
 367     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 368     Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt));
 369     res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
 370   }
 371 
 372   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 373   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 374 
 375   // Wrap it up in VectorBox to keep object type information.
 376   res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
 377   set_result(res);
 378   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 379   return true;
 380 }
 381 
 382 // <VM ,Sh extends VectorShuffle<E>, E>
 383 // VM shuffleToVector(Class<VM> VecClass, Class<?>E , Class<?> ShuffleClass, Sh s, int length,
 384 //                    ShuffleToVectorOperation<VM,Sh,E> defaultImpl)
 385 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
 386   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->is_instptr();
 387   const TypeInstPtr* elem_klass    = gvn().type(argument(1))->is_instptr();
 388   const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->is_instptr();
 389   Node* shuffle                    = argument(3);
 390   const TypeInt* vlen              = gvn().type(argument(4))->is_int();
 391 
 392   if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) {
 393     return false; // not enough info for intrinsification
 394   }
 395   if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
 396     if (C->print_intrinsics()) {
 397       tty->print_cr("  ** klass argument not initialized");
 398     }
 399     return false;
 400   }
 401 
 402   int num_elem = vlen->get_con();
 403   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 404   BasicType elem_bt = elem_type->basic_type();
 405 
 406   if (num_elem < 4) {
 407     return false;
 408   }
 409 
 410   int cast_vopc = VectorCastNode::opcode(T_BYTE); // from shuffle of type T_BYTE
 411   // Make sure that cast is implemented to particular type/size combination.
 412   if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
 413     if (C->print_intrinsics()) {
 414       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 415         cast_vopc, num_elem, type2name(elem_bt));
 416     }
 417     return false;
 418   }
 419 
 420   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 421   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 422 
 423   // Unbox shuffle with true flag to indicate its load shuffle to vector
 424   Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, elem_bt, num_elem, true);
 425 
 426   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 427   const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 428 
 429   // Box vector
 430   Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
 431   set_result(res);
 432   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 433   return true;
 434 }
 435 
 436 // <V extends Vector<?,?>>
 437 // V broadcastCoerced(Class<?> vectorClass, Class<?> elementType, int vlen,
 438 //                    long bits,
 439 //                    LongFunction<V> defaultImpl)
 440 bool LibraryCallKit::inline_vector_broadcast_coerced() {
 441   const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
 442   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->is_instptr();
 443   const TypeInt* vlen             = gvn().type(argument(2))->is_int();
 444 
 445   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 446     if (C->print_intrinsics()) {
 447       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 448                     NodeClassNames[argument(0)->Opcode()],
 449                     NodeClassNames[argument(1)->Opcode()],
 450                     NodeClassNames[argument(2)->Opcode()]);
 451     }
 452     return false; // not enough info for intrinsification
 453   }
 454 
 455   if (!is_klass_initialized(vector_klass)) {
 456     if (C->print_intrinsics()) {
 457       tty->print_cr("  ** klass argument not initialized");
 458     }
 459     return false;
 460   }
 461   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 462   if (!elem_type->is_primitive_type()) {
 463     if (C->print_intrinsics()) {
 464       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 465     }
 466     return false; // should be primitive type
 467   }
 468   BasicType elem_bt = elem_type->basic_type();
 469   int num_elem = vlen->get_con();
 470   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 471   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 472 
 473   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 474   if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt,
 475                             (is_vector_mask(vbox_klass) ? VecMaskUseStore : VecMaskNotUsed), true /*has_scalar_args*/)) {
 476     if (C->print_intrinsics()) {
 477       tty->print_cr("  ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d",
 478                     num_elem, type2name(elem_bt),
 479                     is_vector_mask(vbox_klass) ? 1 : 0);
 480     }
 481     return false; // not supported
 482   }
 483 
 484   Node* bits = argument(3); // long
 485 
 486   Node* elem = NULL;
 487   switch (elem_bt) {
 488     case T_BOOLEAN: // fall-through
 489     case T_BYTE:    // fall-through
 490     case T_SHORT:   // fall-through
 491     case T_CHAR:    // fall-through
 492     case T_INT: {
 493       elem = gvn().transform(new ConvL2INode(bits));
 494       break;
 495     }
 496     case T_DOUBLE: {
 497       elem = gvn().transform(new MoveL2DNode(bits));
 498       break;
 499     }
 500     case T_FLOAT: {
 501       bits = gvn().transform(new ConvL2INode(bits));
 502       elem = gvn().transform(new MoveI2FNode(bits));
 503       break;
 504     }
 505     case T_LONG: {
 506       elem = bits; // no conversion needed
 507       break;
 508     }
 509     default: fatal("%s", type2name(elem_bt));
 510   }
 511 
 512   Node* broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt));
 513   broadcast = gvn().transform(broadcast);
 514 
 515   Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
 516   set_result(box);
 517   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 518   return true;
 519 }
 520 
 521 //    <C, V extends Vector<?,?>>
 522 //    V load(Class<?> vectorClass, Class<?> elementType, int vlen,
 523 //           Object base, long offset,
 524 //           /* Vector.Mask<E,S> m*/
 525 //           Object container, int index,
 526 //           LoadOperation<C, VM> defaultImpl) {
 527 //
 528 //    <C, V extends Vector<?,?>>
 529 //    void store(Class<?> vectorClass, Class<?> elementType, int vlen,
 530 //               Object base, long offset,
 531 //               V v, /*Vector.Mask<E,S> m*/
 532 //               Object container, int index,
 533 //               StoreVectorOperation<C, V> defaultImpl) {
 534 
 535 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 536   const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
 537   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->is_instptr();
 538   const TypeInt* vlen             = gvn().type(argument(2))->is_int();
 539 
 540   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 541     if (C->print_intrinsics()) {
 542       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 543                     NodeClassNames[argument(0)->Opcode()],
 544                     NodeClassNames[argument(1)->Opcode()],
 545                     NodeClassNames[argument(2)->Opcode()]);
 546     }
 547     return false; // not enough info for intrinsification
 548   }
 549   if (!is_klass_initialized(vector_klass)) {
 550     if (C->print_intrinsics()) {
 551       tty->print_cr("  ** klass argument not initialized");
 552     }
 553     return false;
 554   }
 555 
 556   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 557   if (!elem_type->is_primitive_type()) {
 558     if (C->print_intrinsics()) {
 559       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 560     }
 561     return false; // should be primitive type
 562   }
 563   BasicType elem_bt = elem_type->basic_type();
 564   int num_elem = vlen->get_con();
 565 
 566   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 567   if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
 568     if (C->print_intrinsics()) {
 569       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
 570                     is_store, is_store ? "store" : "load",
 571                     num_elem, type2name(elem_bt));
 572     }
 573     return false; // not supported
 574   }
 575 
 576   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 577   bool is_mask = is_vector_mask(vbox_klass);
 578 
 579   Node* base = argument(3);
 580   Node* offset = ConvL2X(argument(4));
 581   DecoratorSet decorators = C2_UNSAFE_ACCESS;
 582   Node* addr = make_unsafe_address(base, offset, decorators, (is_mask ? T_BOOLEAN : elem_bt), true);
 583 
 584   // Can base be NULL? Otherwise, always on-heap access.
 585   bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
 586 
 587   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
 588   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
 589 
 590   // Now handle special case where load/store happens from/to byte array but element type is not byte.
 591   bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
 592   // Handle loading masks.
 593   // If there is no consistency between array and vector element types, it must be special byte array case or loading masks
 594   if (arr_type != NULL && !using_byte_array && elem_bt != arr_type->elem()->array_element_basic_type() && !is_mask) {
 595     return false;
 596   }
 597   // Since we are using byte array, we need to double check that the byte operations are supported by backend.
 598   if (using_byte_array) {
 599     int byte_num_elem = num_elem * type2aelembytes(elem_bt);
 600     if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)) {
 601       if (C->print_intrinsics()) {
 602         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
 603                       is_store, is_store ? "store" : "load",
 604                       byte_num_elem, type2name(elem_bt));
 605       }
 606       return false; // not supported
 607     }
 608   }
 609   if (is_mask) {
 610     if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
 611       if (C->print_intrinsics()) {
 612         tty->print_cr("  ** not supported: arity=%d op=%s/mask vlen=%d etype=bit ismask=no",
 613                       is_store, is_store ? "store" : "load",
 614                       num_elem);
 615       }
 616       return false; // not supported
 617     }
 618     if (!is_store) {
 619       if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
 620         return false; // not supported
 621       }
 622     } else {
 623          if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
 624            return false; // not supported
 625          }
 626     }
 627   }
 628 
 629   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 630 
 631   if (can_access_non_heap) {
 632     insert_mem_bar(Op_MemBarCPUOrder);
 633   }
 634 
 635   if (is_store) {
 636     Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
 637     if (val == NULL) {
 638       return false; // operand unboxing failed
 639     }
 640     set_all_memory(reset_memory());
 641 
 642     // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
 643     int store_num_elem = num_elem;
 644     if (using_byte_array) {
 645       store_num_elem = num_elem * type2aelembytes(elem_bt);
 646       const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
 647       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
 648     }
 649 
 650     Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
 651     set_memory(vstore, addr_type);
 652   } else {
 653     // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
 654     Node* vload = NULL;
 655     if (using_byte_array) {
 656       int load_num_elem = num_elem * type2aelembytes(elem_bt);
 657       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE));
 658       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
 659       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
 660     } else {
 661       // Special handle for masks
 662       if (is_mask) {
 663           vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
 664           const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
 665           vload = gvn().transform(new VectorLoadMaskNode(vload, to_vect_type));
 666       } else {
 667           vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
 668       }
 669     }
 670     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
 671     set_result(box);
 672   }
 673 
 674   if (can_access_non_heap) {
 675     insert_mem_bar(Op_MemBarCPUOrder);
 676   }
 677 
 678   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 679   return true;
 680 }
 681 
 682 //   <C, V extends Vector<?>, W extends IntVector, E, S extends VectorSpecies<E>>
 683 //   void loadWithMap(Class<?> vectorClass, Class<E> E, int length, Class<?> vectorIndexClass,
 684 //                    Object base, long offset, // Unsafe addressing
 685 //                    W index_vector,
 686 //                    C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
 687 //                    LoadVectorOperationWithMap<C, V, E, S> defaultImpl)
 688 //
 689 //    <C, V extends Vector<?>, W extends IntVector>
 690 //    void storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass,
 691 //                      Object base, long offset,    // Unsafe addressing
 692 //                      W index_vector, V v,
 693 //                      C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
 694 //                      StoreVectorOperationWithMap<C, V> defaultImpl) {
 695 //
 696 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
 697   const TypeInstPtr* vector_klass     = gvn().type(argument(0))->is_instptr();
 698   const TypeInstPtr* elem_klass       = gvn().type(argument(1))->is_instptr();
 699   const TypeInt* vlen                 = gvn().type(argument(2))->is_int();
 700   const TypeInstPtr* vector_idx_klass = gvn().type(argument(3))->is_instptr();
 701 
 702   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) {
 703     if (C->print_intrinsics()) {
 704       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
 705                     NodeClassNames[argument(0)->Opcode()],
 706                     NodeClassNames[argument(1)->Opcode()],
 707                     NodeClassNames[argument(2)->Opcode()],
 708                     NodeClassNames[argument(3)->Opcode()]);
 709     }
 710     return false; // not enough info for intrinsification
 711   }
 712 
 713   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
 714     if (C->print_intrinsics()) {
 715       tty->print_cr("  ** klass argument not initialized");
 716     }
 717     return false;
 718   }
 719   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 720   if (!elem_type->is_primitive_type()) {
 721     if (C->print_intrinsics()) {
 722       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 723     }
 724     return false; // should be primitive type
 725   }
 726   BasicType elem_bt = elem_type->basic_type();
 727   int num_elem = vlen->get_con();
 728 
 729   if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
 730     if (C->print_intrinsics()) {
 731       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
 732                     is_scatter, is_scatter ? "scatter" : "gather",
 733                     num_elem, type2name(elem_bt));
 734     }
 735     return false; // not supported
 736   }
 737 
 738   // Check that the vector holding indices is supported by architecture
 739   if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
 740       if (C->print_intrinsics()) {
 741         tty->print_cr("  ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int ismask=no",
 742                       is_scatter, is_scatter ? "scatter" : "gather",
 743                       num_elem);
 744       }
 745       return false; // not supported
 746     }
 747 
 748   Node* base = argument(4);
 749   Node* offset = ConvL2X(argument(5));
 750   Node* addr = make_unsafe_address(base, offset, C2_UNSAFE_ACCESS, elem_bt, true);
 751 
 752   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
 753   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
 754 
 755   // The array must be consistent with vector type
 756   if (arr_type == NULL || (arr_type != NULL && elem_bt != arr_type->elem()->array_element_basic_type())) {
 757     return false;
 758   }
 759   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 760   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 761 
 762   ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
 763 
 764   if (vbox_idx_klass == NULL) {
 765     return false;
 766   }
 767 
 768   const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
 769 
 770   Node* index_vect = unbox_vector(argument(7), vbox_idx_type, T_INT, num_elem);
 771   if (index_vect == NULL) {
 772     return false;
 773   }
 774   const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
 775   if (is_scatter) {
 776     Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
 777     if (val == NULL) {
 778       return false; // operand unboxing failed
 779     }
 780     set_all_memory(reset_memory());
 781 
 782     Node* vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
 783     set_memory(vstore, addr_type);
 784   } else {
 785     Node* vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
 786 
 787     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
 788     set_result(box);
 789   }
 790 
 791   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 792   return true;
 793 }
 794 
 795 // <V extends Vector<?,?>>
 796 // long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int vlen,
 797 //                       V v,
 798 //                       Function<V,Long> defaultImpl)
 799 
 800 bool LibraryCallKit::inline_vector_reduction() {
 801   const TypeInt* opr              = gvn().type(argument(0))->is_int();
 802   const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr();
 803   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->is_instptr();
 804   const TypeInt* vlen             = gvn().type(argument(3))->is_int();
 805 
 806   if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 807     if (C->print_intrinsics()) {
 808       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
 809                     NodeClassNames[argument(0)->Opcode()],
 810                     NodeClassNames[argument(1)->Opcode()],
 811                     NodeClassNames[argument(2)->Opcode()],
 812                     NodeClassNames[argument(3)->Opcode()]);
 813     }
 814     return false; // not enough info for intrinsification
 815   }
 816   if (!is_klass_initialized(vector_klass)) {
 817     if (C->print_intrinsics()) {
 818       tty->print_cr("  ** klass argument not initialized");
 819     }
 820     return false;
 821   }
 822   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 823   if (!elem_type->is_primitive_type()) {
 824     if (C->print_intrinsics()) {
 825       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 826     }
 827     return false; // should be primitive type
 828   }
 829   BasicType elem_bt = elem_type->basic_type();
 830   int num_elem = vlen->get_con();
 831 
 832   int opc  = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
 833   int sopc = ReductionNode::opcode(opc, elem_bt);
 834 
 835   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 836   if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed)) {
 837     if (C->print_intrinsics()) {
 838       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s ismask=no",
 839                     sopc, num_elem, type2name(elem_bt));
 840     }
 841     return false;
 842   }
 843 
 844   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 845   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 846 
 847   Node* opd = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
 848   if (opd == NULL) {
 849     return false; // operand unboxing failed
 850   }
 851 
 852   Node* init = ReductionNode::make_reduction_input(gvn(), opc, elem_bt);
 853   Node* rn = gvn().transform(ReductionNode::make(opc, NULL, init, opd, elem_bt));
 854 
 855   Node* bits = NULL;
 856   switch (elem_bt) {
 857     case T_BYTE:
 858     case T_SHORT:
 859     case T_INT: {
 860       bits = gvn().transform(new ConvI2LNode(rn));
 861       break;
 862     }
 863     case T_FLOAT: {
 864       rn   = gvn().transform(new MoveF2INode(rn));
 865       bits = gvn().transform(new ConvI2LNode(rn));
 866       break;
 867     }
 868     case T_DOUBLE: {
 869       bits = gvn().transform(new MoveD2LNode(rn));
 870       break;
 871     }
 872     case T_LONG: {
 873       bits = rn; // no conversion needed
 874       break;
 875     }
 876     default: fatal("%s", type2name(elem_bt));
 877   }
 878   set_result(bits);
 879   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 880   return true;
 881 }
 882 
 883 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
 884 //                                V v1, V v2,
 885 //                                BiFunction<V, V, Boolean> defaultImpl) {
 886 //
 887 bool LibraryCallKit::inline_vector_test() {
 888   const TypeInt* cond             = gvn().type(argument(0))->is_int();
 889   const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr();
 890   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->is_instptr();
 891   const TypeInt* vlen             = gvn().type(argument(3))->is_int();
 892 
 893   if (!cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 894     if (C->print_intrinsics()) {
 895       tty->print_cr("  ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
 896                     NodeClassNames[argument(0)->Opcode()],
 897                     NodeClassNames[argument(1)->Opcode()],
 898                     NodeClassNames[argument(2)->Opcode()],
 899                     NodeClassNames[argument(3)->Opcode()]);
 900     }
 901     return false; // not enough info for intrinsification
 902   }
 903   if (!is_klass_initialized(vector_klass)) {
 904     if (C->print_intrinsics()) {
 905       tty->print_cr("  ** klass argument not initialized");
 906     }
 907     return false;
 908   }
 909   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 910   if (!elem_type->is_primitive_type()) {
 911     if (C->print_intrinsics()) {
 912       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 913     }
 914     return false; // should be primitive type
 915   }
 916   BasicType elem_bt = elem_type->basic_type();
 917   int num_elem = vlen->get_con();
 918   BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
 919   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 920   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 921 
 922   if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
 923     if (C->print_intrinsics()) {
 924       tty->print_cr("  ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
 925                     cond->get_con(), num_elem, type2name(elem_bt),
 926                     is_vector_mask(vbox_klass));
 927     }
 928     return false;
 929   }
 930 
 931   Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
 932   Node* opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
 933   if (opd1 == NULL || opd2 == NULL) {
 934     return false; // operand unboxing failed
 935   }
 936   Node* test = new VectorTestNode(opd1, opd2, booltest);
 937   test = gvn().transform(test);
 938 
 939   set_result(test);
 940   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 941   return true;
 942 }
 943 
 944 // public static
 945 // <V extends Vector, M extends Mask>
 946 // V blend(Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int vlen,
 947 //         V v1, V v2, M m,
 948 //         VectorBlendOp<V,M> defaultImpl) { ...
 949 //
 950 bool LibraryCallKit::inline_vector_blend() {
 951   const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
 952   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->is_instptr();
 953   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->is_instptr();
 954   const TypeInt*     vlen         = gvn().type(argument(3))->is_int();
 955 
 956   if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
 957       elem_klass->const_oop() == NULL || !vlen->is_con()) {
 958     if (C->print_intrinsics()) {
 959       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
 960                     NodeClassNames[argument(0)->Opcode()],
 961                     NodeClassNames[argument(1)->Opcode()],
 962                     NodeClassNames[argument(2)->Opcode()],
 963                     NodeClassNames[argument(3)->Opcode()]);
 964     }
 965     return false; // not enough info for intrinsification
 966   }
 967   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
 968     if (C->print_intrinsics()) {
 969       tty->print_cr("  ** klass argument not initialized");
 970     }
 971     return false;
 972   }
 973   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 974   if (!elem_type->is_primitive_type()) {
 975     if (C->print_intrinsics()) {
 976       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 977     }
 978     return false; // should be primitive type
 979   }
 980   BasicType elem_bt = elem_type->basic_type();
 981   BasicType mask_bt = elem_bt;
 982   int num_elem = vlen->get_con();
 983 
 984   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 985     if (C->print_intrinsics()) {
 986       tty->print_cr("  ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
 987                     num_elem, type2name(elem_bt));
 988     }
 989     return false; // not supported
 990   }
 991   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 992   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 993 
 994   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 995   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 996 
 997   Node* v1   = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
 998   Node* v2   = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
 999   Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1000 
1001   if (v1 == NULL || v2 == NULL || mask == NULL) {
1002     return false; // operand unboxing failed
1003   }
1004 
1005   Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1006 
1007   Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1008   set_result(box);
1009   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1010   return true;
1011 }
1012 
1013 //  public static <V extends Vector<E,S>,
1014 //          M extends Vector.Mask<E,S>,
1015 //          S extends Vector.Shape, E>
1016 //  M compare(int cond, Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int vlen,
1017 //            V v1, V v2,
1018 //            VectorCompareOp<V,M> defaultImpl) { ...
1019 //
1020 bool LibraryCallKit::inline_vector_compare() {
1021   const TypeInt*     cond         = gvn().type(argument(0))->is_int();
1022   const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr();
1023   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->is_instptr();
1024   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->is_instptr();
1025   const TypeInt*     vlen         = gvn().type(argument(4))->is_int();
1026 
1027   if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1028       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1029     if (C->print_intrinsics()) {
1030       tty->print_cr("  ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1031                     NodeClassNames[argument(0)->Opcode()],
1032                     NodeClassNames[argument(1)->Opcode()],
1033                     NodeClassNames[argument(2)->Opcode()],
1034                     NodeClassNames[argument(3)->Opcode()],
1035                     NodeClassNames[argument(4)->Opcode()]);
1036     }
1037     return false; // not enough info for intrinsification
1038   }
1039   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1040     if (C->print_intrinsics()) {
1041       tty->print_cr("  ** klass argument not initialized");
1042     }
1043     return false;
1044   }
1045   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1046   if (!elem_type->is_primitive_type()) {
1047     if (C->print_intrinsics()) {
1048       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1049     }
1050     return false; // should be primitive type
1051   }
1052 
1053   int num_elem = vlen->get_con();
1054   BasicType elem_bt = elem_type->basic_type();
1055   BasicType mask_bt = elem_bt;
1056 
1057   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1058     if (C->print_intrinsics()) {
1059       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1060                     cond->get_con(), num_elem, type2name(elem_bt));
1061     }
1062     return false;
1063   }
1064 
1065   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1066   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1067 
1068   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1069   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1070 
1071   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1072   Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1073 
1074   if (v1 == NULL || v2 == NULL) {
1075     return false; // operand unboxing failed
1076   }
1077   BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1078   ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1079 
1080   const TypeVect* vt = TypeVect::make(mask_bt, num_elem);
1081   Node* operation = gvn().transform(new VectorMaskCmpNode(pred, v1, v2, pred_node, vt));
1082 
1083   Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1084   set_result(box);
1085   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1086   return true;
1087 }
1088 
1089 // public static
1090 // <V extends Vector, Sh extends Shuffle>
1091 //  V rearrangeOp(Class<V> vectorClass, Class<Sh> shuffleClass, Class< ? > elementType, int vlen,
1092 //    V v1, Sh sh,
1093 //    VectorSwizzleOp<V, Sh, S, E> defaultImpl) { ...
1094 
1095 bool LibraryCallKit::inline_vector_rearrange() {
1096   const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
1097   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->is_instptr();
1098   const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
1099   const TypeInt*     vlen = gvn().type(argument(3))->is_int();
1100 
1101   if (shuffle_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1102     elem_klass->const_oop() == NULL || !vlen->is_con()) {
1103     if (C->print_intrinsics()) {
1104       tty->print_cr("  ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1105                     NodeClassNames[argument(0)->Opcode()],
1106                     NodeClassNames[argument(1)->Opcode()],
1107                     NodeClassNames[argument(2)->Opcode()],
1108                     NodeClassNames[argument(3)->Opcode()]);
1109     }
1110     return false; // not enough info for intrinsification
1111   }
1112   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(shuffle_klass)) {
1113     if (C->print_intrinsics()) {
1114       tty->print_cr("  ** klass argument not initialized");
1115     }
1116     return false;
1117   }
1118   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1119   if (!elem_type->is_primitive_type()) {
1120     if (C->print_intrinsics()) {
1121       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1122     }
1123     return false; // should be primitive type
1124   }
1125   BasicType elem_bt = elem_type->basic_type();
1126   BasicType shuffle_bt = elem_bt;
1127   int num_elem = vlen->get_con();
1128 
1129   if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1130     if (C->print_intrinsics()) {
1131       tty->print_cr("  ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1132                     num_elem, type2name(elem_bt));
1133     }
1134     return false; // not supported
1135   }
1136   if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed)) {
1137     if (C->print_intrinsics()) {
1138       tty->print_cr("  ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1139                     num_elem, type2name(elem_bt));
1140     }
1141     return false; // not supported
1142   }
1143   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1144   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1145 
1146   ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1147   const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1148 
1149   Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1150   Node* shuffle = unbox_vector(argument(5), shbox_type, shuffle_bt, num_elem);
1151 
1152   if (v1 == NULL || shuffle == NULL) {
1153     return false; // operand unboxing failed
1154   }
1155 
1156   Node* rearrange = gvn().transform(new VectorRearrangeNode(v1, shuffle));
1157 
1158   Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
1159   set_result(box);
1160   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1161   return true;
1162 }
1163 
1164 Node* LibraryCallKit::shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
1165   assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
1166   juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
1167   Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
1168   Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
1169   return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
1170 }
1171 
1172 //  public static
1173 //  <V extends Vector<?,?>>
1174 //  V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int vlen,
1175 //                 V v, int i,
1176 //                 VectorBroadcastIntOp<V> defaultImpl) {
1177 //
1178 bool LibraryCallKit::inline_vector_broadcast_int() {
1179   const TypeInt* opr              = gvn().type(argument(0))->is_int();
1180   const TypeInstPtr* vector_klass = gvn().type(argument(1))->is_instptr();
1181   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->is_instptr();
1182   const TypeInt* vlen             = gvn().type(argument(3))->is_int();
1183 
1184   if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1185     if (C->print_intrinsics()) {
1186       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1187                     NodeClassNames[argument(0)->Opcode()],
1188                     NodeClassNames[argument(1)->Opcode()],
1189                     NodeClassNames[argument(2)->Opcode()],
1190                     NodeClassNames[argument(3)->Opcode()]);
1191     }
1192     return false; // not enough info for intrinsification
1193   }
1194   if (!is_klass_initialized(vector_klass)) {
1195     if (C->print_intrinsics()) {
1196       tty->print_cr("  ** klass argument not initialized");
1197     }
1198     return false;
1199   }
1200   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1201   if (!elem_type->is_primitive_type()) {
1202     if (C->print_intrinsics()) {
1203       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1204     }
1205     return false; // should be primitive type
1206   }
1207   BasicType elem_bt = elem_type->basic_type();
1208   int num_elem = vlen->get_con();
1209   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1210   int sopc = VectorNode::opcode(opc, elem_bt);
1211   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1212   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1213 
1214   if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, true /*has_scalar_args*/)) {
1215     if (C->print_intrinsics()) {
1216       tty->print_cr("  ** not supported: arity=0 op=int/%d vlen=%d etype=%s ismask=no",
1217                     sopc, num_elem, type2name(elem_bt));
1218     }
1219     return false; // not supported
1220   }
1221   Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1222   Node* opd2 = shift_count(argument(5), opc, elem_bt, num_elem);
1223   if (opd1 == NULL || opd2 == NULL) {
1224     return false;
1225   }
1226   Node* operation = gvn().transform(VectorNode::make(opc, opd1, opd2, num_elem, elem_bt));
1227 
1228   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
1229   set_result(vbox);
1230   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1231   return true;
1232 }
1233 
1234 // public static <VOUT extends VectorPayload,
1235 //                 VIN extends VectorPayload,
1236 //                   S extends VectorSpecies>
1237 // VOUT convert(int oprId,
1238 //           Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
1239 //           Class<?>   toVectorClass, Class<?>   toElementType, int   toVLen,
1240 //           VIN v, S s,
1241 //           VectorConvertOp<VOUT, VIN, S> defaultImpl) {
1242 //
1243 bool LibraryCallKit::inline_vector_convert() {
1244   const TypeInt*     opr               = gvn().type(argument(0))->is_int();
1245 
1246   const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->is_instptr();
1247   const TypeInstPtr* elem_klass_from   = gvn().type(argument(2))->is_instptr();
1248   const TypeInt*     vlen_from         = gvn().type(argument(3))->is_int();
1249 
1250   const TypeInstPtr* vector_klass_to   = gvn().type(argument(4))->is_instptr();
1251   const TypeInstPtr* elem_klass_to     = gvn().type(argument(5))->is_instptr();
1252   const TypeInt*     vlen_to           = gvn().type(argument(6))->is_int();
1253 
1254   if (!opr->is_con() ||
1255       vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() ||
1256       vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) {
1257     if (C->print_intrinsics()) {
1258       tty->print_cr("  ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
1259                     NodeClassNames[argument(0)->Opcode()],
1260                     NodeClassNames[argument(1)->Opcode()],
1261                     NodeClassNames[argument(2)->Opcode()],
1262                     NodeClassNames[argument(3)->Opcode()],
1263                     NodeClassNames[argument(4)->Opcode()],
1264                     NodeClassNames[argument(5)->Opcode()],
1265                     NodeClassNames[argument(6)->Opcode()]);
1266     }
1267     return false; // not enough info for intrinsification
1268   }
1269   if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
1270     if (C->print_intrinsics()) {
1271       tty->print_cr("  ** klass argument not initialized");
1272     }
1273     return false;
1274   }
1275 
1276   assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
1277          opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
1278   bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST);
1279 
1280   ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
1281   ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
1282   if (is_vector_shuffle(vbox_klass_from) || is_vector_shuffle(vbox_klass_to)) {
1283     return false; // vector shuffles aren't supported
1284   }
1285   bool is_mask = is_vector_mask(vbox_klass_from);
1286 
1287   ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
1288   if (!elem_type_from->is_primitive_type()) {
1289     return false; // should be primitive type
1290   }
1291   BasicType elem_bt_from = elem_type_from->basic_type();
1292   ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
1293   if (!elem_type_to->is_primitive_type()) {
1294     return false; // should be primitive type
1295   }
1296   BasicType elem_bt_to = elem_type_to->basic_type();
1297   if (is_mask && elem_bt_from != elem_bt_to) {
1298     return false; // type mismatch
1299   }
1300   int num_elem_from = vlen_from->get_con();
1301   int num_elem_to = vlen_to->get_con();
1302 
1303   // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
1304   // since we may need to change size.
1305   if (!arch_supports_vector(Op_VectorReinterpret,
1306                             num_elem_from,
1307                             elem_bt_from,
1308                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
1309     if (C->print_intrinsics()) {
1310       tty->print_cr("  ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
1311                     is_cast ? "cast" : "reinterpret",
1312                     num_elem_from, type2name(elem_bt_from), is_mask);
1313     }
1314     return false;
1315   }
1316 
1317   // Check whether we can support resizing/reinterpreting to the new size.
1318   if (!arch_supports_vector(Op_VectorReinterpret,
1319                             num_elem_to,
1320                             elem_bt_to,
1321                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
1322     if (C->print_intrinsics()) {
1323       tty->print_cr("  ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
1324                     is_cast ? "cast" : "reinterpret",
1325                     num_elem_to, type2name(elem_bt_to), is_mask);
1326     }
1327     return false;
1328   }
1329 
1330   // At this point, we know that both input and output vector registers are supported
1331   // by the architecture. Next check if the casted type is simply to same type - which means
1332   // that it is actually a resize and not a cast.
1333   if (is_cast && elem_bt_from == elem_bt_to) {
1334     is_cast = false;
1335   }
1336 
1337   const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
1338 
1339   Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
1340   if (opd1 == NULL) {
1341     return false;
1342   }
1343 
1344   const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from);
1345   const TypeVect* dst_type = TypeVect::make(elem_bt_to,   num_elem_to);
1346 
1347   Node* op = opd1;
1348   if (is_cast) {
1349     assert(!is_mask, "masks cannot be casted");
1350     int cast_vopc = VectorCastNode::opcode(elem_bt_from);
1351     // Make sure that cast is implemented to particular type/size combination.
1352     if (!arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
1353       if (C->print_intrinsics()) {
1354         tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
1355                       cast_vopc,
1356                       num_elem_to, type2name(elem_bt_to), is_mask);
1357       }
1358       return false;
1359     }
1360 
1361     if (num_elem_from < num_elem_to) {
1362       // Since input and output number of elements are not consistent, we need to make sure we
1363       // properly size. Thus, first make a cast that retains the number of elements from source.
1364       // In case the size exceeds the arch size, we do the minimum.
1365       int num_elem_for_cast = MIN2(num_elem_from, Matcher::max_vector_size(elem_bt_to));
1366 
1367       // It is possible that arch does not support this intermediate vector size
1368       // TODO More complex logic required here to handle this corner case for the sizes.
1369       if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
1370         if (C->print_intrinsics()) {
1371           tty->print_cr("  ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
1372                         cast_vopc,
1373                         num_elem_for_cast, type2name(elem_bt_to), is_mask);
1374         }
1375         return false;
1376       }
1377 
1378       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
1379       // Now ensure that the destination gets properly resized to needed size.
1380       op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
1381     } else if (num_elem_from > num_elem_to) {
1382       // Since number elements from input is larger than output, simply reduce size of input (we are supposed to
1383       // drop top elements anyway).
1384       int num_elem_for_resize = MAX2(num_elem_to, Matcher::min_vector_size(elem_bt_to));
1385 
1386       // It is possible that arch does not support this intermediate vector size
1387       // TODO More complex logic required here to handle this corner case for the sizes.
1388       if (!arch_supports_vector(Op_VectorReinterpret,
1389                                 num_elem_for_resize,
1390                                 elem_bt_from,
1391                                 VecMaskNotUsed)) {
1392         if (C->print_intrinsics()) {
1393           tty->print_cr("  ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
1394                         num_elem_for_resize, type2name(elem_bt_from), is_mask);
1395         }
1396         return false;
1397       }
1398 
1399       op = gvn().transform(new VectorReinterpretNode(op,
1400                                                      src_type,
1401                                                      TypeVect::make(elem_bt_from,
1402                                                                     num_elem_for_resize)));
1403       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
1404     } else {
1405       // Since input and output number of elements match, and since we know this vector size is
1406       // supported, simply do a cast with no resize needed.
1407       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
1408     }
1409   } else if (Type::cmp(src_type, dst_type) != 0) {
1410     assert(!is_cast, "must be reinterpret");
1411     op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
1412   }
1413 
1414   const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
1415   Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
1416   set_result(vbox);
1417   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
1418   return true;
1419 }
1420 
1421 //  public static
1422 //  <V extends Vector<?>>
1423 //  V insert(Class<? extends V> vectorClass, Class<?> elementType, int vlen,
1424 //           V vec, int ix, long val,
1425 //           VecInsertOp<V> defaultImpl) {
1426 //
1427 bool LibraryCallKit::inline_vector_insert() {
1428   const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
1429   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->is_instptr();
1430   const TypeInt* vlen             = gvn().type(argument(2))->is_int();
1431   const TypeInt* idx              = gvn().type(argument(4))->is_int();
1432 
1433   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
1434     if (C->print_intrinsics()) {
1435       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
1436                     NodeClassNames[argument(0)->Opcode()],
1437                     NodeClassNames[argument(1)->Opcode()],
1438                     NodeClassNames[argument(2)->Opcode()],
1439                     NodeClassNames[argument(4)->Opcode()]);
1440     }
1441     return false; // not enough info for intrinsification
1442   }
1443   if (!is_klass_initialized(vector_klass)) {
1444     if (C->print_intrinsics()) {
1445       tty->print_cr("  ** klass argument not initialized");
1446     }
1447     return false;
1448   }
1449   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1450   if (!elem_type->is_primitive_type()) {
1451     if (C->print_intrinsics()) {
1452       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1453     }
1454     return false; // should be primitive type
1455   }
1456   BasicType elem_bt = elem_type->basic_type();
1457   int num_elem = vlen->get_con();
1458   if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
1459     if (C->print_intrinsics()) {
1460       tty->print_cr("  ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
1461                     num_elem, type2name(elem_bt));
1462     }
1463     return false; // not supported
1464   }
1465 
1466   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1467   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1468 
1469   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
1470   if (opd == NULL) {
1471     return false;
1472   }
1473 
1474   Node* insert_val = argument(5);
1475   assert(gvn().type(insert_val)->isa_long() != NULL, "expected to be long");
1476 
1477   // Convert insert value back to its appropriate type.
1478   switch (elem_bt) {
1479     case T_BYTE:
1480       insert_val = gvn().transform(new ConvL2INode(insert_val));
1481       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
1482       break;
1483     case T_SHORT:
1484       insert_val = gvn().transform(new ConvL2INode(insert_val));
1485       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
1486       break;
1487     case T_INT:
1488       insert_val = gvn().transform(new ConvL2INode(insert_val));
1489       break;
1490     case T_FLOAT:
1491       insert_val = gvn().transform(new ConvL2INode(insert_val));
1492       insert_val = gvn().transform(new MoveI2FNode(insert_val));
1493       break;
1494     case T_DOUBLE:
1495       insert_val = gvn().transform(new MoveL2DNode(insert_val));
1496       break;
1497     case T_LONG:
1498       // no conversion needed
1499       break;
1500     default: fatal("%s", type2name(elem_bt)); break;
1501   }
1502 
1503   Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con()));
1504 
1505   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
1506   set_result(vbox);
1507   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1508   return true;
1509 }
1510 
1511 //  public static
1512 //  <V extends Vector<?>>
1513 //  long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
1514 //               V vec, int ix,
1515 //               VecExtractOp<V> defaultImpl) {
1516 //
1517 bool LibraryCallKit::inline_vector_extract() {
1518   const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
1519   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->is_instptr();
1520   const TypeInt* vlen             = gvn().type(argument(2))->is_int();
1521   const TypeInt* idx              = gvn().type(argument(4))->is_int();
1522 
1523   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
1524     if (C->print_intrinsics()) {
1525       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
1526                     NodeClassNames[argument(0)->Opcode()],
1527                     NodeClassNames[argument(1)->Opcode()],
1528                     NodeClassNames[argument(2)->Opcode()],
1529                     NodeClassNames[argument(4)->Opcode()]);
1530     }
1531     return false; // not enough info for intrinsification
1532   }
1533   if (!is_klass_initialized(vector_klass)) {
1534     if (C->print_intrinsics()) {
1535       tty->print_cr("  ** klass argument not initialized");
1536     }
1537     return false;
1538   }
1539   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1540   if (!elem_type->is_primitive_type()) {
1541     if (C->print_intrinsics()) {
1542       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1543     }
1544     return false; // should be primitive type
1545   }
1546   BasicType elem_bt = elem_type->basic_type();
1547   int num_elem = vlen->get_con();
1548   int vopc = ExtractNode::opcode(elem_bt);
1549   if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
1550     if (C->print_intrinsics()) {
1551       tty->print_cr("  ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
1552                     num_elem, type2name(elem_bt));
1553     }
1554     return false; // not supported
1555   }
1556 
1557   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1558   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1559 
1560   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
1561   if (opd == NULL) {
1562     return false;
1563   }
1564 
1565   Node* operation = gvn().transform(ExtractNode::make(opd, idx->get_con(), elem_bt));
1566 
1567   Node* bits = NULL;
1568   switch (elem_bt) {
1569     case T_BYTE:
1570     case T_SHORT:
1571     case T_INT: {
1572       bits = gvn().transform(new ConvI2LNode(operation));
1573       break;
1574     }
1575     case T_FLOAT: {
1576       bits = gvn().transform(new MoveF2INode(operation));
1577       bits = gvn().transform(new ConvI2LNode(bits));
1578       break;
1579     }
1580     case T_DOUBLE: {
1581       bits = gvn().transform(new MoveD2LNode(operation));
1582       break;
1583     }
1584     case T_LONG: {
1585       bits = operation; // no conversion needed
1586       break;
1587     }
1588     default: fatal("%s", type2name(elem_bt));
1589   }
1590 
1591   set_result(bits);
1592   return true;
1593 }
1594