1 /*
   2  * Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "memory/allocation.inline.hpp"
  26 #include "opto/connode.hpp"
  27 #include "opto/cfgnode.hpp"
  28 #include "opto/vectornode.hpp"
  29 
  30 //------------------------------VectorNode--------------------------------------
  31 
  32 // Return the vector operator for the specified scalar operation
  33 // and vector length.
  34 int VectorNode::opcode(int sopc, BasicType bt) {
  35   switch (sopc) {
  36   case Op_AddI:
  37     switch (bt) {
  38     case T_BOOLEAN:
  39     case T_BYTE:      return Op_AddVB;
  40     case T_CHAR:
  41     case T_SHORT:     return Op_AddVS;
  42     case T_INT:       return Op_AddVI;
  43     default:          ShouldNotReachHere(); return 0;
  44     }
  45   case Op_AddL:
  46     assert(bt == T_LONG, "must be");
  47     return Op_AddVL;
  48   case Op_AddF:
  49     assert(bt == T_FLOAT, "must be");
  50     return Op_AddVF;
  51   case Op_AddD:
  52     assert(bt == T_DOUBLE, "must be");
  53     return Op_AddVD;
  54   case Op_SubI:
  55     switch (bt) {
  56     case T_BOOLEAN:
  57     case T_BYTE:   return Op_SubVB;
  58     case T_CHAR:
  59     case T_SHORT:  return Op_SubVS;
  60     case T_INT:    return Op_SubVI;
  61     default:       ShouldNotReachHere(); return 0;
  62     }
  63   case Op_SubL:
  64     assert(bt == T_LONG, "must be");
  65     return Op_SubVL;
  66   case Op_SubF:
  67     assert(bt == T_FLOAT, "must be");
  68     return Op_SubVF;
  69   case Op_SubD:
  70     assert(bt == T_DOUBLE, "must be");
  71     return Op_SubVD;
  72   case Op_MulI:
  73     switch (bt) {
  74     case T_BOOLEAN:
  75     case T_BYTE:   return Op_MulVB;
  76     case T_CHAR:
  77     case T_SHORT:  return Op_MulVS;
  78     case T_INT:    return Op_MulVI;
  79     default:       ShouldNotReachHere(); return 0;
  80     }
  81   case Op_MulL:
  82     assert(bt == T_LONG, "must be");
  83     return Op_MulVL;
  84   case Op_MulF:
  85     assert(bt == T_FLOAT, "must be");
  86     return Op_MulVF;
  87   case Op_MulD:
  88     assert(bt == T_DOUBLE, "must be");
  89     return Op_MulVD;
  90   case Op_FmaD:
  91     assert(bt == T_DOUBLE, "must be");
  92     return Op_FmaVD;
  93   case Op_FmaF:
  94     assert(bt == T_FLOAT, "must be");
  95     return Op_FmaVF;
  96   case Op_CMoveF:
  97     assert(bt == T_FLOAT, "must be");
  98     return Op_CMoveVF;
  99   case Op_CMoveD:
 100     assert(bt == T_DOUBLE, "must be");
 101     return Op_CMoveVD;
 102   case Op_DivF:
 103     assert(bt == T_FLOAT, "must be");
 104     return Op_DivVF;
 105   case Op_DivD:
 106     assert(bt == T_DOUBLE, "must be");
 107     return Op_DivVD;
 108   case Op_MinI:
 109     switch (bt) {
 110     case T_BOOLEAN:
 111     case T_CHAR:   return 0;
 112     case T_BYTE:
 113     case T_SHORT:
 114     case T_INT:    return Op_MinV;
 115     default:       ShouldNotReachHere(); return 0;
 116     }
 117   case Op_MinL:
 118     assert(bt == T_LONG, "must be");
 119     return Op_MinV;
 120   case Op_MinF:
 121     assert(bt == T_FLOAT, "must be");
 122     return Op_MinV;
 123   case Op_MinD:
 124     assert(bt == T_DOUBLE, "must be");
 125     return Op_MinV;
 126   case Op_MaxI:
 127     switch (bt) {
 128     case T_BOOLEAN:
 129     case T_CHAR:   return 0;
 130     case T_BYTE:
 131     case T_SHORT:
 132     case T_INT:    return Op_MaxV;
 133     default:       ShouldNotReachHere(); return 0;
 134     }
 135   case Op_MaxL:
 136     assert(bt == T_LONG, "must be");
 137     return Op_MaxV;
 138   case Op_MaxF:
 139     assert(bt == T_FLOAT, "must be");
 140     return Op_MaxV;
 141   case Op_MaxD:
 142     assert(bt == T_DOUBLE, "must be");
 143     return Op_MaxV;
 144   case Op_AbsI:
 145     switch (bt) {
 146     case T_BOOLEAN:
 147     case T_CHAR:  return 0; // abs does not make sense for unsigned
 148     case T_BYTE:
 149     case T_SHORT:
 150     case T_INT:
 151     case T_LONG:  return Op_AbsV;
 152     default: ShouldNotReachHere(); return 0;
 153     }
 154   case Op_AbsF:
 155     assert(bt == T_FLOAT, "must be");
 156     return Op_AbsVF;
 157   case Op_AbsD:
 158     assert(bt == T_DOUBLE, "must be");
 159     return Op_AbsVD;
 160   case Op_NegI:
 161     assert(bt == T_INT, "must be");
 162     return Op_NegVI;
 163   case Op_NegF:
 164     assert(bt == T_FLOAT, "must be");
 165     return Op_NegVF;
 166   case Op_NegD:
 167     assert(bt == T_DOUBLE, "must be");
 168     return Op_NegVD;
 169   case Op_SqrtF:
 170     assert(bt == T_FLOAT, "must be");
 171     return Op_SqrtVF;
 172   case Op_SqrtD:
 173     assert(bt == T_DOUBLE, "must be");
 174     return Op_SqrtVD;
 175   case Op_Not:
 176     return Op_NotV;
 177   case Op_PopCountI:
 178     if (bt == T_INT) {
 179       return Op_PopCountVI;
 180     }
 181     // Unimplemented for subword types since bit count changes
 182     // depending on size of lane (and sign bit).
 183     return 0;
 184   case Op_LShiftI:
 185     switch (bt) {
 186     case T_BOOLEAN:
 187     case T_BYTE:   return Op_LShiftVB;
 188     case T_CHAR:
 189     case T_SHORT:  return Op_LShiftVS;
 190     case T_INT:    return Op_LShiftVI;
 191       default:       ShouldNotReachHere(); return 0;
 192     }
 193   case Op_LShiftL:
 194     assert(bt == T_LONG, "must be");
 195     return Op_LShiftVL;
 196   case Op_RShiftI:
 197     switch (bt) {
 198     case T_BOOLEAN:return Op_URShiftVB; // boolean is unsigned value
 199     case T_CHAR:   return Op_URShiftVS; // char is unsigned value
 200     case T_BYTE:   return Op_RShiftVB;
 201     case T_SHORT:  return Op_RShiftVS;
 202     case T_INT:    return Op_RShiftVI;
 203     default:       ShouldNotReachHere(); return 0;
 204     }
 205   case Op_RShiftL:
 206     assert(bt == T_LONG, "must be");
 207     return Op_RShiftVL;
 208   case Op_URShiftB:
 209     assert(bt == T_BYTE, "must be");
 210     return Op_URShiftVB;
 211   case Op_URShiftS:
 212     assert(bt == T_SHORT, "must be");
 213     return Op_URShiftVS;
 214   case Op_URShiftI:
 215     switch (bt) {
 216     case T_BOOLEAN:return Op_URShiftVB;
 217     case T_CHAR:   return Op_URShiftVS;
 218     case T_BYTE:
 219     case T_SHORT:  return 0; // Vector logical right shift for signed short
 220                              // values produces incorrect Java result for
 221                              // negative data because java code should convert
 222                              // a short value into int value with sign
 223                              // extension before a shift.
 224     case T_INT:    return Op_URShiftVI;
 225     default:       ShouldNotReachHere(); return 0;
 226     }
 227   case Op_URShiftL:
 228     assert(bt == T_LONG, "must be");
 229     return Op_URShiftVL;
 230   case Op_AndI:
 231   case Op_AndL:
 232     return Op_AndV;
 233   case Op_OrI:
 234   case Op_OrL:
 235     return Op_OrV;
 236   case Op_XorI:
 237   case Op_XorL:
 238     return Op_XorV;
 239 
 240   case Op_LoadB:
 241   case Op_LoadUB:
 242   case Op_LoadUS:
 243   case Op_LoadS:
 244   case Op_LoadI:
 245   case Op_LoadL:
 246   case Op_LoadF:
 247   case Op_LoadD:
 248     return Op_LoadVector;
 249 
 250   case Op_StoreB:
 251   case Op_StoreC:
 252   case Op_StoreI:
 253   case Op_StoreL:
 254   case Op_StoreF:
 255   case Op_StoreD:
 256     return Op_StoreVector;
 257   case Op_MulAddS2I:
 258     return Op_MulAddVS2VI;
 259 
 260   default:
 261     return 0; // Unimplemented
 262   }
 263 }
 264 
 265 int VectorNode::replicate_opcode(BasicType bt) {
 266   switch(bt) {
 267     case T_BOOLEAN:
 268     case T_BYTE:
 269       return Op_ReplicateB;
 270     case T_SHORT:
 271     case T_CHAR:
 272       return Op_ReplicateS;
 273     case T_INT:
 274       return Op_ReplicateI;
 275     case T_LONG:
 276       return Op_ReplicateL;
 277     case T_FLOAT:
 278       return Op_ReplicateF;
 279     case T_DOUBLE:
 280       return Op_ReplicateD;
 281     default:
 282       break;
 283   }
 284 
 285   return 0;
 286 }
 287 
 288 // Also used to check if the code generator
 289 // supports the vector operation.
 290 bool VectorNode::implemented(int opc, uint vlen, BasicType bt) {
 291   if (is_java_primitive(bt) &&
 292       (vlen > 1) && is_power_of_2(vlen) &&
 293       Matcher::vector_size_supported(bt, vlen)) {
 294     int vopc = VectorNode::opcode(opc, bt);
 295     return vopc > 0 && Matcher::match_rule_supported_vector(vopc, vlen, bt);
 296   }
 297   return false;
 298 }
 299 
 300 bool VectorNode::is_type_transition_short_to_int(Node* n) {
 301   switch (n->Opcode()) {
 302   case Op_MulAddS2I:
 303     return true;
 304   }
 305   return false;
 306 }
 307 
 308 bool VectorNode::is_type_transition_to_int(Node* n) {
 309   return is_type_transition_short_to_int(n);
 310 }
 311 
 312 bool VectorNode::is_muladds2i(Node* n) {
 313   if (n->Opcode() == Op_MulAddS2I) {
 314     return true;
 315   }
 316   return false;
 317 }
 318 
 319 bool VectorNode::is_shift(Node* n) {
 320   switch (n->Opcode()) {
 321   case Op_LShiftI:
 322   case Op_LShiftL:
 323   case Op_RShiftI:
 324   case Op_RShiftL:
 325   case Op_URShiftI:
 326   case Op_URShiftL:
 327     return true;
 328   default:
 329     return false;
 330   }
 331 }
 332 
 333 bool VectorNode::is_vshift(Node* n) {
 334   switch (n->Opcode()) {
 335   case Op_LShiftVB:
 336   case Op_LShiftVS:
 337   case Op_LShiftVI:
 338   case Op_LShiftVL:
 339 
 340   case Op_RShiftVB:
 341   case Op_RShiftVS:
 342   case Op_RShiftVI:
 343   case Op_RShiftVL:
 344 
 345   case Op_URShiftVB:
 346   case Op_URShiftVS:
 347   case Op_URShiftVI:
 348   case Op_URShiftVL:
 349     return true;
 350   default:
 351     return false;
 352   }
 353 }
 354 
 355 bool VectorNode::is_vshift_cnt(Node* n) {
 356   switch (n->Opcode()) {
 357   case Op_LShiftCntV:
 358   case Op_RShiftCntV:
 359     return true;
 360   default:
 361     return false;
 362   }
 363 }
 364 
 365 // Check if input is loop invariant vector.
 366 bool VectorNode::is_invariant_vector(Node* n) {
 367   // Only Replicate vector nodes are loop invariant for now.
 368   switch (n->Opcode()) {
 369   case Op_ReplicateB:
 370   case Op_ReplicateS:
 371   case Op_ReplicateI:
 372   case Op_ReplicateL:
 373   case Op_ReplicateF:
 374   case Op_ReplicateD:
 375     return true;
 376   default:
 377     return false;
 378   }
 379 }
 380 
 381 // [Start, end) half-open range defining which operands are vectors
 382 void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
 383   switch (n->Opcode()) {
 384   case Op_LoadB:   case Op_LoadUB:
 385   case Op_LoadS:   case Op_LoadUS:
 386   case Op_LoadI:   case Op_LoadL:
 387   case Op_LoadF:   case Op_LoadD:
 388   case Op_LoadP:   case Op_LoadN:
 389   case Op_LoadBarrierSlowReg:
 390   case Op_LoadBarrierWeakSlowReg:
 391     *start = 0;
 392     *end   = 0; // no vector operands
 393     break;
 394   case Op_StoreB:  case Op_StoreC:
 395   case Op_StoreI:  case Op_StoreL:
 396   case Op_StoreF:  case Op_StoreD:
 397   case Op_StoreP:  case Op_StoreN:
 398     *start = MemNode::ValueIn;
 399     *end   = MemNode::ValueIn + 1; // 1 vector operand
 400     break;
 401   case Op_LShiftI:  case Op_LShiftL:
 402   case Op_RShiftI:  case Op_RShiftL:
 403   case Op_URShiftI: case Op_URShiftL:
 404     *start = 1;
 405     *end   = 2; // 1 vector operand
 406     break;
 407   case Op_AddI: case Op_AddL: case Op_AddF: case Op_AddD:
 408   case Op_SubI: case Op_SubL: case Op_SubF: case Op_SubD:
 409   case Op_MulI: case Op_MulL: case Op_MulF: case Op_MulD:
 410   case Op_DivF: case Op_DivD:
 411   case Op_AndI: case Op_AndL:
 412   case Op_OrI:  case Op_OrL:
 413   case Op_XorI: case Op_XorL:
 414   case Op_MulAddS2I:
 415     *start = 1;
 416     *end   = 3; // 2 vector operands
 417     break;
 418   case Op_CMoveI:  case Op_CMoveL:  case Op_CMoveF:  case Op_CMoveD:
 419     *start = 2;
 420     *end   = n->req();
 421     break;
 422   case Op_FmaD:
 423   case Op_FmaF:
 424     *start = 1;
 425     *end   = 4; // 3 vector operands
 426     break;
 427   default:
 428     *start = 1;
 429     *end   = n->req(); // default is all operands
 430   }
 431 }
 432 
 433 // Return the vector version of a scalar operation node.
 434 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt) {
 435   const TypeVect* vt = TypeVect::make(bt, vlen);
 436   int vopc = VectorNode::opcode(opc, bt);
 437   // This method should not be called for unimplemented vectors.
 438   guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
 439   switch (vopc) {
 440   case Op_AddVB: return new AddVBNode(n1, n2, vt);
 441   case Op_AddVS: return new AddVSNode(n1, n2, vt);
 442   case Op_AddVI: return new AddVINode(n1, n2, vt);
 443   case Op_AddVL: return new AddVLNode(n1, n2, vt);
 444   case Op_AddVF: return new AddVFNode(n1, n2, vt);
 445   case Op_AddVD: return new AddVDNode(n1, n2, vt);
 446 
 447   case Op_SubVB: return new SubVBNode(n1, n2, vt);
 448   case Op_SubVS: return new SubVSNode(n1, n2, vt);
 449   case Op_SubVI: return new SubVINode(n1, n2, vt);
 450   case Op_SubVL: return new SubVLNode(n1, n2, vt);
 451   case Op_SubVF: return new SubVFNode(n1, n2, vt);
 452   case Op_SubVD: return new SubVDNode(n1, n2, vt);
 453 
 454   case Op_MulVB: return new MulVBNode(n1, n2, vt);
 455   case Op_MulVS: return new MulVSNode(n1, n2, vt);
 456   case Op_MulVI: return new MulVINode(n1, n2, vt);
 457   case Op_MulVL: return new MulVLNode(n1, n2, vt);
 458   case Op_MulVF: return new MulVFNode(n1, n2, vt);
 459   case Op_MulVD: return new MulVDNode(n1, n2, vt);
 460 
 461   case Op_DivVF: return new DivVFNode(n1, n2, vt);
 462   case Op_DivVD: return new DivVDNode(n1, n2, vt);
 463 
 464   case Op_MinV: return new MinVNode(n1, n2, vt);
 465   case Op_MaxV: return new MaxVNode(n1, n2, vt);
 466 
 467   case Op_AbsV: return new AbsVNode(n1, vt);
 468   case Op_AbsVF: return new AbsVFNode(n1, vt);
 469   case Op_AbsVD: return new AbsVDNode(n1, vt);
 470 
 471   case Op_NegVI: return new NegVINode(n1, vt);
 472   case Op_NegVF: return new NegVFNode(n1, vt);
 473   case Op_NegVD: return new NegVDNode(n1, vt);
 474 
 475   case Op_SqrtVF: return new SqrtVFNode(n1, vt);
 476   case Op_SqrtVD: return new SqrtVDNode(n1, vt);
 477 
 478   case Op_PopCountVI: return new PopCountVINode(n1, vt);
 479   case Op_NotV: return new NotVNode(n1, vt);
 480 
 481   case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt);
 482   case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt);
 483   case Op_LShiftVI: return new LShiftVINode(n1, n2, vt);
 484   case Op_LShiftVL: return new LShiftVLNode(n1, n2, vt);
 485 
 486   case Op_RShiftVB: return new RShiftVBNode(n1, n2, vt);
 487   case Op_RShiftVS: return new RShiftVSNode(n1, n2, vt);
 488   case Op_RShiftVI: return new RShiftVINode(n1, n2, vt);
 489   case Op_RShiftVL: return new RShiftVLNode(n1, n2, vt);
 490 
 491   case Op_URShiftVB: return new URShiftVBNode(n1, n2, vt);
 492   case Op_URShiftVS: return new URShiftVSNode(n1, n2, vt);
 493   case Op_URShiftVI: return new URShiftVINode(n1, n2, vt);
 494   case Op_URShiftVL: return new URShiftVLNode(n1, n2, vt);
 495 
 496   case Op_AndV: return new AndVNode(n1, n2, vt);
 497   case Op_OrV:  return new OrVNode (n1, n2, vt);
 498   case Op_XorV: return new XorVNode(n1, n2, vt);
 499 
 500   case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt);
 501   default:
 502     fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
 503     return NULL;
 504   }
 505 }
 506 
 507 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) {
 508   const TypeVect* vt = TypeVect::make(bt, vlen);
 509   int vopc = VectorNode::opcode(opc, bt);
 510   // This method should not be called for unimplemented vectors.
 511   guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
 512   switch (vopc) {
 513   case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt);
 514   case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt);
 515   default:
 516     fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
 517     return NULL;
 518   }
 519 }
 520 
 521 // Scalar promotion
 522 VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t) {
 523   BasicType bt = opd_t->array_element_basic_type();
 524   const TypeVect* vt = opd_t->singleton() ? TypeVect::make(opd_t, vlen)
 525                                           : TypeVect::make(bt, vlen);
 526   switch (bt) {
 527   case T_BOOLEAN:
 528   case T_BYTE:
 529     return new ReplicateBNode(s, vt);
 530   case T_CHAR:
 531   case T_SHORT:
 532     return new ReplicateSNode(s, vt);
 533   case T_INT:
 534     return new ReplicateINode(s, vt);
 535   case T_LONG:
 536     return new ReplicateLNode(s, vt);
 537   case T_FLOAT:
 538     return new ReplicateFNode(s, vt);
 539   case T_DOUBLE:
 540     return new ReplicateDNode(s, vt);
 541   default:
 542     fatal("Type '%s' is not supported for vectors", type2name(bt));
 543     return NULL;
 544   }
 545 }
 546 
 547 VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt) {
 548   // Match shift count type with shift vector type.
 549   const TypeVect* vt = TypeVect::make(bt, vlen);
 550   switch (opc) {
 551   case Op_LShiftI:
 552   case Op_LShiftL:
 553     return new LShiftCntVNode(cnt, vt);
 554   case Op_RShiftI:
 555   case Op_RShiftL:
 556   case Op_URShiftB:
 557   case Op_URShiftS:
 558   case Op_URShiftI:
 559   case Op_URShiftL:
 560     return new RShiftCntVNode(cnt, vt);
 561   default:
 562     fatal("Missed vector creation for '%s'", NodeClassNames[opc]);
 563     return NULL;
 564   }
 565 }
 566 
 567 // Return initial Pack node. Additional operands added with add_opd() calls.
 568 PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) {
 569   const TypeVect* vt = TypeVect::make(bt, vlen);
 570   switch (bt) {
 571   case T_BOOLEAN:
 572   case T_BYTE:
 573     return new PackBNode(s, vt);
 574   case T_CHAR:
 575   case T_SHORT:
 576     return new PackSNode(s, vt);
 577   case T_INT:
 578     return new PackINode(s, vt);
 579   case T_LONG:
 580     return new PackLNode(s, vt);
 581   case T_FLOAT:
 582     return new PackFNode(s, vt);
 583   case T_DOUBLE:
 584     return new PackDNode(s, vt);
 585   default:
 586     fatal("Type '%s' is not supported for vectors", type2name(bt));
 587     return NULL;
 588   }
 589 }
 590 
 591 // Create a binary tree form for Packs. [lo, hi) (half-open) range
 592 PackNode* PackNode::binary_tree_pack(int lo, int hi) {
 593   int ct = hi - lo;
 594   assert(is_power_of_2(ct), "power of 2");
 595   if (ct == 2) {
 596     PackNode* pk = PackNode::make(in(lo), 2, vect_type()->element_basic_type());
 597     pk->add_opd(in(lo+1));
 598     return pk;
 599   } else {
 600     int mid = lo + ct/2;
 601     PackNode* n1 = binary_tree_pack(lo,  mid);
 602     PackNode* n2 = binary_tree_pack(mid, hi );
 603 
 604     BasicType bt = n1->vect_type()->element_basic_type();
 605     assert(bt == n2->vect_type()->element_basic_type(), "should be the same");
 606     switch (bt) {
 607     case T_BOOLEAN:
 608     case T_BYTE:
 609       return new PackSNode(n1, n2, TypeVect::make(T_SHORT, 2));
 610     case T_CHAR:
 611     case T_SHORT:
 612       return new PackINode(n1, n2, TypeVect::make(T_INT, 2));
 613     case T_INT:
 614       return new PackLNode(n1, n2, TypeVect::make(T_LONG, 2));
 615     case T_LONG:
 616       return new Pack2LNode(n1, n2, TypeVect::make(T_LONG, 2));
 617     case T_FLOAT:
 618       return new PackDNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
 619     case T_DOUBLE:
 620       return new Pack2DNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
 621     default:
 622       fatal("Type '%s' is not supported for vectors", type2name(bt));
 623       return NULL;
 624     }
 625   }
 626 }
 627 
 628 // Return the vector version of a scalar load node.
 629 LoadVectorNode* LoadVectorNode::make(int opc, Node* ctl, Node* mem,
 630                                      Node* adr, const TypePtr* atyp,
 631                                      uint vlen, BasicType bt,
 632                                      ControlDependency control_dependency) {
 633   const TypeVect* vt = TypeVect::make(bt, vlen);
 634   return new LoadVectorNode(ctl, mem, adr, atyp, vt, control_dependency);
 635 }
 636 
 637 // Return the vector version of a scalar store node.
 638 StoreVectorNode* StoreVectorNode::make(int opc, Node* ctl, Node* mem,
 639                                        Node* adr, const TypePtr* atyp, Node* val,
 640                                        uint vlen) {
 641   return new StoreVectorNode(ctl, mem, adr, atyp, val);
 642 }
 643 
 644 int ExtractNode::opcode(BasicType bt) {
 645   switch (bt) {
 646     case T_BOOLEAN:
 647       return Op_ExtractUB;
 648     case T_BYTE:
 649       return Op_ExtractB;
 650     case T_CHAR:
 651       return Op_ExtractC;
 652     case T_SHORT:
 653       return Op_ExtractS;
 654     case T_INT:
 655       return Op_ExtractI;
 656     case T_LONG:
 657       return Op_ExtractL;
 658     case T_FLOAT:
 659       return Op_ExtractF;
 660     case T_DOUBLE:
 661       return Op_ExtractD;
 662     default:
 663       fatal("Type '%s' is not supported for vectors", type2name(bt));
 664       return 0;
 665   }
 666 }
 667 
 668 // Extract a scalar element of vector.
 669 Node* ExtractNode::make(Node* v, uint position, BasicType bt) {
 670   assert((int)position < Matcher::max_vector_size(bt), "pos in range");
 671   ConINode* pos = ConINode::make((int)position);
 672   switch (bt) {
 673   case T_BOOLEAN:
 674     return new ExtractUBNode(v, pos);
 675   case T_BYTE:
 676     return new ExtractBNode(v, pos);
 677   case T_CHAR:
 678     return new ExtractCNode(v, pos);
 679   case T_SHORT:
 680     return new ExtractSNode(v, pos);
 681   case T_INT:
 682     return new ExtractINode(v, pos);
 683   case T_LONG:
 684     return new ExtractLNode(v, pos);
 685   case T_FLOAT:
 686     return new ExtractFNode(v, pos);
 687   case T_DOUBLE:
 688     return new ExtractDNode(v, pos);
 689   default:
 690     fatal("Type '%s' is not supported for vectors", type2name(bt));
 691     return NULL;
 692   }
 693 }
 694 
 695 int ReductionNode::opcode(int opc, BasicType bt) {
 696   int vopc = opc;
 697   switch (opc) {
 698     case Op_AddI:
 699       switch (bt) {
 700         case T_BOOLEAN:
 701         case T_CHAR: return 0;
 702         case T_BYTE:
 703         case T_SHORT:
 704         case T_INT:
 705           vopc = Op_AddReductionVI;
 706           break;
 707         default:          ShouldNotReachHere(); return 0;
 708       }
 709       break;
 710     case Op_AddL:
 711       assert(bt == T_LONG, "must be");
 712       vopc = Op_AddReductionVL;
 713       break;
 714     case Op_AddF:
 715       assert(bt == T_FLOAT, "must be");
 716       vopc = Op_AddReductionVF;
 717       break;
 718     case Op_AddD:
 719       assert(bt == T_DOUBLE, "must be");
 720       vopc = Op_AddReductionVD;
 721       break;
 722     case Op_MulI:
 723       switch (bt) {
 724         case T_BOOLEAN:
 725         case T_CHAR: return 0;
 726         case T_BYTE:
 727         case T_SHORT:
 728         case T_INT:
 729           vopc = Op_MulReductionVI;
 730           break;
 731         default:          ShouldNotReachHere(); return 0;
 732       }
 733       break;
 734     case Op_MulL:
 735       assert(bt == T_LONG, "must be");
 736       vopc = Op_MulReductionVL;
 737       break;
 738     case Op_MulF:
 739       assert(bt == T_FLOAT, "must be");
 740       vopc = Op_MulReductionVF;
 741       break;
 742     case Op_MulD:
 743       assert(bt == T_DOUBLE, "must be");
 744       vopc = Op_MulReductionVD;
 745       break;
 746     case Op_MinI:
 747       switch (bt) {
 748         case T_BOOLEAN:
 749         case T_CHAR: return 0;
 750         case T_BYTE:
 751         case T_SHORT:
 752         case T_INT:
 753           vopc = Op_MinReductionV;
 754           break;
 755         default:          ShouldNotReachHere(); return 0;
 756       }
 757       break;
 758     case Op_MinL:
 759       assert(bt == T_LONG, "must be");
 760       vopc = Op_MinReductionV;
 761       break;
 762     case Op_MinF:
 763       assert(bt == T_FLOAT, "must be");
 764       vopc = Op_MinReductionV;
 765       break;
 766     case Op_MinD:
 767       assert(bt == T_DOUBLE, "must be");
 768       vopc = Op_MinReductionV;
 769       break;
 770     case Op_MaxI:
 771       switch (bt) {
 772         case T_BOOLEAN:
 773         case T_CHAR: return 0;
 774         case T_BYTE:
 775         case T_SHORT:
 776         case T_INT:
 777           vopc = Op_MaxReductionV;
 778           break;
 779         default:          ShouldNotReachHere(); return 0;
 780       }
 781       break;
 782     case Op_MaxL:
 783       assert(bt == T_LONG, "must be");
 784       vopc = Op_MaxReductionV;
 785       break;
 786     case Op_MaxF:
 787       assert(bt == T_FLOAT, "must be");
 788       vopc = Op_MaxReductionV;
 789       break;
 790     case Op_MaxD:
 791       assert(bt == T_DOUBLE, "must be");
 792       vopc = Op_MaxReductionV;
 793       break;
 794     case Op_AndI:
 795       switch (bt) {
 796       case T_BOOLEAN:
 797       case T_CHAR: return 0;
 798       case T_BYTE:
 799       case T_SHORT:
 800       case T_INT:
 801         vopc = Op_AndReductionV;
 802         break;
 803       default:          ShouldNotReachHere(); return 0;
 804       }
 805       break;
 806     case Op_AndL:
 807       assert(bt == T_LONG, "must be");
 808       vopc = Op_AndReductionV;
 809       break;
 810     case Op_OrI:
 811       switch(bt) {
 812       case T_BOOLEAN:
 813       case T_CHAR: return 0;
 814       case T_BYTE:
 815       case T_SHORT:
 816       case T_INT:
 817         vopc = Op_OrReductionV;
 818         break;
 819       default:  ShouldNotReachHere(); return 0;
 820       }
 821       break;
 822     case Op_OrL:
 823       assert(bt == T_LONG, "must be");
 824       vopc = Op_OrReductionV;
 825       break;
 826     case Op_XorI:
 827       switch(bt) {
 828       case T_BOOLEAN:
 829       case T_CHAR: return 0;
 830       case T_BYTE:
 831       case T_SHORT:
 832       case T_INT:
 833         vopc = Op_XorReductionV;
 834         break;
 835       default:  ShouldNotReachHere(); return 0;
 836       }
 837       break;
 838     case Op_XorL:
 839       assert(bt == T_LONG, "must be");
 840       vopc = Op_XorReductionV;
 841       break;
 842     case Op_SubI:
 843       switch(bt) {
 844       case T_BYTE:
 845       case T_SHORT:
 846       case T_INT:
 847         vopc = Op_SubReductionV;
 848         break;
 849       default:  ShouldNotReachHere(); return 0;
 850       }
 851       break;
 852     case Op_SubL:
 853       assert(bt == T_LONG, "must be");
 854       vopc = Op_SubReductionV;
 855       break;
 856     case Op_SubF:
 857       assert(bt == T_FLOAT, "must be");
 858       vopc = Op_SubReductionVFP;
 859       break;
 860     case Op_SubD:
 861       assert(bt == T_DOUBLE, "must be");
 862       vopc = Op_SubReductionVFP;
 863       break;
 864     // TODO: add MulL for targets that support it
 865     default:
 866       break;
 867   }
 868   return vopc;
 869 }
 870 
 871 // Return the appropriate reduction node.
 872 ReductionNode* ReductionNode::make(int opc, Node *ctrl, Node* n1, Node* n2, BasicType bt) {
 873 
 874   int vopc = opcode(opc, bt);
 875 
 876   // This method should not be called for unimplemented vectors.
 877   guarantee(vopc != opc, "Vector for '%s' is not implemented", NodeClassNames[opc]);
 878 
 879   switch (vopc) {
 880   case Op_AddReductionVI: return new AddReductionVINode(ctrl, n1, n2);
 881   case Op_AddReductionVL: return new AddReductionVLNode(ctrl, n1, n2);
 882   case Op_AddReductionVF: return new AddReductionVFNode(ctrl, n1, n2);
 883   case Op_AddReductionVD: return new AddReductionVDNode(ctrl, n1, n2);
 884   case Op_MulReductionVI: return new MulReductionVINode(ctrl, n1, n2);
 885   case Op_MulReductionVL: return new MulReductionVLNode(ctrl, n1, n2);
 886   case Op_MulReductionVF: return new MulReductionVFNode(ctrl, n1, n2);
 887   case Op_MulReductionVD: return new MulReductionVDNode(ctrl, n1, n2);
 888   case Op_MinReductionV: return new MinReductionVNode(ctrl, n1, n2);
 889   case Op_MaxReductionV: return new MaxReductionVNode(ctrl, n1, n2);
 890   case Op_AndReductionV: return new AndReductionVNode(ctrl, n1, n2);
 891   case Op_OrReductionV: return new OrReductionVNode(ctrl, n1, n2);
 892   case Op_XorReductionV: return new XorReductionVNode(ctrl, n1, n2);
 893   case Op_SubReductionV: return new SubReductionVNode(ctrl, n1, n2);
 894   case Op_SubReductionVFP: return new SubReductionVFPNode(ctrl, n1, n2);
 895   default:
 896     fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
 897     return NULL;
 898   }
 899 }
 900 
 901 VectorCastNode* VectorCastNode::make(int vopc, Node* n1, BasicType bt, uint vlen) {
 902   const TypeVect* vt = TypeVect::make(bt, vlen);
 903   switch (vopc) {
 904     case Op_VectorCastB2X: return new VectorCastB2XNode(n1, vt);
 905     case Op_VectorCastS2X: return new VectorCastS2XNode(n1, vt);
 906     case Op_VectorCastI2X: return new VectorCastI2XNode(n1, vt);
 907     case Op_VectorCastL2X: return new VectorCastL2XNode(n1, vt);
 908     case Op_VectorCastF2X: return new VectorCastF2XNode(n1, vt);
 909     case Op_VectorCastD2X: return new VectorCastD2XNode(n1, vt);
 910     default: fatal("unknown node: %s", NodeClassNames[vopc]);
 911   }
 912   return NULL;
 913 }
 914 
 915 int VectorCastNode::opcode(BasicType bt) {
 916   switch (bt) {
 917     case T_BYTE:   return Op_VectorCastB2X;
 918     case T_SHORT:  return Op_VectorCastS2X;
 919     case T_INT:    return Op_VectorCastI2X;
 920     case T_LONG:   return Op_VectorCastL2X;
 921     case T_FLOAT:  return Op_VectorCastF2X;
 922     case T_DOUBLE: return Op_VectorCastD2X;
 923     default: Unimplemented();
 924   }
 925   return 0;
 926 }
 927 
 928 Node* ReductionNode::make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt) {
 929   int vopc = opcode(opc, bt);
 930   guarantee(vopc != opc, "Vector reduction for '%s' is not implemented", NodeClassNames[opc]);
 931 
 932   switch (vopc) {
 933     case Op_AndReductionV:
 934       switch (bt) {
 935         case T_BYTE:
 936         case T_SHORT:
 937         case T_INT:
 938           return gvn.makecon(TypeInt::MINUS_1);
 939         case T_LONG:
 940           return gvn.makecon(TypeLong::MINUS_1);
 941         default:
 942           fatal("Missed vector creation for '%s' as the basic type is not correct.", NodeClassNames[vopc]);
 943           return NULL;
 944       }
 945       break;
 946     case Op_AddReductionVI: // fallthrough
 947     case Op_AddReductionVL: // fallthrough
 948     case Op_AddReductionVF: // fallthrough
 949     case Op_AddReductionVD:
 950     case Op_OrReductionV:
 951     case Op_XorReductionV:
 952     case Op_SubReductionV:
 953     case Op_SubReductionVFP:
 954       return gvn.zerocon(bt);
 955     case Op_MulReductionVI:
 956       return gvn.makecon(TypeInt::ONE);
 957     case Op_MulReductionVL:
 958       return gvn.makecon(TypeLong::ONE);
 959     case Op_MulReductionVF:
 960       return gvn.makecon(TypeF::ONE);
 961     case Op_MulReductionVD:
 962       return gvn.makecon(TypeD::ONE);
 963     case Op_MinReductionV:
 964       switch (bt) {
 965         case T_BYTE:
 966         case T_SHORT:
 967         case T_INT:
 968           return gvn.makecon(TypeInt::MAX);
 969         case T_LONG:
 970           return gvn.makecon(TypeLong::MAX);
 971         case T_FLOAT:
 972           return gvn.makecon(TypeF::POS_INF);
 973         case T_DOUBLE:
 974           return gvn.makecon(TypeD::POS_INF);
 975           default: Unimplemented(); return NULL;
 976       }
 977       break;
 978     case Op_MaxReductionV:
 979       switch (bt) {
 980         case T_BYTE:
 981         case T_SHORT:
 982         case T_INT:
 983           return gvn.makecon(TypeInt::MIN);
 984         case T_LONG:
 985           return gvn.makecon(TypeLong::MIN);
 986         case T_FLOAT:
 987           return gvn.makecon(TypeF::NEG_INF);
 988         case T_DOUBLE:
 989           return gvn.makecon(TypeD::NEG_INF);
 990           default: Unimplemented(); return NULL;
 991       }
 992       break;
 993     default:
 994       fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
 995       return NULL;
 996   }
 997 }
 998 
 999 bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) {
1000   if (is_java_primitive(bt) &&
1001       (vlen > 1) && is_power_of_2(vlen) &&
1002       Matcher::vector_size_supported(bt, vlen)) {
1003     int vopc = ReductionNode::opcode(opc, bt);
1004     return vopc != opc && Matcher::match_rule_supported(vopc);
1005   }
1006   return false;
1007 }
1008 
1009 #ifndef PRODUCT
1010 void VectorBoxAllocateNode::dump_spec(outputStream *st) const {
1011   CallStaticJavaNode::dump_spec(st);
1012 }
1013 
1014 void VectorMaskCmpNode::dump_spec(outputStream *st) const {
1015   st->print(" %d #", _predicate); _type->dump_on(st);
1016 }
1017 #endif // PRODUCT
1018 
1019 Node* VectorUnboxNode::Identity(PhaseGVN *phase) {
1020   Node* n = obj()->uncast();
1021   if (n->Opcode() == Op_VectorBox) {
1022     if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) {
1023       return n->in(VectorBoxNode::Value);
1024     }
1025   }
1026   return this;
1027 }
1028 
1029 Node* VectorReinterpretNode::Identity(PhaseGVN *phase) {
1030   Node* n = in(1);
1031   if (n->Opcode() == Op_VectorReinterpret) {
1032     if (Type::cmp(bottom_type(), n->in(1)->bottom_type()) == 0) {
1033       return n->in(1);
1034     }
1035   }
1036   return this;
1037 }
1038 
1039 const TypeFunc* VectorBoxNode::vec_box_type(const TypeInstPtr* box_type) {
1040   const Type** fields = TypeTuple::fields(0);
1041   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields);
1042 
1043   fields = TypeTuple::fields(1);
1044   fields[TypeFunc::Parms+0] = box_type;
1045   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
1046 
1047   return TypeFunc::make(domain, range);
1048 }
1049 
1050 Node* SubReductionVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1051   Node* ctrl = in(0);
1052   Node* in1 = in(1);
1053   Node* in2 = in(2);
1054 
1055   if (phase->type(in1)->isa_int()) {
1056                 assert(phase->type(in2)->is_vect()->element_type()->is_int(), "must be consistent");
1057     return new NegINode(phase->transform(new AddReductionVINode(ctrl,in1,in2)));
1058   } else if (phase->type(in1)->isa_long()) {
1059     return new NegLNode(phase->transform(new AddReductionVLNode(ctrl,in1,in2)));
1060   } else {
1061     Unimplemented();
1062     return NULL;
1063   }
1064 }
1065 
1066 Node* VectorInsertNode::make(Node* vec, Node* new_val, int position) {
1067   assert(position < (int)vec->bottom_type()->is_vect()->length(), "pos in range");
1068   ConINode* pos = ConINode::make(position);
1069   return new VectorInsertNode(vec, new_val, pos, vec->bottom_type()->is_vect());
1070 }