1 /*
   2  * Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "memory/allocation.inline.hpp"
  26 #include "opto/connode.hpp"
  27 #include "opto/mulnode.hpp"
  28 #include "opto/subnode.hpp"
  29 #include "opto/vectornode.hpp"
  30 #include "utilities/powerOfTwo.hpp"
  31 #include "utilities/globalDefinitions.hpp"
  32 
  33 //------------------------------VectorNode--------------------------------------
  34 
  35 // Return the vector operator for the specified scalar operation
  36 // and vector length.
  37 int VectorNode::opcode(int sopc, BasicType bt) {
  38   switch (sopc) {
  39   case Op_AddI:
  40     switch (bt) {
  41     case T_BOOLEAN:
  42     case T_BYTE:      return Op_AddVB;
  43     case T_CHAR:
  44     case T_SHORT:     return Op_AddVS;
  45     case T_INT:       return Op_AddVI;
  46     default:          ShouldNotReachHere(); return 0;
  47     }
  48   case Op_AddL:
  49     assert(bt == T_LONG, "must be");
  50     return Op_AddVL;
  51   case Op_AddF:
  52     assert(bt == T_FLOAT, "must be");
  53     return Op_AddVF;
  54   case Op_AddD:
  55     assert(bt == T_DOUBLE, "must be");
  56     return Op_AddVD;
  57   case Op_SubI:
  58     switch (bt) {
  59     case T_BOOLEAN:
  60     case T_BYTE:   return Op_SubVB;
  61     case T_CHAR:
  62     case T_SHORT:  return Op_SubVS;
  63     case T_INT:    return Op_SubVI;
  64     default:       ShouldNotReachHere(); return 0;
  65     }
  66   case Op_SubL:
  67     assert(bt == T_LONG, "must be");
  68     return Op_SubVL;
  69   case Op_SubF:
  70     assert(bt == T_FLOAT, "must be");
  71     return Op_SubVF;
  72   case Op_SubD:
  73     assert(bt == T_DOUBLE, "must be");
  74     return Op_SubVD;
  75   case Op_MulI:
  76     switch (bt) {
  77     case T_BOOLEAN:return 0;
  78     case T_BYTE:   return Op_MulVB;
  79     case T_CHAR:
  80     case T_SHORT:  return Op_MulVS;
  81     case T_INT:    return Op_MulVI;
  82     default:       ShouldNotReachHere(); return 0;
  83     }
  84   case Op_MulL:
  85     assert(bt == T_LONG, "must be");
  86     return Op_MulVL;
  87   case Op_MulF:
  88     assert(bt == T_FLOAT, "must be");
  89     return Op_MulVF;
  90   case Op_MulD:
  91     assert(bt == T_DOUBLE, "must be");
  92     return Op_MulVD;
  93   case Op_FmaD:
  94     assert(bt == T_DOUBLE, "must be");
  95     return Op_FmaVD;
  96   case Op_FmaF:
  97     assert(bt == T_FLOAT, "must be");
  98     return Op_FmaVF;
  99   case Op_CMoveF:
 100     assert(bt == T_FLOAT, "must be");
 101     return Op_CMoveVF;
 102   case Op_CMoveD:
 103     assert(bt == T_DOUBLE, "must be");
 104     return Op_CMoveVD;
 105   case Op_DivF:
 106     assert(bt == T_FLOAT, "must be");
 107     return Op_DivVF;
 108   case Op_DivD:
 109     assert(bt == T_DOUBLE, "must be");
 110     return Op_DivVD;
 111   case Op_AbsI:
 112     switch (bt) {
 113     case T_BOOLEAN:
 114     case T_CHAR:  return 0; // abs does not make sense for unsigned
 115     case T_BYTE:  return Op_AbsVB;
 116     case T_SHORT: return Op_AbsVS;
 117     case T_INT:   return Op_AbsVI;
 118     default: ShouldNotReachHere(); return 0;
 119     }
 120   case Op_AbsL:
 121     assert(bt == T_LONG, "must be");
 122     return Op_AbsVL;
 123   case Op_AbsF:
 124     assert(bt == T_FLOAT, "must be");
 125     return Op_AbsVF;
 126   case Op_AbsD:
 127     assert(bt == T_DOUBLE, "must be");
 128     return Op_AbsVD;
 129   case Op_NegF:
 130     assert(bt == T_FLOAT, "must be");
 131     return Op_NegVF;
 132   case Op_NegD:
 133     assert(bt == T_DOUBLE, "must be");
 134     return Op_NegVD;
 135   case Op_RoundDoubleMode:
 136     assert(bt == T_DOUBLE, "must be");
 137     return Op_RoundDoubleModeV;
 138   case Op_RotateLeft:
 139     assert(bt == T_LONG || bt == T_INT, "must be");
 140     return Op_RotateLeftV;
 141   case Op_RotateRight:
 142     assert(bt == T_LONG || bt == T_INT, "must be");
 143     return Op_RotateRightV;
 144   case Op_SqrtF:
 145     assert(bt == T_FLOAT, "must be");
 146     return Op_SqrtVF;
 147   case Op_SqrtD:
 148     assert(bt == T_DOUBLE, "must be");
 149     return Op_SqrtVD;
 150   case Op_PopCountI:
 151     if (bt == T_INT) {
 152       return Op_PopCountVI;
 153     }
 154     // Unimplemented for subword types since bit count changes
 155     // depending on size of lane (and sign bit).
 156     return 0;
 157   case Op_LShiftI:
 158     switch (bt) {
 159     case T_BOOLEAN:
 160     case T_BYTE:   return Op_LShiftVB;
 161     case T_CHAR:
 162     case T_SHORT:  return Op_LShiftVS;
 163     case T_INT:    return Op_LShiftVI;
 164       default:       ShouldNotReachHere(); return 0;
 165     }
 166   case Op_LShiftL:
 167     assert(bt == T_LONG, "must be");
 168     return Op_LShiftVL;
 169   case Op_RShiftI:
 170     switch (bt) {
 171     case T_BOOLEAN:return Op_URShiftVB; // boolean is unsigned value
 172     case T_CHAR:   return Op_URShiftVS; // char is unsigned value
 173     case T_BYTE:   return Op_RShiftVB;
 174     case T_SHORT:  return Op_RShiftVS;
 175     case T_INT:    return Op_RShiftVI;
 176     default:       ShouldNotReachHere(); return 0;
 177     }
 178   case Op_RShiftL:
 179     assert(bt == T_LONG, "must be");
 180     return Op_RShiftVL;
 181   case Op_URShiftI:
 182     switch (bt) {
 183     case T_BOOLEAN:return Op_URShiftVB;
 184     case T_CHAR:   return Op_URShiftVS;
 185     case T_BYTE:
 186     case T_SHORT:  return 0; // Vector logical right shift for signed short
 187                              // values produces incorrect Java result for
 188                              // negative data because java code should convert
 189                              // a short value into int value with sign
 190                              // extension before a shift.
 191     case T_INT:    return Op_URShiftVI;
 192     default:       ShouldNotReachHere(); return 0;
 193     }
 194   case Op_URShiftL:
 195     assert(bt == T_LONG, "must be");
 196     return Op_URShiftVL;
 197   case Op_AndI:
 198   case Op_AndL:
 199     return Op_AndV;
 200   case Op_OrI:
 201   case Op_OrL:
 202     return Op_OrV;
 203   case Op_XorI:
 204   case Op_XorL:
 205     return Op_XorV;
 206   case Op_MinF:
 207     assert(bt == T_FLOAT, "must be");
 208     return Op_MinV;
 209   case Op_MinD:
 210     assert(bt == T_DOUBLE, "must be");
 211     return Op_MinV;
 212   case Op_MaxF:
 213     assert(bt == T_FLOAT, "must be");
 214     return Op_MaxV;
 215   case Op_MaxD:
 216     assert(bt == T_DOUBLE, "must be");
 217     return Op_MaxV;
 218 
 219   case Op_LoadB:
 220   case Op_LoadUB:
 221   case Op_LoadUS:
 222   case Op_LoadS:
 223   case Op_LoadI:
 224   case Op_LoadL:
 225   case Op_LoadF:
 226   case Op_LoadD:
 227     return Op_LoadVector;
 228 
 229   case Op_StoreB:
 230   case Op_StoreC:
 231   case Op_StoreI:
 232   case Op_StoreL:
 233   case Op_StoreF:
 234   case Op_StoreD:
 235     return Op_StoreVector;
 236   case Op_MulAddS2I:
 237     return Op_MulAddVS2VI;
 238 
 239   default:
 240     return 0; // Unimplemented
 241   }
 242 }
 243 
 244 // Also used to check if the code generator
 245 // supports the vector operation.
 246 bool VectorNode::implemented(int opc, uint vlen, BasicType bt) {
 247   if (is_java_primitive(bt) &&
 248       (vlen > 1) && is_power_of_2(vlen) &&
 249       Matcher::vector_size_supported(bt, vlen)) {
 250     int vopc = VectorNode::opcode(opc, bt);
 251     // For rotate operation we will do a lazy de-generation into
 252     // OrV/LShiftV/URShiftV pattern if the target does not support
 253     // vector rotation instruction.
 254     if (vopc == Op_RotateLeftV || vopc == Op_RotateRightV) {
 255       return is_vector_rotate_supported(vopc, vlen, bt);
 256     }
 257     return vopc > 0 && Matcher::match_rule_supported_vector(vopc, vlen, bt);
 258   }
 259   return false;
 260 }
 261 
 262 bool VectorNode::is_type_transition_short_to_int(Node* n) {
 263   switch (n->Opcode()) {
 264   case Op_MulAddS2I:
 265     return true;
 266   }
 267   return false;
 268 }
 269 
 270 bool VectorNode::is_type_transition_to_int(Node* n) {
 271   return is_type_transition_short_to_int(n);
 272 }
 273 
 274 bool VectorNode::is_muladds2i(Node* n) {
 275   if (n->Opcode() == Op_MulAddS2I) {
 276     return true;
 277   }
 278   return false;
 279 }
 280 
 281 bool VectorNode::is_roundopD(Node* n) {
 282   if (n->Opcode() == Op_RoundDoubleMode) {
 283     return true;
 284   }
 285   return false;
 286 }
 287 
 288 bool VectorNode::is_scalar_rotate(Node* n) {
 289   if (n->Opcode() == Op_RotateLeft || n->Opcode() == Op_RotateRight) {
 290     return true;
 291   }
 292   return false;
 293 }
 294 
 295 bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) {
 296   assert(vopc == Op_RotateLeftV || vopc == Op_RotateRightV, "wrong opcode");
 297 
 298   // If target defines vector rotation patterns then no
 299   // need for degeneration.
 300   if (Matcher::match_rule_supported_vector(vopc, vlen, bt)) {
 301     return true;
 302   }
 303 
 304   // Validate existence of nodes created in case of rotate degeneration.
 305   switch (bt) {
 306     case T_INT:
 307       return Matcher::match_rule_supported_vector(Op_OrV,       vlen, bt) &&
 308              Matcher::match_rule_supported_vector(Op_LShiftVI,  vlen, bt) &&
 309              Matcher::match_rule_supported_vector(Op_URShiftVI, vlen, bt);
 310     case T_LONG:
 311       return Matcher::match_rule_supported_vector(Op_OrV,       vlen, bt) &&
 312              Matcher::match_rule_supported_vector(Op_LShiftVL,  vlen, bt) &&
 313              Matcher::match_rule_supported_vector(Op_URShiftVL, vlen, bt);
 314     default:
 315       assert(false, "not supported: %s", type2name(bt));
 316       return false;
 317   }
 318 }
 319 
 320 bool VectorNode::is_shift(Node* n) {
 321   switch (n->Opcode()) {
 322   case Op_LShiftI:
 323   case Op_LShiftL:
 324   case Op_RShiftI:
 325   case Op_RShiftL:
 326   case Op_URShiftI:
 327   case Op_URShiftL:
 328     return true;
 329   default:
 330     return false;
 331   }
 332 }
 333 
 334 // Check if input is loop invariant vector.
 335 bool VectorNode::is_invariant_vector(Node* n) {
 336   // Only Replicate vector nodes are loop invariant for now.
 337   switch (n->Opcode()) {
 338   case Op_ReplicateB:
 339   case Op_ReplicateS:
 340   case Op_ReplicateI:
 341   case Op_ReplicateL:
 342   case Op_ReplicateF:
 343   case Op_ReplicateD:
 344     return true;
 345   default:
 346     return false;
 347   }
 348 }
 349 
 350 // [Start, end) half-open range defining which operands are vectors
 351 void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
 352   switch (n->Opcode()) {
 353   case Op_LoadB:   case Op_LoadUB:
 354   case Op_LoadS:   case Op_LoadUS:
 355   case Op_LoadI:   case Op_LoadL:
 356   case Op_LoadF:   case Op_LoadD:
 357   case Op_LoadP:   case Op_LoadN:
 358     *start = 0;
 359     *end   = 0; // no vector operands
 360     break;
 361   case Op_StoreB:  case Op_StoreC:
 362   case Op_StoreI:  case Op_StoreL:
 363   case Op_StoreF:  case Op_StoreD:
 364   case Op_StoreP:  case Op_StoreN:
 365     *start = MemNode::ValueIn;
 366     *end   = MemNode::ValueIn + 1; // 1 vector operand
 367     break;
 368   case Op_LShiftI:  case Op_LShiftL:
 369   case Op_RShiftI:  case Op_RShiftL:
 370   case Op_URShiftI: case Op_URShiftL:
 371     *start = 1;
 372     *end   = 2; // 1 vector operand
 373     break;
 374   case Op_AddI: case Op_AddL: case Op_AddF: case Op_AddD:
 375   case Op_SubI: case Op_SubL: case Op_SubF: case Op_SubD:
 376   case Op_MulI: case Op_MulL: case Op_MulF: case Op_MulD:
 377   case Op_DivF: case Op_DivD:
 378   case Op_AndI: case Op_AndL:
 379   case Op_OrI:  case Op_OrL:
 380   case Op_XorI: case Op_XorL:
 381   case Op_MulAddS2I:
 382     *start = 1;
 383     *end   = 3; // 2 vector operands
 384     break;
 385   case Op_CMoveI:  case Op_CMoveL:  case Op_CMoveF:  case Op_CMoveD:
 386     *start = 2;
 387     *end   = n->req();
 388     break;
 389   case Op_FmaD:
 390   case Op_FmaF:
 391     *start = 1;
 392     *end   = 4; // 3 vector operands
 393     break;
 394   default:
 395     *start = 1;
 396     *end   = n->req(); // default is all operands
 397   }
 398 }
 399 
 400 // Return the vector version of a scalar operation node.
 401 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt) {
 402   const TypeVect* vt = TypeVect::make(bt, vlen);
 403   int vopc = VectorNode::opcode(opc, bt);
 404   // This method should not be called for unimplemented vectors.
 405   guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
 406   switch (vopc) {
 407   case Op_AddVB: return new AddVBNode(n1, n2, vt);
 408   case Op_AddVS: return new AddVSNode(n1, n2, vt);
 409   case Op_AddVI: return new AddVINode(n1, n2, vt);
 410   case Op_AddVL: return new AddVLNode(n1, n2, vt);
 411   case Op_AddVF: return new AddVFNode(n1, n2, vt);
 412   case Op_AddVD: return new AddVDNode(n1, n2, vt);
 413 
 414   case Op_SubVB: return new SubVBNode(n1, n2, vt);
 415   case Op_SubVS: return new SubVSNode(n1, n2, vt);
 416   case Op_SubVI: return new SubVINode(n1, n2, vt);
 417   case Op_SubVL: return new SubVLNode(n1, n2, vt);
 418   case Op_SubVF: return new SubVFNode(n1, n2, vt);
 419   case Op_SubVD: return new SubVDNode(n1, n2, vt);
 420 
 421   case Op_MulVB: return new MulVBNode(n1, n2, vt);
 422   case Op_MulVS: return new MulVSNode(n1, n2, vt);
 423   case Op_MulVI: return new MulVINode(n1, n2, vt);
 424   case Op_MulVL: return new MulVLNode(n1, n2, vt);
 425   case Op_MulVF: return new MulVFNode(n1, n2, vt);
 426   case Op_MulVD: return new MulVDNode(n1, n2, vt);
 427 
 428   case Op_DivVF: return new DivVFNode(n1, n2, vt);
 429   case Op_DivVD: return new DivVDNode(n1, n2, vt);
 430 
 431   case Op_AbsVB: return new AbsVBNode(n1, vt);
 432   case Op_AbsVS: return new AbsVSNode(n1, vt);
 433   case Op_AbsVI: return new AbsVINode(n1, vt);
 434   case Op_AbsVL: return new AbsVLNode(n1, vt);
 435   case Op_AbsVF: return new AbsVFNode(n1, vt);
 436   case Op_AbsVD: return new AbsVDNode(n1, vt);
 437 
 438   case Op_NegVF: return new NegVFNode(n1, vt);
 439   case Op_NegVD: return new NegVDNode(n1, vt);
 440 
 441   case Op_SqrtVF: return new SqrtVFNode(n1, vt);
 442   case Op_SqrtVD: return new SqrtVDNode(n1, vt);
 443 
 444   case Op_PopCountVI: return new PopCountVINode(n1, vt);
 445   case Op_RotateLeftV: return new RotateLeftVNode(n1, n2, vt);
 446   case Op_RotateRightV: return new RotateRightVNode(n1, n2, vt);
 447 
 448   case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt);
 449   case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt);
 450   case Op_LShiftVI: return new LShiftVINode(n1, n2, vt);
 451   case Op_LShiftVL: return new LShiftVLNode(n1, n2, vt);
 452 
 453   case Op_RShiftVB: return new RShiftVBNode(n1, n2, vt);
 454   case Op_RShiftVS: return new RShiftVSNode(n1, n2, vt);
 455   case Op_RShiftVI: return new RShiftVINode(n1, n2, vt);
 456   case Op_RShiftVL: return new RShiftVLNode(n1, n2, vt);
 457 
 458   case Op_URShiftVB: return new URShiftVBNode(n1, n2, vt);
 459   case Op_URShiftVS: return new URShiftVSNode(n1, n2, vt);
 460   case Op_URShiftVI: return new URShiftVINode(n1, n2, vt);
 461   case Op_URShiftVL: return new URShiftVLNode(n1, n2, vt);
 462 
 463   case Op_AndV: return new AndVNode(n1, n2, vt);
 464   case Op_OrV:  return new OrVNode (n1, n2, vt);
 465   case Op_XorV: return new XorVNode(n1, n2, vt);
 466 
 467   case Op_MinV: return new MinVNode(n1, n2, vt);
 468   case Op_MaxV: return new MaxVNode(n1, n2, vt);
 469 
 470   case Op_RoundDoubleModeV: return new RoundDoubleModeVNode(n1, n2, vt);
 471 
 472   case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt);
 473   default:
 474     fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
 475     return NULL;
 476   }
 477 }
 478 
 479 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) {
 480   const TypeVect* vt = TypeVect::make(bt, vlen);
 481   int vopc = VectorNode::opcode(opc, bt);
 482   // This method should not be called for unimplemented vectors.
 483   guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
 484   switch (vopc) {
 485   case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt);
 486   case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt);
 487   default:
 488     fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
 489     return NULL;
 490   }
 491 }
 492 
 493 // Scalar promotion
 494 VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t) {
 495   BasicType bt = opd_t->array_element_basic_type();
 496   const TypeVect* vt = opd_t->singleton() ? TypeVect::make(opd_t, vlen)
 497                                           : TypeVect::make(bt, vlen);
 498   switch (bt) {
 499   case T_BOOLEAN:
 500   case T_BYTE:
 501     return new ReplicateBNode(s, vt);
 502   case T_CHAR:
 503   case T_SHORT:
 504     return new ReplicateSNode(s, vt);
 505   case T_INT:
 506     return new ReplicateINode(s, vt);
 507   case T_LONG:
 508     return new ReplicateLNode(s, vt);
 509   case T_FLOAT:
 510     return new ReplicateFNode(s, vt);
 511   case T_DOUBLE:
 512     return new ReplicateDNode(s, vt);
 513   default:
 514     fatal("Type '%s' is not supported for vectors", type2name(bt));
 515     return NULL;
 516   }
 517 }
 518 
 519 VectorNode* VectorNode::shift_count(Node* shift, Node* cnt, uint vlen, BasicType bt) {
 520   assert(VectorNode::is_shift(shift), "sanity");
 521   // Match shift count type with shift vector type.
 522   const TypeVect* vt = TypeVect::make(bt, vlen);
 523   switch (shift->Opcode()) {
 524   case Op_LShiftI:
 525   case Op_LShiftL:
 526     return new LShiftCntVNode(cnt, vt);
 527   case Op_RShiftI:
 528   case Op_RShiftL:
 529   case Op_URShiftI:
 530   case Op_URShiftL:
 531     return new RShiftCntVNode(cnt, vt);
 532   default:
 533     fatal("Missed vector creation for '%s'", NodeClassNames[shift->Opcode()]);
 534     return NULL;
 535   }
 536 }
 537 
 538 bool VectorNode::is_vector_shift(int opc) {
 539   assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
 540   switch (opc) {
 541   case Op_LShiftVB:
 542   case Op_LShiftVS:
 543   case Op_LShiftVI:
 544   case Op_LShiftVL:
 545   case Op_RShiftVB:
 546   case Op_RShiftVS:
 547   case Op_RShiftVI:
 548   case Op_RShiftVL:
 549   case Op_URShiftVB:
 550   case Op_URShiftVS:
 551   case Op_URShiftVI:
 552   case Op_URShiftVL:
 553     return true;
 554   default:
 555     return false;
 556   }
 557 }
 558 
 559 bool VectorNode::is_vector_shift_count(int opc) {
 560   assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
 561   switch (opc) {
 562   case Op_RShiftCntV:
 563   case Op_LShiftCntV:
 564     return true;
 565   default:
 566     return false;
 567   }
 568 }
 569 
 570 static bool is_con_M1(Node* n) {
 571   if (n->is_Con()) {
 572     const Type* t = n->bottom_type();
 573     if (t->isa_int() && t->is_int()->get_con() == -1) {
 574       return true;
 575     }
 576     if (t->isa_long() && t->is_long()->get_con() == -1) {
 577       return true;
 578     }
 579   }
 580   return false;
 581 }
 582 
 583 bool VectorNode::is_all_ones_vector(Node* n) {
 584   switch (n->Opcode()) {
 585   case Op_ReplicateB:
 586   case Op_ReplicateS:
 587   case Op_ReplicateI:
 588   case Op_ReplicateL:
 589     return is_con_M1(n->in(1));
 590   default:
 591     return false;
 592   }
 593 }
 594 
 595 bool VectorNode::is_vector_bitwise_not_pattern(Node* n) {
 596   if (n->Opcode() == Op_XorV) {
 597     return is_all_ones_vector(n->in(1)) ||
 598            is_all_ones_vector(n->in(2));
 599   }
 600   return false;
 601 }
 602 
 603 // Return initial Pack node. Additional operands added with add_opd() calls.
 604 PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) {
 605   const TypeVect* vt = TypeVect::make(bt, vlen);
 606   switch (bt) {
 607   case T_BOOLEAN:
 608   case T_BYTE:
 609     return new PackBNode(s, vt);
 610   case T_CHAR:
 611   case T_SHORT:
 612     return new PackSNode(s, vt);
 613   case T_INT:
 614     return new PackINode(s, vt);
 615   case T_LONG:
 616     return new PackLNode(s, vt);
 617   case T_FLOAT:
 618     return new PackFNode(s, vt);
 619   case T_DOUBLE:
 620     return new PackDNode(s, vt);
 621   default:
 622     fatal("Type '%s' is not supported for vectors", type2name(bt));
 623     return NULL;
 624   }
 625 }
 626 
 627 // Create a binary tree form for Packs. [lo, hi) (half-open) range
 628 PackNode* PackNode::binary_tree_pack(int lo, int hi) {
 629   int ct = hi - lo;
 630   assert(is_power_of_2(ct), "power of 2");
 631   if (ct == 2) {
 632     PackNode* pk = PackNode::make(in(lo), 2, vect_type()->element_basic_type());
 633     pk->add_opd(in(lo+1));
 634     return pk;
 635   } else {
 636     int mid = lo + ct/2;
 637     PackNode* n1 = binary_tree_pack(lo,  mid);
 638     PackNode* n2 = binary_tree_pack(mid, hi );
 639 
 640     BasicType bt = n1->vect_type()->element_basic_type();
 641     assert(bt == n2->vect_type()->element_basic_type(), "should be the same");
 642     switch (bt) {
 643     case T_BOOLEAN:
 644     case T_BYTE:
 645       return new PackSNode(n1, n2, TypeVect::make(T_SHORT, 2));
 646     case T_CHAR:
 647     case T_SHORT:
 648       return new PackINode(n1, n2, TypeVect::make(T_INT, 2));
 649     case T_INT:
 650       return new PackLNode(n1, n2, TypeVect::make(T_LONG, 2));
 651     case T_LONG:
 652       return new Pack2LNode(n1, n2, TypeVect::make(T_LONG, 2));
 653     case T_FLOAT:
 654       return new PackDNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
 655     case T_DOUBLE:
 656       return new Pack2DNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
 657     default:
 658       fatal("Type '%s' is not supported for vectors", type2name(bt));
 659       return NULL;
 660     }
 661   }
 662 }
 663 
 664 // Return the vector version of a scalar load node.
 665 LoadVectorNode* LoadVectorNode::make(int opc, Node* ctl, Node* mem,
 666                                      Node* adr, const TypePtr* atyp,
 667                                      uint vlen, BasicType bt,
 668                                      ControlDependency control_dependency) {
 669   const TypeVect* vt = TypeVect::make(bt, vlen);
 670   return new LoadVectorNode(ctl, mem, adr, atyp, vt, control_dependency);
 671 }
 672 
 673 // Return the vector version of a scalar store node.
 674 StoreVectorNode* StoreVectorNode::make(int opc, Node* ctl, Node* mem,
 675                                        Node* adr, const TypePtr* atyp, Node* val,
 676                                        uint vlen) {
 677   return new StoreVectorNode(ctl, mem, adr, atyp, val);
 678 }
 679 
 680 // Extract a scalar element of vector.
 681 Node* ExtractNode::make(Node* v, uint position, BasicType bt) {
 682   assert((int)position < Matcher::max_vector_size(bt), "pos in range");
 683   ConINode* pos = ConINode::make((int)position);
 684   switch (bt) {
 685   case T_BOOLEAN:
 686     return new ExtractUBNode(v, pos);
 687   case T_BYTE:
 688     return new ExtractBNode(v, pos);
 689   case T_CHAR:
 690     return new ExtractCNode(v, pos);
 691   case T_SHORT:
 692     return new ExtractSNode(v, pos);
 693   case T_INT:
 694     return new ExtractINode(v, pos);
 695   case T_LONG:
 696     return new ExtractLNode(v, pos);
 697   case T_FLOAT:
 698     return new ExtractFNode(v, pos);
 699   case T_DOUBLE:
 700     return new ExtractDNode(v, pos);
 701   default:
 702     fatal("Type '%s' is not supported for vectors", type2name(bt));
 703     return NULL;
 704   }
 705 }
 706 
 707 int ReductionNode::opcode(int opc, BasicType bt) {
 708   int vopc = opc;
 709   switch (opc) {
 710     case Op_AddI:
 711       assert(bt == T_INT, "must be");
 712       vopc = Op_AddReductionVI;
 713       break;
 714     case Op_AddL:
 715       assert(bt == T_LONG, "must be");
 716       vopc = Op_AddReductionVL;
 717       break;
 718     case Op_AddF:
 719       assert(bt == T_FLOAT, "must be");
 720       vopc = Op_AddReductionVF;
 721       break;
 722     case Op_AddD:
 723       assert(bt == T_DOUBLE, "must be");
 724       vopc = Op_AddReductionVD;
 725       break;
 726     case Op_MulI:
 727       assert(bt == T_INT, "must be");
 728       vopc = Op_MulReductionVI;
 729       break;
 730     case Op_MulL:
 731       assert(bt == T_LONG, "must be");
 732       vopc = Op_MulReductionVL;
 733       break;
 734     case Op_MulF:
 735       assert(bt == T_FLOAT, "must be");
 736       vopc = Op_MulReductionVF;
 737       break;
 738     case Op_MulD:
 739       assert(bt == T_DOUBLE, "must be");
 740       vopc = Op_MulReductionVD;
 741       break;
 742     case Op_MinF:
 743       assert(bt == T_FLOAT, "must be");
 744       vopc = Op_MinReductionV;
 745       break;
 746     case Op_MinD:
 747       assert(bt == T_DOUBLE, "must be");
 748       vopc = Op_MinReductionV;
 749       break;
 750     case Op_MaxF:
 751       assert(bt == T_FLOAT, "must be");
 752       vopc = Op_MaxReductionV;
 753       break;
 754     case Op_MaxD:
 755       assert(bt == T_DOUBLE, "must be");
 756       vopc = Op_MaxReductionV;
 757       break;
 758     case Op_AndI:
 759       assert(bt == T_INT, "must be");
 760       vopc = Op_AndReductionV;
 761       break;
 762     case Op_AndL:
 763       assert(bt == T_LONG, "must be");
 764       vopc = Op_AndReductionV;
 765       break;
 766     case Op_OrI:
 767       assert(bt == T_INT, "must be");
 768       vopc = Op_OrReductionV;
 769       break;
 770     case Op_OrL:
 771       assert(bt == T_LONG, "must be");
 772       vopc = Op_OrReductionV;
 773       break;
 774     case Op_XorI:
 775       assert(bt == T_INT, "must be");
 776       vopc = Op_XorReductionV;
 777       break;
 778     case Op_XorL:
 779       assert(bt == T_LONG, "must be");
 780       vopc = Op_XorReductionV;
 781       break;
 782     default:
 783       break;
 784   }
 785   return vopc;
 786 }
 787 
 788 // Return the appropriate reduction node.
 789 ReductionNode* ReductionNode::make(int opc, Node *ctrl, Node* n1, Node* n2, BasicType bt) {
 790 
 791   int vopc = opcode(opc, bt);
 792 
 793   // This method should not be called for unimplemented vectors.
 794   guarantee(vopc != opc, "Vector for '%s' is not implemented", NodeClassNames[opc]);
 795 
 796   switch (vopc) {
 797   case Op_AddReductionVI: return new AddReductionVINode(ctrl, n1, n2);
 798   case Op_AddReductionVL: return new AddReductionVLNode(ctrl, n1, n2);
 799   case Op_AddReductionVF: return new AddReductionVFNode(ctrl, n1, n2);
 800   case Op_AddReductionVD: return new AddReductionVDNode(ctrl, n1, n2);
 801   case Op_MulReductionVI: return new MulReductionVINode(ctrl, n1, n2);
 802   case Op_MulReductionVL: return new MulReductionVLNode(ctrl, n1, n2);
 803   case Op_MulReductionVF: return new MulReductionVFNode(ctrl, n1, n2);
 804   case Op_MulReductionVD: return new MulReductionVDNode(ctrl, n1, n2);
 805   case Op_MinReductionV:  return new MinReductionVNode(ctrl, n1, n2);
 806   case Op_MaxReductionV:  return new MaxReductionVNode(ctrl, n1, n2);
 807   case Op_AndReductionV:  return new AndReductionVNode(ctrl, n1, n2);
 808   case Op_OrReductionV:   return new OrReductionVNode(ctrl, n1, n2);
 809   case Op_XorReductionV:  return new XorReductionVNode(ctrl, n1, n2);
 810   default:
 811     fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
 812     return NULL;
 813   }
 814 }
 815 
 816 bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) {
 817   if (is_java_primitive(bt) &&
 818       (vlen > 1) && is_power_of_2(vlen) &&
 819       Matcher::vector_size_supported(bt, vlen)) {
 820     int vopc = ReductionNode::opcode(opc, bt);
 821     return vopc != opc && Matcher::match_rule_supported(vopc);
 822   }
 823   return false;
 824 }
 825 
 826 MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node* in3,
 827                                       uint truth_table, const TypeVect* vt) {
 828   assert(truth_table <= 0xFF, "invalid");
 829   assert(in1->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
 830   assert(in2->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
 831   assert(in3->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
 832   Node* fn = gvn.intcon(truth_table);
 833   return new MacroLogicVNode(in1, in2, in3, fn, vt);
 834 }
 835 
 836 Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_left,
 837                                            int vlen, BasicType bt, PhaseGVN* phase) {
 838   int shiftLOpc;
 839   int shiftROpc;
 840   Node* shiftLCnt = NULL;
 841   Node* shiftRCnt = NULL;
 842   const TypeVect* vt = TypeVect::make(bt, vlen);
 843 
 844   // Compute shift values for right rotation and
 845   // later swap them in case of left rotation.
 846   if (cnt->is_Con()) {
 847     // Constant shift case.
 848     if (bt == T_INT) {
 849       int shift = cnt->get_int() & 31;
 850       shiftRCnt = phase->intcon(shift);
 851       shiftLCnt = phase->intcon(32 - shift);
 852       shiftLOpc = Op_LShiftI;
 853       shiftROpc = Op_URShiftI;
 854     } else {
 855       int shift = cnt->get_int() & 63;
 856       shiftRCnt = phase->intcon(shift);
 857       shiftLCnt = phase->intcon(64 - shift);
 858       shiftLOpc = Op_LShiftL;
 859       shiftROpc = Op_URShiftL;
 860     }
 861   } else {
 862     // Variable shift case.
 863     assert(VectorNode::is_invariant_vector(cnt), "Broadcast expected");
 864     cnt = cnt->in(1);
 865     if (bt == T_INT) {
 866       shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(31)));
 867       shiftLCnt = phase->transform(new SubINode(phase->intcon(32), shiftRCnt));
 868       shiftLOpc = Op_LShiftI;
 869       shiftROpc = Op_URShiftI;
 870     } else {
 871       assert(cnt->Opcode() == Op_ConvI2L, "ConvI2L expected");
 872       cnt = cnt->in(1);
 873       shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(63)));
 874       shiftLCnt = phase->transform(new SubINode(phase->intcon(64), shiftRCnt));
 875       shiftLOpc = Op_LShiftL;
 876       shiftROpc = Op_URShiftL;
 877     }
 878   }
 879 
 880   // Swap the computed left and right shift counts.
 881   if (is_rotate_left) {
 882     swap(shiftRCnt,shiftLCnt);
 883   }
 884 
 885   shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
 886   shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
 887 
 888   return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt)),
 889                      phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt)),
 890                      vt);
 891 }
 892 
 893 Node* RotateLeftVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
 894   int vlen = length();
 895   BasicType bt = vect_type()->element_basic_type();
 896   if (!Matcher::match_rule_supported_vector(Op_RotateLeftV, vlen, bt)) {
 897     return VectorNode::degenerate_vector_rotate(in(1), in(2), true, vlen, bt, phase);
 898   }
 899   return NULL;
 900 }
 901 
 902 Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
 903   int vlen = length();
 904   BasicType bt = vect_type()->element_basic_type();
 905   if (!Matcher::match_rule_supported_vector(Op_RotateRightV, vlen, bt)) {
 906     return VectorNode::degenerate_vector_rotate(in(1), in(2), false, vlen, bt, phase);
 907   }
 908   return NULL;
 909 }
 910 
 911 Node* OrVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
 912   int lopcode = in(1)->Opcode();
 913   int ropcode = in(2)->Opcode();
 914   const TypeVect* vt = bottom_type()->is_vect();
 915   int vec_len = vt->length();
 916   BasicType bt = vt->element_basic_type();
 917 
 918   // Vector Rotate operations inferencing, this will be useful when vector
 919   // operations are created via non-SLP route i.e. (VectorAPI).
 920   if (Matcher::match_rule_supported_vector(Op_RotateLeftV, vec_len, bt) &&
 921       ((ropcode == Op_LShiftVI && lopcode == Op_URShiftVI) ||
 922        (ropcode == Op_LShiftVL && lopcode == Op_URShiftVL)) &&
 923       in(1)->in(1) == in(2)->in(1)) {
 924     assert(Op_RShiftCntV == in(1)->in(2)->Opcode(), "LShiftCntV operand expected");
 925     assert(Op_LShiftCntV == in(2)->in(2)->Opcode(), "RShiftCntV operand expected");
 926     Node* lshift = in(1)->in(2)->in(1);
 927     Node* rshift = in(2)->in(2)->in(1);
 928     int mod_val = bt == T_LONG ? 64 : 32;
 929     int shift_mask = bt == T_LONG ? 0x3F : 0x1F;
 930     // val >> norm_con_shift | val << (32 - norm_con_shift) => rotate_right val ,
 931     // norm_con_shift
 932     if (lshift->is_Con() && rshift->is_Con() &&
 933         ((lshift->get_int() & shift_mask) ==
 934          (mod_val - (rshift->get_int() & shift_mask)))) {
 935       return new RotateRightVNode(
 936           in(1)->in(1), phase->intcon(lshift->get_int() & shift_mask), vt);
 937     }
 938     if (lshift->Opcode() == Op_AndI && rshift->Opcode() == Op_AndI &&
 939         lshift->in(2)->is_Con() && rshift->in(2)->is_Con() &&
 940         lshift->in(2)->get_int() == (mod_val - 1) &&
 941         rshift->in(2)->get_int() == (mod_val - 1)) {
 942       lshift = lshift->in(1);
 943       rshift = rshift->in(1);
 944       // val << var_shift | val >> (0/32 - var_shift) => rotate_left val ,
 945       // var_shift
 946       if (lshift->Opcode() == Op_SubI && lshift->in(2) == rshift &&
 947           lshift->in(1)->is_Con() &&
 948           (lshift->in(1)->get_int() == 0 ||
 949            lshift->in(1)->get_int() == mod_val)) {
 950         Node* rotate_cnt = phase->transform(new ReplicateINode(rshift, vt));
 951         return new RotateLeftVNode(in(1)->in(1), rotate_cnt, vt);
 952       }
 953     }
 954   }
 955 
 956   if (Matcher::match_rule_supported_vector(Op_RotateRightV, vec_len, bt) &&
 957       ((ropcode == Op_URShiftVI && lopcode == Op_LShiftVI) ||
 958        (ropcode == Op_URShiftVL && lopcode == Op_LShiftVL)) &&
 959       in(1)->in(1) == in(2)->in(1)) {
 960     assert(Op_LShiftCntV == in(1)->in(2)->Opcode(), "RShiftCntV operand expected");
 961     assert(Op_RShiftCntV == in(2)->in(2)->Opcode(), "LShiftCntV operand expected");
 962     Node* rshift = in(1)->in(2)->in(1);
 963     Node* lshift = in(2)->in(2)->in(1);
 964     int mod_val = bt == T_LONG ? 64 : 32;
 965     int shift_mask = bt == T_LONG ? 0x3F : 0x1F;
 966     // val << norm_con_shift | val >> (32 - norm_con_shift) => rotate_left val
 967     // , norm_con_shift
 968     if (rshift->is_Con() && lshift->is_Con() &&
 969         ((rshift->get_int() & shift_mask) ==
 970          (mod_val - (lshift->get_int() & shift_mask)))) {
 971       return new RotateLeftVNode(
 972           in(1)->in(1), phase->intcon(rshift->get_int() & shift_mask), vt);
 973     }
 974     if (lshift->Opcode() == Op_AndI && rshift->Opcode() == Op_AndI &&
 975         lshift->in(2)->is_Con() && rshift->in(2)->is_Con() &&
 976         rshift->in(2)->get_int() == (mod_val - 1) &&
 977         lshift->in(2)->get_int() == (mod_val - 1)) {
 978       rshift = rshift->in(1);
 979       lshift = lshift->in(1);
 980       // val >> var_shift | val << (0/32 - var_shift) => rotate_right val ,
 981       // var_shift
 982       if (rshift->Opcode() == Op_SubI && rshift->in(2) == lshift &&
 983           rshift->in(1)->is_Con() &&
 984           (rshift->in(1)->get_int() == 0 ||
 985            rshift->in(1)->get_int() == mod_val)) {
 986         Node* rotate_cnt = phase->transform(new ReplicateINode(lshift, vt));
 987         return new RotateRightVNode(in(1)->in(1), rotate_cnt, vt);
 988       }
 989     }
 990   }
 991   return NULL;
 992 }
 993