1 /* 2 * Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #include "precompiled.hpp" 25 #include "memory/allocation.inline.hpp" 26 #include "opto/connode.hpp" 27 #include "opto/mulnode.hpp" 28 #include "opto/subnode.hpp" 29 #include "opto/vectornode.hpp" 30 #include "utilities/powerOfTwo.hpp" 31 #include "utilities/globalDefinitions.hpp" 32 33 //------------------------------VectorNode-------------------------------------- 34 35 // Return the vector operator for the specified scalar operation 36 // and vector length. 37 int VectorNode::opcode(int sopc, BasicType bt) { 38 switch (sopc) { 39 case Op_AddI: 40 switch (bt) { 41 case T_BOOLEAN: 42 case T_BYTE: return Op_AddVB; 43 case T_CHAR: 44 case T_SHORT: return Op_AddVS; 45 case T_INT: return Op_AddVI; 46 default: ShouldNotReachHere(); return 0; 47 } 48 case Op_AddL: 49 assert(bt == T_LONG, "must be"); 50 return Op_AddVL; 51 case Op_AddF: 52 assert(bt == T_FLOAT, "must be"); 53 return Op_AddVF; 54 case Op_AddD: 55 assert(bt == T_DOUBLE, "must be"); 56 return Op_AddVD; 57 case Op_SubI: 58 switch (bt) { 59 case T_BOOLEAN: 60 case T_BYTE: return Op_SubVB; 61 case T_CHAR: 62 case T_SHORT: return Op_SubVS; 63 case T_INT: return Op_SubVI; 64 default: ShouldNotReachHere(); return 0; 65 } 66 case Op_SubL: 67 assert(bt == T_LONG, "must be"); 68 return Op_SubVL; 69 case Op_SubF: 70 assert(bt == T_FLOAT, "must be"); 71 return Op_SubVF; 72 case Op_SubD: 73 assert(bt == T_DOUBLE, "must be"); 74 return Op_SubVD; 75 case Op_MulI: 76 switch (bt) { 77 case T_BOOLEAN:return 0; 78 case T_BYTE: return Op_MulVB; 79 case T_CHAR: 80 case T_SHORT: return Op_MulVS; 81 case T_INT: return Op_MulVI; 82 default: ShouldNotReachHere(); return 0; 83 } 84 case Op_MulL: 85 assert(bt == T_LONG, "must be"); 86 return Op_MulVL; 87 case Op_MulF: 88 assert(bt == T_FLOAT, "must be"); 89 return Op_MulVF; 90 case Op_MulD: 91 assert(bt == T_DOUBLE, "must be"); 92 return Op_MulVD; 93 case Op_FmaD: 94 assert(bt == T_DOUBLE, "must be"); 95 return Op_FmaVD; 96 case Op_FmaF: 97 assert(bt == T_FLOAT, "must be"); 98 return Op_FmaVF; 99 case Op_CMoveF: 100 assert(bt == T_FLOAT, "must be"); 101 return Op_CMoveVF; 102 case Op_CMoveD: 103 assert(bt == T_DOUBLE, "must be"); 104 return Op_CMoveVD; 105 case Op_DivF: 106 assert(bt == T_FLOAT, "must be"); 107 return Op_DivVF; 108 case Op_DivD: 109 assert(bt == T_DOUBLE, "must be"); 110 return Op_DivVD; 111 case Op_AbsI: 112 switch (bt) { 113 case T_BOOLEAN: 114 case T_CHAR: return 0; // abs does not make sense for unsigned 115 case T_BYTE: return Op_AbsVB; 116 case T_SHORT: return Op_AbsVS; 117 case T_INT: return Op_AbsVI; 118 default: ShouldNotReachHere(); return 0; 119 } 120 case Op_AbsL: 121 assert(bt == T_LONG, "must be"); 122 return Op_AbsVL; 123 case Op_AbsF: 124 assert(bt == T_FLOAT, "must be"); 125 return Op_AbsVF; 126 case Op_AbsD: 127 assert(bt == T_DOUBLE, "must be"); 128 return Op_AbsVD; 129 case Op_NegF: 130 assert(bt == T_FLOAT, "must be"); 131 return Op_NegVF; 132 case Op_NegD: 133 assert(bt == T_DOUBLE, "must be"); 134 return Op_NegVD; 135 case Op_RoundDoubleMode: 136 assert(bt == T_DOUBLE, "must be"); 137 return Op_RoundDoubleModeV; 138 case Op_RotateLeft: 139 assert(bt == T_LONG || bt == T_INT, "must be"); 140 return Op_RotateLeftV; 141 case Op_RotateRight: 142 assert(bt == T_LONG || bt == T_INT, "must be"); 143 return Op_RotateRightV; 144 case Op_SqrtF: 145 assert(bt == T_FLOAT, "must be"); 146 return Op_SqrtVF; 147 case Op_SqrtD: 148 assert(bt == T_DOUBLE, "must be"); 149 return Op_SqrtVD; 150 case Op_PopCountI: 151 if (bt == T_INT) { 152 return Op_PopCountVI; 153 } 154 // Unimplemented for subword types since bit count changes 155 // depending on size of lane (and sign bit). 156 return 0; 157 case Op_LShiftI: 158 switch (bt) { 159 case T_BOOLEAN: 160 case T_BYTE: return Op_LShiftVB; 161 case T_CHAR: 162 case T_SHORT: return Op_LShiftVS; 163 case T_INT: return Op_LShiftVI; 164 default: ShouldNotReachHere(); return 0; 165 } 166 case Op_LShiftL: 167 assert(bt == T_LONG, "must be"); 168 return Op_LShiftVL; 169 case Op_RShiftI: 170 switch (bt) { 171 case T_BOOLEAN:return Op_URShiftVB; // boolean is unsigned value 172 case T_CHAR: return Op_URShiftVS; // char is unsigned value 173 case T_BYTE: return Op_RShiftVB; 174 case T_SHORT: return Op_RShiftVS; 175 case T_INT: return Op_RShiftVI; 176 default: ShouldNotReachHere(); return 0; 177 } 178 case Op_RShiftL: 179 assert(bt == T_LONG, "must be"); 180 return Op_RShiftVL; 181 case Op_URShiftI: 182 switch (bt) { 183 case T_BOOLEAN:return Op_URShiftVB; 184 case T_CHAR: return Op_URShiftVS; 185 case T_BYTE: 186 case T_SHORT: return 0; // Vector logical right shift for signed short 187 // values produces incorrect Java result for 188 // negative data because java code should convert 189 // a short value into int value with sign 190 // extension before a shift. 191 case T_INT: return Op_URShiftVI; 192 default: ShouldNotReachHere(); return 0; 193 } 194 case Op_URShiftL: 195 assert(bt == T_LONG, "must be"); 196 return Op_URShiftVL; 197 case Op_AndI: 198 case Op_AndL: 199 return Op_AndV; 200 case Op_OrI: 201 case Op_OrL: 202 return Op_OrV; 203 case Op_XorI: 204 case Op_XorL: 205 return Op_XorV; 206 case Op_MinF: 207 assert(bt == T_FLOAT, "must be"); 208 return Op_MinV; 209 case Op_MinD: 210 assert(bt == T_DOUBLE, "must be"); 211 return Op_MinV; 212 case Op_MaxF: 213 assert(bt == T_FLOAT, "must be"); 214 return Op_MaxV; 215 case Op_MaxD: 216 assert(bt == T_DOUBLE, "must be"); 217 return Op_MaxV; 218 219 case Op_LoadB: 220 case Op_LoadUB: 221 case Op_LoadUS: 222 case Op_LoadS: 223 case Op_LoadI: 224 case Op_LoadL: 225 case Op_LoadF: 226 case Op_LoadD: 227 return Op_LoadVector; 228 229 case Op_StoreB: 230 case Op_StoreC: 231 case Op_StoreI: 232 case Op_StoreL: 233 case Op_StoreF: 234 case Op_StoreD: 235 return Op_StoreVector; 236 case Op_MulAddS2I: 237 return Op_MulAddVS2VI; 238 239 default: 240 return 0; // Unimplemented 241 } 242 } 243 244 // Also used to check if the code generator 245 // supports the vector operation. 246 bool VectorNode::implemented(int opc, uint vlen, BasicType bt) { 247 if (is_java_primitive(bt) && 248 (vlen > 1) && is_power_of_2(vlen) && 249 Matcher::vector_size_supported(bt, vlen)) { 250 int vopc = VectorNode::opcode(opc, bt); 251 // For rotate operation we will do a lazy de-generation into 252 // OrV/LShiftV/URShiftV pattern if the target does not support 253 // vector rotation instruction. 254 if (vopc == Op_RotateLeftV || vopc == Op_RotateRightV) { 255 return is_vector_rotate_supported(vopc, vlen, bt); 256 } 257 return vopc > 0 && Matcher::match_rule_supported_vector(vopc, vlen, bt); 258 } 259 return false; 260 } 261 262 bool VectorNode::is_type_transition_short_to_int(Node* n) { 263 switch (n->Opcode()) { 264 case Op_MulAddS2I: 265 return true; 266 } 267 return false; 268 } 269 270 bool VectorNode::is_type_transition_to_int(Node* n) { 271 return is_type_transition_short_to_int(n); 272 } 273 274 bool VectorNode::is_muladds2i(Node* n) { 275 if (n->Opcode() == Op_MulAddS2I) { 276 return true; 277 } 278 return false; 279 } 280 281 bool VectorNode::is_roundopD(Node* n) { 282 if (n->Opcode() == Op_RoundDoubleMode) { 283 return true; 284 } 285 return false; 286 } 287 288 bool VectorNode::is_scalar_rotate(Node* n) { 289 if (n->Opcode() == Op_RotateLeft || n->Opcode() == Op_RotateRight) { 290 return true; 291 } 292 return false; 293 } 294 295 bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) { 296 assert(vopc == Op_RotateLeftV || vopc == Op_RotateRightV, "wrong opcode"); 297 298 // If target defines vector rotation patterns then no 299 // need for degeneration. 300 if (Matcher::match_rule_supported_vector(vopc, vlen, bt)) { 301 return true; 302 } 303 304 // Validate existence of nodes created in case of rotate degeneration. 305 switch (bt) { 306 case T_INT: 307 return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) && 308 Matcher::match_rule_supported_vector(Op_LShiftVI, vlen, bt) && 309 Matcher::match_rule_supported_vector(Op_URShiftVI, vlen, bt); 310 case T_LONG: 311 return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) && 312 Matcher::match_rule_supported_vector(Op_LShiftVL, vlen, bt) && 313 Matcher::match_rule_supported_vector(Op_URShiftVL, vlen, bt); 314 default: 315 assert(false, "not supported: %s", type2name(bt)); 316 return false; 317 } 318 } 319 320 bool VectorNode::is_shift(Node* n) { 321 switch (n->Opcode()) { 322 case Op_LShiftI: 323 case Op_LShiftL: 324 case Op_RShiftI: 325 case Op_RShiftL: 326 case Op_URShiftI: 327 case Op_URShiftL: 328 return true; 329 default: 330 return false; 331 } 332 } 333 334 // Check if input is loop invariant vector. 335 bool VectorNode::is_invariant_vector(Node* n) { 336 // Only Replicate vector nodes are loop invariant for now. 337 switch (n->Opcode()) { 338 case Op_ReplicateB: 339 case Op_ReplicateS: 340 case Op_ReplicateI: 341 case Op_ReplicateL: 342 case Op_ReplicateF: 343 case Op_ReplicateD: 344 return true; 345 default: 346 return false; 347 } 348 } 349 350 // [Start, end) half-open range defining which operands are vectors 351 void VectorNode::vector_operands(Node* n, uint* start, uint* end) { 352 switch (n->Opcode()) { 353 case Op_LoadB: case Op_LoadUB: 354 case Op_LoadS: case Op_LoadUS: 355 case Op_LoadI: case Op_LoadL: 356 case Op_LoadF: case Op_LoadD: 357 case Op_LoadP: case Op_LoadN: 358 *start = 0; 359 *end = 0; // no vector operands 360 break; 361 case Op_StoreB: case Op_StoreC: 362 case Op_StoreI: case Op_StoreL: 363 case Op_StoreF: case Op_StoreD: 364 case Op_StoreP: case Op_StoreN: 365 *start = MemNode::ValueIn; 366 *end = MemNode::ValueIn + 1; // 1 vector operand 367 break; 368 case Op_LShiftI: case Op_LShiftL: 369 case Op_RShiftI: case Op_RShiftL: 370 case Op_URShiftI: case Op_URShiftL: 371 *start = 1; 372 *end = 2; // 1 vector operand 373 break; 374 case Op_AddI: case Op_AddL: case Op_AddF: case Op_AddD: 375 case Op_SubI: case Op_SubL: case Op_SubF: case Op_SubD: 376 case Op_MulI: case Op_MulL: case Op_MulF: case Op_MulD: 377 case Op_DivF: case Op_DivD: 378 case Op_AndI: case Op_AndL: 379 case Op_OrI: case Op_OrL: 380 case Op_XorI: case Op_XorL: 381 case Op_MulAddS2I: 382 *start = 1; 383 *end = 3; // 2 vector operands 384 break; 385 case Op_CMoveI: case Op_CMoveL: case Op_CMoveF: case Op_CMoveD: 386 *start = 2; 387 *end = n->req(); 388 break; 389 case Op_FmaD: 390 case Op_FmaF: 391 *start = 1; 392 *end = 4; // 3 vector operands 393 break; 394 default: 395 *start = 1; 396 *end = n->req(); // default is all operands 397 } 398 } 399 400 // Return the vector version of a scalar operation node. 401 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt) { 402 const TypeVect* vt = TypeVect::make(bt, vlen); 403 int vopc = VectorNode::opcode(opc, bt); 404 // This method should not be called for unimplemented vectors. 405 guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]); 406 switch (vopc) { 407 case Op_AddVB: return new AddVBNode(n1, n2, vt); 408 case Op_AddVS: return new AddVSNode(n1, n2, vt); 409 case Op_AddVI: return new AddVINode(n1, n2, vt); 410 case Op_AddVL: return new AddVLNode(n1, n2, vt); 411 case Op_AddVF: return new AddVFNode(n1, n2, vt); 412 case Op_AddVD: return new AddVDNode(n1, n2, vt); 413 414 case Op_SubVB: return new SubVBNode(n1, n2, vt); 415 case Op_SubVS: return new SubVSNode(n1, n2, vt); 416 case Op_SubVI: return new SubVINode(n1, n2, vt); 417 case Op_SubVL: return new SubVLNode(n1, n2, vt); 418 case Op_SubVF: return new SubVFNode(n1, n2, vt); 419 case Op_SubVD: return new SubVDNode(n1, n2, vt); 420 421 case Op_MulVB: return new MulVBNode(n1, n2, vt); 422 case Op_MulVS: return new MulVSNode(n1, n2, vt); 423 case Op_MulVI: return new MulVINode(n1, n2, vt); 424 case Op_MulVL: return new MulVLNode(n1, n2, vt); 425 case Op_MulVF: return new MulVFNode(n1, n2, vt); 426 case Op_MulVD: return new MulVDNode(n1, n2, vt); 427 428 case Op_DivVF: return new DivVFNode(n1, n2, vt); 429 case Op_DivVD: return new DivVDNode(n1, n2, vt); 430 431 case Op_AbsVB: return new AbsVBNode(n1, vt); 432 case Op_AbsVS: return new AbsVSNode(n1, vt); 433 case Op_AbsVI: return new AbsVINode(n1, vt); 434 case Op_AbsVL: return new AbsVLNode(n1, vt); 435 case Op_AbsVF: return new AbsVFNode(n1, vt); 436 case Op_AbsVD: return new AbsVDNode(n1, vt); 437 438 case Op_NegVF: return new NegVFNode(n1, vt); 439 case Op_NegVD: return new NegVDNode(n1, vt); 440 441 case Op_SqrtVF: return new SqrtVFNode(n1, vt); 442 case Op_SqrtVD: return new SqrtVDNode(n1, vt); 443 444 case Op_PopCountVI: return new PopCountVINode(n1, vt); 445 case Op_RotateLeftV: return new RotateLeftVNode(n1, n2, vt); 446 case Op_RotateRightV: return new RotateRightVNode(n1, n2, vt); 447 448 case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt); 449 case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt); 450 case Op_LShiftVI: return new LShiftVINode(n1, n2, vt); 451 case Op_LShiftVL: return new LShiftVLNode(n1, n2, vt); 452 453 case Op_RShiftVB: return new RShiftVBNode(n1, n2, vt); 454 case Op_RShiftVS: return new RShiftVSNode(n1, n2, vt); 455 case Op_RShiftVI: return new RShiftVINode(n1, n2, vt); 456 case Op_RShiftVL: return new RShiftVLNode(n1, n2, vt); 457 458 case Op_URShiftVB: return new URShiftVBNode(n1, n2, vt); 459 case Op_URShiftVS: return new URShiftVSNode(n1, n2, vt); 460 case Op_URShiftVI: return new URShiftVINode(n1, n2, vt); 461 case Op_URShiftVL: return new URShiftVLNode(n1, n2, vt); 462 463 case Op_AndV: return new AndVNode(n1, n2, vt); 464 case Op_OrV: return new OrVNode (n1, n2, vt); 465 case Op_XorV: return new XorVNode(n1, n2, vt); 466 467 case Op_MinV: return new MinVNode(n1, n2, vt); 468 case Op_MaxV: return new MaxVNode(n1, n2, vt); 469 470 case Op_RoundDoubleModeV: return new RoundDoubleModeVNode(n1, n2, vt); 471 472 case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt); 473 default: 474 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]); 475 return NULL; 476 } 477 } 478 479 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) { 480 const TypeVect* vt = TypeVect::make(bt, vlen); 481 int vopc = VectorNode::opcode(opc, bt); 482 // This method should not be called for unimplemented vectors. 483 guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]); 484 switch (vopc) { 485 case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt); 486 case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt); 487 default: 488 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]); 489 return NULL; 490 } 491 } 492 493 // Scalar promotion 494 VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t) { 495 BasicType bt = opd_t->array_element_basic_type(); 496 const TypeVect* vt = opd_t->singleton() ? TypeVect::make(opd_t, vlen) 497 : TypeVect::make(bt, vlen); 498 switch (bt) { 499 case T_BOOLEAN: 500 case T_BYTE: 501 return new ReplicateBNode(s, vt); 502 case T_CHAR: 503 case T_SHORT: 504 return new ReplicateSNode(s, vt); 505 case T_INT: 506 return new ReplicateINode(s, vt); 507 case T_LONG: 508 return new ReplicateLNode(s, vt); 509 case T_FLOAT: 510 return new ReplicateFNode(s, vt); 511 case T_DOUBLE: 512 return new ReplicateDNode(s, vt); 513 default: 514 fatal("Type '%s' is not supported for vectors", type2name(bt)); 515 return NULL; 516 } 517 } 518 519 VectorNode* VectorNode::shift_count(Node* shift, Node* cnt, uint vlen, BasicType bt) { 520 assert(VectorNode::is_shift(shift), "sanity"); 521 // Match shift count type with shift vector type. 522 const TypeVect* vt = TypeVect::make(bt, vlen); 523 switch (shift->Opcode()) { 524 case Op_LShiftI: 525 case Op_LShiftL: 526 return new LShiftCntVNode(cnt, vt); 527 case Op_RShiftI: 528 case Op_RShiftL: 529 case Op_URShiftI: 530 case Op_URShiftL: 531 return new RShiftCntVNode(cnt, vt); 532 default: 533 fatal("Missed vector creation for '%s'", NodeClassNames[shift->Opcode()]); 534 return NULL; 535 } 536 } 537 538 bool VectorNode::is_vector_shift(int opc) { 539 assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode"); 540 switch (opc) { 541 case Op_LShiftVB: 542 case Op_LShiftVS: 543 case Op_LShiftVI: 544 case Op_LShiftVL: 545 case Op_RShiftVB: 546 case Op_RShiftVS: 547 case Op_RShiftVI: 548 case Op_RShiftVL: 549 case Op_URShiftVB: 550 case Op_URShiftVS: 551 case Op_URShiftVI: 552 case Op_URShiftVL: 553 return true; 554 default: 555 return false; 556 } 557 } 558 559 bool VectorNode::is_vector_shift_count(int opc) { 560 assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode"); 561 switch (opc) { 562 case Op_RShiftCntV: 563 case Op_LShiftCntV: 564 return true; 565 default: 566 return false; 567 } 568 } 569 570 static bool is_con_M1(Node* n) { 571 if (n->is_Con()) { 572 const Type* t = n->bottom_type(); 573 if (t->isa_int() && t->is_int()->get_con() == -1) { 574 return true; 575 } 576 if (t->isa_long() && t->is_long()->get_con() == -1) { 577 return true; 578 } 579 } 580 return false; 581 } 582 583 bool VectorNode::is_all_ones_vector(Node* n) { 584 switch (n->Opcode()) { 585 case Op_ReplicateB: 586 case Op_ReplicateS: 587 case Op_ReplicateI: 588 case Op_ReplicateL: 589 return is_con_M1(n->in(1)); 590 default: 591 return false; 592 } 593 } 594 595 bool VectorNode::is_vector_bitwise_not_pattern(Node* n) { 596 if (n->Opcode() == Op_XorV) { 597 return is_all_ones_vector(n->in(1)) || 598 is_all_ones_vector(n->in(2)); 599 } 600 return false; 601 } 602 603 // Return initial Pack node. Additional operands added with add_opd() calls. 604 PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) { 605 const TypeVect* vt = TypeVect::make(bt, vlen); 606 switch (bt) { 607 case T_BOOLEAN: 608 case T_BYTE: 609 return new PackBNode(s, vt); 610 case T_CHAR: 611 case T_SHORT: 612 return new PackSNode(s, vt); 613 case T_INT: 614 return new PackINode(s, vt); 615 case T_LONG: 616 return new PackLNode(s, vt); 617 case T_FLOAT: 618 return new PackFNode(s, vt); 619 case T_DOUBLE: 620 return new PackDNode(s, vt); 621 default: 622 fatal("Type '%s' is not supported for vectors", type2name(bt)); 623 return NULL; 624 } 625 } 626 627 // Create a binary tree form for Packs. [lo, hi) (half-open) range 628 PackNode* PackNode::binary_tree_pack(int lo, int hi) { 629 int ct = hi - lo; 630 assert(is_power_of_2(ct), "power of 2"); 631 if (ct == 2) { 632 PackNode* pk = PackNode::make(in(lo), 2, vect_type()->element_basic_type()); 633 pk->add_opd(in(lo+1)); 634 return pk; 635 } else { 636 int mid = lo + ct/2; 637 PackNode* n1 = binary_tree_pack(lo, mid); 638 PackNode* n2 = binary_tree_pack(mid, hi ); 639 640 BasicType bt = n1->vect_type()->element_basic_type(); 641 assert(bt == n2->vect_type()->element_basic_type(), "should be the same"); 642 switch (bt) { 643 case T_BOOLEAN: 644 case T_BYTE: 645 return new PackSNode(n1, n2, TypeVect::make(T_SHORT, 2)); 646 case T_CHAR: 647 case T_SHORT: 648 return new PackINode(n1, n2, TypeVect::make(T_INT, 2)); 649 case T_INT: 650 return new PackLNode(n1, n2, TypeVect::make(T_LONG, 2)); 651 case T_LONG: 652 return new Pack2LNode(n1, n2, TypeVect::make(T_LONG, 2)); 653 case T_FLOAT: 654 return new PackDNode(n1, n2, TypeVect::make(T_DOUBLE, 2)); 655 case T_DOUBLE: 656 return new Pack2DNode(n1, n2, TypeVect::make(T_DOUBLE, 2)); 657 default: 658 fatal("Type '%s' is not supported for vectors", type2name(bt)); 659 return NULL; 660 } 661 } 662 } 663 664 // Return the vector version of a scalar load node. 665 LoadVectorNode* LoadVectorNode::make(int opc, Node* ctl, Node* mem, 666 Node* adr, const TypePtr* atyp, 667 uint vlen, BasicType bt, 668 ControlDependency control_dependency) { 669 const TypeVect* vt = TypeVect::make(bt, vlen); 670 return new LoadVectorNode(ctl, mem, adr, atyp, vt, control_dependency); 671 } 672 673 // Return the vector version of a scalar store node. 674 StoreVectorNode* StoreVectorNode::make(int opc, Node* ctl, Node* mem, 675 Node* adr, const TypePtr* atyp, Node* val, 676 uint vlen) { 677 return new StoreVectorNode(ctl, mem, adr, atyp, val); 678 } 679 680 // Extract a scalar element of vector. 681 Node* ExtractNode::make(Node* v, uint position, BasicType bt) { 682 assert((int)position < Matcher::max_vector_size(bt), "pos in range"); 683 ConINode* pos = ConINode::make((int)position); 684 switch (bt) { 685 case T_BOOLEAN: 686 return new ExtractUBNode(v, pos); 687 case T_BYTE: 688 return new ExtractBNode(v, pos); 689 case T_CHAR: 690 return new ExtractCNode(v, pos); 691 case T_SHORT: 692 return new ExtractSNode(v, pos); 693 case T_INT: 694 return new ExtractINode(v, pos); 695 case T_LONG: 696 return new ExtractLNode(v, pos); 697 case T_FLOAT: 698 return new ExtractFNode(v, pos); 699 case T_DOUBLE: 700 return new ExtractDNode(v, pos); 701 default: 702 fatal("Type '%s' is not supported for vectors", type2name(bt)); 703 return NULL; 704 } 705 } 706 707 int ReductionNode::opcode(int opc, BasicType bt) { 708 int vopc = opc; 709 switch (opc) { 710 case Op_AddI: 711 assert(bt == T_INT, "must be"); 712 vopc = Op_AddReductionVI; 713 break; 714 case Op_AddL: 715 assert(bt == T_LONG, "must be"); 716 vopc = Op_AddReductionVL; 717 break; 718 case Op_AddF: 719 assert(bt == T_FLOAT, "must be"); 720 vopc = Op_AddReductionVF; 721 break; 722 case Op_AddD: 723 assert(bt == T_DOUBLE, "must be"); 724 vopc = Op_AddReductionVD; 725 break; 726 case Op_MulI: 727 assert(bt == T_INT, "must be"); 728 vopc = Op_MulReductionVI; 729 break; 730 case Op_MulL: 731 assert(bt == T_LONG, "must be"); 732 vopc = Op_MulReductionVL; 733 break; 734 case Op_MulF: 735 assert(bt == T_FLOAT, "must be"); 736 vopc = Op_MulReductionVF; 737 break; 738 case Op_MulD: 739 assert(bt == T_DOUBLE, "must be"); 740 vopc = Op_MulReductionVD; 741 break; 742 case Op_MinF: 743 assert(bt == T_FLOAT, "must be"); 744 vopc = Op_MinReductionV; 745 break; 746 case Op_MinD: 747 assert(bt == T_DOUBLE, "must be"); 748 vopc = Op_MinReductionV; 749 break; 750 case Op_MaxF: 751 assert(bt == T_FLOAT, "must be"); 752 vopc = Op_MaxReductionV; 753 break; 754 case Op_MaxD: 755 assert(bt == T_DOUBLE, "must be"); 756 vopc = Op_MaxReductionV; 757 break; 758 case Op_AndI: 759 assert(bt == T_INT, "must be"); 760 vopc = Op_AndReductionV; 761 break; 762 case Op_AndL: 763 assert(bt == T_LONG, "must be"); 764 vopc = Op_AndReductionV; 765 break; 766 case Op_OrI: 767 assert(bt == T_INT, "must be"); 768 vopc = Op_OrReductionV; 769 break; 770 case Op_OrL: 771 assert(bt == T_LONG, "must be"); 772 vopc = Op_OrReductionV; 773 break; 774 case Op_XorI: 775 assert(bt == T_INT, "must be"); 776 vopc = Op_XorReductionV; 777 break; 778 case Op_XorL: 779 assert(bt == T_LONG, "must be"); 780 vopc = Op_XorReductionV; 781 break; 782 default: 783 break; 784 } 785 return vopc; 786 } 787 788 // Return the appropriate reduction node. 789 ReductionNode* ReductionNode::make(int opc, Node *ctrl, Node* n1, Node* n2, BasicType bt) { 790 791 int vopc = opcode(opc, bt); 792 793 // This method should not be called for unimplemented vectors. 794 guarantee(vopc != opc, "Vector for '%s' is not implemented", NodeClassNames[opc]); 795 796 switch (vopc) { 797 case Op_AddReductionVI: return new AddReductionVINode(ctrl, n1, n2); 798 case Op_AddReductionVL: return new AddReductionVLNode(ctrl, n1, n2); 799 case Op_AddReductionVF: return new AddReductionVFNode(ctrl, n1, n2); 800 case Op_AddReductionVD: return new AddReductionVDNode(ctrl, n1, n2); 801 case Op_MulReductionVI: return new MulReductionVINode(ctrl, n1, n2); 802 case Op_MulReductionVL: return new MulReductionVLNode(ctrl, n1, n2); 803 case Op_MulReductionVF: return new MulReductionVFNode(ctrl, n1, n2); 804 case Op_MulReductionVD: return new MulReductionVDNode(ctrl, n1, n2); 805 case Op_MinReductionV: return new MinReductionVNode(ctrl, n1, n2); 806 case Op_MaxReductionV: return new MaxReductionVNode(ctrl, n1, n2); 807 case Op_AndReductionV: return new AndReductionVNode(ctrl, n1, n2); 808 case Op_OrReductionV: return new OrReductionVNode(ctrl, n1, n2); 809 case Op_XorReductionV: return new XorReductionVNode(ctrl, n1, n2); 810 default: 811 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]); 812 return NULL; 813 } 814 } 815 816 bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) { 817 if (is_java_primitive(bt) && 818 (vlen > 1) && is_power_of_2(vlen) && 819 Matcher::vector_size_supported(bt, vlen)) { 820 int vopc = ReductionNode::opcode(opc, bt); 821 return vopc != opc && Matcher::match_rule_supported(vopc); 822 } 823 return false; 824 } 825 826 MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node* in3, 827 uint truth_table, const TypeVect* vt) { 828 assert(truth_table <= 0xFF, "invalid"); 829 assert(in1->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch"); 830 assert(in2->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch"); 831 assert(in3->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch"); 832 Node* fn = gvn.intcon(truth_table); 833 return new MacroLogicVNode(in1, in2, in3, fn, vt); 834 } 835 836 Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_left, 837 int vlen, BasicType bt, PhaseGVN* phase) { 838 int shiftLOpc; 839 int shiftROpc; 840 Node* shiftLCnt = NULL; 841 Node* shiftRCnt = NULL; 842 const TypeVect* vt = TypeVect::make(bt, vlen); 843 844 // Compute shift values for right rotation and 845 // later swap them in case of left rotation. 846 if (cnt->is_Con()) { 847 // Constant shift case. 848 if (bt == T_INT) { 849 int shift = cnt->get_int() & 31; 850 shiftRCnt = phase->intcon(shift); 851 shiftLCnt = phase->intcon(32 - shift); 852 shiftLOpc = Op_LShiftI; 853 shiftROpc = Op_URShiftI; 854 } else { 855 int shift = cnt->get_int() & 63; 856 shiftRCnt = phase->intcon(shift); 857 shiftLCnt = phase->intcon(64 - shift); 858 shiftLOpc = Op_LShiftL; 859 shiftROpc = Op_URShiftL; 860 } 861 } else { 862 // Variable shift case. 863 assert(VectorNode::is_invariant_vector(cnt), "Broadcast expected"); 864 cnt = cnt->in(1); 865 if (bt == T_INT) { 866 shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(31))); 867 shiftLCnt = phase->transform(new SubINode(phase->intcon(32), shiftRCnt)); 868 shiftLOpc = Op_LShiftI; 869 shiftROpc = Op_URShiftI; 870 } else { 871 assert(cnt->Opcode() == Op_ConvI2L, "ConvI2L expected"); 872 cnt = cnt->in(1); 873 shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(63))); 874 shiftLCnt = phase->transform(new SubINode(phase->intcon(64), shiftRCnt)); 875 shiftLOpc = Op_LShiftL; 876 shiftROpc = Op_URShiftL; 877 } 878 } 879 880 // Swap the computed left and right shift counts. 881 if (is_rotate_left) { 882 swap(shiftRCnt,shiftLCnt); 883 } 884 885 shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt)); 886 shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt)); 887 888 return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt)), 889 phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt)), 890 vt); 891 } 892 893 Node* RotateLeftVNode::Ideal(PhaseGVN* phase, bool can_reshape) { 894 int vlen = length(); 895 BasicType bt = vect_type()->element_basic_type(); 896 if (!Matcher::match_rule_supported_vector(Op_RotateLeftV, vlen, bt)) { 897 return VectorNode::degenerate_vector_rotate(in(1), in(2), true, vlen, bt, phase); 898 } 899 return NULL; 900 } 901 902 Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) { 903 int vlen = length(); 904 BasicType bt = vect_type()->element_basic_type(); 905 if (!Matcher::match_rule_supported_vector(Op_RotateRightV, vlen, bt)) { 906 return VectorNode::degenerate_vector_rotate(in(1), in(2), false, vlen, bt, phase); 907 } 908 return NULL; 909 } 910 911 Node* OrVNode::Ideal(PhaseGVN* phase, bool can_reshape) { 912 int lopcode = in(1)->Opcode(); 913 int ropcode = in(2)->Opcode(); 914 const TypeVect* vt = bottom_type()->is_vect(); 915 int vec_len = vt->length(); 916 BasicType bt = vt->element_basic_type(); 917 918 // Vector Rotate operations inferencing, this will be useful when vector 919 // operations are created via non-SLP route i.e. (VectorAPI). 920 if (Matcher::match_rule_supported_vector(Op_RotateLeftV, vec_len, bt) && 921 ((ropcode == Op_LShiftVI && lopcode == Op_URShiftVI) || 922 (ropcode == Op_LShiftVL && lopcode == Op_URShiftVL)) && 923 in(1)->in(1) == in(2)->in(1)) { 924 assert(Op_RShiftCntV == in(1)->in(2)->Opcode(), "LShiftCntV operand expected"); 925 assert(Op_LShiftCntV == in(2)->in(2)->Opcode(), "RShiftCntV operand expected"); 926 Node* lshift = in(1)->in(2)->in(1); 927 Node* rshift = in(2)->in(2)->in(1); 928 int mod_val = bt == T_LONG ? 64 : 32; 929 int shift_mask = bt == T_LONG ? 0x3F : 0x1F; 930 // val >> norm_con_shift | val << (32 - norm_con_shift) => rotate_right val , 931 // norm_con_shift 932 if (lshift->is_Con() && rshift->is_Con() && 933 ((lshift->get_int() & shift_mask) == 934 (mod_val - (rshift->get_int() & shift_mask)))) { 935 return new RotateRightVNode( 936 in(1)->in(1), phase->intcon(lshift->get_int() & shift_mask), vt); 937 } 938 if (lshift->Opcode() == Op_AndI && rshift->Opcode() == Op_AndI && 939 lshift->in(2)->is_Con() && rshift->in(2)->is_Con() && 940 lshift->in(2)->get_int() == (mod_val - 1) && 941 rshift->in(2)->get_int() == (mod_val - 1)) { 942 lshift = lshift->in(1); 943 rshift = rshift->in(1); 944 // val << var_shift | val >> (0/32 - var_shift) => rotate_left val , 945 // var_shift 946 if (lshift->Opcode() == Op_SubI && lshift->in(2) == rshift && 947 lshift->in(1)->is_Con() && 948 (lshift->in(1)->get_int() == 0 || 949 lshift->in(1)->get_int() == mod_val)) { 950 Node* rotate_cnt = phase->transform(new ReplicateINode(rshift, vt)); 951 return new RotateLeftVNode(in(1)->in(1), rotate_cnt, vt); 952 } 953 } 954 } 955 956 if (Matcher::match_rule_supported_vector(Op_RotateRightV, vec_len, bt) && 957 ((ropcode == Op_URShiftVI && lopcode == Op_LShiftVI) || 958 (ropcode == Op_URShiftVL && lopcode == Op_LShiftVL)) && 959 in(1)->in(1) == in(2)->in(1)) { 960 assert(Op_LShiftCntV == in(1)->in(2)->Opcode(), "RShiftCntV operand expected"); 961 assert(Op_RShiftCntV == in(2)->in(2)->Opcode(), "LShiftCntV operand expected"); 962 Node* rshift = in(1)->in(2)->in(1); 963 Node* lshift = in(2)->in(2)->in(1); 964 int mod_val = bt == T_LONG ? 64 : 32; 965 int shift_mask = bt == T_LONG ? 0x3F : 0x1F; 966 // val << norm_con_shift | val >> (32 - norm_con_shift) => rotate_left val 967 // , norm_con_shift 968 if (rshift->is_Con() && lshift->is_Con() && 969 ((rshift->get_int() & shift_mask) == 970 (mod_val - (lshift->get_int() & shift_mask)))) { 971 return new RotateLeftVNode( 972 in(1)->in(1), phase->intcon(rshift->get_int() & shift_mask), vt); 973 } 974 if (lshift->Opcode() == Op_AndI && rshift->Opcode() == Op_AndI && 975 lshift->in(2)->is_Con() && rshift->in(2)->is_Con() && 976 rshift->in(2)->get_int() == (mod_val - 1) && 977 lshift->in(2)->get_int() == (mod_val - 1)) { 978 rshift = rshift->in(1); 979 lshift = lshift->in(1); 980 // val >> var_shift | val << (0/32 - var_shift) => rotate_right val , 981 // var_shift 982 if (rshift->Opcode() == Op_SubI && rshift->in(2) == lshift && 983 rshift->in(1)->is_Con() && 984 (rshift->in(1)->get_int() == 0 || 985 rshift->in(1)->get_int() == mod_val)) { 986 Node* rotate_cnt = phase->transform(new ReplicateINode(lshift, vt)); 987 return new RotateRightVNode(in(1)->in(1), rotate_cnt, vt); 988 } 989 } 990 } 991 return NULL; 992 } 993