1 /*
2 * Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.inline.hpp"
26 #include "opto/connode.hpp"
27 #include "opto/mulnode.hpp"
28 #include "opto/subnode.hpp"
29 #include "opto/vectornode.hpp"
30 #include "utilities/powerOfTwo.hpp"
31 #include "utilities/globalDefinitions.hpp"
32
33 //------------------------------VectorNode--------------------------------------
34
35 // Return the vector operator for the specified scalar operation
36 // and vector length.
37 int VectorNode::opcode(int sopc, BasicType bt) {
38 switch (sopc) {
39 case Op_AddI:
40 switch (bt) {
41 case T_BOOLEAN:
42 case T_BYTE: return Op_AddVB;
43 case T_CHAR:
44 case T_SHORT: return Op_AddVS;
45 case T_INT: return Op_AddVI;
46 default: ShouldNotReachHere(); return 0;
47 }
48 case Op_AddL:
49 assert(bt == T_LONG, "must be");
50 return Op_AddVL;
51 case Op_AddF:
52 assert(bt == T_FLOAT, "must be");
53 return Op_AddVF;
54 case Op_AddD:
55 assert(bt == T_DOUBLE, "must be");
56 return Op_AddVD;
57 case Op_SubI:
58 switch (bt) {
59 case T_BOOLEAN:
60 case T_BYTE: return Op_SubVB;
61 case T_CHAR:
62 case T_SHORT: return Op_SubVS;
63 case T_INT: return Op_SubVI;
64 default: ShouldNotReachHere(); return 0;
65 }
66 case Op_SubL:
67 assert(bt == T_LONG, "must be");
68 return Op_SubVL;
69 case Op_SubF:
70 assert(bt == T_FLOAT, "must be");
71 return Op_SubVF;
72 case Op_SubD:
73 assert(bt == T_DOUBLE, "must be");
74 return Op_SubVD;
75 case Op_MulI:
76 switch (bt) {
77 case T_BOOLEAN:return 0;
78 case T_BYTE: return Op_MulVB;
79 case T_CHAR:
80 case T_SHORT: return Op_MulVS;
81 case T_INT: return Op_MulVI;
82 default: ShouldNotReachHere(); return 0;
83 }
84 case Op_MulL:
85 assert(bt == T_LONG, "must be");
86 return Op_MulVL;
87 case Op_MulF:
88 assert(bt == T_FLOAT, "must be");
89 return Op_MulVF;
90 case Op_MulD:
91 assert(bt == T_DOUBLE, "must be");
92 return Op_MulVD;
93 case Op_FmaD:
94 assert(bt == T_DOUBLE, "must be");
95 return Op_FmaVD;
96 case Op_FmaF:
97 assert(bt == T_FLOAT, "must be");
98 return Op_FmaVF;
99 case Op_CMoveF:
100 assert(bt == T_FLOAT, "must be");
101 return Op_CMoveVF;
102 case Op_CMoveD:
103 assert(bt == T_DOUBLE, "must be");
104 return Op_CMoveVD;
105 case Op_DivF:
106 assert(bt == T_FLOAT, "must be");
107 return Op_DivVF;
108 case Op_DivD:
109 assert(bt == T_DOUBLE, "must be");
110 return Op_DivVD;
111 case Op_AbsI:
112 switch (bt) {
113 case T_BOOLEAN:
114 case T_CHAR: return 0; // abs does not make sense for unsigned
115 case T_BYTE: return Op_AbsVB;
116 case T_SHORT: return Op_AbsVS;
117 case T_INT: return Op_AbsVI;
118 default: ShouldNotReachHere(); return 0;
119 }
120 case Op_AbsL:
121 assert(bt == T_LONG, "must be");
122 return Op_AbsVL;
123 case Op_AbsF:
124 assert(bt == T_FLOAT, "must be");
125 return Op_AbsVF;
126 case Op_AbsD:
127 assert(bt == T_DOUBLE, "must be");
128 return Op_AbsVD;
129 case Op_NegF:
130 assert(bt == T_FLOAT, "must be");
131 return Op_NegVF;
132 case Op_NegD:
133 assert(bt == T_DOUBLE, "must be");
134 return Op_NegVD;
135 case Op_RoundDoubleMode:
136 assert(bt == T_DOUBLE, "must be");
137 return Op_RoundDoubleModeV;
138 case Op_RotateLeft:
139 assert(bt == T_LONG || bt == T_INT, "must be");
140 return Op_RotateLeftV;
141 case Op_RotateRight:
142 assert(bt == T_LONG || bt == T_INT, "must be");
143 return Op_RotateRightV;
144 case Op_SqrtF:
145 assert(bt == T_FLOAT, "must be");
146 return Op_SqrtVF;
147 case Op_SqrtD:
148 assert(bt == T_DOUBLE, "must be");
149 return Op_SqrtVD;
150 case Op_PopCountI:
151 if (bt == T_INT) {
152 return Op_PopCountVI;
153 }
154 // Unimplemented for subword types since bit count changes
155 // depending on size of lane (and sign bit).
156 return 0;
157 case Op_LShiftI:
158 switch (bt) {
159 case T_BOOLEAN:
160 case T_BYTE: return Op_LShiftVB;
161 case T_CHAR:
162 case T_SHORT: return Op_LShiftVS;
163 case T_INT: return Op_LShiftVI;
164 default: ShouldNotReachHere(); return 0;
165 }
166 case Op_LShiftL:
167 assert(bt == T_LONG, "must be");
168 return Op_LShiftVL;
169 case Op_RShiftI:
170 switch (bt) {
171 case T_BOOLEAN:return Op_URShiftVB; // boolean is unsigned value
172 case T_CHAR: return Op_URShiftVS; // char is unsigned value
173 case T_BYTE: return Op_RShiftVB;
174 case T_SHORT: return Op_RShiftVS;
175 case T_INT: return Op_RShiftVI;
176 default: ShouldNotReachHere(); return 0;
177 }
178 case Op_RShiftL:
179 assert(bt == T_LONG, "must be");
180 return Op_RShiftVL;
181 case Op_URShiftI:
182 switch (bt) {
183 case T_BOOLEAN:return Op_URShiftVB;
184 case T_CHAR: return Op_URShiftVS;
185 case T_BYTE:
186 case T_SHORT: return 0; // Vector logical right shift for signed short
187 // values produces incorrect Java result for
188 // negative data because java code should convert
189 // a short value into int value with sign
190 // extension before a shift.
191 case T_INT: return Op_URShiftVI;
192 default: ShouldNotReachHere(); return 0;
193 }
194 case Op_URShiftL:
195 assert(bt == T_LONG, "must be");
196 return Op_URShiftVL;
197 case Op_AndI:
198 case Op_AndL:
199 return Op_AndV;
200 case Op_OrI:
201 case Op_OrL:
202 return Op_OrV;
203 case Op_XorI:
204 case Op_XorL:
205 return Op_XorV;
206 case Op_MinF:
207 assert(bt == T_FLOAT, "must be");
208 return Op_MinV;
209 case Op_MinD:
210 assert(bt == T_DOUBLE, "must be");
211 return Op_MinV;
212 case Op_MaxF:
213 assert(bt == T_FLOAT, "must be");
214 return Op_MaxV;
215 case Op_MaxD:
216 assert(bt == T_DOUBLE, "must be");
217 return Op_MaxV;
218
219 case Op_LoadB:
220 case Op_LoadUB:
221 case Op_LoadUS:
222 case Op_LoadS:
223 case Op_LoadI:
224 case Op_LoadL:
225 case Op_LoadF:
226 case Op_LoadD:
227 return Op_LoadVector;
228
229 case Op_StoreB:
230 case Op_StoreC:
231 case Op_StoreI:
232 case Op_StoreL:
233 case Op_StoreF:
234 case Op_StoreD:
235 return Op_StoreVector;
236 case Op_MulAddS2I:
237 return Op_MulAddVS2VI;
238
239 default:
240 return 0; // Unimplemented
241 }
242 }
243
244 // Also used to check if the code generator
245 // supports the vector operation.
246 bool VectorNode::implemented(int opc, uint vlen, BasicType bt) {
247 if (is_java_primitive(bt) &&
248 (vlen > 1) && is_power_of_2(vlen) &&
249 Matcher::vector_size_supported(bt, vlen)) {
250 int vopc = VectorNode::opcode(opc, bt);
251 // For rotate operation we will do a lazy de-generation into
252 // OrV/LShiftV/URShiftV pattern if the target does not support
253 // vector rotation instruction.
254 if (vopc == Op_RotateLeftV || vopc == Op_RotateRightV) {
255 return is_vector_rotate_supported(vopc, vlen, bt);
256 }
257 return vopc > 0 && Matcher::match_rule_supported_vector(vopc, vlen, bt);
258 }
259 return false;
260 }
261
262 bool VectorNode::is_type_transition_short_to_int(Node* n) {
263 switch (n->Opcode()) {
264 case Op_MulAddS2I:
265 return true;
266 }
267 return false;
268 }
269
270 bool VectorNode::is_type_transition_to_int(Node* n) {
271 return is_type_transition_short_to_int(n);
272 }
273
274 bool VectorNode::is_muladds2i(Node* n) {
275 if (n->Opcode() == Op_MulAddS2I) {
276 return true;
277 }
278 return false;
279 }
280
281 bool VectorNode::is_roundopD(Node* n) {
282 if (n->Opcode() == Op_RoundDoubleMode) {
283 return true;
284 }
285 return false;
286 }
287
288 bool VectorNode::is_scalar_rotate(Node* n) {
289 if (n->Opcode() == Op_RotateLeft || n->Opcode() == Op_RotateRight) {
290 return true;
291 }
292 return false;
293 }
294
295 bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) {
296 assert(vopc == Op_RotateLeftV || vopc == Op_RotateRightV, "wrong opcode");
297
298 // If target defines vector rotation patterns then no
299 // need for degeneration.
300 if (Matcher::match_rule_supported_vector(vopc, vlen, bt)) {
301 return true;
302 }
303
304 // Validate existence of nodes created in case of rotate degeneration.
305 switch (bt) {
306 case T_INT:
307 return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) &&
308 Matcher::match_rule_supported_vector(Op_LShiftVI, vlen, bt) &&
309 Matcher::match_rule_supported_vector(Op_URShiftVI, vlen, bt);
310 case T_LONG:
311 return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) &&
312 Matcher::match_rule_supported_vector(Op_LShiftVL, vlen, bt) &&
313 Matcher::match_rule_supported_vector(Op_URShiftVL, vlen, bt);
314 default:
315 assert(false, "not supported: %s", type2name(bt));
316 return false;
317 }
318 }
319
320 bool VectorNode::is_shift(Node* n) {
321 switch (n->Opcode()) {
322 case Op_LShiftI:
323 case Op_LShiftL:
324 case Op_RShiftI:
325 case Op_RShiftL:
326 case Op_URShiftI:
327 case Op_URShiftL:
328 return true;
329 default:
330 return false;
331 }
332 }
333
334 // Check if input is loop invariant vector.
335 bool VectorNode::is_invariant_vector(Node* n) {
336 // Only Replicate vector nodes are loop invariant for now.
337 switch (n->Opcode()) {
338 case Op_ReplicateB:
339 case Op_ReplicateS:
340 case Op_ReplicateI:
341 case Op_ReplicateL:
342 case Op_ReplicateF:
343 case Op_ReplicateD:
344 return true;
345 default:
346 return false;
347 }
348 }
349
350 // [Start, end) half-open range defining which operands are vectors
351 void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
352 switch (n->Opcode()) {
353 case Op_LoadB: case Op_LoadUB:
354 case Op_LoadS: case Op_LoadUS:
355 case Op_LoadI: case Op_LoadL:
356 case Op_LoadF: case Op_LoadD:
357 case Op_LoadP: case Op_LoadN:
358 *start = 0;
359 *end = 0; // no vector operands
360 break;
361 case Op_StoreB: case Op_StoreC:
362 case Op_StoreI: case Op_StoreL:
363 case Op_StoreF: case Op_StoreD:
364 case Op_StoreP: case Op_StoreN:
365 *start = MemNode::ValueIn;
366 *end = MemNode::ValueIn + 1; // 1 vector operand
367 break;
368 case Op_LShiftI: case Op_LShiftL:
369 case Op_RShiftI: case Op_RShiftL:
370 case Op_URShiftI: case Op_URShiftL:
371 *start = 1;
372 *end = 2; // 1 vector operand
373 break;
374 case Op_AddI: case Op_AddL: case Op_AddF: case Op_AddD:
375 case Op_SubI: case Op_SubL: case Op_SubF: case Op_SubD:
376 case Op_MulI: case Op_MulL: case Op_MulF: case Op_MulD:
377 case Op_DivF: case Op_DivD:
378 case Op_AndI: case Op_AndL:
379 case Op_OrI: case Op_OrL:
380 case Op_XorI: case Op_XorL:
381 case Op_MulAddS2I:
382 *start = 1;
383 *end = 3; // 2 vector operands
384 break;
385 case Op_CMoveI: case Op_CMoveL: case Op_CMoveF: case Op_CMoveD:
386 *start = 2;
387 *end = n->req();
388 break;
389 case Op_FmaD:
390 case Op_FmaF:
391 *start = 1;
392 *end = 4; // 3 vector operands
393 break;
394 default:
395 *start = 1;
396 *end = n->req(); // default is all operands
397 }
398 }
399
400 // Return the vector version of a scalar operation node.
401 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt) {
402 const TypeVect* vt = TypeVect::make(bt, vlen);
403 int vopc = VectorNode::opcode(opc, bt);
404 // This method should not be called for unimplemented vectors.
405 guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
406 switch (vopc) {
407 case Op_AddVB: return new AddVBNode(n1, n2, vt);
408 case Op_AddVS: return new AddVSNode(n1, n2, vt);
409 case Op_AddVI: return new AddVINode(n1, n2, vt);
410 case Op_AddVL: return new AddVLNode(n1, n2, vt);
411 case Op_AddVF: return new AddVFNode(n1, n2, vt);
412 case Op_AddVD: return new AddVDNode(n1, n2, vt);
413
414 case Op_SubVB: return new SubVBNode(n1, n2, vt);
415 case Op_SubVS: return new SubVSNode(n1, n2, vt);
416 case Op_SubVI: return new SubVINode(n1, n2, vt);
417 case Op_SubVL: return new SubVLNode(n1, n2, vt);
418 case Op_SubVF: return new SubVFNode(n1, n2, vt);
419 case Op_SubVD: return new SubVDNode(n1, n2, vt);
420
421 case Op_MulVB: return new MulVBNode(n1, n2, vt);
422 case Op_MulVS: return new MulVSNode(n1, n2, vt);
423 case Op_MulVI: return new MulVINode(n1, n2, vt);
424 case Op_MulVL: return new MulVLNode(n1, n2, vt);
425 case Op_MulVF: return new MulVFNode(n1, n2, vt);
426 case Op_MulVD: return new MulVDNode(n1, n2, vt);
427
428 case Op_DivVF: return new DivVFNode(n1, n2, vt);
429 case Op_DivVD: return new DivVDNode(n1, n2, vt);
430
431 case Op_AbsVB: return new AbsVBNode(n1, vt);
432 case Op_AbsVS: return new AbsVSNode(n1, vt);
433 case Op_AbsVI: return new AbsVINode(n1, vt);
434 case Op_AbsVL: return new AbsVLNode(n1, vt);
435 case Op_AbsVF: return new AbsVFNode(n1, vt);
436 case Op_AbsVD: return new AbsVDNode(n1, vt);
437
438 case Op_NegVF: return new NegVFNode(n1, vt);
439 case Op_NegVD: return new NegVDNode(n1, vt);
440
441 case Op_SqrtVF: return new SqrtVFNode(n1, vt);
442 case Op_SqrtVD: return new SqrtVDNode(n1, vt);
443
444 case Op_PopCountVI: return new PopCountVINode(n1, vt);
445 case Op_RotateLeftV: return new RotateLeftVNode(n1, n2, vt);
446 case Op_RotateRightV: return new RotateRightVNode(n1, n2, vt);
447
448 case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt);
449 case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt);
450 case Op_LShiftVI: return new LShiftVINode(n1, n2, vt);
451 case Op_LShiftVL: return new LShiftVLNode(n1, n2, vt);
452
453 case Op_RShiftVB: return new RShiftVBNode(n1, n2, vt);
454 case Op_RShiftVS: return new RShiftVSNode(n1, n2, vt);
455 case Op_RShiftVI: return new RShiftVINode(n1, n2, vt);
456 case Op_RShiftVL: return new RShiftVLNode(n1, n2, vt);
457
458 case Op_URShiftVB: return new URShiftVBNode(n1, n2, vt);
459 case Op_URShiftVS: return new URShiftVSNode(n1, n2, vt);
460 case Op_URShiftVI: return new URShiftVINode(n1, n2, vt);
461 case Op_URShiftVL: return new URShiftVLNode(n1, n2, vt);
462
463 case Op_AndV: return new AndVNode(n1, n2, vt);
464 case Op_OrV: return new OrVNode (n1, n2, vt);
465 case Op_XorV: return new XorVNode(n1, n2, vt);
466
467 case Op_MinV: return new MinVNode(n1, n2, vt);
468 case Op_MaxV: return new MaxVNode(n1, n2, vt);
469
470 case Op_RoundDoubleModeV: return new RoundDoubleModeVNode(n1, n2, vt);
471
472 case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt);
473 default:
474 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
475 return NULL;
476 }
477 }
478
479 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) {
480 const TypeVect* vt = TypeVect::make(bt, vlen);
481 int vopc = VectorNode::opcode(opc, bt);
482 // This method should not be called for unimplemented vectors.
483 guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
484 switch (vopc) {
485 case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt);
486 case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt);
487 default:
488 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
489 return NULL;
490 }
491 }
492
493 // Scalar promotion
494 VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t) {
495 BasicType bt = opd_t->array_element_basic_type();
496 const TypeVect* vt = opd_t->singleton() ? TypeVect::make(opd_t, vlen)
497 : TypeVect::make(bt, vlen);
498 switch (bt) {
499 case T_BOOLEAN:
500 case T_BYTE:
501 return new ReplicateBNode(s, vt);
502 case T_CHAR:
503 case T_SHORT:
504 return new ReplicateSNode(s, vt);
505 case T_INT:
506 return new ReplicateINode(s, vt);
507 case T_LONG:
508 return new ReplicateLNode(s, vt);
509 case T_FLOAT:
510 return new ReplicateFNode(s, vt);
511 case T_DOUBLE:
512 return new ReplicateDNode(s, vt);
513 default:
514 fatal("Type '%s' is not supported for vectors", type2name(bt));
515 return NULL;
516 }
517 }
518
519 VectorNode* VectorNode::shift_count(Node* shift, Node* cnt, uint vlen, BasicType bt) {
520 assert(VectorNode::is_shift(shift), "sanity");
521 // Match shift count type with shift vector type.
522 const TypeVect* vt = TypeVect::make(bt, vlen);
523 switch (shift->Opcode()) {
524 case Op_LShiftI:
525 case Op_LShiftL:
526 return new LShiftCntVNode(cnt, vt);
527 case Op_RShiftI:
528 case Op_RShiftL:
529 case Op_URShiftI:
530 case Op_URShiftL:
531 return new RShiftCntVNode(cnt, vt);
532 default:
533 fatal("Missed vector creation for '%s'", NodeClassNames[shift->Opcode()]);
534 return NULL;
535 }
536 }
537
538 bool VectorNode::is_vector_shift(int opc) {
539 assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
540 switch (opc) {
541 case Op_LShiftVB:
542 case Op_LShiftVS:
543 case Op_LShiftVI:
544 case Op_LShiftVL:
545 case Op_RShiftVB:
546 case Op_RShiftVS:
547 case Op_RShiftVI:
548 case Op_RShiftVL:
549 case Op_URShiftVB:
550 case Op_URShiftVS:
551 case Op_URShiftVI:
552 case Op_URShiftVL:
553 return true;
554 default:
555 return false;
556 }
557 }
558
559 bool VectorNode::is_vector_shift_count(int opc) {
560 assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
561 switch (opc) {
562 case Op_RShiftCntV:
563 case Op_LShiftCntV:
564 return true;
565 default:
566 return false;
567 }
568 }
569
570 static bool is_con_M1(Node* n) {
571 if (n->is_Con()) {
572 const Type* t = n->bottom_type();
573 if (t->isa_int() && t->is_int()->get_con() == -1) {
574 return true;
575 }
576 if (t->isa_long() && t->is_long()->get_con() == -1) {
577 return true;
578 }
579 }
580 return false;
581 }
582
583 bool VectorNode::is_all_ones_vector(Node* n) {
584 switch (n->Opcode()) {
585 case Op_ReplicateB:
586 case Op_ReplicateS:
587 case Op_ReplicateI:
588 case Op_ReplicateL:
589 return is_con_M1(n->in(1));
590 default:
591 return false;
592 }
593 }
594
595 bool VectorNode::is_vector_bitwise_not_pattern(Node* n) {
596 if (n->Opcode() == Op_XorV) {
597 return is_all_ones_vector(n->in(1)) ||
598 is_all_ones_vector(n->in(2));
599 }
600 return false;
601 }
602
603 // Return initial Pack node. Additional operands added with add_opd() calls.
604 PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) {
605 const TypeVect* vt = TypeVect::make(bt, vlen);
606 switch (bt) {
607 case T_BOOLEAN:
608 case T_BYTE:
609 return new PackBNode(s, vt);
610 case T_CHAR:
611 case T_SHORT:
612 return new PackSNode(s, vt);
613 case T_INT:
614 return new PackINode(s, vt);
615 case T_LONG:
616 return new PackLNode(s, vt);
617 case T_FLOAT:
618 return new PackFNode(s, vt);
619 case T_DOUBLE:
620 return new PackDNode(s, vt);
621 default:
622 fatal("Type '%s' is not supported for vectors", type2name(bt));
623 return NULL;
624 }
625 }
626
627 // Create a binary tree form for Packs. [lo, hi) (half-open) range
628 PackNode* PackNode::binary_tree_pack(int lo, int hi) {
629 int ct = hi - lo;
630 assert(is_power_of_2(ct), "power of 2");
631 if (ct == 2) {
632 PackNode* pk = PackNode::make(in(lo), 2, vect_type()->element_basic_type());
633 pk->add_opd(in(lo+1));
634 return pk;
635 } else {
636 int mid = lo + ct/2;
637 PackNode* n1 = binary_tree_pack(lo, mid);
638 PackNode* n2 = binary_tree_pack(mid, hi );
639
640 BasicType bt = n1->vect_type()->element_basic_type();
641 assert(bt == n2->vect_type()->element_basic_type(), "should be the same");
642 switch (bt) {
643 case T_BOOLEAN:
644 case T_BYTE:
645 return new PackSNode(n1, n2, TypeVect::make(T_SHORT, 2));
646 case T_CHAR:
647 case T_SHORT:
648 return new PackINode(n1, n2, TypeVect::make(T_INT, 2));
649 case T_INT:
650 return new PackLNode(n1, n2, TypeVect::make(T_LONG, 2));
651 case T_LONG:
652 return new Pack2LNode(n1, n2, TypeVect::make(T_LONG, 2));
653 case T_FLOAT:
654 return new PackDNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
655 case T_DOUBLE:
656 return new Pack2DNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
657 default:
658 fatal("Type '%s' is not supported for vectors", type2name(bt));
659 return NULL;
660 }
661 }
662 }
663
664 // Return the vector version of a scalar load node.
665 LoadVectorNode* LoadVectorNode::make(int opc, Node* ctl, Node* mem,
666 Node* adr, const TypePtr* atyp,
667 uint vlen, BasicType bt,
668 ControlDependency control_dependency) {
669 const TypeVect* vt = TypeVect::make(bt, vlen);
670 return new LoadVectorNode(ctl, mem, adr, atyp, vt, control_dependency);
671 }
672
673 // Return the vector version of a scalar store node.
674 StoreVectorNode* StoreVectorNode::make(int opc, Node* ctl, Node* mem,
675 Node* adr, const TypePtr* atyp, Node* val,
676 uint vlen) {
677 return new StoreVectorNode(ctl, mem, adr, atyp, val);
678 }
679
680 // Extract a scalar element of vector.
681 Node* ExtractNode::make(Node* v, uint position, BasicType bt) {
682 assert((int)position < Matcher::max_vector_size(bt), "pos in range");
683 ConINode* pos = ConINode::make((int)position);
684 switch (bt) {
685 case T_BOOLEAN:
686 return new ExtractUBNode(v, pos);
687 case T_BYTE:
688 return new ExtractBNode(v, pos);
689 case T_CHAR:
690 return new ExtractCNode(v, pos);
691 case T_SHORT:
692 return new ExtractSNode(v, pos);
693 case T_INT:
694 return new ExtractINode(v, pos);
695 case T_LONG:
696 return new ExtractLNode(v, pos);
697 case T_FLOAT:
698 return new ExtractFNode(v, pos);
699 case T_DOUBLE:
700 return new ExtractDNode(v, pos);
701 default:
702 fatal("Type '%s' is not supported for vectors", type2name(bt));
703 return NULL;
704 }
705 }
706
707 int ReductionNode::opcode(int opc, BasicType bt) {
708 int vopc = opc;
709 switch (opc) {
710 case Op_AddI:
711 assert(bt == T_INT, "must be");
712 vopc = Op_AddReductionVI;
713 break;
714 case Op_AddL:
715 assert(bt == T_LONG, "must be");
716 vopc = Op_AddReductionVL;
717 break;
718 case Op_AddF:
719 assert(bt == T_FLOAT, "must be");
720 vopc = Op_AddReductionVF;
721 break;
722 case Op_AddD:
723 assert(bt == T_DOUBLE, "must be");
724 vopc = Op_AddReductionVD;
725 break;
726 case Op_MulI:
727 assert(bt == T_INT, "must be");
728 vopc = Op_MulReductionVI;
729 break;
730 case Op_MulL:
731 assert(bt == T_LONG, "must be");
732 vopc = Op_MulReductionVL;
733 break;
734 case Op_MulF:
735 assert(bt == T_FLOAT, "must be");
736 vopc = Op_MulReductionVF;
737 break;
738 case Op_MulD:
739 assert(bt == T_DOUBLE, "must be");
740 vopc = Op_MulReductionVD;
741 break;
742 case Op_MinF:
743 assert(bt == T_FLOAT, "must be");
744 vopc = Op_MinReductionV;
745 break;
746 case Op_MinD:
747 assert(bt == T_DOUBLE, "must be");
748 vopc = Op_MinReductionV;
749 break;
750 case Op_MaxF:
751 assert(bt == T_FLOAT, "must be");
752 vopc = Op_MaxReductionV;
753 break;
754 case Op_MaxD:
755 assert(bt == T_DOUBLE, "must be");
756 vopc = Op_MaxReductionV;
757 break;
758 case Op_AndI:
759 assert(bt == T_INT, "must be");
760 vopc = Op_AndReductionV;
761 break;
762 case Op_AndL:
763 assert(bt == T_LONG, "must be");
764 vopc = Op_AndReductionV;
765 break;
766 case Op_OrI:
767 assert(bt == T_INT, "must be");
768 vopc = Op_OrReductionV;
769 break;
770 case Op_OrL:
771 assert(bt == T_LONG, "must be");
772 vopc = Op_OrReductionV;
773 break;
774 case Op_XorI:
775 assert(bt == T_INT, "must be");
776 vopc = Op_XorReductionV;
777 break;
778 case Op_XorL:
779 assert(bt == T_LONG, "must be");
780 vopc = Op_XorReductionV;
781 break;
782 default:
783 break;
784 }
785 return vopc;
786 }
787
788 // Return the appropriate reduction node.
789 ReductionNode* ReductionNode::make(int opc, Node *ctrl, Node* n1, Node* n2, BasicType bt) {
790
791 int vopc = opcode(opc, bt);
792
793 // This method should not be called for unimplemented vectors.
794 guarantee(vopc != opc, "Vector for '%s' is not implemented", NodeClassNames[opc]);
795
796 switch (vopc) {
797 case Op_AddReductionVI: return new AddReductionVINode(ctrl, n1, n2);
798 case Op_AddReductionVL: return new AddReductionVLNode(ctrl, n1, n2);
799 case Op_AddReductionVF: return new AddReductionVFNode(ctrl, n1, n2);
800 case Op_AddReductionVD: return new AddReductionVDNode(ctrl, n1, n2);
801 case Op_MulReductionVI: return new MulReductionVINode(ctrl, n1, n2);
802 case Op_MulReductionVL: return new MulReductionVLNode(ctrl, n1, n2);
803 case Op_MulReductionVF: return new MulReductionVFNode(ctrl, n1, n2);
804 case Op_MulReductionVD: return new MulReductionVDNode(ctrl, n1, n2);
805 case Op_MinReductionV: return new MinReductionVNode(ctrl, n1, n2);
806 case Op_MaxReductionV: return new MaxReductionVNode(ctrl, n1, n2);
807 case Op_AndReductionV: return new AndReductionVNode(ctrl, n1, n2);
808 case Op_OrReductionV: return new OrReductionVNode(ctrl, n1, n2);
809 case Op_XorReductionV: return new XorReductionVNode(ctrl, n1, n2);
810 default:
811 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
812 return NULL;
813 }
814 }
815
816 bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) {
817 if (is_java_primitive(bt) &&
818 (vlen > 1) && is_power_of_2(vlen) &&
819 Matcher::vector_size_supported(bt, vlen)) {
820 int vopc = ReductionNode::opcode(opc, bt);
821 return vopc != opc && Matcher::match_rule_supported(vopc);
822 }
823 return false;
824 }
825
826 MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node* in3,
827 uint truth_table, const TypeVect* vt) {
828 assert(truth_table <= 0xFF, "invalid");
829 assert(in1->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
830 assert(in2->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
831 assert(in3->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
832 Node* fn = gvn.intcon(truth_table);
833 return new MacroLogicVNode(in1, in2, in3, fn, vt);
834 }
835
836 Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_left,
837 int vlen, BasicType bt, PhaseGVN* phase) {
838 int shiftLOpc;
839 int shiftROpc;
840 Node* shiftLCnt = NULL;
841 Node* shiftRCnt = NULL;
842 const TypeVect* vt = TypeVect::make(bt, vlen);
843
844 // Compute shift values for right rotation and
845 // later swap them in case of left rotation.
846 if (cnt->is_Con()) {
847 // Constant shift case.
848 if (bt == T_INT) {
849 int shift = cnt->get_int() & 31;
850 shiftRCnt = phase->intcon(shift);
851 shiftLCnt = phase->intcon(32 - shift);
852 shiftLOpc = Op_LShiftI;
853 shiftROpc = Op_URShiftI;
854 } else {
855 int shift = cnt->get_int() & 63;
856 shiftRCnt = phase->intcon(shift);
857 shiftLCnt = phase->intcon(64 - shift);
858 shiftLOpc = Op_LShiftL;
859 shiftROpc = Op_URShiftL;
860 }
861 } else {
862 // Variable shift case.
863 assert(VectorNode::is_invariant_vector(cnt), "Broadcast expected");
864 cnt = cnt->in(1);
865 if (bt == T_INT) {
866 shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(31)));
867 shiftLCnt = phase->transform(new SubINode(phase->intcon(32), shiftRCnt));
868 shiftLOpc = Op_LShiftI;
869 shiftROpc = Op_URShiftI;
870 } else {
871 assert(cnt->Opcode() == Op_ConvI2L, "ConvI2L expected");
872 cnt = cnt->in(1);
873 shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(63)));
874 shiftLCnt = phase->transform(new SubINode(phase->intcon(64), shiftRCnt));
875 shiftLOpc = Op_LShiftL;
876 shiftROpc = Op_URShiftL;
877 }
878 }
879
880 // Swap the computed left and right shift counts.
881 if (is_rotate_left) {
882 swap(shiftRCnt,shiftLCnt);
883 }
884
885 shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
886 shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
887
888 return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt)),
889 phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt)),
890 vt);
891 }
892
893 Node* RotateLeftVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
894 int vlen = length();
895 BasicType bt = vect_type()->element_basic_type();
896 if (!Matcher::match_rule_supported_vector(Op_RotateLeftV, vlen, bt)) {
897 return VectorNode::degenerate_vector_rotate(in(1), in(2), true, vlen, bt, phase);
898 }
899 return NULL;
900 }
901
902 Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
903 int vlen = length();
904 BasicType bt = vect_type()->element_basic_type();
905 if (!Matcher::match_rule_supported_vector(Op_RotateRightV, vlen, bt)) {
906 return VectorNode::degenerate_vector_rotate(in(1), in(2), false, vlen, bt, phase);
907 }
908 return NULL;
909 }
910
911 Node* OrVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
912 int lopcode = in(1)->Opcode();
913 int ropcode = in(2)->Opcode();
914 const TypeVect* vt = bottom_type()->is_vect();
915 int vec_len = vt->length();
916 BasicType bt = vt->element_basic_type();
917
918 // Vector Rotate operations inferencing, this will be useful when vector
919 // operations are created via non-SLP route i.e. (VectorAPI).
920 if (Matcher::match_rule_supported_vector(Op_RotateLeftV, vec_len, bt) &&
921 ((ropcode == Op_LShiftVI && lopcode == Op_URShiftVI) ||
922 (ropcode == Op_LShiftVL && lopcode == Op_URShiftVL)) &&
923 in(1)->in(1) == in(2)->in(1)) {
924 assert(Op_RShiftCntV == in(1)->in(2)->Opcode(), "LShiftCntV operand expected");
925 assert(Op_LShiftCntV == in(2)->in(2)->Opcode(), "RShiftCntV operand expected");
926 Node* lshift = in(1)->in(2)->in(1);
927 Node* rshift = in(2)->in(2)->in(1);
928 int mod_val = bt == T_LONG ? 64 : 32;
929 int shift_mask = bt == T_LONG ? 0x3F : 0x1F;
930 // val >> norm_con_shift | val << (32 - norm_con_shift) => rotate_right val ,
931 // norm_con_shift
932 if (lshift->is_Con() && rshift->is_Con() &&
933 ((lshift->get_int() & shift_mask) ==
934 (mod_val - (rshift->get_int() & shift_mask)))) {
935 return new RotateRightVNode(
936 in(1)->in(1), phase->intcon(lshift->get_int() & shift_mask), vt);
937 }
938 if (lshift->Opcode() == Op_AndI && rshift->Opcode() == Op_AndI &&
939 lshift->in(2)->is_Con() && rshift->in(2)->is_Con() &&
940 lshift->in(2)->get_int() == (mod_val - 1) &&
941 rshift->in(2)->get_int() == (mod_val - 1)) {
942 lshift = lshift->in(1);
943 rshift = rshift->in(1);
944 // val << var_shift | val >> (0/32 - var_shift) => rotate_left val ,
945 // var_shift
946 if (lshift->Opcode() == Op_SubI && lshift->in(2) == rshift &&
947 lshift->in(1)->is_Con() &&
948 (lshift->in(1)->get_int() == 0 ||
949 lshift->in(1)->get_int() == mod_val)) {
950 Node* rotate_cnt = phase->transform(new ReplicateINode(rshift, vt));
951 return new RotateLeftVNode(in(1)->in(1), rotate_cnt, vt);
952 }
953 }
954 }
955
956 if (Matcher::match_rule_supported_vector(Op_RotateRightV, vec_len, bt) &&
957 ((ropcode == Op_URShiftVI && lopcode == Op_LShiftVI) ||
958 (ropcode == Op_URShiftVL && lopcode == Op_LShiftVL)) &&
959 in(1)->in(1) == in(2)->in(1)) {
960 assert(Op_LShiftCntV == in(1)->in(2)->Opcode(), "RShiftCntV operand expected");
961 assert(Op_RShiftCntV == in(2)->in(2)->Opcode(), "LShiftCntV operand expected");
962 Node* rshift = in(1)->in(2)->in(1);
963 Node* lshift = in(2)->in(2)->in(1);
964 int mod_val = bt == T_LONG ? 64 : 32;
965 int shift_mask = bt == T_LONG ? 0x3F : 0x1F;
966 // val << norm_con_shift | val >> (32 - norm_con_shift) => rotate_left val
967 // , norm_con_shift
968 if (rshift->is_Con() && lshift->is_Con() &&
969 ((rshift->get_int() & shift_mask) ==
970 (mod_val - (lshift->get_int() & shift_mask)))) {
971 return new RotateLeftVNode(
972 in(1)->in(1), phase->intcon(rshift->get_int() & shift_mask), vt);
973 }
974 if (lshift->Opcode() == Op_AndI && rshift->Opcode() == Op_AndI &&
975 lshift->in(2)->is_Con() && rshift->in(2)->is_Con() &&
976 rshift->in(2)->get_int() == (mod_val - 1) &&
977 lshift->in(2)->get_int() == (mod_val - 1)) {
978 rshift = rshift->in(1);
979 lshift = lshift->in(1);
980 // val >> var_shift | val << (0/32 - var_shift) => rotate_right val ,
981 // var_shift
982 if (rshift->Opcode() == Op_SubI && rshift->in(2) == lshift &&
983 rshift->in(1)->is_Con() &&
984 (rshift->in(1)->get_int() == 0 ||
985 rshift->in(1)->get_int() == mod_val)) {
986 Node* rotate_cnt = phase->transform(new ReplicateINode(lshift, vt));
987 return new RotateRightVNode(in(1)->in(1), rotate_cnt, vt);
988 }
989 }
990 }
991 return NULL;
992 }
993