1340 Node* first = executed_first(p);
1341 int opc = n->Opcode();
1342 if (n->is_Load()) {
1343 Node* ctl = n->in(MemNode::Control);
1344 Node* mem = first->in(MemNode::Memory);
1345 Node* adr = low_adr->in(MemNode::Address);
1346 const TypePtr* atyp = n->adr_type();
1347 vn = LoadVectorNode::make(_phase->C, opc, ctl, mem, adr, atyp, vlen, velt_basic_type(n));
1348 } else if (n->is_Store()) {
1349 // Promote value to be stored to vector
1350 Node* val = vector_opd(p, MemNode::ValueIn);
1351 Node* ctl = n->in(MemNode::Control);
1352 Node* mem = first->in(MemNode::Memory);
1353 Node* adr = low_adr->in(MemNode::Address);
1354 const TypePtr* atyp = n->adr_type();
1355 vn = StoreVectorNode::make(_phase->C, opc, ctl, mem, adr, atyp, val, vlen);
1356 } else if (n->req() == 3) {
1357 // Promote operands to vector
1358 Node* in1 = vector_opd(p, 1);
1359 Node* in2 = vector_opd(p, 2);
1360 vn = VectorNode::make(_phase->C, opc, in1, in2, vlen, velt_basic_type(n));
1361 } else {
1362 ShouldNotReachHere();
1363 }
1364 assert(vn != NULL, "sanity");
1365 _phase->_igvn.register_new_node_with_optimizer(vn);
1366 _phase->set_ctrl(vn, _phase->get_ctrl(p->at(0)));
1367 for (uint j = 0; j < p->size(); j++) {
1368 Node* pm = p->at(j);
1369 _igvn.replace_node(pm, vn);
1370 }
1371 _igvn._worklist.push(vn);
1372 #ifdef ASSERT
1373 if (TraceNewVectors) {
1374 tty->print("new Vector node: ");
1375 vn->dump();
1376 }
1377 #endif
1378 }
1379 }
1383 // Create a vector operand for the nodes in pack p for operand: in(opd_idx)
1384 Node* SuperWord::vector_opd(Node_List* p, int opd_idx) {
1385 Node* p0 = p->at(0);
1386 uint vlen = p->size();
1387 Node* opd = p0->in(opd_idx);
1388
1389 bool same_opd = true;
1390 for (uint i = 1; i < vlen; i++) {
1391 Node* pi = p->at(i);
1392 Node* in = pi->in(opd_idx);
1393 if (opd != in) {
1394 same_opd = false;
1395 break;
1396 }
1397 }
1398
1399 if (same_opd) {
1400 if (opd->is_Vector() || opd->is_LoadVector()) {
1401 return opd; // input is matching vector
1402 }
1403 assert(!opd->is_StoreVector(), "such vector is not expected here");
1404 // Convert scalar input to vector with the same number of elements as
1405 // p0's vector. Use p0's type because size of operand's container in
1406 // vector should match p0's size regardless operand's size.
1407 const Type* p0_t = velt_type(p0);
1408 VectorNode* vn = VectorNode::scalar2vector(_phase->C, opd, vlen, p0_t);
1409
1410 _phase->_igvn.register_new_node_with_optimizer(vn);
1411 _phase->set_ctrl(vn, _phase->get_ctrl(opd));
1412 #ifdef ASSERT
1413 if (TraceNewVectors) {
1414 tty->print("new Vector node: ");
1415 vn->dump();
1416 }
1417 #endif
1418 return vn;
1419 }
1420
1421 // Insert pack operation
1422 BasicType bt = velt_basic_type(p0);
1701 // Example: char a,b,c; a = b + c;
1702 // Normally the type of the add is integer, but for packed character
1703 // operations the type of the add needs to be char.
1704 void SuperWord::compute_vector_element_type() {
1705 #ifndef PRODUCT
1706 if (TraceSuperWord && Verbose)
1707 tty->print_cr("\ncompute_velt_type:");
1708 #endif
1709
1710 // Initial type
1711 for (int i = 0; i < _block.length(); i++) {
1712 Node* n = _block.at(i);
1713 set_velt_type(n, container_type(n));
1714 }
1715
1716 // Propagate narrowed type backwards through operations
1717 // that don't depend on higher order bits
1718 for (int i = _block.length() - 1; i >= 0; i--) {
1719 Node* n = _block.at(i);
1720 // Only integer types need be examined
1721 if (n->bottom_type()->isa_int()) {
1722 uint start, end;
1723 vector_opd_range(n, &start, &end);
1724 const Type* vt = velt_type(n);
1725
1726 for (uint j = start; j < end; j++) {
1727 Node* in = n->in(j);
1728 // Don't propagate through a type conversion
1729 if (n->bottom_type() != in->bottom_type())
1730 continue;
1731 switch(in->Opcode()) {
1732 case Op_AddI: case Op_AddL:
1733 case Op_SubI: case Op_SubL:
1734 case Op_MulI: case Op_MulL:
1735 case Op_AndI: case Op_AndL:
1736 case Op_OrI: case Op_OrL:
1737 case Op_XorI: case Op_XorL:
1738 case Op_LShiftI: case Op_LShiftL:
1739 case Op_CMoveI: case Op_CMoveL:
1740 if (in_bb(in)) {
1741 bool same_type = true;
1742 for (DUIterator_Fast kmax, k = in->fast_outs(kmax); k < kmax; k++) {
1743 Node *use = in->fast_out(k);
1744 if (!in_bb(use) || !same_velt_type(use, n)) {
1745 same_type = false;
1746 break;
1747 }
1748 }
1749 if (same_type) {
1750 set_velt_type(in, vt);
1751 }
1752 }
1753 }
1754 }
1755 }
1756 }
1757 #ifndef PRODUCT
1758 if (TraceSuperWord && Verbose) {
1759 for (int i = 0; i < _block.length(); i++) {
1760 Node* n = _block.at(i);
1761 velt_type(n)->dump();
1762 tty->print("\t");
1763 n->dump();
1764 }
1765 }
1766 #endif
1767 }
1768
1769 //------------------------------memory_alignment---------------------------
1770 // Alignment within a vector memory reference
1771 int SuperWord::memory_alignment(MemNode* s, int iv_adjust_in_bytes) {
1772 SWPointer p(s, this);
1773 if (!p.valid()) {
1774 return bottom_align;
1775 }
1776 int vw = vector_width_in_bytes(s);
1777 if (vw < 2) {
1778 return bottom_align; // No vectors for this type
1779 }
1780 int offset = p.offset_in_bytes();
1781 offset += iv_adjust_in_bytes;
1782 int off_rem = offset % vw;
1783 int off_mod = off_rem >= 0 ? off_rem : off_rem + vw;
1784 return off_mod;
1785 }
1786
1787 //---------------------------container_type---------------------------
1788 // Smallest type containing range of values
1789 const Type* SuperWord::container_type(Node* n) {
1790 if (n->is_Mem()) {
1791 return Type::get_const_basic_type(n->as_Mem()->memory_type());
1792 }
1793 const Type* t = _igvn.type(n);
1794 if (t->basic_type() == T_INT) {
1795 if (t->higher_equal(TypeInt::BOOL)) return TypeInt::BOOL;
1796 if (t->higher_equal(TypeInt::BYTE)) return TypeInt::BYTE;
1797 if (t->higher_equal(TypeInt::CHAR)) return TypeInt::CHAR;
1798 if (t->higher_equal(TypeInt::SHORT)) return TypeInt::SHORT;
1799 return TypeInt::INT;
1800 }
1801 return t;
1802 }
1803
1804 bool SuperWord::same_velt_type(Node* n1, Node* n2) {
1805 const Type* vt1 = velt_type(n1);
1806 const Type* vt2 = velt_type(n1);
1807 if (vt1->basic_type() == T_INT && vt2->basic_type() == T_INT) {
1808 // Compare vectors element sizes for integer types.
1809 return data_size(n1) == data_size(n2);
1810 }
1811 return vt1 == vt2;
1812 }
1813
1814 //-------------------------vector_opd_range-----------------------
1815 // (Start, end] half-open range defining which operands are vector
1816 void SuperWord::vector_opd_range(Node* n, uint* start, uint* end) {
1817 switch (n->Opcode()) {
1818 case Op_LoadB: case Op_LoadUB:
|
1340 Node* first = executed_first(p);
1341 int opc = n->Opcode();
1342 if (n->is_Load()) {
1343 Node* ctl = n->in(MemNode::Control);
1344 Node* mem = first->in(MemNode::Memory);
1345 Node* adr = low_adr->in(MemNode::Address);
1346 const TypePtr* atyp = n->adr_type();
1347 vn = LoadVectorNode::make(_phase->C, opc, ctl, mem, adr, atyp, vlen, velt_basic_type(n));
1348 } else if (n->is_Store()) {
1349 // Promote value to be stored to vector
1350 Node* val = vector_opd(p, MemNode::ValueIn);
1351 Node* ctl = n->in(MemNode::Control);
1352 Node* mem = first->in(MemNode::Memory);
1353 Node* adr = low_adr->in(MemNode::Address);
1354 const TypePtr* atyp = n->adr_type();
1355 vn = StoreVectorNode::make(_phase->C, opc, ctl, mem, adr, atyp, val, vlen);
1356 } else if (n->req() == 3) {
1357 // Promote operands to vector
1358 Node* in1 = vector_opd(p, 1);
1359 Node* in2 = vector_opd(p, 2);
1360 if (VectorNode::is_invariant_vector(in1) && (n->is_Add() || n->is_Mul())) {
1361 // Move invariant vector input into second position to avoid register spilling.
1362 Node* tmp = in1;
1363 in1 = in2;
1364 in2 = tmp;
1365 }
1366 vn = VectorNode::make(_phase->C, opc, in1, in2, vlen, velt_basic_type(n));
1367 } else {
1368 ShouldNotReachHere();
1369 }
1370 assert(vn != NULL, "sanity");
1371 _phase->_igvn.register_new_node_with_optimizer(vn);
1372 _phase->set_ctrl(vn, _phase->get_ctrl(p->at(0)));
1373 for (uint j = 0; j < p->size(); j++) {
1374 Node* pm = p->at(j);
1375 _igvn.replace_node(pm, vn);
1376 }
1377 _igvn._worklist.push(vn);
1378 #ifdef ASSERT
1379 if (TraceNewVectors) {
1380 tty->print("new Vector node: ");
1381 vn->dump();
1382 }
1383 #endif
1384 }
1385 }
1389 // Create a vector operand for the nodes in pack p for operand: in(opd_idx)
1390 Node* SuperWord::vector_opd(Node_List* p, int opd_idx) {
1391 Node* p0 = p->at(0);
1392 uint vlen = p->size();
1393 Node* opd = p0->in(opd_idx);
1394
1395 bool same_opd = true;
1396 for (uint i = 1; i < vlen; i++) {
1397 Node* pi = p->at(i);
1398 Node* in = pi->in(opd_idx);
1399 if (opd != in) {
1400 same_opd = false;
1401 break;
1402 }
1403 }
1404
1405 if (same_opd) {
1406 if (opd->is_Vector() || opd->is_LoadVector()) {
1407 return opd; // input is matching vector
1408 }
1409 if ((opd_idx == 2) && VectorNode::is_shift(p0)) {
1410 // No vector is needed for shift count.
1411 // Vector instructions do not mask shift count, do it here.
1412 Compile* C = _phase->C;
1413 Node* cnt = opd;
1414 juint mask = (p0->bottom_type() == TypeInt::INT) ? (BitsPerInt - 1) : (BitsPerLong - 1);
1415 const TypeInt* t = opd->find_int_type();
1416 if (t != NULL && t->is_con()) {
1417 juint shift = t->get_con();
1418 if (shift > mask) { // Unsigned cmp
1419 cnt = ConNode::make(C, TypeInt::make(shift & mask));
1420 }
1421 } else {
1422 if (t == NULL || t->_lo < 0 || t->_hi > (int)mask) {
1423 cnt = ConNode::make(C, TypeInt::make(mask));
1424 _phase->_igvn.register_new_node_with_optimizer(cnt);
1425 cnt = new (C, 3) AndINode(opd, cnt);
1426 _phase->_igvn.register_new_node_with_optimizer(cnt);
1427 _phase->set_ctrl(cnt, _phase->get_ctrl(opd));
1428 }
1429 assert(opd->bottom_type()->isa_int(), "int type only");
1430 // Move non constant shift count into XMM register.
1431 cnt = new (_phase->C, 2) MoveI2FNode(cnt);
1432 }
1433 if (cnt != opd) {
1434 _phase->_igvn.register_new_node_with_optimizer(cnt);
1435 _phase->set_ctrl(cnt, _phase->get_ctrl(opd));
1436 }
1437 return cnt;
1438 }
1439 assert(!opd->is_StoreVector(), "such vector is not expected here");
1440 // Convert scalar input to vector with the same number of elements as
1441 // p0's vector. Use p0's type because size of operand's container in
1442 // vector should match p0's size regardless operand's size.
1443 const Type* p0_t = velt_type(p0);
1444 VectorNode* vn = VectorNode::scalar2vector(_phase->C, opd, vlen, p0_t);
1445
1446 _phase->_igvn.register_new_node_with_optimizer(vn);
1447 _phase->set_ctrl(vn, _phase->get_ctrl(opd));
1448 #ifdef ASSERT
1449 if (TraceNewVectors) {
1450 tty->print("new Vector node: ");
1451 vn->dump();
1452 }
1453 #endif
1454 return vn;
1455 }
1456
1457 // Insert pack operation
1458 BasicType bt = velt_basic_type(p0);
1737 // Example: char a,b,c; a = b + c;
1738 // Normally the type of the add is integer, but for packed character
1739 // operations the type of the add needs to be char.
1740 void SuperWord::compute_vector_element_type() {
1741 #ifndef PRODUCT
1742 if (TraceSuperWord && Verbose)
1743 tty->print_cr("\ncompute_velt_type:");
1744 #endif
1745
1746 // Initial type
1747 for (int i = 0; i < _block.length(); i++) {
1748 Node* n = _block.at(i);
1749 set_velt_type(n, container_type(n));
1750 }
1751
1752 // Propagate narrowed type backwards through operations
1753 // that don't depend on higher order bits
1754 for (int i = _block.length() - 1; i >= 0; i--) {
1755 Node* n = _block.at(i);
1756 // Only integer types need be examined
1757 const Type* vt = velt_type(n);
1758 if (vt->basic_type() == T_INT) {
1759 uint start, end;
1760 vector_opd_range(n, &start, &end);
1761 const Type* vt = velt_type(n);
1762
1763 for (uint j = start; j < end; j++) {
1764 Node* in = n->in(j);
1765 // Don't propagate through a memory
1766 if (!in->is_Mem() && in_bb(in) && velt_type(in)->basic_type() == T_INT &&
1767 data_size(n) < data_size(in)) {
1768 bool same_type = true;
1769 for (DUIterator_Fast kmax, k = in->fast_outs(kmax); k < kmax; k++) {
1770 Node *use = in->fast_out(k);
1771 if (!in_bb(use) || !same_velt_type(use, n)) {
1772 same_type = false;
1773 break;
1774 }
1775 }
1776 if (same_type) {
1777 set_velt_type(in, vt);
1778 }
1779 }
1780 }
1781 }
1782 }
1783 #ifndef PRODUCT
1784 if (TraceSuperWord && Verbose) {
1785 for (int i = 0; i < _block.length(); i++) {
1786 Node* n = _block.at(i);
1787 velt_type(n)->dump();
1788 tty->print("\t");
1789 n->dump();
1790 }
1791 }
1792 #endif
1793 }
1794
1795 //------------------------------memory_alignment---------------------------
1796 // Alignment within a vector memory reference
1797 int SuperWord::memory_alignment(MemNode* s, int iv_adjust_in_bytes) {
1798 SWPointer p(s, this);
1799 if (!p.valid()) {
1800 return bottom_align;
1801 }
1802 int vw = vector_width_in_bytes(s);
1803 if (vw < 2) {
1804 return bottom_align; // No vectors for this type
1805 }
1806 int offset = p.offset_in_bytes();
1807 offset += iv_adjust_in_bytes;
1808 int off_rem = offset % vw;
1809 int off_mod = off_rem >= 0 ? off_rem : off_rem + vw;
1810 return off_mod;
1811 }
1812
1813 //---------------------------container_type---------------------------
1814 // Smallest type containing range of values
1815 const Type* SuperWord::container_type(Node* n) {
1816 if (n->is_Mem()) {
1817 return Type::get_const_basic_type(n->as_Mem()->memory_type());
1818 }
1819 const Type* t = _igvn.type(n);
1820 if (t->basic_type() == T_INT) {
1821 // A narrow type of arithmetic operations will be determined by
1822 // propagating the type of memory operations.
1823 return TypeInt::INT;
1824 }
1825 return t;
1826 }
1827
1828 bool SuperWord::same_velt_type(Node* n1, Node* n2) {
1829 const Type* vt1 = velt_type(n1);
1830 const Type* vt2 = velt_type(n1);
1831 if (vt1->basic_type() == T_INT && vt2->basic_type() == T_INT) {
1832 // Compare vectors element sizes for integer types.
1833 return data_size(n1) == data_size(n2);
1834 }
1835 return vt1 == vt2;
1836 }
1837
1838 //-------------------------vector_opd_range-----------------------
1839 // (Start, end] half-open range defining which operands are vector
1840 void SuperWord::vector_opd_range(Node* n, uint* start, uint* end) {
1841 switch (n->Opcode()) {
1842 case Op_LoadB: case Op_LoadUB:
|