1 /*
2 * Copyright (c) 2007, 2013, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
1363 insert_extracts(_packset.at(i));
1364 }
1365
1366 Compile* C = _phase->C;
1367 uint max_vlen_in_bytes = 0;
1368 for (int i = 0; i < _block.length(); i++) {
1369 Node* n = _block.at(i);
1370 Node_List* p = my_pack(n);
1371 if (p && n == executed_last(p)) {
1372 uint vlen = p->size();
1373 uint vlen_in_bytes = 0;
1374 Node* vn = NULL;
1375 Node* low_adr = p->at(0);
1376 Node* first = executed_first(p);
1377 int opc = n->Opcode();
1378 if (n->is_Load()) {
1379 Node* ctl = n->in(MemNode::Control);
1380 Node* mem = first->in(MemNode::Memory);
1381 Node* adr = low_adr->in(MemNode::Address);
1382 const TypePtr* atyp = n->adr_type();
1383 vn = LoadVectorNode::make(C, opc, ctl, mem, adr, atyp, vlen, velt_basic_type(n));
1384 vlen_in_bytes = vn->as_LoadVector()->memory_size();
1385 } else if (n->is_Store()) {
1386 // Promote value to be stored to vector
1387 Node* val = vector_opd(p, MemNode::ValueIn);
1388 Node* ctl = n->in(MemNode::Control);
1389 Node* mem = first->in(MemNode::Memory);
1390 Node* adr = low_adr->in(MemNode::Address);
1391 const TypePtr* atyp = n->adr_type();
1392 vn = StoreVectorNode::make(C, opc, ctl, mem, adr, atyp, val, vlen);
1393 vlen_in_bytes = vn->as_StoreVector()->memory_size();
1394 } else if (n->req() == 3) {
1395 // Promote operands to vector
1396 Node* in1 = vector_opd(p, 1);
1397 Node* in2 = vector_opd(p, 2);
1398 if (VectorNode::is_invariant_vector(in1) && (n->is_Add() || n->is_Mul())) {
1399 // Move invariant vector input into second position to avoid register spilling.
1400 Node* tmp = in1;
1401 in1 = in2;
1402 in2 = tmp;
1403 }
1404 vn = VectorNode::make(C, opc, in1, in2, vlen, velt_basic_type(n));
1405 vlen_in_bytes = vn->as_Vector()->length_in_bytes();
1406 } else {
1407 ShouldNotReachHere();
1408 }
1409 assert(vn != NULL, "sanity");
1410 _igvn.register_new_node_with_optimizer(vn);
1411 _phase->set_ctrl(vn, _phase->get_ctrl(p->at(0)));
1412 for (uint j = 0; j < p->size(); j++) {
1413 Node* pm = p->at(j);
1414 _igvn.replace_node(pm, vn);
1415 }
1416 _igvn._worklist.push(vn);
1417
1418 if (vlen_in_bytes > max_vlen_in_bytes) {
1419 max_vlen_in_bytes = vlen_in_bytes;
1420 }
1421 #ifdef ASSERT
1422 if (TraceNewVectors) {
1423 tty->print("new Vector node: ");
1424 vn->dump();
1433 // Create a vector operand for the nodes in pack p for operand: in(opd_idx)
1434 Node* SuperWord::vector_opd(Node_List* p, int opd_idx) {
1435 Node* p0 = p->at(0);
1436 uint vlen = p->size();
1437 Node* opd = p0->in(opd_idx);
1438
1439 if (same_inputs(p, opd_idx)) {
1440 if (opd->is_Vector() || opd->is_LoadVector()) {
1441 assert(((opd_idx != 2) || !VectorNode::is_shift(p0)), "shift's count can't be vector");
1442 return opd; // input is matching vector
1443 }
1444 if ((opd_idx == 2) && VectorNode::is_shift(p0)) {
1445 Compile* C = _phase->C;
1446 Node* cnt = opd;
1447 // Vector instructions do not mask shift count, do it here.
1448 juint mask = (p0->bottom_type() == TypeInt::INT) ? (BitsPerInt - 1) : (BitsPerLong - 1);
1449 const TypeInt* t = opd->find_int_type();
1450 if (t != NULL && t->is_con()) {
1451 juint shift = t->get_con();
1452 if (shift > mask) { // Unsigned cmp
1453 cnt = ConNode::make(C, TypeInt::make(shift & mask));
1454 }
1455 } else {
1456 if (t == NULL || t->_lo < 0 || t->_hi > (int)mask) {
1457 cnt = ConNode::make(C, TypeInt::make(mask));
1458 _igvn.register_new_node_with_optimizer(cnt);
1459 cnt = new AndINode(opd, cnt);
1460 _igvn.register_new_node_with_optimizer(cnt);
1461 _phase->set_ctrl(cnt, _phase->get_ctrl(opd));
1462 }
1463 assert(opd->bottom_type()->isa_int(), "int type only");
1464 // Move non constant shift count into vector register.
1465 cnt = VectorNode::shift_count(C, p0, cnt, vlen, velt_basic_type(p0));
1466 }
1467 if (cnt != opd) {
1468 _igvn.register_new_node_with_optimizer(cnt);
1469 _phase->set_ctrl(cnt, _phase->get_ctrl(opd));
1470 }
1471 return cnt;
1472 }
1473 assert(!opd->is_StoreVector(), "such vector is not expected here");
1474 // Convert scalar input to vector with the same number of elements as
1475 // p0's vector. Use p0's type because size of operand's container in
1476 // vector should match p0's size regardless operand's size.
1477 const Type* p0_t = velt_type(p0);
1478 VectorNode* vn = VectorNode::scalar2vector(_phase->C, opd, vlen, p0_t);
1479
1480 _igvn.register_new_node_with_optimizer(vn);
1481 _phase->set_ctrl(vn, _phase->get_ctrl(opd));
1482 #ifdef ASSERT
1483 if (TraceNewVectors) {
1484 tty->print("new Vector node: ");
1485 vn->dump();
1486 }
1487 #endif
1488 return vn;
1489 }
1490
1491 // Insert pack operation
1492 BasicType bt = velt_basic_type(p0);
1493 PackNode* pk = PackNode::make(_phase->C, opd, vlen, bt);
1494 DEBUG_ONLY( const BasicType opd_bt = opd->bottom_type()->basic_type(); )
1495
1496 for (uint i = 1; i < vlen; i++) {
1497 Node* pi = p->at(i);
1498 Node* in = pi->in(opd_idx);
1499 assert(my_pack(in) == NULL, "Should already have been unpacked");
1500 assert(opd_bt == in->bottom_type()->basic_type(), "all same type");
1501 pk->add_opd(in);
1502 }
1503 _igvn.register_new_node_with_optimizer(pk);
1504 _phase->set_ctrl(pk, _phase->get_ctrl(opd));
1505 #ifdef ASSERT
1506 if (TraceNewVectors) {
1507 tty->print("new Vector node: ");
1508 pk->dump();
1509 }
1510 #endif
1511 return pk;
1512 }
1513
1529 Node* n = use->in(k);
1530 if (def == n) {
1531 if (!is_vector_use(use, k)) {
1532 _n_idx_list.push(use, k);
1533 }
1534 }
1535 }
1536 }
1537 }
1538
1539 while (_n_idx_list.is_nonempty()) {
1540 Node* use = _n_idx_list.node();
1541 int idx = _n_idx_list.index();
1542 _n_idx_list.pop();
1543 Node* def = use->in(idx);
1544
1545 // Insert extract operation
1546 _igvn.hash_delete(def);
1547 int def_pos = alignment(def) / data_size(def);
1548
1549 Node* ex = ExtractNode::make(_phase->C, def, def_pos, velt_basic_type(def));
1550 _igvn.register_new_node_with_optimizer(ex);
1551 _phase->set_ctrl(ex, _phase->get_ctrl(def));
1552 _igvn.replace_input_of(use, idx, ex);
1553 _igvn._worklist.push(def);
1554
1555 bb_insert_after(ex, bb_idx(def));
1556 set_velt_type(ex, velt_type(def));
1557 }
1558 }
1559
1560 //------------------------------is_vector_use---------------------------
1561 // Is use->in(u_idx) a vector use?
1562 bool SuperWord::is_vector_use(Node* use, int u_idx) {
1563 Node_List* u_pk = my_pack(use);
1564 if (u_pk == NULL) return false;
1565 Node* def = use->in(u_idx);
1566 Node_List* d_pk = my_pack(def);
1567 if (d_pk == NULL) {
1568 // check for scalar promotion
1569 Node* n = u_pk->at(0)->in(u_idx);
|
1 /*
2 * Copyright (c) 2007, 2014, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
1363 insert_extracts(_packset.at(i));
1364 }
1365
1366 Compile* C = _phase->C;
1367 uint max_vlen_in_bytes = 0;
1368 for (int i = 0; i < _block.length(); i++) {
1369 Node* n = _block.at(i);
1370 Node_List* p = my_pack(n);
1371 if (p && n == executed_last(p)) {
1372 uint vlen = p->size();
1373 uint vlen_in_bytes = 0;
1374 Node* vn = NULL;
1375 Node* low_adr = p->at(0);
1376 Node* first = executed_first(p);
1377 int opc = n->Opcode();
1378 if (n->is_Load()) {
1379 Node* ctl = n->in(MemNode::Control);
1380 Node* mem = first->in(MemNode::Memory);
1381 Node* adr = low_adr->in(MemNode::Address);
1382 const TypePtr* atyp = n->adr_type();
1383 vn = LoadVectorNode::make(opc, ctl, mem, adr, atyp, vlen, velt_basic_type(n));
1384 vlen_in_bytes = vn->as_LoadVector()->memory_size();
1385 } else if (n->is_Store()) {
1386 // Promote value to be stored to vector
1387 Node* val = vector_opd(p, MemNode::ValueIn);
1388 Node* ctl = n->in(MemNode::Control);
1389 Node* mem = first->in(MemNode::Memory);
1390 Node* adr = low_adr->in(MemNode::Address);
1391 const TypePtr* atyp = n->adr_type();
1392 vn = StoreVectorNode::make(opc, ctl, mem, adr, atyp, val, vlen);
1393 vlen_in_bytes = vn->as_StoreVector()->memory_size();
1394 } else if (n->req() == 3) {
1395 // Promote operands to vector
1396 Node* in1 = vector_opd(p, 1);
1397 Node* in2 = vector_opd(p, 2);
1398 if (VectorNode::is_invariant_vector(in1) && (n->is_Add() || n->is_Mul())) {
1399 // Move invariant vector input into second position to avoid register spilling.
1400 Node* tmp = in1;
1401 in1 = in2;
1402 in2 = tmp;
1403 }
1404 vn = VectorNode::make(opc, in1, in2, vlen, velt_basic_type(n));
1405 vlen_in_bytes = vn->as_Vector()->length_in_bytes();
1406 } else {
1407 ShouldNotReachHere();
1408 }
1409 assert(vn != NULL, "sanity");
1410 _igvn.register_new_node_with_optimizer(vn);
1411 _phase->set_ctrl(vn, _phase->get_ctrl(p->at(0)));
1412 for (uint j = 0; j < p->size(); j++) {
1413 Node* pm = p->at(j);
1414 _igvn.replace_node(pm, vn);
1415 }
1416 _igvn._worklist.push(vn);
1417
1418 if (vlen_in_bytes > max_vlen_in_bytes) {
1419 max_vlen_in_bytes = vlen_in_bytes;
1420 }
1421 #ifdef ASSERT
1422 if (TraceNewVectors) {
1423 tty->print("new Vector node: ");
1424 vn->dump();
1433 // Create a vector operand for the nodes in pack p for operand: in(opd_idx)
1434 Node* SuperWord::vector_opd(Node_List* p, int opd_idx) {
1435 Node* p0 = p->at(0);
1436 uint vlen = p->size();
1437 Node* opd = p0->in(opd_idx);
1438
1439 if (same_inputs(p, opd_idx)) {
1440 if (opd->is_Vector() || opd->is_LoadVector()) {
1441 assert(((opd_idx != 2) || !VectorNode::is_shift(p0)), "shift's count can't be vector");
1442 return opd; // input is matching vector
1443 }
1444 if ((opd_idx == 2) && VectorNode::is_shift(p0)) {
1445 Compile* C = _phase->C;
1446 Node* cnt = opd;
1447 // Vector instructions do not mask shift count, do it here.
1448 juint mask = (p0->bottom_type() == TypeInt::INT) ? (BitsPerInt - 1) : (BitsPerLong - 1);
1449 const TypeInt* t = opd->find_int_type();
1450 if (t != NULL && t->is_con()) {
1451 juint shift = t->get_con();
1452 if (shift > mask) { // Unsigned cmp
1453 cnt = ConNode::make(TypeInt::make(shift & mask));
1454 }
1455 } else {
1456 if (t == NULL || t->_lo < 0 || t->_hi > (int)mask) {
1457 cnt = ConNode::make(TypeInt::make(mask));
1458 _igvn.register_new_node_with_optimizer(cnt);
1459 cnt = new AndINode(opd, cnt);
1460 _igvn.register_new_node_with_optimizer(cnt);
1461 _phase->set_ctrl(cnt, _phase->get_ctrl(opd));
1462 }
1463 assert(opd->bottom_type()->isa_int(), "int type only");
1464 // Move non constant shift count into vector register.
1465 cnt = VectorNode::shift_count(p0, cnt, vlen, velt_basic_type(p0));
1466 }
1467 if (cnt != opd) {
1468 _igvn.register_new_node_with_optimizer(cnt);
1469 _phase->set_ctrl(cnt, _phase->get_ctrl(opd));
1470 }
1471 return cnt;
1472 }
1473 assert(!opd->is_StoreVector(), "such vector is not expected here");
1474 // Convert scalar input to vector with the same number of elements as
1475 // p0's vector. Use p0's type because size of operand's container in
1476 // vector should match p0's size regardless operand's size.
1477 const Type* p0_t = velt_type(p0);
1478 VectorNode* vn = VectorNode::scalar2vector(opd, vlen, p0_t);
1479
1480 _igvn.register_new_node_with_optimizer(vn);
1481 _phase->set_ctrl(vn, _phase->get_ctrl(opd));
1482 #ifdef ASSERT
1483 if (TraceNewVectors) {
1484 tty->print("new Vector node: ");
1485 vn->dump();
1486 }
1487 #endif
1488 return vn;
1489 }
1490
1491 // Insert pack operation
1492 BasicType bt = velt_basic_type(p0);
1493 PackNode* pk = PackNode::make(opd, vlen, bt);
1494 DEBUG_ONLY( const BasicType opd_bt = opd->bottom_type()->basic_type(); )
1495
1496 for (uint i = 1; i < vlen; i++) {
1497 Node* pi = p->at(i);
1498 Node* in = pi->in(opd_idx);
1499 assert(my_pack(in) == NULL, "Should already have been unpacked");
1500 assert(opd_bt == in->bottom_type()->basic_type(), "all same type");
1501 pk->add_opd(in);
1502 }
1503 _igvn.register_new_node_with_optimizer(pk);
1504 _phase->set_ctrl(pk, _phase->get_ctrl(opd));
1505 #ifdef ASSERT
1506 if (TraceNewVectors) {
1507 tty->print("new Vector node: ");
1508 pk->dump();
1509 }
1510 #endif
1511 return pk;
1512 }
1513
1529 Node* n = use->in(k);
1530 if (def == n) {
1531 if (!is_vector_use(use, k)) {
1532 _n_idx_list.push(use, k);
1533 }
1534 }
1535 }
1536 }
1537 }
1538
1539 while (_n_idx_list.is_nonempty()) {
1540 Node* use = _n_idx_list.node();
1541 int idx = _n_idx_list.index();
1542 _n_idx_list.pop();
1543 Node* def = use->in(idx);
1544
1545 // Insert extract operation
1546 _igvn.hash_delete(def);
1547 int def_pos = alignment(def) / data_size(def);
1548
1549 Node* ex = ExtractNode::make(def, def_pos, velt_basic_type(def));
1550 _igvn.register_new_node_with_optimizer(ex);
1551 _phase->set_ctrl(ex, _phase->get_ctrl(def));
1552 _igvn.replace_input_of(use, idx, ex);
1553 _igvn._worklist.push(def);
1554
1555 bb_insert_after(ex, bb_idx(def));
1556 set_velt_type(ex, velt_type(def));
1557 }
1558 }
1559
1560 //------------------------------is_vector_use---------------------------
1561 // Is use->in(u_idx) a vector use?
1562 bool SuperWord::is_vector_use(Node* use, int u_idx) {
1563 Node_List* u_pk = my_pack(use);
1564 if (u_pk == NULL) return false;
1565 Node* def = use->in(u_idx);
1566 Node_List* d_pk = my_pack(def);
1567 if (d_pk == NULL) {
1568 // check for scalar promotion
1569 Node* n = u_pk->at(0)->in(u_idx);
|