1414 int opc = n->Opcode();
1415 if (n->is_Load()) {
1416 Node* ctl = n->in(MemNode::Control);
1417 Node* mem = first->in(MemNode::Memory);
1418 SWPointer p1(n->as_Mem(), this);
1419 // Identify the memory dependency for the new loadVector node by
1420 // walking up through memory chain.
1421 // This is done to give flexibility to the new loadVector node so that
1422 // it can move above independent storeVector nodes.
1423 while (mem->is_StoreVector()) {
1424 SWPointer p2(mem->as_Mem(), this);
1425 int cmp = p1.cmp(p2);
1426 if (SWPointer::not_equal(cmp) || !SWPointer::comparable(cmp)) {
1427 mem = mem->in(MemNode::Memory);
1428 } else {
1429 break; // dependent memory
1430 }
1431 }
1432 Node* adr = low_adr->in(MemNode::Address);
1433 const TypePtr* atyp = n->adr_type();
1434 vn = LoadVectorNode::make(C, opc, ctl, mem, adr, atyp, vlen, velt_basic_type(n));
1435 vlen_in_bytes = vn->as_LoadVector()->memory_size();
1436 } else if (n->is_Store()) {
1437 // Promote value to be stored to vector
1438 Node* val = vector_opd(p, MemNode::ValueIn);
1439 Node* ctl = n->in(MemNode::Control);
1440 Node* mem = first->in(MemNode::Memory);
1441 Node* adr = low_adr->in(MemNode::Address);
1442 const TypePtr* atyp = n->adr_type();
1443 vn = StoreVectorNode::make(C, opc, ctl, mem, adr, atyp, val, vlen);
1444 vlen_in_bytes = vn->as_StoreVector()->memory_size();
1445 } else if (n->req() == 3) {
1446 // Promote operands to vector
1447 Node* in1 = vector_opd(p, 1);
1448 Node* in2 = vector_opd(p, 2);
1449 if (VectorNode::is_invariant_vector(in1) && (n->is_Add() || n->is_Mul())) {
1450 // Move invariant vector input into second position to avoid register spilling.
1451 Node* tmp = in1;
1452 in1 = in2;
1453 in2 = tmp;
1454 }
2011 }
2012 }
2013 return n;
2014 }
2015
2016 //------------------------------executed_last---------------------------
2017 // Return the node executed last in pack p.
2018 Node* SuperWord::executed_last(Node_List* p) {
2019 Node* n = p->at(0);
2020 int n_rpo = bb_idx(n);
2021 for (uint i = 1; i < p->size(); i++) {
2022 Node* s = p->at(i);
2023 int s_rpo = bb_idx(s);
2024 if (s_rpo > n_rpo) {
2025 n = s;
2026 n_rpo = s_rpo;
2027 }
2028 }
2029 return n;
2030 }
2031
2032 //----------------------------align_initial_loop_index---------------------------
2033 // Adjust pre-loop limit so that in main loop, a load/store reference
2034 // to align_to_ref will be a position zero in the vector.
2035 // (iv + k) mod vector_align == 0
2036 void SuperWord::align_initial_loop_index(MemNode* align_to_ref) {
2037 CountedLoopNode *main_head = lp()->as_CountedLoop();
2038 assert(main_head->is_main_loop(), "");
2039 CountedLoopEndNode* pre_end = get_pre_loop_end(main_head);
2040 assert(pre_end != NULL, "");
2041 Node *pre_opaq1 = pre_end->limit();
2042 assert(pre_opaq1->Opcode() == Op_Opaque1, "");
2043 Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
2044 Node *lim0 = pre_opaq->in(1);
2045
2046 // Where we put new limit calculations
2047 Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl);
2048
2049 // Ensure the original loop limit is available from the
2050 // pre-loop Opaque1 node.
|
1414 int opc = n->Opcode();
1415 if (n->is_Load()) {
1416 Node* ctl = n->in(MemNode::Control);
1417 Node* mem = first->in(MemNode::Memory);
1418 SWPointer p1(n->as_Mem(), this);
1419 // Identify the memory dependency for the new loadVector node by
1420 // walking up through memory chain.
1421 // This is done to give flexibility to the new loadVector node so that
1422 // it can move above independent storeVector nodes.
1423 while (mem->is_StoreVector()) {
1424 SWPointer p2(mem->as_Mem(), this);
1425 int cmp = p1.cmp(p2);
1426 if (SWPointer::not_equal(cmp) || !SWPointer::comparable(cmp)) {
1427 mem = mem->in(MemNode::Memory);
1428 } else {
1429 break; // dependent memory
1430 }
1431 }
1432 Node* adr = low_adr->in(MemNode::Address);
1433 const TypePtr* atyp = n->adr_type();
1434 vn = LoadVectorNode::make(C, opc, ctl, mem, adr, atyp, vlen, velt_basic_type(n), control_dependency(p));
1435 vlen_in_bytes = vn->as_LoadVector()->memory_size();
1436 } else if (n->is_Store()) {
1437 // Promote value to be stored to vector
1438 Node* val = vector_opd(p, MemNode::ValueIn);
1439 Node* ctl = n->in(MemNode::Control);
1440 Node* mem = first->in(MemNode::Memory);
1441 Node* adr = low_adr->in(MemNode::Address);
1442 const TypePtr* atyp = n->adr_type();
1443 vn = StoreVectorNode::make(C, opc, ctl, mem, adr, atyp, val, vlen);
1444 vlen_in_bytes = vn->as_StoreVector()->memory_size();
1445 } else if (n->req() == 3) {
1446 // Promote operands to vector
1447 Node* in1 = vector_opd(p, 1);
1448 Node* in2 = vector_opd(p, 2);
1449 if (VectorNode::is_invariant_vector(in1) && (n->is_Add() || n->is_Mul())) {
1450 // Move invariant vector input into second position to avoid register spilling.
1451 Node* tmp = in1;
1452 in1 = in2;
1453 in2 = tmp;
1454 }
2011 }
2012 }
2013 return n;
2014 }
2015
2016 //------------------------------executed_last---------------------------
2017 // Return the node executed last in pack p.
2018 Node* SuperWord::executed_last(Node_List* p) {
2019 Node* n = p->at(0);
2020 int n_rpo = bb_idx(n);
2021 for (uint i = 1; i < p->size(); i++) {
2022 Node* s = p->at(i);
2023 int s_rpo = bb_idx(s);
2024 if (s_rpo > n_rpo) {
2025 n = s;
2026 n_rpo = s_rpo;
2027 }
2028 }
2029 return n;
2030 }
2031
2032 LoadNode::ControlDependency SuperWord::control_dependency(Node_List* p) {
2033 LoadNode::ControlDependency dep = LoadNode::DependsOnlyOnTest;
2034 for (uint i = 0; i < p->size(); i++) {
2035 Node* n = p->at(i);
2036 assert(n->is_Load(), "only meaningful for loads");
2037 if (!n->depends_only_on_test()) {
2038 dep = LoadNode::Pinned;
2039 }
2040 }
2041 return dep;
2042 }
2043
2044
2045 //----------------------------align_initial_loop_index---------------------------
2046 // Adjust pre-loop limit so that in main loop, a load/store reference
2047 // to align_to_ref will be a position zero in the vector.
2048 // (iv + k) mod vector_align == 0
2049 void SuperWord::align_initial_loop_index(MemNode* align_to_ref) {
2050 CountedLoopNode *main_head = lp()->as_CountedLoop();
2051 assert(main_head->is_main_loop(), "");
2052 CountedLoopEndNode* pre_end = get_pre_loop_end(main_head);
2053 assert(pre_end != NULL, "");
2054 Node *pre_opaq1 = pre_end->limit();
2055 assert(pre_opaq1->Opcode() == Op_Opaque1, "");
2056 Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
2057 Node *lim0 = pre_opaq->in(1);
2058
2059 // Where we put new limit calculations
2060 Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl);
2061
2062 // Ensure the original loop limit is available from the
2063 // pre-loop Opaque1 node.
|