2122 } else { // !in_pack(current, pk) ==> a sandwiched store
2123 remove_and_insert(current, previous, lower_insert_pt, upper_insert_pt, schedule_before_pack);
2124 }
2125
2126 if (current == first) break;
2127 current = my_mem->as_Mem();
2128 } // end while
2129
2130 // Reconnect loads back to upper_insert_pt.
2131 for (uint i = 0; i < memops.size(); i++) {
2132 Node *ld = memops.at(i);
2133 if (ld->in(MemNode::Memory) != upper_insert_pt) {
2134 _igvn.replace_input_of(ld, MemNode::Memory, upper_insert_pt);
2135 }
2136 }
2137 } else if (pk->at(0)->is_Load()) { //load
2138 // all loads in the pack should have the same memory state. By default,
2139 // we use the memory state of the last load. However, if any load could
2140 // not be moved down due to the dependence constraint, we use the memory
2141 // state of the first load.
2142 Node* last_mem = executed_last(pk)->in(MemNode::Memory);
2143 Node* first_mem = executed_first(pk)->in(MemNode::Memory);
2144 bool schedule_last = true;
2145 for (uint i = 0; i < pk->size(); i++) {
2146 Node* ld = pk->at(i);
2147 for (Node* current = last_mem; current != ld->in(MemNode::Memory);
2148 current=current->in(MemNode::Memory)) {
2149 assert(current != first_mem, "corrupted memory graph");
2150 if(current->is_Mem() && !independent(current, ld)){
2151 schedule_last = false; // a later store depends on this load
2152 break;
2153 }
2154 }
2155 }
2156
2157 Node* mem_input = schedule_last ? last_mem : first_mem;
2158 _igvn.hash_delete(mem_input);
2159 // Give each load the same memory state
2160 for (uint i = 0; i < pk->size(); i++) {
2161 LoadNode* ld = pk->at(i)->as_Load();
2162 _igvn.replace_input_of(ld, MemNode::Memory, mem_input);
2163 }
2164 }
2165 }
2166
2167 #ifndef PRODUCT
2168 void SuperWord::print_loop(bool whole) {
2169 Node_Stack stack(_arena, _phase->C->unique() >> 2);
2170 Node_List rpo_list;
2171 VectorSet visited(_arena);
2172 visited.set(lpt()->_head->_idx);
|
2122 } else { // !in_pack(current, pk) ==> a sandwiched store
2123 remove_and_insert(current, previous, lower_insert_pt, upper_insert_pt, schedule_before_pack);
2124 }
2125
2126 if (current == first) break;
2127 current = my_mem->as_Mem();
2128 } // end while
2129
2130 // Reconnect loads back to upper_insert_pt.
2131 for (uint i = 0; i < memops.size(); i++) {
2132 Node *ld = memops.at(i);
2133 if (ld->in(MemNode::Memory) != upper_insert_pt) {
2134 _igvn.replace_input_of(ld, MemNode::Memory, upper_insert_pt);
2135 }
2136 }
2137 } else if (pk->at(0)->is_Load()) { //load
2138 // all loads in the pack should have the same memory state. By default,
2139 // we use the memory state of the last load. However, if any load could
2140 // not be moved down due to the dependence constraint, we use the memory
2141 // state of the first load.
2142
2143 // We first need to find the "first" and "last" loads. Start with
2144 // arbitrary values:
2145 Node* last_mem = pk->at(0)->in(MemNode::Memory);
2146 Node* first_mem = last_mem;
2147 // Walk the memory graph from the current first load until the
2148 // start of the loop and check if nodes on the way are memory
2149 // edges of loads in the pack. The last one we encounter is the
2150 // first load.
2151 for (Node* current = first_mem; in_bb(current); current = current->is_Phi() ? current->in(LoopNode::EntryControl) : current->in(MemNode::Memory)) {
2152 assert(current->is_Mem() || (current->is_Phi() && current->in(0) == bb()), "unexpected memory");
2153 for (uint i = 1; i < pk->size(); i++) {
2154 Node* ld = pk->at(i);
2155 if (ld->in(MemNode::Memory) == current) {
2156 first_mem = current;
2157 break;
2158 }
2159 }
2160 }
2161 // Find the last load by going over the pack again, and walking
2162 // the memory graph from the loads of the pack to the memory of
2163 // the first load. If we encounter the memory of the current last
2164 // load, then we started from further down in the memory graph and
2165 // the load we started from is the last load. Check for dependence
2166 // constraints in that loop as well.
2167 bool schedule_last = true;
2168 for (uint i = 0; i < pk->size(); i++) {
2169 Node* ld = pk->at(i);
2170 for (Node* current = ld->in(MemNode::Memory); current != first_mem; current = current->in(MemNode::Memory)) {
2171 assert(current->is_Mem() && in_bb(current), "unexpected memory");
2172 if (current->in(MemNode::Memory) == last_mem) {
2173 last_mem = ld->in(MemNode::Memory);
2174 }
2175 if (!independent(current, ld)) {
2176 schedule_last = false; // a later store depends on this load
2177 }
2178 }
2179 }
2180
2181 Node* mem_input = schedule_last ? last_mem : first_mem;
2182 _igvn.hash_delete(mem_input);
2183 // Give each load the same memory state
2184 for (uint i = 0; i < pk->size(); i++) {
2185 LoadNode* ld = pk->at(i)->as_Load();
2186 _igvn.replace_input_of(ld, MemNode::Memory, mem_input);
2187 }
2188 }
2189 }
2190
2191 #ifndef PRODUCT
2192 void SuperWord::print_loop(bool whole) {
2193 Node_Stack stack(_arena, _phase->C->unique() >> 2);
2194 Node_List rpo_list;
2195 VectorSet visited(_arena);
2196 visited.set(lpt()->_head->_idx);
|