973 //connect current to insert_pt 974 current->set_req(MemNode::Memory, insert_pt); 975 _igvn._worklist.push(current); 976 } 977 978 //------------------------------co_locate_pack---------------------------------- 979 // To schedule a store pack, we need to move any sandwiched memory ops either before 980 // or after the pack, based upon dependence information: 981 // (1) If any store in the pack depends on the sandwiched memory op, the 982 // sandwiched memory op must be scheduled BEFORE the pack; 983 // (2) If a sandwiched memory op depends on any store in the pack, the 984 // sandwiched memory op must be scheduled AFTER the pack; 985 // (3) If a sandwiched memory op (say, memA) depends on another sandwiched 986 // memory op (say memB), memB must be scheduled before memA. So, if memA is 987 // scheduled before the pack, memB must also be scheduled before the pack; 988 // (4) If there is no dependence restriction for a sandwiched memory op, we simply 989 // schedule this store AFTER the pack 990 // (5) We know there is no dependence cycle, so there in no other case; 991 // (6) Finally, all memory ops in another single pack should be moved in the same direction. 992 // 993 // To schedule a load pack: the memory edge of every loads in the pack must be 994 // the same as the memory edge of the last executed load in the pack 995 void SuperWord::co_locate_pack(Node_List* pk) { 996 if (pk->at(0)->is_Store()) { 997 MemNode* first = executed_first(pk)->as_Mem(); 998 MemNode* last = executed_last(pk)->as_Mem(); 999 Unique_Node_List schedule_before_pack; 1000 Unique_Node_List memops; 1001 1002 MemNode* current = last->in(MemNode::Memory)->as_Mem(); 1003 MemNode* previous = last; 1004 while (true) { 1005 assert(in_bb(current), "stay in block"); 1006 memops.push(previous); 1007 for (DUIterator i = current->outs(); current->has_out(i); i++) { 1008 Node* use = current->out(i); 1009 if (use->is_Mem() && use != previous) 1010 memops.push(use); 1011 } 1012 if(current == first) break; 1013 previous = current; 1014 current = current->in(MemNode::Memory)->as_Mem(); 1059 if (schedule_before_pack.member(use)) { 1060 _igvn.hash_delete(upper_insert_pt); 1061 use->set_req(MemNode::Memory, upper_insert_pt); 1062 } else { 1063 _igvn.hash_delete(lower_insert_pt); 1064 use->set_req(MemNode::Memory, lower_insert_pt); 1065 } 1066 _igvn._worklist.push(use); 1067 --i; // deleted this edge; rescan position 1068 } 1069 } 1070 previous = current; 1071 } else { // !in_pack(current, pk) ==> a sandwiched store 1072 remove_and_insert(current, previous, lower_insert_pt, upper_insert_pt, schedule_before_pack); 1073 } 1074 1075 if (current == first) break; 1076 current = my_mem->as_Mem(); 1077 } // end while 1078 } else if (pk->at(0)->is_Load()) { //load 1079 // all use the memory state that the last executed load uses 1080 LoadNode* last_load = executed_last(pk)->as_Load(); 1081 Node* last_mem = last_load->in(MemNode::Memory); 1082 _igvn.hash_delete(last_mem); 1083 // Give each load same memory state as last 1084 for (uint i = 0; i < pk->size(); i++) { 1085 LoadNode* ld = pk->at(i)->as_Load(); 1086 _igvn.hash_delete(ld); 1087 ld->set_req(MemNode::Memory, last_mem); 1088 _igvn._worklist.push(ld); 1089 } 1090 } 1091 } 1092 1093 //------------------------------output--------------------------- 1094 // Convert packs into vector node operations 1095 void SuperWord::output() { 1096 if (_packset.length() == 0) return; 1097 1098 // MUST ENSURE main loop's initial value is properly aligned: 1099 // (iv_initial_value + min_iv_offset) % vector_width_in_bytes() == 0 1100 1101 align_initial_loop_index(align_to_ref()); 1102 1103 // Insert extract (unpack) operations for scalar uses 1104 for (int i = 0; i < _packset.length(); i++) { 1105 insert_extracts(_packset.at(i)); 1106 } 1107 | 973 //connect current to insert_pt 974 current->set_req(MemNode::Memory, insert_pt); 975 _igvn._worklist.push(current); 976 } 977 978 //------------------------------co_locate_pack---------------------------------- 979 // To schedule a store pack, we need to move any sandwiched memory ops either before 980 // or after the pack, based upon dependence information: 981 // (1) If any store in the pack depends on the sandwiched memory op, the 982 // sandwiched memory op must be scheduled BEFORE the pack; 983 // (2) If a sandwiched memory op depends on any store in the pack, the 984 // sandwiched memory op must be scheduled AFTER the pack; 985 // (3) If a sandwiched memory op (say, memA) depends on another sandwiched 986 // memory op (say memB), memB must be scheduled before memA. So, if memA is 987 // scheduled before the pack, memB must also be scheduled before the pack; 988 // (4) If there is no dependence restriction for a sandwiched memory op, we simply 989 // schedule this store AFTER the pack 990 // (5) We know there is no dependence cycle, so there in no other case; 991 // (6) Finally, all memory ops in another single pack should be moved in the same direction. 992 // 993 // To schedule a load pack, we use the memory state of either the first or the last load in 994 // the pack, based on the dependence constraint. 995 void SuperWord::co_locate_pack(Node_List* pk) { 996 if (pk->at(0)->is_Store()) { 997 MemNode* first = executed_first(pk)->as_Mem(); 998 MemNode* last = executed_last(pk)->as_Mem(); 999 Unique_Node_List schedule_before_pack; 1000 Unique_Node_List memops; 1001 1002 MemNode* current = last->in(MemNode::Memory)->as_Mem(); 1003 MemNode* previous = last; 1004 while (true) { 1005 assert(in_bb(current), "stay in block"); 1006 memops.push(previous); 1007 for (DUIterator i = current->outs(); current->has_out(i); i++) { 1008 Node* use = current->out(i); 1009 if (use->is_Mem() && use != previous) 1010 memops.push(use); 1011 } 1012 if(current == first) break; 1013 previous = current; 1014 current = current->in(MemNode::Memory)->as_Mem(); 1059 if (schedule_before_pack.member(use)) { 1060 _igvn.hash_delete(upper_insert_pt); 1061 use->set_req(MemNode::Memory, upper_insert_pt); 1062 } else { 1063 _igvn.hash_delete(lower_insert_pt); 1064 use->set_req(MemNode::Memory, lower_insert_pt); 1065 } 1066 _igvn._worklist.push(use); 1067 --i; // deleted this edge; rescan position 1068 } 1069 } 1070 previous = current; 1071 } else { // !in_pack(current, pk) ==> a sandwiched store 1072 remove_and_insert(current, previous, lower_insert_pt, upper_insert_pt, schedule_before_pack); 1073 } 1074 1075 if (current == first) break; 1076 current = my_mem->as_Mem(); 1077 } // end while 1078 } else if (pk->at(0)->is_Load()) { //load 1079 // all loads in the pack should have the same memory state. By default, 1080 // we use the memory state of the last load. However, if any load could 1081 // not be moved down due to the dependence constraint, we use the memory 1082 // state of the first load. 1083 Node* last_mem = executed_last(pk)->in(MemNode::Memory); 1084 Node* first_mem = executed_first(pk)->in(MemNode::Memory); 1085 bool schedule_last = true; 1086 for (uint i = 0; i < pk->size(); i++) { 1087 Node* ld = pk->at(i); 1088 for (Node* current = last_mem; current != ld->in(MemNode::Memory); 1089 current=current->in(MemNode::Memory)) { 1090 if(current->is_Mem() && !independent(current, ld)){ 1091 schedule_last = false; // a later store depends on this load 1092 break; 1093 } 1094 } 1095 } 1096 1097 Node* mem_input = schedule_last ? last_mem : first_mem; 1098 _igvn.hash_delete(mem_input); 1099 // Give each load the same memory state 1100 for (uint i = 0; i < pk->size(); i++) { 1101 LoadNode* ld = pk->at(i)->as_Load(); 1102 _igvn.hash_delete(ld); 1103 ld->set_req(MemNode::Memory, mem_input); 1104 _igvn._worklist.push(ld); 1105 } 1106 } 1107 } 1108 1109 //------------------------------output--------------------------- 1110 // Convert packs into vector node operations 1111 void SuperWord::output() { 1112 if (_packset.length() == 0) return; 1113 1114 // MUST ENSURE main loop's initial value is properly aligned: 1115 // (iv_initial_value + min_iv_offset) % vector_width_in_bytes() == 0 1116 1117 align_initial_loop_index(align_to_ref()); 1118 1119 // Insert extract (unpack) operations for scalar uses 1120 for (int i = 0; i < _packset.length(); i++) { 1121 insert_extracts(_packset.at(i)); 1122 } 1123 |