1150 } 1151 1152 for (int i = 0; i < _block.length(); i++) { 1153 Node* n = _block.at(i); 1154 Node_List* p = my_pack(n); 1155 if (p && n == executed_last(p)) { 1156 uint vlen = p->size(); 1157 Node* vn = NULL; 1158 Node* low_adr = p->at(0); 1159 Node* first = executed_first(p); 1160 if (n->is_Load()) { 1161 int opc = n->Opcode(); 1162 Node* ctl = n->in(MemNode::Control); 1163 Node* mem = first->in(MemNode::Memory); 1164 Node* adr = low_adr->in(MemNode::Address); 1165 const TypePtr* atyp = n->adr_type(); 1166 vn = VectorLoadNode::make(_phase->C, opc, ctl, mem, adr, atyp, vlen); 1167 1168 } else if (n->is_Store()) { 1169 // Promote value to be stored to vector 1170 VectorNode* val = vector_opd(p, MemNode::ValueIn); 1171 1172 int opc = n->Opcode(); 1173 Node* ctl = n->in(MemNode::Control); 1174 Node* mem = first->in(MemNode::Memory); 1175 Node* adr = low_adr->in(MemNode::Address); 1176 const TypePtr* atyp = n->adr_type(); 1177 vn = VectorStoreNode::make(_phase->C, opc, ctl, mem, adr, atyp, val, vlen); 1178 1179 } else if (n->req() == 3) { 1180 // Promote operands to vector 1181 Node* in1 = vector_opd(p, 1); 1182 Node* in2 = vector_opd(p, 2); 1183 vn = VectorNode::make(_phase->C, n->Opcode(), in1, in2, vlen, velt_type(n)); 1184 1185 } else { 1186 ShouldNotReachHere(); 1187 } 1188 1189 _phase->_igvn.register_new_node_with_optimizer(vn); 1190 _phase->set_ctrl(vn, _phase->get_ctrl(p->at(0))); 1191 for (uint j = 0; j < p->size(); j++) { 1192 Node* pm = p->at(j); 1193 _igvn.replace_node(pm, vn); 1194 } 1195 _igvn._worklist.push(vn); 1196 } 1197 } 1198 } 1199 1200 //------------------------------vector_opd--------------------------- 1201 // Create a vector operand for the nodes in pack p for operand: in(opd_idx) 1202 VectorNode* SuperWord::vector_opd(Node_List* p, int opd_idx) { 1203 Node* p0 = p->at(0); 1204 uint vlen = p->size(); 1205 Node* opd = p0->in(opd_idx); 1206 1207 bool same_opd = true; 1208 for (uint i = 1; i < vlen; i++) { 1209 Node* pi = p->at(i); 1210 Node* in = pi->in(opd_idx); 1211 if (opd != in) { 1212 same_opd = false; 1213 break; 1214 } 1215 } 1216 1217 if (same_opd) { 1218 if (opd->is_Vector()) { 1219 return (VectorNode*)opd; // input is matching vector 1220 } 1221 // Convert scalar input to vector. Use p0's type because it's container 1222 // maybe smaller than the operand's container. 1223 const Type* opd_t = velt_type(!in_bb(opd) ? p0 : opd); 1224 const Type* p0_t = velt_type(p0); 1225 if (p0_t->higher_equal(opd_t)) opd_t = p0_t; 1226 VectorNode* vn = VectorNode::scalar2vector(_phase->C, opd, vlen, opd_t); 1227 1228 _phase->_igvn.register_new_node_with_optimizer(vn); 1229 _phase->set_ctrl(vn, _phase->get_ctrl(opd)); 1230 return vn; 1231 } 1232 1233 // Insert pack operation 1234 const Type* opd_t = velt_type(!in_bb(opd) ? p0 : opd); 1235 PackNode* pk = PackNode::make(_phase->C, opd, opd_t); 1236 1237 for (uint i = 1; i < vlen; i++) { 1238 Node* pi = p->at(i); 1239 Node* in = pi->in(opd_idx); 1240 assert(my_pack(in) == NULL, "Should already have been unpacked"); | 1150 } 1151 1152 for (int i = 0; i < _block.length(); i++) { 1153 Node* n = _block.at(i); 1154 Node_List* p = my_pack(n); 1155 if (p && n == executed_last(p)) { 1156 uint vlen = p->size(); 1157 Node* vn = NULL; 1158 Node* low_adr = p->at(0); 1159 Node* first = executed_first(p); 1160 if (n->is_Load()) { 1161 int opc = n->Opcode(); 1162 Node* ctl = n->in(MemNode::Control); 1163 Node* mem = first->in(MemNode::Memory); 1164 Node* adr = low_adr->in(MemNode::Address); 1165 const TypePtr* atyp = n->adr_type(); 1166 vn = VectorLoadNode::make(_phase->C, opc, ctl, mem, adr, atyp, vlen); 1167 1168 } else if (n->is_Store()) { 1169 // Promote value to be stored to vector 1170 Node* val = vector_opd(p, MemNode::ValueIn); 1171 1172 int opc = n->Opcode(); 1173 Node* ctl = n->in(MemNode::Control); 1174 Node* mem = first->in(MemNode::Memory); 1175 Node* adr = low_adr->in(MemNode::Address); 1176 const TypePtr* atyp = n->adr_type(); 1177 vn = VectorStoreNode::make(_phase->C, opc, ctl, mem, adr, atyp, val, vlen); 1178 1179 } else if (n->req() == 3) { 1180 // Promote operands to vector 1181 Node* in1 = vector_opd(p, 1); 1182 Node* in2 = vector_opd(p, 2); 1183 vn = VectorNode::make(_phase->C, n->Opcode(), in1, in2, vlen, velt_type(n)); 1184 1185 } else { 1186 ShouldNotReachHere(); 1187 } 1188 1189 _phase->_igvn.register_new_node_with_optimizer(vn); 1190 _phase->set_ctrl(vn, _phase->get_ctrl(p->at(0))); 1191 for (uint j = 0; j < p->size(); j++) { 1192 Node* pm = p->at(j); 1193 _igvn.replace_node(pm, vn); 1194 } 1195 _igvn._worklist.push(vn); 1196 } 1197 } 1198 } 1199 1200 //------------------------------vector_opd--------------------------- 1201 // Create a vector operand for the nodes in pack p for operand: in(opd_idx) 1202 Node* SuperWord::vector_opd(Node_List* p, int opd_idx) { 1203 Node* p0 = p->at(0); 1204 uint vlen = p->size(); 1205 Node* opd = p0->in(opd_idx); 1206 1207 bool same_opd = true; 1208 for (uint i = 1; i < vlen; i++) { 1209 Node* pi = p->at(i); 1210 Node* in = pi->in(opd_idx); 1211 if (opd != in) { 1212 same_opd = false; 1213 break; 1214 } 1215 } 1216 1217 if (same_opd) { 1218 if (opd->is_Vector() || opd->is_VectorLoad()) { 1219 return opd; // input is matching vector 1220 } 1221 assert(!opd->is_VectorStore(), "such vector is not expected here"); 1222 // Convert scalar input to vector. Use p0's type because it's container 1223 // maybe smaller than the operand's container. 1224 const Type* opd_t = velt_type(!in_bb(opd) ? p0 : opd); 1225 const Type* p0_t = velt_type(p0); 1226 if (p0_t->higher_equal(opd_t)) opd_t = p0_t; 1227 VectorNode* vn = VectorNode::scalar2vector(_phase->C, opd, vlen, opd_t); 1228 1229 _phase->_igvn.register_new_node_with_optimizer(vn); 1230 _phase->set_ctrl(vn, _phase->get_ctrl(opd)); 1231 return vn; 1232 } 1233 1234 // Insert pack operation 1235 const Type* opd_t = velt_type(!in_bb(opd) ? p0 : opd); 1236 PackNode* pk = PackNode::make(_phase->C, opd, opd_t); 1237 1238 for (uint i = 1; i < vlen; i++) { 1239 Node* pi = p->at(i); 1240 Node* in = pi->in(opd_idx); 1241 assert(my_pack(in) == NULL, "Should already have been unpacked"); |