--- old/src/share/vm/opto/superword.cpp 2016-04-12 19:50:47.098635400 -0700 +++ new/src/share/vm/opto/superword.cpp 2016-04-12 19:50:46.759635400 -0700 @@ -51,6 +51,7 @@ _packset(arena(), 8, 0, NULL), // packs for the current block _bb_idx(arena(), (int)(1.10 * phase->C->unique()), 0, 0), // node idx to index in bb _block(arena(), 8, 0, NULL), // nodes in current block + _post_block(arena(), 8, 0, NULL), // nodes common to current block which are marked as post loop vectorizable _data_entry(arena(), 8, 0, NULL), // nodes with all inputs from outside _mem_slice_head(arena(), 8, 0, NULL), // memory slice heads _mem_slice_tail(arena(), 8, 0, NULL), // memory slice tails @@ -99,10 +100,30 @@ if (!cl->is_valid_counted_loop()) return; // skip malformed counted loop - if (!cl->is_main_loop() ) return; // skip normal, pre, and post loops + bool post_loop_allowed = (PostLoopMultiversioning && Matcher::has_predicated_vectors() && cl->is_post_loop()); + if (post_loop_allowed) { + if (cl->is_reduction_loop()) return; // no predication mapping + Node *limit = cl->limit(); + if (limit->is_Con()) return; // non constant limits only + // Now check the limit for expressions we do not handle + if (limit->is_Add()) { + Node *in2 = limit->in(2); + if (in2->is_Con()) { + int val = in2->get_int(); + // should not try to program these cases + if (val < 0) return; + } + } + } + + // skip any loop that has not been assigned max unroll by analysis + if (do_optimization) { + if (cl->slp_max_unroll() == 0) return; + } + // Check for no control flow in body (other than exit) Node *cl_exit = cl->loopexit(); - if (cl_exit->in(0) != lpt->_head) { + if (cl->is_main_loop() && (cl_exit->in(0) != lpt->_head)) { #ifndef PRODUCT if (TraceSuperWord) { tty->print_cr("SuperWord::transform_loop: loop too complicated, cl_exit->in(0) != lpt->_head"); @@ -120,15 +141,16 @@ return; } - // We only re-enter slp when we vector mapped a queried loop and we want to - // continue unrolling, in this case, slp is not subsequently done. - if (cl->do_unroll_only()) return; - - // Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit)))) - CountedLoopEndNode* pre_end = get_pre_loop_end(cl); - if (pre_end == NULL) return; - Node *pre_opaq1 = pre_end->limit(); - if (pre_opaq1->Opcode() != Op_Opaque1) return; + // Skip any loops already optimized by slp + if (cl->is_vectorized_loop()) return; + + if (cl->is_main_loop()) { + // Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit)))) + CountedLoopEndNode* pre_end = get_pre_loop_end(cl); + if (pre_end == NULL) return; + Node *pre_opaq1 = pre_end->limit(); + if (pre_opaq1->Opcode() != Op_Opaque1) return; + } init(); // initialize data structures @@ -141,6 +163,19 @@ if (do_optimization) { assert(_packset.length() == 0, "packset must be empty"); SLP_extract(); + if (PostLoopMultiversioning && Matcher::has_predicated_vectors()) { + if (cl->is_vectorized_loop() && cl->is_main_loop() && !cl->is_reduction_loop()) { + IdealLoopTree *lpt_next = lpt->_next; + CountedLoopNode *cl_next = lpt_next->_head->as_CountedLoop(); + _phase->has_range_checks(lpt_next); + if (cl_next->is_post_loop() && !cl_next->range_checks_present()) { + if (!cl_next->is_vectorized_loop()) { + int slp_max_unroll_factor = cl->slp_max_unroll(); + cl_next->set_slp_max_unroll(slp_max_unroll_factor); + } + } + } + } } } @@ -153,6 +188,9 @@ Node_Stack nstack((int)ignored_size); CountedLoopNode *cl = lpt()->_head->as_CountedLoop(); Node *cl_exit = cl->loopexit(); + int rpo_idx = _post_block.length(); + + assert(rpo_idx == 0, "post loop block is empty"); // First clear the entries for (uint i = 0; i < lpt()->_body.size(); i++) { @@ -160,6 +198,7 @@ } int max_vector = Matcher::max_vector_size(T_INT); + bool post_loop_allowed = (PostLoopMultiversioning && Matcher::has_predicated_vectors() && cl->is_post_loop()); // Process the loop, some/all of the stack entries will not be in order, ergo // need to preprocess the ignored initial state before we process the loop @@ -258,6 +297,7 @@ if (is_slp) { // Now we try to find the maximum supported consistent vector which the machine // description can use + bool small_basic_type = false; for (uint i = 0; i < lpt()->_body.size(); i++) { if (ignored_loop_nodes[i] != -1) continue; @@ -268,6 +308,24 @@ } else { bt = n->bottom_type()->basic_type(); } + + if (post_loop_allowed) { + if (!small_basic_type) { + switch (bt) { + case T_CHAR: + case T_BYTE: + case T_SHORT: + small_basic_type = true; + break; + case T_LONG: + // TODO: Remove when support completed for mask context with LONG. + // The ad files need to be augmented. + small_basic_type = true; + break; + } + } + } + if (is_java_primitive(bt) == false) continue; int cur_max_vector = Matcher::max_vector_size(bt); @@ -287,6 +345,12 @@ if (cur_max_vector < max_vector) { max_vector = cur_max_vector; } + + // We only process post loops on predicated targets where we want to + // mask map the loop to a single iteration + if (post_loop_allowed) { + _post_block.at_put_grow(rpo_idx++, n); + } } } if (is_slp) { @@ -294,7 +358,14 @@ cl->mark_passed_slp(); } cl->mark_was_slp(); - cl->set_slp_max_unroll(local_loop_unroll_factor); + if (cl->is_main_loop()) { + cl->set_slp_max_unroll(local_loop_unroll_factor); + } else if (post_loop_allowed) { + if (!small_basic_type) { + // avoid replication context for small basic types in programmable masked loops + cl->set_slp_max_unroll(local_loop_unroll_factor); + } + } } } @@ -349,66 +420,103 @@ if (!construct_bb()) { return; // Exit if no interesting nodes or complex graph. } + // build _dg, _disjoint_ptrs dependence_graph(); // compute function depth(Node*) compute_max_depth(); - if (_do_vector_loop) { - if (mark_generations() != -1) { - hoist_loads_in_graph(); // this only rebuild the graph; all basic structs need rebuild explicitly + CountedLoopNode *cl = lpt()->_head->as_CountedLoop(); + bool post_loop_allowed = (PostLoopMultiversioning && Matcher::has_predicated_vectors() && cl->is_post_loop()); + if (cl->is_main_loop()) { + if (_do_vector_loop) { + if (mark_generations() != -1) { + hoist_loads_in_graph(); // this only rebuild the graph; all basic structs need rebuild explicitly - if (!construct_bb()) { - return; // Exit if no interesting nodes or complex graph. + if (!construct_bb()) { + return; // Exit if no interesting nodes or complex graph. + } + dependence_graph(); + compute_max_depth(); } - dependence_graph(); - compute_max_depth(); - } #ifndef PRODUCT - if (TraceSuperWord) { - tty->print_cr("\nSuperWord::_do_vector_loop: graph after hoist_loads_in_graph"); - _lpt->dump_head(); - for (int j = 0; j < _block.length(); j++) { - Node* n = _block.at(j); - int d = depth(n); - for (int i = 0; i < d; i++) tty->print("%s", " "); - tty->print("%d :", d); - n->dump(); + if (TraceSuperWord) { + tty->print_cr("\nSuperWord::_do_vector_loop: graph after hoist_loads_in_graph"); + _lpt->dump_head(); + for (int j = 0; j < _block.length(); j++) { + Node* n = _block.at(j); + int d = depth(n); + for (int i = 0; i < d; i++) tty->print("%s", " "); + tty->print("%d :", d); + n->dump(); + } } - } #endif - } + } - compute_vector_element_type(); + compute_vector_element_type(); - // Attempt vectorization + // Attempt vectorization - find_adjacent_refs(); + find_adjacent_refs(); - extend_packlist(); + extend_packlist(); - if (_do_vector_loop) { - if (_packset.length() == 0) { - if (TraceSuperWord) { - tty->print_cr("\nSuperWord::_do_vector_loop DFA could not build packset, now trying to build anyway"); + if (_do_vector_loop) { + if (_packset.length() == 0) { + if (TraceSuperWord) { + tty->print_cr("\nSuperWord::_do_vector_loop DFA could not build packset, now trying to build anyway"); + } + pack_parallel(); } - pack_parallel(); } - } - combine_packs(); + combine_packs(); - construct_my_pack_map(); + construct_my_pack_map(); - if (_do_vector_loop) { - merge_packs_to_cmovd(); - } + if (_do_vector_loop) { + merge_packs_to_cmovd(); + } + + filter_packs(); - filter_packs(); + schedule(); + } else if (post_loop_allowed) { + int saved_mapped_unroll_factor = cl->slp_max_unroll(); + if (saved_mapped_unroll_factor) { + int vector_mapped_unroll_factor = saved_mapped_unroll_factor; - schedule(); + // now reset the slp_unroll_factor so that we can check the analysis mapped + // what the vector loop was mapped to + cl->set_slp_max_unroll(0); + + // do the analysis on the post loop + unrolling_analysis(vector_mapped_unroll_factor); + + // if our analyzed loop is a canonical fit, start processing it + if (vector_mapped_unroll_factor == saved_mapped_unroll_factor) { + // now add the vector nodes to packsets + for (int i = 0; i < _post_block.length(); i++) { + Node* n = _post_block.at(i); + Node_List* singleton = new Node_List(); + singleton->push(n); + _packset.append(singleton); + set_my_pack(n, singleton); + } + + // map base types for vector usage + compute_vector_element_type(); + } else { + return; + } + } else { + // for some reason we could not map the slp analysis state of the vectorized loop + return; + } + } output(); } @@ -810,6 +918,7 @@ // Add dependence edges to load/store nodes for memory dependence // A.out()->DependNode.in(1) and DependNode.out()->B.prec(x) void SuperWord::dependence_graph() { + CountedLoopNode *cl = lpt()->_head->as_CountedLoop(); // First, assign a dependence node to each memory node for (int i = 0; i < _block.length(); i++ ) { Node *n = _block.at(i); @@ -824,7 +933,9 @@ Node* n_tail = _mem_slice_tail.at(i); // Get slice in predecessor order (last is first) - mem_slice_preds(n_tail, n, _nlist); + if (cl->is_main_loop()) { + mem_slice_preds(n_tail, n, _nlist); + } #ifndef PRODUCT if(TraceSuperWord && Verbose) { @@ -2028,20 +2139,23 @@ } #endif - // MUST ENSURE main loop's initial value is properly aligned: - // (iv_initial_value + min_iv_offset) % vector_width_in_bytes() == 0 + CountedLoopNode *cl = lpt()->_head->as_CountedLoop(); + if (cl->is_main_loop()) { + // MUST ENSURE main loop's initial value is properly aligned: + // (iv_initial_value + min_iv_offset) % vector_width_in_bytes() == 0 - align_initial_loop_index(align_to_ref()); + align_initial_loop_index(align_to_ref()); - // Insert extract (unpack) operations for scalar uses - for (int i = 0; i < _packset.length(); i++) { - insert_extracts(_packset.at(i)); + // Insert extract (unpack) operations for scalar uses + for (int i = 0; i < _packset.length(); i++) { + insert_extracts(_packset.at(i)); + } } Compile* C = _phase->C; - CountedLoopNode *cl = lpt()->_head->as_CountedLoop(); uint max_vlen_in_bytes = 0; uint max_vlen = 0; + bool can_process_post_loop = (PostLoopMultiversioning && Matcher::has_predicated_vectors() && cl->is_post_loop()); NOT_PRODUCT(if(is_trace_loop_reverse()) {tty->print_cr("SWPointer::output: print loop before create_reserve_version_of_loop"); print_loop(true);}) @@ -2063,6 +2177,10 @@ Node* vn = NULL; Node* low_adr = p->at(0); Node* first = executed_first(p); + if (can_process_post_loop) { + // override vlen with the main loops vector length + vlen = cl->slp_max_unroll(); + } NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::output: %d executed first, %d executed last in pack", first->_idx, n->_idx); print_pack(p);}) int opc = n->Opcode(); if (n->is_Load()) { @@ -2152,6 +2270,10 @@ vn = VectorNode::make(opc, in, NULL, vlen, velt_basic_type(n)); vlen_in_bytes = vn->as_Vector()->length_in_bytes(); } else if (is_cmov_pack(p)) { + if (can_process_post_loop) { + // do not refactor of flow in post loop context + return; + } if (!n->is_CMove()) { continue; } @@ -2216,6 +2338,7 @@ ShouldNotReachHere(); } + _block.at_put(i, vn); _igvn.register_new_node_with_optimizer(vn); _phase->set_ctrl(vn, _phase->get_ctrl(p->at(0))); for (uint j = 0; j < p->size(); j++) { @@ -2224,6 +2347,14 @@ } _igvn._worklist.push(vn); + if (can_process_post_loop) { + // first check if the vector size if the maximum vector which we can use on the machine, + // other vector size have reduced values for predicated data mapping. + if (vlen_in_bytes != (uint)MaxVectorSize) { + return; + } + } + if (vlen_in_bytes > max_vlen_in_bytes) { max_vlen = vlen; max_vlen_in_bytes = vlen_in_bytes; @@ -2246,15 +2377,37 @@ if (TraceSuperWordLoopUnrollAnalysis) { tty->print_cr("vector loop(unroll=%d, len=%d)\n", max_vlen, max_vlen_in_bytes*BitsPerByte); } - // For atomic unrolled loops which are vector mapped, instigate more unrolling. + + // For atomic unrolled loops which are vector mapped, instigate more unrolling cl->set_notpassed_slp(); - // if vector resources are limited, do not allow additional unrolling - if (FLOATPRESSURE > 8) { - C->set_major_progress(); + if (cl->is_main_loop()) { + // if vector resources are limited, do not allow additional unrolling, also + // do not unroll more on pure vector loops which were not reduced so that we can + // program the post loop to single iteration execution. + if (FLOATPRESSURE > 8) { + C->set_major_progress(); + cl->mark_do_unroll_only(); + } } - cl->mark_do_unroll_only(); + if (do_reserve_copy()) { cl->mark_loop_vectorized(); + if (can_process_post_loop) { + // Now create the difference of trip and limit and use it as our mask index. + // Note: We limited the unroll of the vectorized loop so that + // only vlen-1 size iterations can remain to be mask programmed. + Node *incr = cl->incr(); + SubINode *index = new SubINode(cl->limit(), cl->init_trip()); + _igvn.register_new_node_with_optimizer(index); + MaskCreateINode *mask = new MaskCreateINode(_phase->get_ctrl(cl->init_trip()), index); + _igvn.register_new_node_with_optimizer(mask); + // make this a single iteration loop + AddINode *new_incr = new AddINode(incr->in(1), mask); + _igvn.register_new_node_with_optimizer(new_incr); + _phase->set_ctrl(new_incr, _phase->get_ctrl(incr)); + _igvn.replace_node(incr, new_incr); + cl->mark_is_multiversioned(); + } } } } @@ -2273,6 +2426,12 @@ Node* p0 = p->at(0); uint vlen = p->size(); Node* opd = p0->in(opd_idx); + CountedLoopNode *cl = lpt()->_head->as_CountedLoop(); + + if (PostLoopMultiversioning && Matcher::has_predicated_vectors() && cl->is_post_loop()) { + // override vlen with the main loops vector length + vlen = cl->slp_max_unroll(); + } if (same_inputs(p, opd_idx)) { if (opd->is_Vector() || opd->is_LoadVector()) { @@ -3089,13 +3248,13 @@ return pre_end; } - //------------------------------init--------------------------- void SuperWord::init() { _dg.init(); _packset.clear(); _disjoint_ptrs.clear(); _block.clear(); + _post_block.clear(); _data_entry.clear(); _mem_slice_head.clear(); _mem_slice_tail.clear(); @@ -3119,6 +3278,7 @@ _packset.clear(); _disjoint_ptrs.clear(); _block.clear(); + _post_block.clear(); _data_entry.clear(); _mem_slice_head.clear(); _mem_slice_tail.clear();