src/share/vm/opto/superword.cpp

*** 98,107 ****
--- 98,111 ----
    // Make sure the are no extra control users of the loop backedge
    if (cl->back_control()->outcnt() != 1) {
      return;
    }
  
+   // We only re-enter slp when we vector mapped a queried loop and we want to
+   // continue unrolling, in this case, slp is not subsequently done.
+   if (cl->ignore_slp()) return;
+ 
    // Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit))))
    CountedLoopEndNode* pre_end = get_pre_loop_end(cl);
    if (pre_end == NULL) return;
    Node *pre_opaq1 = pre_end->limit();
    if (pre_opaq1->Opcode() != Op_Opaque1) return;
*** 119,134 ****
      SLP_extract();
    }
  }
  
  //------------------------------early unrolling analysis------------------------------
! void SuperWord::unrolling_analysis(CountedLoopNode *cl, int &local_loop_unroll_factor) {
    bool is_slp = true;
    ResourceMark rm;
    size_t ignored_size = lpt()->_body.size();
    int *ignored_loop_nodes = NEW_RESOURCE_ARRAY(int, ignored_size);
    Node_Stack nstack((int)ignored_size);
    Node *cl_exit = cl->loopexit();
  
    // First clear the entries
    for (uint i = 0; i < lpt()->_body.size(); i++) {
      ignored_loop_nodes[i] = -1;
--- 123,139 ----
      SLP_extract();
    }
  }
  
  //------------------------------early unrolling analysis------------------------------
! void SuperWord::unrolling_analysis(int &local_loop_unroll_factor) {
    bool is_slp = true;
    ResourceMark rm;
    size_t ignored_size = lpt()->_body.size();
    int *ignored_loop_nodes = NEW_RESOURCE_ARRAY(int, ignored_size);
    Node_Stack nstack((int)ignored_size);
+   CountedLoopNode *cl = lpt()->_head->as_CountedLoop();
    Node *cl_exit = cl->loopexit();
  
    // First clear the entries
    for (uint i = 0; i < lpt()->_body.size(); i++) {
      ignored_loop_nodes[i] = -1;
*** 239,255 ****
  
        int cur_max_vector = Matcher::max_vector_size(bt);
  
        // If a max vector exists which is not larger than _local_loop_unroll_factor
        // stop looking, we already have the max vector to map to.
!       if (cur_max_vector <= local_loop_unroll_factor) {
          is_slp = false;
! #ifndef PRODUCT
!         if (TraceSuperWordLoopUnrollAnalysis) {
!           tty->print_cr("slp analysis fails: unroll limit equals max vector\n");
!         }
! #endif
          break;
        }
  
        // Map the maximal common vector
        if (VectorNode::implemented(n->Opcode(), cur_max_vector, bt)) {
--- 244,256 ----
  
        int cur_max_vector = Matcher::max_vector_size(bt);
  
        // If a max vector exists which is not larger than _local_loop_unroll_factor
        // stop looking, we already have the max vector to map to.
!       if (cur_max_vector < local_loop_unroll_factor) {
          is_slp = false;
!         NOT_PRODUCT(if (TraceSuperWordLoopUnrollAnalysis) tty->print_cr("slp analysis fails: unroll limit greater than max vector\n"));
          break;
        }
  
        // Map the maximal common vector
        if (VectorNode::implemented(n->Opcode(), cur_max_vector, bt)) {
*** 258,269 ****
          }
        }
      }
      if (is_slp) {
        local_loop_unroll_factor = max_vector;
-     }
      cl->mark_passed_slp();
      cl->set_slp_max_unroll(local_loop_unroll_factor);
    }
  }
  
  //------------------------------SLP_extract---------------------------
--- 259,271 ----
          }
        }
      }
      if (is_slp) {
        local_loop_unroll_factor = max_vector;
        cl->mark_passed_slp();
+     }
+     cl->mark_was_slp();
      cl->set_slp_max_unroll(local_loop_unroll_factor);
    }
  }
  
  //------------------------------SLP_extract---------------------------
*** 1748,1758 ****
--- 1750,1762 ----
    for (int i = 0; i < _packset.length(); i++) {
      insert_extracts(_packset.at(i));
    }
  
    Compile* C = _phase->C;
+   CountedLoopNode *cl = lpt()->_head->as_CountedLoop();
    uint max_vlen_in_bytes = 0;
+   uint max_vlen = 0;
    for (int i = 0; i < _block.length(); i++) {
      Node* n = _block.at(i);
      Node_List* p = my_pack(n);
      if (p && n == executed_last(p)) {
        uint vlen = p->size();
*** 1831,1840 ****
--- 1835,1845 ----
          _igvn.replace_node(pm, vn);
        }
        _igvn._worklist.push(vn);
  
        if (vlen_in_bytes > max_vlen_in_bytes) {
+         max_vlen = vlen;
          max_vlen_in_bytes = vlen_in_bytes;
        }
  #ifdef ASSERT
        if (TraceNewVectors) {
          tty->print("new Vector node: ");
*** 1842,1851 ****
--- 1847,1868 ----
        }
  #endif
      }
    }
    C->set_max_vector_size(max_vlen_in_bytes);
+   if (SuperWordLoopUnrollAnalysis) {
+     if (cl->has_passed_slp()) {
+       int slp_max_unroll_factor = cl->slp_max_unroll();
+       if (slp_max_unroll_factor == max_vlen) {
+         NOT_PRODUCT(if (TraceSuperWordLoopUnrollAnalysis) tty->print_cr("vector loop(unroll=%d, len=%d)\n", max_vlen, max_vlen_in_bytes*BitsPerByte));
+         // For atomic unrolled loops which are vector mapped, instigate more unrolling.
+         cl->set_notpassed_slp();
+         C->set_major_progress();
+         cl->mark_no_slp();
+       }
+     }
+   }
  }
  
  //------------------------------vector_opd---------------------------
  // Create a vector operand for the nodes in pack p for operand: in(opd_idx)
  Node* SuperWord::vector_opd(Node_List* p, int opd_idx) {