src/share/vm/opto/superword.cpp

@@ -98,10 +98,14 @@
   // Make sure the are no extra control users of the loop backedge
   if (cl->back_control()->outcnt() != 1) {
     return;
   }
 
+  // We only re-enter slp when we vector mapped a queried loop and we want to
+  // continue unrolling, in this case, slp is not subsequently done.
+  if (cl->ignore_slp()) return;
+
   // Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit))))
   CountedLoopEndNode* pre_end = get_pre_loop_end(cl);
   if (pre_end == NULL) return;
   Node *pre_opaq1 = pre_end->limit();
   if (pre_opaq1->Opcode() != Op_Opaque1) return;

@@ -119,16 +123,17 @@
     SLP_extract();
   }
 }
 
 //------------------------------early unrolling analysis------------------------------
-void SuperWord::unrolling_analysis(CountedLoopNode *cl, int &local_loop_unroll_factor) {
+void SuperWord::unrolling_analysis(int &local_loop_unroll_factor) {
   bool is_slp = true;
   ResourceMark rm;
   size_t ignored_size = lpt()->_body.size();
   int *ignored_loop_nodes = NEW_RESOURCE_ARRAY(int, ignored_size);
   Node_Stack nstack((int)ignored_size);
+  CountedLoopNode *cl = lpt()->_head->as_CountedLoop();
   Node *cl_exit = cl->loopexit();
 
   // First clear the entries
   for (uint i = 0; i < lpt()->_body.size(); i++) {
     ignored_loop_nodes[i] = -1;

@@ -239,17 +244,13 @@
 
       int cur_max_vector = Matcher::max_vector_size(bt);
 
       // If a max vector exists which is not larger than _local_loop_unroll_factor
       // stop looking, we already have the max vector to map to.
-      if (cur_max_vector <= local_loop_unroll_factor) {
+      if (cur_max_vector < local_loop_unroll_factor) {
         is_slp = false;
-#ifndef PRODUCT
-        if (TraceSuperWordLoopUnrollAnalysis) {
-          tty->print_cr("slp analysis fails: unroll limit equals max vector\n");
-        }
-#endif
+        NOT_PRODUCT(if (TraceSuperWordLoopUnrollAnalysis) tty->print_cr("slp analysis fails: unroll limit greater than max vector\n"));
         break;
       }
 
       // Map the maximal common vector
       if (VectorNode::implemented(n->Opcode(), cur_max_vector, bt)) {

@@ -258,12 +259,13 @@
         }
       }
     }
     if (is_slp) {
       local_loop_unroll_factor = max_vector;
-    }
     cl->mark_passed_slp();
+    }
+    cl->mark_was_slp();
     cl->set_slp_max_unroll(local_loop_unroll_factor);
   }
 }
 
 //------------------------------SLP_extract---------------------------

@@ -1748,11 +1750,13 @@
   for (int i = 0; i < _packset.length(); i++) {
     insert_extracts(_packset.at(i));
   }
 
   Compile* C = _phase->C;
+  CountedLoopNode *cl = lpt()->_head->as_CountedLoop();
   uint max_vlen_in_bytes = 0;
+  uint max_vlen = 0;
   for (int i = 0; i < _block.length(); i++) {
     Node* n = _block.at(i);
     Node_List* p = my_pack(n);
     if (p && n == executed_last(p)) {
       uint vlen = p->size();

@@ -1831,10 +1835,11 @@
         _igvn.replace_node(pm, vn);
       }
       _igvn._worklist.push(vn);
 
       if (vlen_in_bytes > max_vlen_in_bytes) {
+        max_vlen = vlen;
         max_vlen_in_bytes = vlen_in_bytes;
       }
 #ifdef ASSERT
       if (TraceNewVectors) {
         tty->print("new Vector node: ");

@@ -1842,10 +1847,22 @@
       }
 #endif
     }
   }
   C->set_max_vector_size(max_vlen_in_bytes);
+  if (SuperWordLoopUnrollAnalysis) {
+    if (cl->has_passed_slp()) {
+      int slp_max_unroll_factor = cl->slp_max_unroll();
+      if (slp_max_unroll_factor == max_vlen) {
+        NOT_PRODUCT(if (TraceSuperWordLoopUnrollAnalysis) tty->print_cr("vector loop(unroll=%d, len=%d)\n", max_vlen, max_vlen_in_bytes*BitsPerByte));
+        // For atomic unrolled loops which are vector mapped, instigate more unrolling.
+        cl->set_notpassed_slp();
+        C->set_major_progress();
+        cl->mark_no_slp();
+      }
+    }
+  }
 }
 
 //------------------------------vector_opd---------------------------
 // Create a vector operand for the nodes in pack p for operand: in(opd_idx)
 Node* SuperWord::vector_opd(Node_List* p, int opd_idx) {