src/hotspot/share/opto/superword.cpp

@@ -2137,21 +2137,45 @@
   } else if (pk->at(0)->is_Load()) { //load
     // all loads in the pack should have the same memory state. By default,
     // we use the memory state of the last load. However, if any load could
     // not be moved down due to the dependence constraint, we use the memory
     // state of the first load.
-    Node* last_mem  = executed_last(pk)->in(MemNode::Memory);
-    Node* first_mem = executed_first(pk)->in(MemNode::Memory);
+
+    // We first need to find the "first" and "last" loads. Start with
+    // arbitrary values:
+    Node* last_mem  = pk->at(0)->in(MemNode::Memory);
+    Node* first_mem = last_mem;
+    // Walk the memory graph from the current first load until the
+    // start of the loop and check if nodes on the way are memory
+    // edges of loads in the pack. The last one we encounter is the
+    // first load.
+    for (Node* current = first_mem; in_bb(current); current = current->is_Phi() ? current->in(LoopNode::EntryControl) : current->in(MemNode::Memory)) {
+      assert(current->is_Mem() || (current->is_Phi() && current->in(0) == bb()), "unexpected memory");
+      for (uint i = 1; i < pk->size(); i++) {
+        Node* ld = pk->at(i);
+        if (ld->in(MemNode::Memory) == current) {
+          first_mem = current;
+          break;
+        }
+      }
+    }
+    // Find the last load by going over the pack again, and walking
+    // the memory graph from the loads of the pack to the memory of
+    // the first load. If we encounter the memory of the current last
+    // load, then we started from further down in the memory graph and
+    // the load we started from is the last load. Check for dependence
+    // constraints in that loop as well.
     bool schedule_last = true;
     for (uint i = 0; i < pk->size(); i++) {
       Node* ld = pk->at(i);
-      for (Node* current = last_mem; current != ld->in(MemNode::Memory);
-           current=current->in(MemNode::Memory)) {
-        assert(current != first_mem, "corrupted memory graph");
-        if(current->is_Mem() && !independent(current, ld)){
+      for (Node* current = ld->in(MemNode::Memory); current != first_mem; current = current->in(MemNode::Memory)) {
+        assert(current->is_Mem() && in_bb(current), "unexpected memory");
+        if (current->in(MemNode::Memory) == last_mem) {
+          last_mem = ld->in(MemNode::Memory);
+        }
+        if (!independent(current, ld)) {
           schedule_last = false; // a later store depends on this load
-          break;
         }
       }
     }
 
     Node* mem_input = schedule_last ? last_mem : first_mem;