src/share/vm/opto/superword.cpp

*** 988,999 ****
  // (4) If there is no dependence restriction for a sandwiched memory op, we simply
  //     schedule this store AFTER the pack
  // (5) We know there is no dependence cycle, so there in no other case;
  // (6) Finally, all memory ops in another single pack should be moved in the same direction.
  //
! // To schedule a load pack: the memory edge of every loads in the pack must be
! // the same as the memory edge of the last executed load in the pack
  void SuperWord::co_locate_pack(Node_List* pk) {
    if (pk->at(0)->is_Store()) {
      MemNode* first     = executed_first(pk)->as_Mem();
      MemNode* last      = executed_last(pk)->as_Mem();
      Unique_Node_List schedule_before_pack;
--- 988,999 ----
  // (4) If there is no dependence restriction for a sandwiched memory op, we simply
  //     schedule this store AFTER the pack
  // (5) We know there is no dependence cycle, so there in no other case;
  // (6) Finally, all memory ops in another single pack should be moved in the same direction.
  //
! // To schedule a load pack, we use the memory state of either the first or the last load in
! // the pack, based on the dependence constraint.
  void SuperWord::co_locate_pack(Node_List* pk) {
    if (pk->at(0)->is_Store()) {
      MemNode* first     = executed_first(pk)->as_Mem();
      MemNode* last      = executed_last(pk)->as_Mem();
      Unique_Node_List schedule_before_pack;
*** 1074,1092 ****
  
        if (current == first) break;
        current = my_mem->as_Mem();
      } // end while
    } else if (pk->at(0)->is_Load()) { //load
!     // all use the memory state that the last executed load uses
!     LoadNode* last_load  = executed_last(pk)->as_Load();
!     Node* last_mem       = last_load->in(MemNode::Memory);
!     _igvn.hash_delete(last_mem);
!     // Give each load same memory state as last
      for (uint i = 0; i < pk->size(); i++) {
        LoadNode* ld = pk->at(i)->as_Load();
        _igvn.hash_delete(ld);
!       ld->set_req(MemNode::Memory, last_mem);
        _igvn._worklist.push(ld);
      }
    }
  }
  
--- 1074,1108 ----
  
        if (current == first) break;
        current = my_mem->as_Mem();
      } // end while
    } else if (pk->at(0)->is_Load()) { //load
!     // all loads in the pack should have the same memory state. By default,
!     // we use the memory state of the last load. However, if any load could
!     // not be moved down due to the dependence constraint, we use the memory
!     // state of the first load.
!     Node* last_mem  = executed_last(pk)->in(MemNode::Memory);
!     Node* first_mem = executed_first(pk)->in(MemNode::Memory);
!     bool schedule_last = true;
      for (uint i = 0; i < pk->size(); i++) {
+       Node* ld = pk->at(i);
+       for (Node* current = last_mem; current != ld->in(MemNode::Memory); 
+            current=current->in(MemNode::Memory)) {
+         if(current->is_Mem() && !independent(current, ld)){
+           schedule_last = false; // a later store depends on this load
+           break;
+         }
+       }
+     }
+ 
+     Node* mem_input = schedule_last ? last_mem : first_mem;
+     _igvn.hash_delete(mem_input);
+     // Give each load the same memory state
+     for (uint i = 0; i < pk->size(); i++) {
        LoadNode* ld = pk->at(i)->as_Load();
        _igvn.hash_delete(ld);
!       ld->set_req(MemNode::Memory, mem_input);
        _igvn._worklist.push(ld);
      }
    }
  }