src/hotspot/share/opto/loopUnswitch.cpp

@@ -22,10 +22,12 @@
  *
  */
 
 #include "precompiled.hpp"
 #include "memory/allocation.inline.hpp"
+#include "opto/mulnode.hpp"
+#include "opto/addnode.hpp"
 #include "opto/connode.hpp"
 #include "opto/convertnode.hpp"
 #include "opto/loopnode.hpp"
 #include "opto/opaquenode.hpp"
 #include "opto/rootnode.hpp"

@@ -49,10 +51,50 @@
 //                                 endloop
 //                               endif
 //
 // Note: the "else" clause may be empty
 
+static bool is_flattened_array_check(Node* iff, PhaseTransform* phase) {
+  if (iff->Opcode() != Op_If) {
+    return false;
+  }
+  Node* bol = iff->in(1);
+  if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {
+    return false;
+  }
+  Node* cmp = bol->in(1);
+  if (cmp->Opcode() != Op_CmpI) {
+    return false;
+  }
+  Node* cmp_in1 = cmp->in(1);
+  Node* cmp_in2 = cmp->in(2);
+  if ((unsigned int)cmp_in2->find_int_con(0) != Klass::_lh_array_tag_vt_value) {
+    return false;
+  }
+  if (cmp_in1->Opcode() != Op_RShiftI) {
+    return false;
+  }
+  Node* shift_in1 = cmp_in1->in(1);
+  Node* shift_in2 = cmp_in1->in(2);
+  if ((unsigned int)shift_in2->find_int_con(0) != Klass::_lh_array_tag_shift) {
+    return false;
+  }
+  if (shift_in1->Opcode() != Op_LoadI) {
+    return false;
+  }
+  intptr_t offset;
+  Node* addr = AddPNode::Ideal_base_and_offset(shift_in1->in(MemNode::Address), phase, offset);
+  if (addr == NULL || offset != in_bytes(Klass::layout_helper_offset())) {
+    return false;
+  }
+  if (!phase->type(addr)->isa_klassptr()) {
+    return false;
+  }
+  
+  return true;
+}
+
 //------------------------------policy_unswitching-----------------------------
 // Return TRUE or FALSE if the loop should be unswitched
 // (ie. clone loop with an invariant test that does not exit the loop)
 bool IdealLoopTree::policy_unswitching( PhaseIdealLoop *phase ) const {
   if (!LoopUnswitching) {

@@ -72,21 +114,27 @@
 
   LoopNode* head = _head->as_Loop();
   if (head->unswitch_count() + 1 > head->unswitch_max()) {
     return false;
   }
-  if (phase->find_unswitching_candidate(this) == NULL) {
+
+  if (head->is_flattened_arrays()) {
+    return false;
+  }
+  
+  Node_List flattened_checks;
+  if (phase->find_unswitching_candidate(this, flattened_checks) == NULL && flattened_checks.size() == 0) {
     return false;
   }
 
   // Too speculative if running low on nodes.
   return phase->may_require_nodes(est_loop_clone_sz(3, _body.size()));
 }
 
 //------------------------------find_unswitching_candidate-----------------------------
 // Find candidate "if" for unswitching
-IfNode* PhaseIdealLoop::find_unswitching_candidate(const IdealLoopTree *loop) const {
+IfNode* PhaseIdealLoop::find_unswitching_candidate(const IdealLoopTree *loop, Node_List& flattened_checks) const {
 
   // Find first invariant test that doesn't exit the loop
   LoopNode *head = loop->_head->as_Loop();
   IfNode* unswitch_iff = NULL;
   Node* n = head->in(LoopNode::LoopBackControl);

@@ -107,10 +155,24 @@
         }
       }
     }
     n = n_dom;
   }
+  
+  if (unswitch_iff == NULL || is_flattened_array_check(unswitch_iff, &_igvn)) {
+    // collect all flattened array checks
+    for (uint i = 0; i < loop->_body.size(); i++) {
+      Node* n = loop->_body.at(i);
+      if (is_flattened_array_check(n, &_igvn) &&
+          loop->is_invariant(n->in(1)) &&
+          !loop->is_loop_exit(n)) {
+        flattened_checks.push(n);
+      }
+    }
+    unswitch_iff = NULL;
+  }
+  
   return unswitch_iff;
 }
 
 //------------------------------do_unswitching-----------------------------
 // Clone loop with an invariant test (that does not exit) and

@@ -119,12 +181,16 @@
 void PhaseIdealLoop::do_unswitching (IdealLoopTree *loop, Node_List &old_new) {
 
   // Find first invariant test that doesn't exit the loop
   LoopNode *head = loop->_head->as_Loop();
 
-  IfNode* unswitch_iff = find_unswitching_candidate((const IdealLoopTree *)loop);
-  assert(unswitch_iff != NULL, "should be at least one");
+  Node_List flattened_checks;
+  IfNode* unswitch_iff = find_unswitching_candidate((const IdealLoopTree *)loop, flattened_checks);
+  assert(unswitch_iff != NULL || flattened_checks.size() > 0, "should be at least one");
+  if (unswitch_iff == NULL) {
+    unswitch_iff = flattened_checks.at(0)->as_If();
+  }
 
 #ifndef PRODUCT
   if (TraceLoopOpts) {
     tty->print("Unswitch   %d ", head->unswitch_count()+1);
     loop->dump_head();

@@ -166,17 +232,49 @@
   // Increment unswitch count
   LoopNode* head_clone = old_new[head->_idx]->as_Loop();
   int nct = head->unswitch_count() + 1;
   head->set_unswitch_count(nct);
   head_clone->set_unswitch_count(nct);
+  head_clone->mark_flattened_arrays();
 
   // Add test to new "if" outside of loop
   IfNode* invar_iff   = proj_true->in(0)->as_If();
   Node* invar_iff_c   = invar_iff->in(0);
+  invar_iff->_prob    = unswitch_iff->_prob;
+  if (flattened_checks.size() > 0) {
+    // Flattened array checks are used in
+    // Parse::array_store()/Parse::array_load() to switch between a
+    // legacy object array access and a flattened value array
+    // access. We want the performance impact on legacy accesses to be
+    // as small as possible so we make 2 copies of the loops: a fast
+    // one where all accesses are known to be legacy, a slow one where
+    // some accesses are to flattened arrays. Flattened array checks
+    // can be removed from the first one but not from the second one
+    // as it can have a mix of flattened/legacy accesses.
+    BoolNode* bol       = unswitch_iff->in(1)->clone()->as_Bool();
+    register_new_node(bol, invar_iff->in(0));
+    Node* cmp = bol->in(1)->clone();
+    register_new_node(cmp, invar_iff->in(0));
+    bol->set_req(1, cmp);
+    Node* in1 = NULL;
+    for (uint i = 0; i < flattened_checks.size(); i++) {
+      Node* v = flattened_checks.at(i)->in(1)->in(1)->in(1);
+      v = new AndINode(v, _igvn.intcon(Klass::_lh_array_tag_vt_value));
+      register_new_node(v, invar_iff->in(0));
+      if (in1 == NULL) {
+        in1 = v;
+      } else {
+        in1 = new OrINode(in1, v);
+        register_new_node(in1, invar_iff->in(0));
+      }
+    }
+    cmp->set_req(1, in1);
+    invar_iff->set_req(1, bol);
+  } else {
   BoolNode* bol       = unswitch_iff->in(1)->as_Bool();
   invar_iff->set_req(1, bol);
-  invar_iff->_prob    = unswitch_iff->_prob;
+  }
 
   ProjNode* proj_false = invar_iff->proj_out(0)->as_Proj();
 
   // Hoist invariant casts out of each loop to the appropriate
   // control projection.

@@ -203,17 +301,25 @@
       Node* use_clone = old_new[use->_idx];
       _igvn.replace_input_of(use_clone, 1, nuse);
     }
   }
 
+  IfNode* unswitch_iff_clone = old_new[unswitch_iff->_idx]->as_If();
+  if (flattened_checks.size() > 0) {
+    for (uint i = 0; i < flattened_checks.size(); i++) {
+      IfNode* iff = flattened_checks.at(i)->as_If();
+      _igvn.rehash_node_delayed(iff);
+      short_circuit_if(iff, proj_true);
+    }
+  } else {
   // Hardwire the control paths in the loops into if(true) and if(false)
   _igvn.rehash_node_delayed(unswitch_iff);
   short_circuit_if(unswitch_iff, proj_true);
 
-  IfNode* unswitch_iff_clone = old_new[unswitch_iff->_idx]->as_If();
   _igvn.rehash_node_delayed(unswitch_iff_clone);
   short_circuit_if(unswitch_iff_clone, proj_false);
+  }
 
   // Reoptimize loops
   loop->record_for_igvn();
   for(int i = loop->_body.size() - 1; i >= 0 ; i--) {
     Node *n = loop->_body[i];