src/share/vm/runtime/deoptimization.cpp

*** 1330,1339 ****
--- 1330,1340 ----
      // recompile relies on a methodDataOop to record heroic opt failures.
  
      // Whether the interpreter is producing MDO data or not, we also need
      // to use the MDO to detect hot deoptimization points and control
      // aggressive optimization.
+     bool inc_recompile_count = false;
      if (ProfileTraps && update_trap_state && trap_mdo.not_null()) {
        assert(trap_mdo() == get_method_data(thread, trap_method, false), "sanity");
        uint this_trap_count = 0;
        bool maybe_prior_trap = false;
        bool maybe_prior_recompile = false;
*** 1372,1400 ****
          }
  
          // Detect repeated recompilation at the same BCI, and enforce a limit.
          if (make_not_entrant && maybe_prior_recompile) {
            // More than one recompile at this point.
!           trap_mdo->inc_overflow_recompile_count();
!           if (maybe_prior_trap
!               && ((uint)trap_mdo->overflow_recompile_count()
!                   > (uint)PerBytecodeRecompilationCutoff)) {
!             // Give up on the method containing the bad BCI.
!             if (trap_method() == nm->method()) {
!               make_not_compilable = true;
              } else {
!               trap_method->set_not_compilable();
!               // But give grace to the enclosing nm->method().
              }
-           }
-         }
        } else {
          // For reasons which are not recorded per-bytecode, we simply
          // force recompiles unconditionally.
          // (Note that PerMethodRecompilationCutoff is enforced elsewhere.)
          make_not_entrant = true;
        }
  
        // Go back to the compiler if there are too many traps in this method.
        if (this_trap_count >= (uint)PerMethodTrapLimit) {
          // If there are too many traps in this method, force a recompile.
          // This will allow the compiler to see the limit overflow, and
--- 1373,1400 ----
          }
  
          // Detect repeated recompilation at the same BCI, and enforce a limit.
          if (make_not_entrant && maybe_prior_recompile) {
            // More than one recompile at this point.
!           inc_recompile_count = maybe_prior_trap;
!         }
        } else {
!         if (reason == Reason_bimorphic) {
!           // This isn't recorded per bci because of MDO limitations
!           // but lets piggy back the invalidation on the
!           // Reason_class_check count.
!           uint prior_trap_count = trap_mdo->trap_count(Reason_bimorphic);
!           if (prior_trap_count >= (uint)PerBytecodeTrapLimit) {
!             make_not_entrant = true;
            }
          } else {
            // For reasons which are not recorded per-bytecode, we simply
            // force recompiles unconditionally.
            // (Note that PerMethodRecompilationCutoff is enforced elsewhere.)
            make_not_entrant = true;
          }
+       }
  
        // Go back to the compiler if there are too many traps in this method.
        if (this_trap_count >= (uint)PerMethodTrapLimit) {
          // If there are too many traps in this method, force a recompile.
          // This will allow the compiler to see the limit overflow, and
*** 1406,1416 ****
        }
  
        // Here's more hysteresis:  If there has been a recompile at
        // this trap point already, run the method in the interpreter
        // for a while to exercise it more thoroughly.
!       if (make_not_entrant && maybe_prior_recompile && maybe_prior_trap) {
          reset_counters = true;
        }
  
        if (make_not_entrant && pdata != NULL) {
          // Record the recompilation event, if any.
--- 1406,1421 ----
        }
  
        // Here's more hysteresis:  If there has been a recompile at
        // this trap point already, run the method in the interpreter
        // for a while to exercise it more thoroughly.
!       // Also do this to collect more data for transition from monomorphic
!       // case to bimorphic. But recompile method immediately if it is already
!       // compiled with bimorphic code.
!       if (make_not_entrant && maybe_prior_trap && 
!           ((maybe_prior_recompile && reason != Reason_bimorphic) ||
!            (UseBimorphicInlining  && reason == Reason_class_check))) {
          reset_counters = true;
        }
  
        if (make_not_entrant && pdata != NULL) {
          // Record the recompilation event, if any.
*** 1421,1445 ****
        }
      }
  
      // Take requested actions on the method:
  
      // Reset invocation counters
      if (reset_counters) {
        if (nm->is_osr_method())
          reset_invocation_counter(trap_scope, CompileThreshold);
        else
          reset_invocation_counter(trap_scope);
      }
  
-     // Recompile
-     if (make_not_entrant) {
-       nm->make_not_entrant();
-     }
- 
      // Give up compiling
!     if (make_not_compilable) {
        assert(make_not_entrant, "consistent");
        nm->method()->set_not_compilable();
      }
  
    } // Free marked resources
--- 1426,1466 ----
        }
      }
  
      // Take requested actions on the method:
  
+     // Recompile
+     if (make_not_entrant) {
+       if (!nm->make_not_entrant()) {
+         return; // the call did not change nmethod's state
+       }
+     }
+ 
+     if (inc_recompile_count) {
+       trap_mdo->inc_overflow_recompile_count();
+       if ((uint)trap_mdo->overflow_recompile_count() >
+           (uint)PerBytecodeRecompilationCutoff) {
+         // Give up on the method containing the bad BCI.
+         if (trap_method() == nm->method()) {
+           make_not_compilable = true;
+         } else {
+           trap_method->set_not_compilable();
+           // But give grace to the enclosing nm->method().
+         }
+       }
+     }
+ 
      // Reset invocation counters
      if (reset_counters) {
        if (nm->is_osr_method())
          reset_invocation_counter(trap_scope, CompileThreshold);
        else
          reset_invocation_counter(trap_scope);
      }
  
      // Give up compiling
!     if (make_not_compilable && !nm->method()->is_not_compilable()) {
        assert(make_not_entrant, "consistent");
        nm->method()->set_not_compilable();
      }
  
    } // Free marked resources
*** 1508,1522 ****
        tstate1 = trap_state_add_reason(tstate1, per_bc_reason);
        // Store the updated state on the MDO, for next time.
        if (tstate1 != tstate0)
          pdata->set_trap_state(tstate1);
      } else {
!       if (LogCompilation && xtty != NULL)
          // Missing MDP?  Leave a small complaint in the log.
          xtty->elem("missing_mdp bci='%d'", trap_bci);
      }
    }
  
    // Return results:
    ret_this_trap_count = this_trap_count;
    ret_maybe_prior_trap = maybe_prior_trap;
    ret_maybe_prior_recompile = maybe_prior_recompile;
--- 1529,1545 ----
        tstate1 = trap_state_add_reason(tstate1, per_bc_reason);
        // Store the updated state on the MDO, for next time.
        if (tstate1 != tstate0)
          pdata->set_trap_state(tstate1);
      } else {
!       if (LogCompilation && xtty != NULL) {
!         ttyLocker ttyl;
          // Missing MDP?  Leave a small complaint in the log.
          xtty->elem("missing_mdp bci='%d'", trap_bci);
        }
      }
+   }
  
    // Return results:
    ret_this_trap_count = this_trap_count;
    ret_maybe_prior_trap = maybe_prior_trap;
    ret_maybe_prior_recompile = maybe_prior_recompile;
*** 1671,1681 ****
    "unreached",
    "unhandled",
    "constraint",
    "div0_check",
    "age",
!   "predicate"
  };
  const char* Deoptimization::_trap_action_name[Action_LIMIT] = {
    // Note:  Keep this in sync. with enum DeoptAction.
    "none",
    "maybe_recompile",
--- 1694,1705 ----
    "unreached",
    "unhandled",
    "constraint",
    "div0_check",
    "age",
!   "predicate",
!   "bimorphic"
  };
  const char* Deoptimization::_trap_action_name[Action_LIMIT] = {
    // Note:  Keep this in sync. with enum DeoptAction.
    "none",
    "maybe_recompile",