src/hotspot/share/prims/jvmtiImpl.cpp

rev 60137 : 8227745: Enable Escape Analysis for Better Performance in the Presence of JVMTI Agents
Reviewed-by: mdoerr, goetz

@@ -439,10 +439,11 @@
   , _depth(depth)
   , _index(index)
   , _type(type)
   , _jvf(NULL)
   , _set(false)
+  , _eb(NULL, NULL, false) // no references escape
   , _result(JVMTI_ERROR_NONE)
 {
 }
 
 // Constructor for object or non-object setter

@@ -453,10 +454,11 @@
   , _index(index)
   , _type(type)
   , _value(value)
   , _jvf(NULL)
   , _set(true)
+  , _eb(JavaThread::current(), thread, type == T_OBJECT)
   , _result(JVMTI_ERROR_NONE)
 {
 }
 
 // Constructor for object getter

@@ -466,10 +468,11 @@
   , _depth(depth)
   , _index(index)
   , _type(T_OBJECT)
   , _jvf(NULL)
   , _set(false)
+  , _eb(calling_thread, thread, true)
   , _result(JVMTI_ERROR_NONE)
 {
 }
 
 vframe *VM_GetOrSetLocal::get_vframe() {

@@ -641,10 +644,54 @@
 
 static bool can_be_deoptimized(vframe* vf) {
   return (vf->is_compiled_frame() && vf->fr().can_be_deoptimized());
 }
 
+// Revert optimizations based on escape analysis if this is an access to a local object
+bool VM_GetOrSetLocal::deoptimize_objects(javaVFrame* jvf) {
+#if COMPILER2_OR_JVMCI
+  if (NOT_JVMCI(DoEscapeAnalysis &&) _type == T_OBJECT) {
+    if (_depth < _thread->frames_to_pop_failed_realloc()) {
+      // cannot access frame with failed reallocations
+      _result = JVMTI_ERROR_OUT_OF_MEMORY;
+      return false;
+    }
+    if (can_be_deoptimized(jvf)) {
+      compiledVFrame* cf = compiledVFrame::cast(jvf);
+      if (cf->not_global_escape_in_scope() && !_eb.deoptimize_objects(cf->fr().id())) {
+        // reallocation of scalar replaced objects failed, because heap is exhausted
+        _result = JVMTI_ERROR_OUT_OF_MEMORY;
+        return false;
+      }
+    }
+
+    // With this access the object could escape the thread changing its escape state from ArgEscape,
+    // to GlobalEscape so we must deoptimize callers which could have optimized on the escape state.
+    vframe* vf = jvf;
+    do {
+      // move to next physical frame
+      while(!vf->is_top()) {
+        vf = vf->sender();
+      }
+      vf = vf->sender();
+
+      if (vf != NULL && vf->is_compiled_frame()) {
+        compiledVFrame* cvf = compiledVFrame::cast(vf);
+        // Deoptimize objects if arg escape is being passed down the stack.
+        // Note that deoptimizing the frame is not enough, because objects need to be relocked
+        if (cvf->arg_escape() && !_eb.deoptimize_objects(cvf->fr().id())) {
+          // reallocation of scalar replaced objects failed, because heap is exhausted
+          _result = JVMTI_ERROR_OUT_OF_MEMORY;
+          return false;
+        }
+      }
+    } while(vf != NULL && !vf->is_entry_frame());
+  }
+#endif // COMPILER2_OR_JVMCI
+  return true;
+}
+
 bool VM_GetOrSetLocal::doit_prologue() {
   _jvf = get_java_vframe();
   NULL_CHECK(_jvf, false);
 
   Method* method_oop = _jvf->method();

@@ -662,13 +709,18 @@
   }
 
   if (!check_slot_type_no_lvt(_jvf)) {
     return false;
   }
-  if (method_oop->has_localvariable_table()) {
-    return check_slot_type_lvt(_jvf);
+  if (method_oop->has_localvariable_table() && !check_slot_type_lvt(_jvf)) {
+    return false;
   }
+
+  if (!deoptimize_objects(_jvf)) {
+    return false;
+  }
+
   return true;
 }
 
 void VM_GetOrSetLocal::doit() {
   InterpreterOopMap oop_mask;