src/hotspot/share/prims/jvmtiEnv.cpp

rev 56101 : 8227745: Enable Escape Analysis for better performance when debugging
Reviewed-by: ???

@@ -1191,10 +1191,15 @@
 
   // growable array of jvmti monitors info on the C-heap
   GrowableArray<jvmtiMonitorStackDepthInfo*> *owned_monitors_list =
       new (ResourceObj::C_HEAP, mtInternal) GrowableArray<jvmtiMonitorStackDepthInfo*>(1, true);
 
+  JVMTIEscapeBarrier eb(calling_thread, java_thread, true);
+  if (!eb.deoptimize_objects(MaxJavaStackTraceDepth)) {
+    return JVMTI_ERROR_OUT_OF_MEMORY;
+  }
+
   // It is only safe to perform the direct operation on the current
   // thread. All other usage needs to use a vm-safepoint-op for safety.
   if (java_thread == calling_thread) {
     err = get_owned_monitors(calling_thread, java_thread, owned_monitors_list);
   } else {

@@ -1237,10 +1242,15 @@
 
   // growable array of jvmti monitors info on the C-heap
   GrowableArray<jvmtiMonitorStackDepthInfo*> *owned_monitors_list =
          new (ResourceObj::C_HEAP, mtInternal) GrowableArray<jvmtiMonitorStackDepthInfo*>(1, true);
 
+  JVMTIEscapeBarrier eb(calling_thread, java_thread, true);
+  if (!eb.deoptimize_objects(MaxJavaStackTraceDepth)) {
+    return JVMTI_ERROR_OUT_OF_MEMORY;
+  }
+
   // It is only safe to perform the direct operation on the current
   // thread. All other usage needs to use a vm-safepoint-op for safety.
   if (java_thread == calling_thread) {
     err = get_owned_monitors(calling_thread, java_thread, owned_monitors_list);
   } else {

@@ -1331,10 +1341,15 @@
 
   Handle thread_hndl(current_thread, thread_oop);
   {
     MutexLocker mu(Threads_lock); // grab Threads_lock
 
+    while (JVMTIEscapeBarrier::deoptimizing_objects_for_all_threads()) {
+      // Must not add new threads that push frames with ea based optimizations
+      Threads_lock->wait(0, Monitor::_as_suspend_equivalent_flag);
+    }
+
     JvmtiAgentThread *new_thread = new JvmtiAgentThread(this, proc, arg);
 
     // At this point it may be possible that no osthread was created for the
     // JavaThread due to lack of memory.
     if (new_thread == NULL || new_thread->osthread() == NULL) {

@@ -1636,10 +1651,17 @@
     if (osThread->get_state() == MONITOR_WAIT) {
       return JVMTI_ERROR_OPAQUE_FRAME;
     }
   }
 
+  if (java_thread->frames_to_pop_failed_realloc() > 0) {
+    // VM is in the process of popping the top frame, because it has scalar replaced objects which
+    // could not be reallocated on the heap.
+    // Return JVMTI_ERROR_OUT_OF_MEMORY to avoid interfering with the VM.
+    return JVMTI_ERROR_OUT_OF_MEMORY;
+  }
+
   {
     ResourceMark rm(current_thread);
     // Check if there are more than one Java frame in this thread, that the top two frames
     // are Java (not native) frames, and that there is no intervening VM frame
     int frame_count = 0;

@@ -1667,13 +1689,19 @@
         return JVMTI_ERROR_OPAQUE_FRAME;
       }
     }
 
     // If any of the top 2 frames is a compiled one, need to deoptimize it
+    JVMTIEscapeBarrier eb(current_thread, java_thread, !is_interpreted[0] || !is_interpreted[1]);
     for (int i = 0; i < 2; i++) {
       if (!is_interpreted[i]) {
         Deoptimization::deoptimize_frame(java_thread, frame_sp[i]);
+        // eagerly reallocate scalar replaced objects
+        if (!eb.deoptimize_objects(frame_sp[i])) {
+          // reallocation of scalar replaced objects failed -> return with error
+          return JVMTI_ERROR_OUT_OF_MEMORY;
+        }
       }
     }
 
     // Update the thread state to reflect that the top frame is popped
     // so that cur_stack_depth is maintained properly and all frameIDs