src/hotspot/share/prims/jvmtiEnvBase.cpp

rev 56101 : 8227745: Enable Escape Analysis for better performance when debugging
Reviewed-by: ???

@@ -1324,10 +1324,17 @@
 jvmtiError
 JvmtiEnvBase::check_top_frame(JavaThread* current_thread, JavaThread* java_thread,
                               jvalue value, TosState tos, Handle* ret_ob_h) {
   ResourceMark rm(current_thread);
 
+  if (java_thread->frames_to_pop_failed_realloc() > 0) {
+    // VM is in the process of popping the top frame, because it has scalar replaced objects
+    // which could not be reallocated on the heap.
+    // Return JVMTI_ERROR_OUT_OF_MEMORY to avoid interfering with the VM.
+    return JVMTI_ERROR_OUT_OF_MEMORY;
+  }
+
   vframe *vf = vframeFor(java_thread, 0);
   NULL_CHECK(vf, JVMTI_ERROR_NO_MORE_FRAMES);
 
   javaVFrame *jvf = (javaVFrame*) vf;
   if (!vf->is_java_frame() || jvf->method()->is_native()) {

@@ -1338,10 +1345,16 @@
   if (vf->is_compiled_frame()) {
     if (!vf->fr().can_be_deoptimized()) {
       return JVMTI_ERROR_OPAQUE_FRAME;
     }
     Deoptimization::deoptimize_frame(java_thread, jvf->fr().id());
+    // eagerly reallocate scalar replaced objects
+    JVMTIEscapeBarrier eb(current_thread, java_thread, true);
+    if (!eb.deoptimize_objects(jvf->fr().id())) {
+      // reallocation of scalar replaced objects failed -> return with error
+      return JVMTI_ERROR_OUT_OF_MEMORY;
+    }
   }
 
   // Get information about method return type
   Symbol* signature = jvf->method()->signature();