src/jdk.jdwp.agent/share/native/libjdwp/invoker.c

@  rev 15550 : 8153711: [REDO] JDWP: Memory Leak: GlobalRefs never deleted when processing invokeMethod command
|  Summary: Delete global references in invoker_completeInvokeRequest()
|  Reviewed-by: duke

@@ -209,10 +209,42 @@
     }
 
     return error;
 }
 
+/*
+ * Delete global argument references from the request which got put there before a
+ * invoke request was carried out. See fillInvokeRequest().
+ */
+static void
+deleteGlobalArgumentRefs(JNIEnv *env, InvokeRequest *request)
+{
+    void *cursor;
+    jint argIndex = 0;
+    jvalue *argument = request->arguments;
+    jbyte argumentTag = firstArgumentTypeTag(request->methodSignature, &cursor);
+
+    if (request->clazz != NULL) {
+        tossGlobalRef(env, &(request->clazz));
+    }
+    if (request->instance != NULL) {
+        tossGlobalRef(env, &(request->instance));
+    }
+    /* Delete global argument references */
+    while (argIndex < request->argumentCount) {
+        if ((argumentTag == JDWP_TAG(OBJECT)) ||
+            (argumentTag == JDWP_TAG(ARRAY))) {
+            if (argument->l != NULL) {
+                tossGlobalRef(env, &(argument->l));
+            }
+        }
+        argument++;
+        argIndex++;
+        argumentTag = nextArgumentTypeTag(&cursor);
+    }
+}
+
 static jvmtiError
 fillInvokeRequest(JNIEnv *env, InvokeRequest *request,
                   jbyte invokeType, jbyte options, jint id,
                   jthread thread, jclass clazz, jmethodID method,
                   jobject instance,

@@ -320,10 +352,12 @@
 
 static void
 invokeConstructor(JNIEnv *env, InvokeRequest *request)
 {
     jobject object;
+    
+    JDI_ASSERT_MSG(request->clazz, "Request clazz null");
     object = JNI_FUNC_PTR(env,NewObjectA)(env, request->clazz,
                                      request->method,
                                      request->arguments);
     request->returnValue.l = NULL;
     if (object != NULL) {

@@ -336,10 +370,11 @@
 {
     switch(returnTypeTag(request->methodSignature)) {
         case JDWP_TAG(OBJECT):
         case JDWP_TAG(ARRAY): {
             jobject object;
+            JDI_ASSERT_MSG(request->clazz, "Request clazz null");
             object = JNI_FUNC_PTR(env,CallStaticObjectMethodA)(env,
                                        request->clazz,
                                        request->method,
                                        request->arguments);
             request->returnValue.l = NULL;

@@ -424,10 +459,11 @@
 {
     switch(returnTypeTag(request->methodSignature)) {
         case JDWP_TAG(OBJECT):
         case JDWP_TAG(ARRAY): {
             jobject object;
+            JDI_ASSERT_MSG(request->instance, "Request instance null");
             object = JNI_FUNC_PTR(env,CallObjectMethodA)(env,
                                  request->instance,
                                  request->method,
                                  request->arguments);
             request->returnValue.l = NULL;

@@ -511,10 +547,12 @@
 {
     switch(returnTypeTag(request->methodSignature)) {
         case JDWP_TAG(OBJECT):
         case JDWP_TAG(ARRAY): {
             jobject object;
+            JDI_ASSERT_MSG(request->clazz, "Request clazz null");
+            JDI_ASSERT_MSG(request->instance, "Request instance null");
             object = JNI_FUNC_PTR(env,CallNonvirtualObjectMethodA)(env,
                                            request->instance,
                                            request->clazz,
                                            request->method,
                                            request->arguments);

@@ -607,10 +645,12 @@
 invoker_doInvoke(jthread thread)
 {
     JNIEnv *env;
     jboolean startNow;
     InvokeRequest *request;
+    jbyte options;
+    jbyte invokeType;
 
     JDI_ASSERT(thread);
 
     debugMonitorEnter(invokerLock);
 

@@ -623,10 +663,13 @@
     startNow = request->pending && !request->started;
 
     if (startNow) {
         request->started = JNI_TRUE;
     }
+    options = request->options;
+    invokeType = request->invokeType;
+
     debugMonitorExit(invokerLock);
 
     if (!startNow) {
         return JNI_FALSE;
     }

@@ -637,19 +680,19 @@
 
         jobject exception;
 
         JNI_FUNC_PTR(env,ExceptionClear)(env);
 
-        switch (request->invokeType) {
+        switch (invokeType) {
             case INVOKE_CONSTRUCTOR:
                 invokeConstructor(env, request);
                 break;
             case INVOKE_STATIC:
                 invokeStatic(env, request);
                 break;
             case INVOKE_INSTANCE:
-                if (request->options & JDWP_INVOKE_OPTIONS(NONVIRTUAL) ) {
+                if (options & JDWP_INVOKE_OPTIONS(NONVIRTUAL) ) {
                     invokeNonvirtual(env, request);
                 } else {
                     invokeVirtual(env, request);
                 }
                 break;

@@ -723,23 +766,55 @@
         exc = request->exception;
         returnValue = request->returnValue;
     }
 
     /*
+     * At this time, there's no need to retain global references on
+     * arguments since the reply is processed. No one will deal with
+     * this request ID anymore, so we must call deleteGlobalArgumentRefs().
+     *
+     * We cannot delete saved exception or return value references
+     * since otherwise a deleted handle would escape when writing
+     * the response to the stream. Instead, we clean those refs up
+     * after writing the respone.
+     */
+    deleteGlobalArgumentRefs(env, request);
+
+    /*
      * Give up the lock before I/O operation
      */
     debugMonitorExit(invokerLock);
     eventHandler_unlock();
 
-
     if (!detached) {
         outStream_initReply(&out, id);
         (void)outStream_writeValue(env, &out, tag, returnValue);
         (void)outStream_writeObjectTag(env, &out, exc);
         (void)outStream_writeObjectRef(env, &out, exc);
         outStream_sendReply(&out);
     }
+
+    /*
+     * Delete potentially saved global references of return value
+     * and exception
+     */
+    eventHandler_lock(); // for proper lock order
+    debugMonitorEnter(invokerLock);
+    /* Delete potentially saved return value */
+    if ((request->invokeType == INVOKE_CONSTRUCTOR) ||
+        (returnTypeTag(request->methodSignature) == JDWP_TAG(OBJECT)) ||
+        (returnTypeTag(request->methodSignature) == JDWP_TAG(ARRAY))) {
+        if (request->returnValue.l != NULL) {
+            tossGlobalRef(env, &(request->returnValue.l));
+        }
+    }
+    /* Delete potentially saved exception */
+    if (request->exception != NULL) {
+        tossGlobalRef(env, &(request->exception));
+    }
+    debugMonitorExit(invokerLock);
+    eventHandler_unlock();
 }
 
 jboolean
 invoker_isEnabled(jthread thread)
 {