src/solaris/native/sun/java2d/opengl/GLXSurfaceData.c

@@ -1,7 +1,7 @@
 /*
- * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 only, as
  * published by the Free Software Foundation.  Oracle designates this

@@ -48,12 +48,10 @@
 extern struct MComponentPeerIDs mComponentPeerIDs;
 
 extern void
     OGLSD_SetNativeDimensions(JNIEnv *env, OGLSDOps *oglsdo, jint w, jint h);
 
-jboolean surfaceCreationFailed = JNI_FALSE;
-
 #endif /* !HEADLESS */
 
 JNIEXPORT void JNICALL
 Java_sun_java2d_opengl_GLXSurfaceData_initOps(JNIEnv *env, jobject glxsd,
                                               jobject peer, jlong aData)

@@ -391,22 +389,10 @@
                 oglsdo->width, oglsdo->height);
 
     return JNI_TRUE;
 }
 
-static int
-GLXSD_BadAllocXErrHandler(Display *display, XErrorEvent *xerr)
-{
-    int ret = 0;
-    if (xerr->error_code == BadAlloc) {
-        surfaceCreationFailed = JNI_TRUE;
-    } else {
-        ret = (*xerror_saved_handler)(display, xerr);
-    }
-    return ret;
-}
-
 JNIEXPORT jboolean JNICALL
 Java_sun_java2d_opengl_GLXSurfaceData_initPbuffer
     (JNIEnv *env, jobject glxsd,
      jlong pData, jlong pConfigInfo,
      jboolean isOpaque,

@@ -418,10 +404,12 @@
     GLXSDOps *glxsdo;
     GLXPbuffer pbuffer;
     int attrlist[] = {GLX_PBUFFER_WIDTH, 0,
                       GLX_PBUFFER_HEIGHT, 0,
                       GLX_PRESERVED_CONTENTS, GL_FALSE, 0};
+    jboolean errorOccurredFlag;
+    jobject errorHandlerRef;
 
     J2dTraceLn3(J2D_TRACE_INFO,
                 "GLXSurfaceData_initPbuffer: w=%d h=%d opq=%d",
                 width, height, isOpaque);
 

@@ -445,16 +433,19 @@
     }
 
     attrlist[1] = width;
     attrlist[3] = height;
 
-    surfaceCreationFailed = JNI_FALSE;
-    EXEC_WITH_XERROR_HANDLER(
-        GLXSD_BadAllocXErrHandler,
-        pbuffer = j2d_glXCreatePbuffer(awt_display,
-                                       glxinfo->fbconfig, attrlist));
-    if ((pbuffer == 0) || surfaceCreationFailed) {
+    WITH_XERROR_HANDLER(env, "sun/awt/X11/XErrorHandler$GLXBadAllocHandler",
+        "()Lsun/awt/X11/XErrorHandler$GLXBadAllocHandler;", JNI_TRUE, errorHandlerRef);
+    pbuffer = j2d_glXCreatePbuffer(awt_display, glxinfo->fbconfig, attrlist);
+    // Call XSync without the acquired AWT lock to avoid a deadlock (see 8015730).
+    XSync(awt_display, False);
+    RESTORE_XERROR_HANDLER(env, JNI_FALSE);
+    errorOccurredFlag = GET_HANDLER_ERROR_OCCURRED_FLAG(env, errorHandlerRef);
+
+    if ((pbuffer == 0) || errorOccurredFlag) {
         J2dRlsTraceLn(J2D_TRACE_ERROR,
             "GLXSurfaceData_initPbuffer: could not create glx pbuffer");
         return JNI_FALSE;
     }