src/java.desktop/unix/classes/sun/awt/X11/XWindowPeer.java

@@ -1060,10 +1060,20 @@
                              XLayerProtocol.LAYER_NORMAL);
     }
 
     public void updateAlwaysOnTopState() {
         this.alwaysOnTop = ((Window) this.target).isAlwaysOnTop();
+        if (ownerPeer != null) {
+            XToolkit.awtLock();
+            try {
+                applyWindowType();
+                restoreTransientFor(this);
+            }
+            finally {
+                XToolkit.awtUnlock();
+            }
+        }
         updateAlwaysOnTop();
     }
 
     boolean isLocationByPlatform() {
         return locationByPlatform;

@@ -1645,10 +1655,15 @@
         }
         long bpw = window.getWindow();
         while (!XlibUtil.isToplevelWindow(bpw) && !XlibUtil.isXAWTToplevelWindow(bpw)) {
             bpw = XlibUtil.getParentWindow(bpw);
         }
+        if (!transientForWindow.isVisible()) {
+            XlibWrapper.XSetTransientFor(XToolkit.getDisplay(), bpw, XToolkit.getDefaultRootWindow());
+            window.curRealTransientFor = null;
+            return;
+        }
         long tpw = transientForWindow.getWindow();
         while (!XlibUtil.isToplevelWindow(tpw) && !XlibUtil.isXAWTToplevelWindow(tpw)) {
             tpw = XlibUtil.getParentWindow(tpw);
         }
         XlibWrapper.XSetTransientFor(XToolkit.getDisplay(), bpw, tpw);

@@ -1944,11 +1959,11 @@
         XAtom typeAtom = null;
 
         switch (getWindowType())
         {
             case NORMAL:
-                typeAtom = (ownerPeer == null) ?
+                typeAtom = ownerPeer == null || !ownerPeer.isVisible() ?
                                protocol.XA_NET_WM_WINDOW_TYPE_NORMAL :
                                protocol.XA_NET_WM_WINDOW_TYPE_DIALOG;
                 break;
             case UTILITY:
                 typeAtom = protocol.XA_NET_WM_WINDOW_TYPE_UTILITY;