src/java.desktop/unix/classes/sun/awt/X11/XWindowPeer.java

@@ -84,11 +84,11 @@
     private XWindowPeer ownerPeer;
 
     // used for modal blocking to keep existing z-order
     protected XWindowPeer prevTransientFor, nextTransientFor;
     // value of WM_TRANSIENT_FOR hint set on this window
-    private XWindowPeer curRealTransientFor;
+    private XBaseWindow curRealTransientFor;
 
     private boolean grab = false; // Whether to do a grab during showing
 
     private boolean isMapped = false; // Is this window mapped or not
     private boolean mustControlStackPosition = false; // Am override-redirect not on top

@@ -1060,10 +1060,20 @@
                              XLayerProtocol.LAYER_NORMAL);
     }
 
     public void updateAlwaysOnTopState() {
         this.alwaysOnTop = ((Window) this.target).isAlwaysOnTop();
+        if (ownerPeer != null) {
+            XToolkit.awtLock();
+            try {
+                restoreTransientFor(this);
+                applyWindowType();
+            }
+            finally {
+                XToolkit.awtUnlock();
+            }
+        }
         updateAlwaysOnTop();
     }
 
     boolean isLocationByPlatform() {
         return locationByPlatform;

@@ -1103,11 +1113,31 @@
         updateFocusability();
         promoteDefaultPosition();
         if (!vis && warningWindow != null) {
             warningWindow.setSecurityWarningVisible(false, false);
         }
+        boolean refreshChildsTransientFor = isVisible() != vis;
         super.setVisible(vis);
+        if (refreshChildsTransientFor) {
+            for (Window child : ((Window) target).getOwnedWindows()) {
+                XToolkit.awtLock();
+                try {
+                    if(!child.isLightweight() && child.isVisible()) {
+                        ComponentPeer childPeer = AWTAccessor.
+                                getComponentAccessor().getPeer(child);
+                        if(childPeer instanceof XWindowPeer) {
+                            XWindowPeer windowPeer = (XWindowPeer) childPeer;
+                            restoreTransientFor(windowPeer);
+                            windowPeer.applyWindowType();
+                        }
+                    }
+                }
+                finally {
+                    XToolkit.awtUnlock();
+                }
+            }
+        }
         if (!vis && !isWithdrawn()) {
             // ICCCM, 4.1.4. Changing Window State:
             // "Iconic -> Withdrawn - The client should unmap the window and follow it
             // with a synthetic UnmapNotify event as described later in this section."
             // The same is true for Normal -> Withdrawn

@@ -1632,13 +1662,10 @@
         }
         if (updateChain) {
             window.prevTransientFor = transientForWindow;
             transientForWindow.nextTransientFor = window;
         }
-        if (window.curRealTransientFor == transientForWindow) {
-            return;
-        }
         if (!allStates && (window.getWMState() != transientForWindow.getWMState())) {
             return;
         }
         if (window.getScreenNumber() != transientForWindow.getScreenNumber()) {
             return;

@@ -1646,15 +1673,18 @@
         long bpw = window.getWindow();
         while (!XlibUtil.isToplevelWindow(bpw) && !XlibUtil.isXAWTToplevelWindow(bpw)) {
             bpw = XlibUtil.getParentWindow(bpw);
         }
         long tpw = transientForWindow.getWindow();
-        while (!XlibUtil.isToplevelWindow(tpw) && !XlibUtil.isXAWTToplevelWindow(tpw)) {
+        XBaseWindow parent = transientForWindow;
+        while (tpw != 0 && ((!XlibUtil.isToplevelWindow(tpw) &&
+                !XlibUtil.isXAWTToplevelWindow(tpw)) || !parent.isVisible())) {
             tpw = XlibUtil.getParentWindow(tpw);
+            parent = XToolkit.windowToXWindow(tpw);
         }
         XlibWrapper.XSetTransientFor(XToolkit.getDisplay(), bpw, tpw);
-        window.curRealTransientFor = transientForWindow;
+        window.curRealTransientFor = parent;
     }
 
     /*
      * This method does nothing if this window is not blocked by any modal dialog.
      * For modal blocked windows this method looks up for the nearest

@@ -1944,11 +1974,11 @@
         XAtom typeAtom = null;
 
         switch (getWindowType())
         {
             case NORMAL:
-                typeAtom = (ownerPeer == null) ?
+                typeAtom = curRealTransientFor == null ?
                                protocol.XA_NET_WM_WINDOW_TYPE_NORMAL :
                                protocol.XA_NET_WM_WINDOW_TYPE_DIALOG;
                 break;
             case UTILITY:
                 typeAtom = protocol.XA_NET_WM_WINDOW_TYPE_UTILITY;