Window decorations on Linux: fixed behavior of maximize/restore button when tiling window to left or right half of screen (issue #647)

CHANGELOG.md

@@ -28,6 +28,8 @@ FlatLaf Change Log
     decorations are used (e.g. Windows 10/11) or not (e.g. macOS). Now the glass
     pane no longer overlaps the FlatLaf window title bar. (issue #630)
   - Linux: Fixed broken window resizing on multi-screen setups. (issue #632)
+  - Linux: Fixed behavior of maximize/restore button when tiling window to left
+    or right half of screen. (issue #647)
 - IntelliJ Themes:
   - Fixed default button hover background in "Solarized Light" theme. (issue
     #628)

flatlaf-core/src/main/java/com/formdev/flatlaf/ui/FlatRootPaneUI.java

@@ -551,7 +551,7 @@ public class FlatRootPaneUI
 
 		protected boolean isWindowMaximized( Component c ) {
 			Container parent = c.getParent();
-			return parent instanceof Frame && (((Frame)parent).getExtendedState() & Frame.MAXIMIZED_BOTH) != 0;
+			return parent instanceof Frame && (((Frame)parent).getExtendedState() & Frame.MAXIMIZED_BOTH) == Frame.MAXIMIZED_BOTH;
 		}
 	}
 

flatlaf-core/src/main/java/com/formdev/flatlaf/ui/FlatTitlePane.java

@@ -360,9 +360,8 @@ public class FlatTitlePane
 
 		if( window instanceof Frame ) {
 			Frame frame = (Frame) window;
-			boolean maximized = ((frame.getExtendedState() & Frame.MAXIMIZED_BOTH) != 0);
 
-			if( maximized &&
+			if( isWindowMaximized() &&
 				!(SystemInfo.isLinux && FlatNativeLinuxLibrary.isWMUtilsSupported( window )) &&
 				rootPane.getClientProperty( "_flatlaf.maximizedBoundsUpToDate" ) == null )
 			{
@@ -393,7 +392,7 @@ public class FlatTitlePane
 		if( window instanceof Frame ) {
 			Frame frame = (Frame) window;
 			boolean maximizable = frame.isResizable() && clientPropertyBoolean( rootPane, TITLE_BAR_SHOW_MAXIMIZE, true );
-			boolean maximized = ((frame.getExtendedState() & Frame.MAXIMIZED_BOTH) != 0);
+			boolean maximized = isWindowMaximized();
 
 			iconifyButton.setVisible( clientPropertyBoolean( rootPane, TITLE_BAR_SHOW_ICONIFFY, true ) );
 			maximizeButton.setVisible( maximizable && !maximized );
@@ -643,7 +642,10 @@ public class FlatTitlePane
 
 	/** @since 2.4 */
 	protected boolean isWindowMaximized() {
-		return window instanceof Frame && (((Frame)window).getExtendedState() & Frame.MAXIMIZED_BOTH) != 0;
+		// Windows and macOS use always MAXIMIZED_BOTH.
+		// Only Linux uses MAXIMIZED_VERT and MAXIMIZED_HORIZ (when dragging window to left or right edge).
+		// (searched jdk source code)
+		return window instanceof Frame && (((Frame)window).getExtendedState() & Frame.MAXIMIZED_BOTH) == Frame.MAXIMIZED_BOTH;
 	}
 
 	/**
@@ -661,8 +663,30 @@ public class FlatTitlePane
 		rootPane.putClientProperty( "_flatlaf.maximizedBoundsUpToDate", true );
 
 		// maximize window
-		if( !FlatNativeWindowBorder.showWindow( frame, FlatNativeWindowBorder.Provider.SW_MAXIMIZE ) )
+		if( !FlatNativeWindowBorder.showWindow( frame, FlatNativeWindowBorder.Provider.SW_MAXIMIZE ) ) {
-			frame.setExtendedState( frame.getExtendedState() | Frame.MAXIMIZED_BOTH );
+			int oldState = frame.getExtendedState();
+			int newState = oldState | Frame.MAXIMIZED_BOTH;
+
+			if( SystemInfo.isLinux ) {
+				// Linux supports vertical and horizontal maximization:
+				//   - dragging a window to left or right edge of screen vertically maximizes
+				//     the window to the left or right half of the screen
+				//   - don't know whether user can do horizontal maximization
+				// (Windows and macOS use only MAXIMIZED_BOTH)
+				//
+				// If a window is maximized vertically or horizontally (but not both),
+				// then Frame.setExtendedState() behaves not as expected on Linux.
+				// E.g. if window state is MAXIMIZED_VERT, calling setExtendedState(MAXIMIZED_BOTH)
+				// changes state to MAXIMIZED_HORIZ. But calling setExtendedState(MAXIMIZED_HORIZ)
+				// changes state from MAXIMIZED_VERT to MAXIMIZED_BOTH.
+				// Seems to be a bug in sun.awt.X11.XNETProtocol.requestState(),
+				// which does some strange state XOR-ing...
+				if( (oldState & Frame.MAXIMIZED_BOTH) == Frame.MAXIMIZED_VERT )
+					newState = oldState & ~Frame.MAXIMIZED_BOTH | Frame.MAXIMIZED_HORIZ;
+			}
+
+			frame.setExtendedState( newState );
+		}
 	}
 
 	protected void updateMaximizedBounds() {
@@ -766,8 +790,7 @@ public class FlatTitlePane
 		if( !(window instanceof Frame) || !((Frame)window).isResizable() )
 			return;
 
-		Frame frame = (Frame) window;
+		if( isWindowMaximized() )
-		if( (frame.getExtendedState() & Frame.MAXIMIZED_BOTH) != 0 )
 			restore();
 		else
 			maximize();
@@ -1188,6 +1211,13 @@ public class FlatTitlePane
 
 		@Override
 		public void windowStateChanged( WindowEvent e ) {
+/*debug
+			System.out.println( "state " + e.getOldState() + " -> " + e.getNewState() + "     "
+				+ ((e.getNewState() & Frame.MAXIMIZED_HORIZ) != 0 ? " HORIZ" : "")
+				+ ((e.getNewState() & Frame.MAXIMIZED_VERT) != 0 ? " VERT" : "")
+			);
+debug*/
+
 			frameStateChanged();
 			updateNativeTitleBarHeightAndHitTestSpots();
 		}
@@ -1195,7 +1225,7 @@ public class FlatTitlePane
 		//---- interface MouseListener ----
 
 		private Point dragOffset;
-		private boolean nativeMove;
+		private boolean linuxNativeMove;
 		private long lastSingleClickWhen;
 
 		@Override
@@ -1241,7 +1271,7 @@ public class FlatTitlePane
 				return;
 
 			dragOffset = SwingUtilities.convertPoint( FlatTitlePane.this, e.getPoint(), window );
-			nativeMove = false;
+			linuxNativeMove = false;
 
 			// on Linux, move or maximize/restore window
 			if( SystemInfo.isLinux && FlatNativeLinuxLibrary.isWMUtilsSupported( window ) ) {
@@ -1261,7 +1291,7 @@ public class FlatTitlePane
 					case 1:
 						// move window via _NET_WM_MOVERESIZE message
 						e.consume();
-						nativeMove = FlatNativeLinuxLibrary.moveOrResizeWindow( window, e, FlatNativeLinuxLibrary.MOVE );
+						linuxNativeMove = FlatNativeLinuxLibrary.moveOrResizeWindow( window, e, FlatNativeLinuxLibrary.MOVE );
 						lastSingleClickWhen = e.getWhen();
 						break;
 
@@ -1291,7 +1321,7 @@ public class FlatTitlePane
 			if( window == null || dragOffset == null )
 				return; // should newer occur
 
-			if( nativeMove )
+			if( linuxNativeMove )
 				return;
 
 			if( !SwingUtilities.isLeftMouseButton( e ) )

flatlaf-testing/src/main/java/com/formdev/flatlaf/testing/SwingFrameStateTest.java

@@ -0,0 +1,56 @@
+package com.formdev.flatlaf.testing;
+
+import java.awt.FlowLayout;
+import java.awt.Frame;
+import javax.swing.JButton;
+import javax.swing.JFrame;
+import javax.swing.JLabel;
+import javax.swing.JPanel;
+import javax.swing.SwingUtilities;
+import com.formdev.flatlaf.FlatLightLaf;
+import com.formdev.flatlaf.util.SystemInfo;
+
+public class SwingFrameStateTest
+{
+	public static void main( String[] args ) {
+		SwingUtilities.invokeLater( () -> {
+			FlatLightLaf.setup();
+
+			if( SystemInfo.isLinux )
+				JFrame.setDefaultLookAndFeelDecorated( true );
+
+			JFrame frame = new JFrame( "SwingFrameStateTest" );
+
+			JButton restoreButton = new JButton( "Restore" );
+			JButton vertButton = new JButton( "Max. Vert." );
+			JButton horizButton = new JButton( "Max. Horiz." );
+			JButton bothButton = new JButton( "Max. Both" );
+			restoreButton.addActionListener( e -> frame.setExtendedState( 0 ) );
+			vertButton.addActionListener( e -> frame.setExtendedState( Frame.MAXIMIZED_VERT ) );
+			horizButton.addActionListener( e -> frame.setExtendedState( Frame.MAXIMIZED_HORIZ ) );
+			bothButton.addActionListener( e -> frame.setExtendedState( Frame.MAXIMIZED_BOTH ) );
+
+			JPanel panel = new JPanel( new FlowLayout() );
+			panel.add( restoreButton );
+			panel.add( vertButton );
+			panel.add( horizButton );
+			panel.add( bothButton );
+
+			JLabel stateInfo = new JLabel();
+			frame.addWindowStateListener( e -> {
+				int state = frame.getExtendedState();
+				stateInfo.setText( " state "
+					+ ((state & Frame.MAXIMIZED_VERT) != 0 ? "VERT " : "")
+					+ ((state & Frame.MAXIMIZED_HORIZ) != 0 ? "HORIZ " : "")
+				);
+			} );
+			panel.add( stateInfo );
+
+			frame.setDefaultCloseOperation( JFrame.EXIT_ON_CLOSE );
+			frame.getContentPane().add( panel );
+			frame.setSize( 700, 300 );
+			frame.setLocationRelativeTo( null );
+			frame.setVisible( true );
+		} );
+	}
+}