Windows: fixed possible deadlock with TabbedPane in window title area in "full window content" mode (issue #909)

CHANGELOG.md

@@ -8,6 +8,8 @@ FlatLaf Change Log
 - HTML: Fixed wrong rendering if HTML text contains `<style>` tag with
   attributes (e.g. `<style type='text/css'>`). (issue #905; regression in 3.5.1)
 - FlatLaf window decorations:
+  - Windows: Fixed possible deadlock with TabbedPane in window title area in
+    "full window content" mode. (issue #909)
   - Linux: Fixed continuous cursor toggling between resize and standard cursor
     when resizing window. (issue #907)
 

flatlaf-core/src/main/java/com/formdev/flatlaf/ui/FlatTabbedPaneUI.java

@@ -2306,8 +2306,23 @@ debug*/
 	/** @since 3.4 */
 	@Override
 	public Boolean isTitleBarCaptionAt( int x, int y ) {
-		if( tabForCoordinate( tabPane, x, y ) >= 0 )
-			return false;
+		// Note: not using tabForCoordinate() here because this may validate layout and cause dead lock
+
+		if( moreTabsButton != null ) {
+			// convert x,y from JTabbedPane coordinate space to ScrollableTabPanel coordinate space
+			Point viewPosition = tabViewport.getViewPosition();
+			x = x - tabViewport.getX() + viewPosition.x;
+			y = y - tabViewport.getY() + viewPosition.y;
+
+			// check whether point is within viewport
+			if( !tabViewport.getViewRect().contains( x, y ) )
+				return null; // check children
+		}
+
+		for( int i = 0; i < rects.length; i++ ) {
+			if( rects[i].contains( x, y ) )
+				return false;
+		}
 
 		return null; // check children
 	}

flatlaf-core/src/main/java/com/formdev/flatlaf/ui/FlatTitlePane.java

@@ -1058,10 +1058,11 @@ public class FlatTitlePane
 	 * <p>
 	 * Note:
 	 * <ul>
-	 *   <li>This method is invoked often when mouse is moved over title bar
+	 *   <li>This method is invoked often when mouse is moved over window title bar area
 	 *       and should therefore return quickly.
 	 *   <li>This method is invoked on 'AWT-Windows' thread (not 'AWT-EventQueue' thread)
 	 *       while processing Windows messages.
+	 *       It <b>must not</b> change any component property or layout because this could cause a dead lock.
 	 * </ul>
 	 */
 	private boolean captionHitTest( Point pt ) {
@@ -1578,6 +1579,15 @@ debug*/
 		 * Useful for components that do not use mouse input on whole component bounds.
 		 * E.g. a tabbed pane with a few tabs has some empty space beside the tabs
 		 * that can be used to move the window.
+		 * <p>
+		 * Note:
+		 * <ul>
+		 *   <li>This method is invoked often when mouse is moved over window title bar area
+		 *       and should therefore return quickly.
+		 *   <li>This method is invoked on 'AWT-Windows' thread (not 'AWT-EventQueue' thread)
+		 *       while processing Windows messages.
+		 *       It <b>must not</b> change any component property or layout because this could cause a dead lock.
+		 * </ul>
 		 *
 		 * @return {@code true} if the component is not interested in mouse input at the given location
 		 *         {@code false} if the component wants process mouse input at the given location