macOS: Fix logic for determining NSWindowStyleMaskTitled/Borderless

We were working under the assumption that the NSWindowStyleMaskBorderless
style mask excluded all others, on account of having the value 0. But in
practice this does not seem to be the case, and you can combine the mask
with many of the other masks. The only mask that is mutually exclusive with
the borderless mask is NSWindowStyleMaskTitled (with a value of 1).

Clarify this be restructuring QCocoaWindow::windowStyleMask().

Task-number: QTBUG-71485
Change-Id: I4bbd603fd2373c11f76e84b72a2a60aa2356b032
Reviewed-by: Doris Verria <doris.verria@qt.io>
Reviewed-by: Øystein Heskestad <oystein.heskestad@qt.io>
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>

src/plugins/platforms/cocoa/qcocoawindow.mm

@@ -485,31 +485,51 @@ NSInteger QCocoaWindow::windowLevel(Qt::WindowFlags flags)
 NSUInteger QCocoaWindow::windowStyleMask(Qt::WindowFlags flags)
 {
     const Qt::WindowType type = static_cast<Qt::WindowType>(int(flags & Qt::WindowType_Mask));
-    const bool frameless = (flags & Qt::FramelessWindowHint) || windowIsPopupType(type);
 
-    // Remove zoom button by disabling resize for CustomizeWindowHint windows, except for
+    // Determine initial style mask based on whether the window should
-    // Qt::Tool windows (e.g. dock windows) which should always be resizable.
+    // have a frame and title or not. The NSWindowStyleMaskBorderless
-    const bool resizable = !(flags & Qt::CustomizeWindowHint) || (type == Qt::Tool);
+    // and NSWindowStyleMaskTitled styles are mutually exclusive, with
+    // values of 0 and 1 correspondingly.
+    NSUInteger styleMask = [&]{
+        // Honor explicit requests for borderless windows
+        if (flags & Qt::FramelessWindowHint)
+            return NSWindowStyleMaskBorderless;
 
-    // Select base window type. Note that the value of NSBorderlessWindowMask is 0.
+        // Popup windows should always be borderless
-    NSUInteger styleMask = (frameless || !resizable) ? NSWindowStyleMaskBorderless : NSWindowStyleMaskResizable;
+        if (windowIsPopupType(type))
+            return NSWindowStyleMaskBorderless;
 
-    if (frameless) {
+        if (flags & Qt::CustomizeWindowHint) {
+            // CustomizeWindowHint turns off the default window title hints,
+            // so the choice is then up to the user via Qt::WindowTitleHint.
+            return flags & Qt::WindowTitleHint
+                ? NSWindowStyleMaskTitled
+                : NSWindowStyleMaskBorderless;
+        } else {
+            // Otherwise, default to using titled windows
+            return NSWindowStyleMaskTitled;
+        }
+    }();
+
+    // FIXME: Control visibility of buttons directly, instead of affecting styleMask
+    if (styleMask == NSWindowStyleMaskBorderless) {
         // Frameless windows do not display the traffic lights buttons for
         // e.g. minimize, however StyleMaskMiniaturizable is required to allow
         // programmatic minimize.
         styleMask |= NSWindowStyleMaskMiniaturizable;
     } else if (flags & Qt::CustomizeWindowHint) {
-        if (flags & Qt::WindowTitleHint)
-            styleMask |= NSWindowStyleMaskTitled;
         if (flags & Qt::WindowCloseButtonHint)
             styleMask |= NSWindowStyleMaskClosable;
         if (flags & Qt::WindowMinimizeButtonHint)
             styleMask |= NSWindowStyleMaskMiniaturizable;
         if (flags & Qt::WindowMaximizeButtonHint)
             styleMask |= NSWindowStyleMaskResizable;
+
+        // Force tool windows to be resizable
+        if (type == Qt::Tool)
+            styleMask |= NSWindowStyleMaskResizable;
     } else {
-        styleMask |= NSWindowStyleMaskClosable | NSWindowStyleMaskTitled;
+        styleMask |= NSWindowStyleMaskClosable | NSWindowStyleMaskResizable;
 
         if (type != Qt::Dialog)
             styleMask |= NSWindowStyleMaskMiniaturizable;
@@ -518,6 +538,7 @@ NSUInteger QCocoaWindow::windowStyleMask(Qt::WindowFlags flags)
     if (type == Qt::Tool)
         styleMask |= NSWindowStyleMaskUtilityWindow;
 
+    // FIXME: Remove use of deprecated style mask
     if (m_drawContentBorderGradient)
         styleMask |= NSWindowStyleMaskTexturedBackground;