macOS: Rework worksWhenModal and update on modal session change

Instead of basing the worksWhenModal property on the window type, which
will include both windows inside the current modal session (as intend),
but also windows below the current modal session (wrongly), we check
to see if the window is a transient child of the current top level
modal window.

Going via NSApp.modalWindow means we also catch cases where the top
level modal session is run by a native window, not part of the modal
session stack in the Cocoa event dispatcher.

The new logic relies on windows such as popups, dialogs, etc to set
the correct transient parent, but this seems to be the case already.

To ensure the window tag is also updated, we call setWorksWhenModal
on modal session changes. We could change worksWhenModal into e.g.
shouldWorkWhenModal and always use the setter, but that would mean
the initial window tag update in [NSWindow _commonAwake] would pick
up the incorrect value. And if the window tag is not updated after
that due to the workaround in [QNSPanel setWorksWhenModal:] being
compiled out, the window would not be possible to order front,
which is worse than being able to order front a window with
worksWhenModal=NO.

Fixes: QTBUG-76654
Task-number: QTBUG-71480
Change-Id: I38b14422d274dcc03b4c7d5ef87066e282ed9111
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

src/plugins/platforms/cocoa/qcocoawindowmanager.mm

@@ -100,6 +100,18 @@ void QCocoaWindowManager::modalSessionChanged()
             }
         }
     }
+
+    // Our worksWhenModal implementation is declarative and will normally be picked
+    // up by AppKit when needed, but to make sure AppKit also reflects the state
+    // in the window tag, so that the window can be ordered front by clicking it,
+    // we need to explicitly call setWorksWhenModal.
+    for (id window in NSApp.windows) {
+        if ([window isKindOfClass:[QNSPanel class]]) {
+            auto *panel = static_cast<QNSPanel *>(window);
+            // Call setter to tell AppKit that our state has changed
+            [panel setWorksWhenModal:panel.worksWhenModal];
+        }
+    }
 }
 
 static void initializeWindowManager() { Q_UNUSED(QCocoaWindowManager::instance()); }

src/plugins/platforms/cocoa/qnswindow.mm

@@ -158,8 +158,79 @@ static bool isMouseEvent(NSEvent *ev)
 #define QNSWINDOW_PROTOCOL_IMPLMENTATION 1
 #include "qnswindow.mm"
 #undef QNSWINDOW_PROTOCOL_IMPLMENTATION
+
+- (BOOL)worksWhenModal
+{
+    if (!m_platformWindow)
+        return NO;
+
+    // Conceptually there are two sets of windows we need consider:
+    //
+    //   - windows 'lower' in the modal session stack
+    //   - windows 'within' the current modal session
+    //
+    // The first set of windows should always be blocked by the current
+    // modal session, regardless of window type. The latter set may contain
+    // windows with a transient parent, which from Qt's point of view makes
+    // them 'child' windows, so we treat them as operable within the current
+    // modal session.
+
+    if (!NSApp.modalWindow)
+        return NO;
+
+    // If the current modal window (top level modal session) is not a Qt window we
+    // have no way of knowing if this window is transient child of the modal window.
+    if (![NSApp.modalWindow conformsToProtocol:@protocol(QNSWindowProtocol)])
+        return NO;
+
+    if (auto *modalWindow = static_cast<QCocoaNSWindow *>(NSApp.modalWindow).platformWindow) {
+        if (modalWindow->window()->isAncestorOf(m_platformWindow->window(), QWindow::IncludeTransients))
+            return YES;
+    }
+
+    return NO;
+}
 @end
 
+#if !defined(QT_APPLE_NO_PRIVATE_APIS)
+// When creating an NSWindow the worksWhenModal function is queried,
+// and the resulting state is used to set the corresponding window tag,
+// which the window server uses to determine whether or not the window
+// should be allowed to activate via mouse clicks in the title-bar.
+// Unfortunately, prior to macOS 10.15, this window tag was never
+// updated after the initial assignment in [NSWindow _commonAwake],
+// which meant that windows that dynamically change their worksWhenModal
+// state will behave as if they were never allowed to work when modal.
+// We work around this by manually updating the window tag when needed.
+
+typedef uint32_t CGSConnectionID;
+typedef uint32_t CGSWindowID;
+
+extern "C" {
+CGSConnectionID CGSMainConnectionID() __attribute__((weak_import));
+OSStatus CGSSetWindowTags(const CGSConnectionID, const CGSWindowID, int *, int) __attribute__((weak_import));
+OSStatus CGSClearWindowTags(const CGSConnectionID, const CGSWindowID, int *, int) __attribute__((weak_import));
+}
+
+@interface QNSPanel (WorksWhenModalWindowTagWorkaround) @end
+@implementation QNSPanel (WorksWhenModalWindowTagWorkaround)
+- (void)setWorksWhenModal:(BOOL)worksWhenModal
+{
+    [super setWorksWhenModal:worksWhenModal];
+
+    if (QOperatingSystemVersion::current() < QOperatingSystemVersion::MacOSCatalina) {
+        if (CGSMainConnectionID && CGSSetWindowTags && CGSClearWindowTags) {
+            static int kWorksWhenModalWindowTag = 0x40;
+            auto *function = worksWhenModal ? CGSSetWindowTags : CGSClearWindowTags;
+            function(CGSMainConnectionID(), self.windowNumber, &kWorksWhenModalWindowTag, 64);
+        } else {
+            qWarning() << "Missing APIs for window tag handling, can not update worksWhenModal state";
+        }
+    }
+}
+@end
+#endif // QT_APPLE_NO_PRIVATE_APIS
+
 #else // QNSWINDOW_PROTOCOL_IMPLMENTATION
 
 // The following content is mixed in to the QNSWindow and QNSPanel classes via includes
@@ -237,17 +308,6 @@ static bool isMouseEvent(NSEvent *ev)
     return canBecomeMain;
 }
 
-- (BOOL)worksWhenModal
-{
-    if (m_platformWindow && [self isKindOfClass:[QNSPanel class]]) {
-        Qt::WindowType type = m_platformWindow->window()->type();
-        if (type == Qt::Popup || type == Qt::Dialog || type == Qt::Tool)
-            return YES;
-    }
-
-    return [super worksWhenModal];
-}
-
 - (BOOL)isOpaque
 {
     return m_platformWindow ? m_platformWindow->isOpaque() : [super isOpaque];