Cocoa: Ensure menus for a dialog's menubar are validated correctly

Although the items were enabled for a dialog's menubar they were not
appearing as such because Cocoa will query the menu item's target to see
if it has a worksWhenModal selector. Therefore to ensure that the menu
item will be enabled, we need to add this selector to our delegate and
return YES from it when the window for the menubar is the dialog.

Task-number: QTBUG-44584
Change-Id: Ic62dc027d563069d2f5c2b7bf9810184bd76de39
Reviewed-by: Morten Johan Sørvig <morten.sorvig@qt.io>
Reviewed-by: Gabriel de Dietrich <gabriel.dedietrich@qt.io>

src/plugins/platforms/cocoa/qcocoamenu.mm

@@ -253,6 +253,18 @@ QT_NAMESPACE_ALIAS_OBJC_CLASS(QCocoaMenuDelegate);
     return nil;
 }
 
+// Cocoa will query the menu item's target for the worksWhenModal selector.
+// So we need to implement this to allow the items to be handled correctly
+// when a modal dialog is visible.
+- (BOOL)worksWhenModal
+{
+    if (!QGuiApplication::modalWindow())
+        return YES;
+    if (auto *mb = qobject_cast<QCocoaMenuBar *>(m_menu->menuParent()))
+        return QGuiApplication::modalWindow()->handle() == mb->cocoaWindow() ? YES : NO;
+    return YES;
+}
+
 @end
 
 QT_BEGIN_NAMESPACE

src/plugins/platforms/cocoa/qcocoamenubar.h

@@ -71,6 +71,7 @@ public:
 
     QList<QCocoaMenuItem*> merged() const;
     NSMenuItem *itemForRole(QPlatformMenuItem::MenuRole r);
+    QCocoaWindow *cocoaWindow() const;
 
     void syncMenu_helper(QPlatformMenu *menu, bool menubarUpdate);
 

src/plugins/platforms/cocoa/qcocoamenubar.mm

@@ -452,5 +452,10 @@ NSMenuItem *QCocoaMenuBar::itemForRole(QPlatformMenuItem::MenuRole r)
     return Q_NULLPTR;
 }
 
+QCocoaWindow *QCocoaMenuBar::cocoaWindow() const
+{
+    return m_window.data();
+}
+
 QT_END_NAMESPACE