Fix in-process QAxServers inside modal windows.

The main window of in-process QAxServers doesn't have QWindow parent,
but it does have native parent that is part of the native window tree
of the application. The lack of Qt parent makes embedded controls look
like toplevel windows, which causes problems e.g. with modality.

Introduced new optional method QPlatformWindow::isEmbedded() to
detect if a window is an embedded window and utilized it in proper
places during modality handling.

Task-number: QTBUG-26871
Change-Id: Iac9a51dae06b8fc15410de7838857e203e4275b8
Reviewed-by: Andreas Holzammer <andreas.holzammer@kdab.com>
Reviewed-by: Joerg Bornemann <joerg.bornemann@nokia.com>

src/gui/kernel/qguiapplication.cpp

@@ -604,8 +604,13 @@ QWindowList QGuiApplication::topLevelWindows()
     const QWindowList &list = QGuiApplicationPrivate::window_list;
     QWindowList topLevelWindows;
     for (int i = 0; i < list.size(); i++) {
-        if (!list.at(i)->parent())
-            topLevelWindows.prepend(list.at(i));
+        if (!list.at(i)->parent()) {
+            // Top windows of embedded QAxServers do not have QWindow parents,
+            // but they are not true top level windows, so do not include them.
+            const bool embedded = list.at(i)->handle() && list.at(i)->handle()->isEmbedded(0);
+            if (!embedded)
+                topLevelWindows.prepend(list.at(i));
+        }
     }
     return topLevelWindows;
 }

src/gui/kernel/qplatformwindow.cpp

@@ -174,6 +174,19 @@ bool QPlatformWindow::isActive() const
     return false;
 }
 
+/*!
+    Returns true if the window is a descendant of an embedded non-Qt window.
+    Example of an embedded non-Qt window is the parent window of an in-process QAxServer.
+
+    If \a parentWindow is nonzero, only check if the window is embedded in the
+    specified \a parentWindow.
+*/
+bool QPlatformWindow::isEmbedded(const QPlatformWindow *parentWindow) const
+{
+    Q_UNUSED(parentWindow);
+    return false;
+}
+
 /*!
     Requests setting the window state of this surface
     to \a type. Returns the actual state set.

src/gui/kernel/qplatformwindow.h

@@ -103,6 +103,7 @@ public:
 
     virtual bool isExposed() const;
     virtual bool isActive() const;
+    virtual bool isEmbedded(const QPlatformWindow *parentWindow) const;
 
     virtual void propagateSizeHints();
 

src/plugins/platforms/windows/qwindowswindow.cpp

@@ -424,6 +424,7 @@ QWindowsWindow::WindowData
     if (desktop) {                        // desktop widget. No frame, hopefully?
         result.hwnd = GetDesktopWindow();
         result.geometry = frameGeometry(result.hwnd, true);
+        result.embedded = false;
         if (QWindowsContext::verboseWindows)
             qDebug().nospace() << "Created desktop window " << w << result.hwnd;
         return result;
@@ -830,6 +831,21 @@ bool QWindowsWindow::isActive() const
     return false;
 }
 
+bool QWindowsWindow::isEmbedded(const QPlatformWindow *parentWindow) const
+{
+    if (parentWindow) {
+        const QWindowsWindow *ww = static_cast<const QWindowsWindow *>(parentWindow);
+        const HWND hwnd = ww->handle();
+        if (!IsChild(hwnd, m_data.hwnd))
+            return false;
+    }
+
+    if (!m_data.embedded && parent())
+        return parent()->isEmbedded(0);
+
+    return m_data.embedded;
+}
+
 // partially from QWidgetPrivate::show_sys()
 void QWindowsWindow::show_sys() const
 {

src/plugins/platforms/windows/qwindowswindow.h

@@ -154,6 +154,7 @@ public:
     virtual void setVisible(bool visible);
     bool isVisible() const;
     virtual bool isActive() const;
+    virtual bool isEmbedded(const QPlatformWindow *parentWindow) const;
     virtual Qt::WindowFlags setWindowFlags(Qt::WindowFlags flags);
     virtual Qt::WindowState setWindowState(Qt::WindowState state);
 

src/widgets/kernel/qapplication.cpp

@@ -106,6 +106,7 @@ extern bool qt_wince_is_pocket_pc();  //qguifunctions_wince.cpp
 #endif
 
 #include "qdatetime.h"
+#include <qpa/qplatformwindow.h>
 
 //#define ALIEN_DEBUG
 
@@ -2317,6 +2318,13 @@ bool QApplicationPrivate::isWindowBlocked(QWindow *window, QWindow **blockingWin
                     p = w->transientParent();
                 w = p;
             }
+
+            // Embedded in-process windows are not visible in normal parent-child chain,
+            // so check the native parent chain, too.
+            const QPlatformWindow *platWin = window->handle();
+            const QPlatformWindow *modalPlatWin = modalWindow->handle();
+            if (platWin && modalPlatWin && platWin->isEmbedded(modalPlatWin))
+                return false;
         }
 
         Qt::WindowModality windowModality = modalWindow->windowModality();