Support transparency in qt5 system tray icons, regardless of tray visual

This ports the Qt4 method of supporting the system tray
window visual that has an alpha channel as well as ones
that do not. We detect whether or not we have a 32-bit
format and either use a Qt::transparent background, or
call xcb_clear_region before painting the icon.

[ChangeLog][Linux/XCB] Fix transparency of tray icons in cases where
there is no alpha channel or system tray visual.

Task-number: QTBUG-35832
Change-Id: I43d500c39846d2916dd39f8c47c8d85e59b49cae
Reviewed-by: Uli Schlachter <psychon@znc.in>
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@digia.com>

src/plugins/platforms/xcb/qxcbnativeinterface.cpp

@@ -93,7 +93,9 @@ static int resourceType(const QByteArray &key)
 }
 
 QXcbNativeInterface::QXcbNativeInterface() :
-    m_genericEventFilterType(QByteArrayLiteral("xcb_generic_event_t"))
+    m_genericEventFilterType(QByteArrayLiteral("xcb_generic_event_t")),
+    m_sysTraySelectionAtom(XCB_ATOM_NONE),
+    m_systrayVisualId(XCB_NONE)
 
 {
 }
@@ -135,6 +137,82 @@ QRect QXcbNativeInterface::systemTrayWindowGlobalGeometry(const QWindow *window)
     return QRect();
 }
 
+xcb_window_t QXcbNativeInterface::locateSystemTray(xcb_connection_t *conn, const QXcbScreen *screen)
+{
+    if (m_sysTraySelectionAtom == XCB_ATOM_NONE) {
+        const QByteArray net_sys_tray = QString::fromLatin1("_NET_SYSTEM_TRAY_S%1").arg(screen->screenNumber()).toLatin1();
+        xcb_intern_atom_cookie_t intern_c =
+            xcb_intern_atom_unchecked(conn, true, net_sys_tray.length(), net_sys_tray);
+
+        xcb_intern_atom_reply_t *intern_r = xcb_intern_atom_reply(conn, intern_c, 0);
+
+        if (!intern_r)
+            return XCB_WINDOW_NONE;
+
+        m_sysTraySelectionAtom = intern_r->atom;
+        free(intern_r);
+    }
+
+    xcb_get_selection_owner_cookie_t sel_owner_c = xcb_get_selection_owner_unchecked(conn, m_sysTraySelectionAtom);
+    xcb_get_selection_owner_reply_t *sel_owner_r = xcb_get_selection_owner_reply(conn, sel_owner_c, 0);
+
+    if (!sel_owner_r)
+        return XCB_WINDOW_NONE;
+
+    xcb_window_t selection_window = sel_owner_r->owner;
+    free(sel_owner_r);
+
+    return selection_window;
+}
+
+bool QXcbNativeInterface::systrayVisualHasAlphaChannel() {
+    const QXcbScreen *screen = static_cast<QXcbScreen *>(QGuiApplication::primaryScreen()->handle());
+
+    if (m_systrayVisualId == XCB_NONE) {
+        xcb_connection_t *xcb_conn = screen->xcb_connection();
+        xcb_atom_t tray_atom = screen->atom(QXcbAtom::_NET_SYSTEM_TRAY_VISUAL);
+
+        xcb_window_t systray_window = locateSystemTray(xcb_conn, screen);
+        if (systray_window == XCB_WINDOW_NONE)
+            return false;
+
+        // Get the xcb property for the _NET_SYSTEM_TRAY_VISUAL atom
+        xcb_get_property_cookie_t systray_atom_cookie;
+        xcb_get_property_reply_t *systray_atom_reply;
+
+        systray_atom_cookie = xcb_get_property_unchecked(xcb_conn, false, systray_window,
+                                                        tray_atom, XCB_ATOM_VISUALID, 0, 1);
+        systray_atom_reply = xcb_get_property_reply(xcb_conn, systray_atom_cookie, 0);
+
+        if (!systray_atom_reply)
+            return false;
+
+        if (systray_atom_reply->value_len > 0 && xcb_get_property_value_length(systray_atom_reply) > 0) {
+            xcb_visualid_t * vids = (uint32_t *)xcb_get_property_value(systray_atom_reply);
+            m_systrayVisualId = vids[0];
+        }
+
+        free(systray_atom_reply);
+    }
+
+    if (m_systrayVisualId != XCB_NONE) {
+        quint8 depth = screen->depthOfVisual(m_systrayVisualId);
+        return depth == 32;
+    } else {
+        return false;
+    }
+}
+
+void QXcbNativeInterface::clearRegion(const QWindow *qwindow, const QRect& rect)
+{
+    if (const QPlatformWindow *platformWindow = qwindow->handle()) {
+        const QXcbWindow *qxwindow = static_cast<const QXcbWindow *>(platformWindow);
+        xcb_connection_t *xcb_conn = qxwindow->xcb_connection();
+
+        xcb_clear_area(xcb_conn, false, qxwindow->xcb_window(), rect.x(), rect.y(), rect.width(), rect.height());
+    }
+}
+
 void *QXcbNativeInterface::nativeResourceForIntegration(const QByteArray &resourceString)
 {
     void *result = 0;

src/plugins/platforms/xcb/qxcbnativeinterface.h

@@ -107,6 +107,8 @@ public:
 
     Q_INVOKABLE void beep();
     Q_INVOKABLE bool systemTrayAvailable(const QScreen *screen) const;
+    Q_INVOKABLE void clearRegion(const QWindow *qwindow, const QRect& rect);
+    Q_INVOKABLE bool systrayVisualHasAlphaChannel();
     Q_INVOKABLE bool requestSystemTrayWindowDock(const QWindow *window);
     Q_INVOKABLE QRect systemTrayWindowGlobalGeometry(const QWindow *window);
 
@@ -114,8 +116,13 @@ signals:
     void systemTrayWindowChanged(QScreen *screen);
 
 private:
+    xcb_window_t locateSystemTray(xcb_connection_t *conn, const QXcbScreen *screen);
+
     const QByteArray m_genericEventFilterType;
 
+    xcb_atom_t m_sysTraySelectionAtom;
+    xcb_visualid_t m_systrayVisualId;
+
     static QXcbScreen *qPlatformScreenForWindow(QWindow *window);
 };
 

src/plugins/platforms/xcb/qxcbscreen.cpp

@@ -200,6 +200,7 @@ QXcbScreen::QXcbScreen(QXcbConnection *connection, xcb_screen_t *scr,
         while (visualtype_iterator.rem) {
             xcb_visualtype_t *visualtype = visualtype_iterator.data;
             m_visuals.insert(visualtype->visual_id, *visualtype);
+            m_visualDepths.insert(visualtype->visual_id, depth->depth);
             xcb_visualtype_next(&visualtype_iterator);
         }
 
@@ -296,6 +297,14 @@ const xcb_visualtype_t *QXcbScreen::visualForId(xcb_visualid_t visualid) const
     return &*it;
 }
 
+quint8 QXcbScreen::depthOfVisual(xcb_visualid_t visualid) const
+{
+    QMap<xcb_visualid_t, quint8>::const_iterator it = m_visualDepths.find(visualid);
+    if (it == m_visualDepths.constEnd())
+        return 0;
+    return *it;
+}
+
 QImage::Format QXcbScreen::format() const
 {
     return QImage::Format_RGB32;

src/plugins/platforms/xcb/qxcbscreen.h

@@ -93,6 +93,7 @@ public:
     bool syncRequestSupported() const { return m_syncRequestSupported; }
 
     const xcb_visualtype_t *visualForId(xcb_visualid_t) const;
+    quint8 depthOfVisual(xcb_visualid_t) const;
 
     QString name() const { return m_outputName; }
 
@@ -127,6 +128,7 @@ private:
     bool m_syncRequestSupported;
     xcb_window_t m_clientLeader;
     QMap<xcb_visualid_t, xcb_visualtype_t> m_visuals;
+    QMap<xcb_visualid_t, quint8> m_visualDepths;
     QXcbCursor *m_cursor;
     int m_refreshRate;
     int m_forcedDpi;

src/widgets/util/qsystemtrayicon_x11.cpp

@@ -104,11 +104,24 @@ QSystemTrayIconSys::QSystemTrayIconSys(QSystemTrayIcon *qIn)
     setObjectName(QStringLiteral("QSystemTrayIconSys"));
     setToolTip(q->toolTip());
     setAttribute(Qt::WA_AlwaysShowToolTips, true);
-    setAttribute(Qt::WA_TranslucentBackground, true);
     setAttribute(Qt::WA_QuitOnClose, false);
     const QSize size(22, 22); // Gnome, standard size
     setGeometry(QRect(QPoint(0, 0), size));
     setMinimumSize(size);
+
+    // We need two different behaviors depending on whether the X11 visual for the system tray
+    // (a) exists and (b) supports an alpha channel, i.e. is 32 bits.
+    // If we have a visual that has an alpha channel, we can paint this widget with a transparent
+    // background and it will work.
+    // However, if there's no alpha channel visual, in order for transparent tray icons to work,
+    // we do not have a transparent background on the widget, but call xcb_clear_region before
+    // painting the icon
+    bool hasAlphaChannel = false;
+    QMetaObject::invokeMethod(QGuiApplication::platformNativeInterface(),
+                              "systrayVisualHasAlphaChannel", Qt::DirectConnection,
+                              Q_RETURN_ARG(bool, hasAlphaChannel));
+    setAttribute(Qt::WA_TranslucentBackground, hasAlphaChannel);
+
     addToTray();
 }
 
@@ -199,12 +212,21 @@ bool QSystemTrayIconSys::event(QEvent *e)
 
 void QSystemTrayIconSys::paintEvent(QPaintEvent *)
 {
-    // Note: Transparent pixels require a particular Visual which XCB
-    // currently does not support yet.
     const QRect rect(QPoint(0, 0), geometry().size());
     QPainter painter(this);
-    painter.setCompositionMode(QPainter::CompositionMode_Source);
-    painter.fillRect(rect, Qt::transparent);
+
+    // If we have Qt::WA_TranslucentBackground set, during widget creation
+    // we detected the systray visual supported an alpha channel
+    if (testAttribute(Qt::WA_TranslucentBackground)) {
+        painter.setCompositionMode(QPainter::CompositionMode_Source);
+        painter.fillRect(rect, Qt::transparent);
+    } else {
+        QMetaObject::invokeMethod(QGuiApplication::platformNativeInterface(),
+                                    "clearRegion", Qt::DirectConnection,
+                                    Q_ARG(const QWindow *, windowHandle()),
+                                    Q_ARG(const QRect&, rect)
+                                 );
+    }
     painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
     q->icon().paint(&painter, rect);
 }