Fix up QOpenGLWidget transparency support

The glColorMask call was troublesome. In addition, the Qt::WA_TranslucentBackground
was misinterpreted and recommended misleadingly in the documentation. The hellogl2
example's --transparent argument was disfunctional in practice.

Replace glColorMask with glBlendFuncSeparate. The hellogl2 example and the docs are
now corrected wrt enabling semi-transparency in a QOpenGLWidget that is not a top-level
(which is the most common case).

Task-number: QTBUG-47276
Change-Id: I6f40e732d455f5efcf158649ac9a52ff9f240e85
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@theqtcompany.com>
Reviewed-by: Gunnar Sletta <gunnar@sletta.org>

examples/opengl/hellogl2/glwidget.cpp

@@ -55,8 +55,11 @@ GLWidget::GLWidget(QWidget *parent)
     // --transparent causes the clear color to be transparent. Therefore, on systems that
     // support it, the widget will become transparent apart from the logo.
     m_transparent = QCoreApplication::arguments().contains(QStringLiteral("--transparent"));
-    if (m_transparent)
-        setAttribute(Qt::WA_TranslucentBackground);
+    if (m_transparent) {
+        QSurfaceFormat fmt = format();
+        fmt.setAlphaBufferSize(8);
+        setFormat(fmt);
+    }
 }
 
 GLWidget::~GLWidget()

examples/opengl/hellogl2/main.cpp

@@ -59,6 +59,10 @@ int main(int argc, char *argv[])
     QSurfaceFormat::setDefaultFormat(fmt);
 
     MainWindow mainWindow;
+    if (QCoreApplication::arguments().contains(QStringLiteral("--transparent"))) {
+        mainWindow.setAttribute(Qt::WA_TranslucentBackground);
+        mainWindow.setAttribute(Qt::WA_NoSystemBackground, false);
+    }
     mainWindow.resize(mainWindow.sizeHint());
     int desktopArea = QApplication::desktop()->width() *
                      QApplication::desktop()->height();

src/gui/painting/qplatformbackingstore.cpp

@@ -300,11 +300,7 @@ void QPlatformBackingStore::composeAndFlush(QWindow *window, const QRegion &regi
     }
 
     funcs->glEnable(GL_BLEND);
-    funcs->glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
-    // Do not write out alpha. We need blending, but only for RGB. The toplevel may have
-    // alpha enabled in which case blending (writing out < 1.0 alpha values) would lead to
-    // semi-transparency even when it is not wanted.
-    funcs->glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_FALSE);
+    funcs->glBlendFuncSeparate(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA, GL_ONE, GL_ONE);
 
     // Backingstore texture with the normal widgets.
     GLuint textureId = 0;
@@ -364,7 +360,6 @@ void QPlatformBackingStore::composeAndFlush(QWindow *window, const QRegion &regi
             blit(textures, i, window, deviceWindowRect, d_ptr->blitter);
     }
 
-    funcs->glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
     funcs->glDisable(GL_BLEND);
     d_ptr->blitter->release();
 

src/widgets/kernel/qopenglwidget.cpp

@@ -425,7 +425,12 @@ QT_BEGIN_NAMESPACE
 
   Note that this does not apply when there are no other widgets underneath and
   the intention is to have a semi-transparent window. In that case the
-  traditional approach of setting Qt::WA_TranslucentBackground is sufficient.
+  traditional approach of setting Qt::WA_TranslucentBackground
+  on the top-level window is sufficient. Note that if the transparent areas are
+  only desired in the QOpenGLWidget, then Qt::WA_NoSystemBackground will need
+  to be turned back to \c false after enabling Qt::WA_TranslucentBackground.
+  Additionally, requesting an alpha channel for the QOpenGLWidget's context via
+  setFormat() may be necessary too, depending on the system.
 
   QOpenGLWidget supports multiple update behaviors, just like QOpenGLWindow. In
   preserved mode the rendered content from the previous paintGL() call is
@@ -607,7 +612,7 @@ void QOpenGLWidgetPaintDevicePrivate::beginPaint()
     // with the palette's background color.
     if (w->autoFillBackground()) {
         QOpenGLFunctions *f = QOpenGLContext::currentContext()->functions();
-        if (w->testAttribute(Qt::WA_TranslucentBackground)) {
+        if (w->format().hasAlpha()) {
             f->glClearColor(0, 0, 0, 0);
         } else {
             QColor c = w->palette().brush(w->backgroundRole()).color();
@@ -955,13 +960,12 @@ QOpenGLWidget::UpdateBehavior QOpenGLWidget::updateBehavior() const
   OpenGL widgets, individual calls to this function can be replaced by one single call to
   QSurfaceFormat::setDefaultFormat() before creating the first widget.
 
-  \note Requesting an alpha buffer via this function, or by setting
-  Qt::WA_TranslucentBackground, will not lead to the desired results when the intention is
-  to make other widgets beneath visible. Instead, use Qt::WA_AlwaysStackOnTop to enable
-  semi-transparent QOpenGLWidget instances with other widgets visible underneath. Keep in
-  mind however that this breaks the stacking order, so it will no longer be possible to
-  have other widgets on top of the QOpenGLWidget. When the intention is to have a
-  semi-transparent top-level window, Qt::WA_TranslucentBackground is sufficient.
+  \note Requesting an alpha buffer via this function will not lead to the
+  desired results when the intention is to make other widgets beneath visible.
+  Instead, use Qt::WA_AlwaysStackOnTop to enable semi-transparent QOpenGLWidget
+  instances with other widgets visible underneath. Keep in mind however that
+  this breaks the stacking order, so it will no longer be possible to have
+  other widgets on top of the QOpenGLWidget.
 
   \sa format(), Qt::WA_AlwaysStackOnTop, QSurfaceFormat::setDefaultFormat()
  */