direct2d: Fix composition mode support

When the composition mode changes to a mode which is not supported by
Direct2D's primitive blending, the rendering follows the emulated (slow)
code path using rasterFill(). This allows the direct2d paint engine to
handle all composition modes supported by QImage.

Task-number: QTBUG-40602
Change-Id: I0ac0b5c89aab2483cb2ef7768d6dec8e16913249
Done-with: Andrew Knight <andrew.knight@digia.com>
Reviewed-by: Andrew Knight <andrew.knight@digia.com>

src/plugins/platforms/direct2d/qwindowsdirect2ddevicecontext.cpp

@@ -125,4 +125,38 @@ bool QWindowsDirect2DDeviceContext::end()
     return d->end();
 }
 
+void QWindowsDirect2DDeviceContext::suspend()
+{
+    Q_D(QWindowsDirect2DDeviceContext);
+    if (d->refCount > 0)
+        d->deviceContext->EndDraw();
+}
+
+void QWindowsDirect2DDeviceContext::resume()
+{
+    Q_D(QWindowsDirect2DDeviceContext);
+    if (d->refCount > 0)
+        d->deviceContext->BeginDraw();
+}
+
+QWindowsDirect2DDeviceContextSuspender::QWindowsDirect2DDeviceContextSuspender(QWindowsDirect2DDeviceContext *dc)
+    : m_dc(dc)
+{
+    Q_ASSERT(m_dc);
+    m_dc->suspend();
+}
+
+QWindowsDirect2DDeviceContextSuspender::~QWindowsDirect2DDeviceContextSuspender()
+{
+    resume();
+}
+
+void QWindowsDirect2DDeviceContextSuspender::resume()
+{
+    if (m_dc) {
+        m_dc->resume();
+        m_dc = Q_NULLPTR;
+    }
+}
+
 QT_END_NAMESPACE

src/plugins/platforms/direct2d/qwindowsdirect2ddevicecontext.h

@@ -69,6 +69,7 @@ class QWindowsDirect2DDeviceContextPrivate;
 class QWindowsDirect2DDeviceContext
 {
     Q_DECLARE_PRIVATE(QWindowsDirect2DDeviceContext)
+    friend class QWindowsDirect2DDeviceContextSuspender;
 public:
     QWindowsDirect2DDeviceContext(ID2D1DeviceContext *dc);
     ~QWindowsDirect2DDeviceContext();
@@ -79,9 +80,23 @@ public:
     bool end();
 
 private:
+    void suspend();
+    void resume();
+
     QScopedPointer<QWindowsDirect2DDeviceContextPrivate> d_ptr;
 };
 
+class QWindowsDirect2DDeviceContextSuspender {
+    Q_DISABLE_COPY(QWindowsDirect2DDeviceContextSuspender)
+
+    QWindowsDirect2DDeviceContext *m_dc;
+public:
+    QWindowsDirect2DDeviceContextSuspender(QWindowsDirect2DDeviceContext *dc);
+    ~QWindowsDirect2DDeviceContextSuspender();
+
+    void resume();
+};
+
 QT_END_NAMESPACE
 
 #endif // QWINDOWSDIRECT2DDEVICECONTEXT_H

src/plugins/platforms/direct2d/qwindowsdirect2dpaintengine.cpp

@@ -230,6 +230,7 @@ public:
     }
 
     QWindowsDirect2DBitmap *bitmap;
+    QImage fallbackImage;
 
     unsigned int clipFlags;
     QStack<ClipType> pushedClips;
@@ -393,7 +394,10 @@ public:
             break;
 
         default:
-            qWarning("Unsupported composition mode: %d", mode);
+            // Activating an unsupported mode at any time will cause the QImage
+            // fallback to be used for the remainder of the active paint session
+            dc()->SetPrimitiveBlend(D2D1_PRIMITIVE_BLEND_COPY);
+            flags |= QWindowsDirect2DPaintEngine::EmulateComposition;
             break;
         }
     }
@@ -968,7 +972,17 @@ bool QWindowsDirect2DPaintEngine::begin(QPaintDevice * pdev)
 bool QWindowsDirect2DPaintEngine::end()
 {
     Q_D(QWindowsDirect2DPaintEngine);
-    // First pop any user-applied clipping
+
+    // Always clear all emulation-related things so we are in a clean state for our next painting run
+    const bool emulatingComposition = d->flags.testFlag(EmulateComposition);
+    d->flags &= ~QWindowsDirect2DPaintEngine::EmulateComposition;
+    if (!d->fallbackImage.isNull()) {
+        if (emulatingComposition)
+            drawImage(d->fallbackImage.rect(), d->fallbackImage, d->fallbackImage.rect());
+        d->fallbackImage = QImage();
+    }
+
+    // Pop any user-applied clipping
     d->clearClips();
     // Now the system clip from begin() above
     if (d->clipFlags & SimpleSystemClip) {
@@ -977,6 +991,7 @@ bool QWindowsDirect2DPaintEngine::end()
     } else {
         d->dc()->PopLayer();
     }
+
     return d->bitmap->deviceContext()->end();
 }
 
@@ -1406,6 +1421,19 @@ void QWindowsDirect2DPaintEngine::drawPixmap(const QRectF &r,
         return;
     }
 
+    if (d->flags.testFlag(EmulateComposition)) {
+        const qreal points[] = {
+            r.x(), r.y(),
+            r.x() + r.width(), r.y(),
+            r.x() + r.width(), r.y() + r.height(),
+            r.x(), r.y() + r.height()
+        };
+        const QVectorPath vp(points, 4, 0, QVectorPath::RectangleHint);
+        const QBrush brush(sr.isValid() ? pm.copy(sr.toRect()) : pm);
+        rasterFill(vp, brush);
+        return;
+    }
+
     QWindowsDirect2DPlatformPixmap *pp = static_cast<QWindowsDirect2DPlatformPixmap *>(pm.handle());
     QWindowsDirect2DBitmap *bitmap = pp->bitmap();
 
@@ -1589,9 +1617,17 @@ void QWindowsDirect2DPaintEngine::rasterFill(const QVectorPath &path, const QBru
 {
     Q_D(QWindowsDirect2DPaintEngine);
 
-    QImage img(d->bitmap->size(), QImage::Format_ARGB32);
-    img.fill(Qt::transparent);
+    if (d->fallbackImage.isNull()) {
+        if (d->flags.testFlag(EmulateComposition)) {
+            QWindowsDirect2DPaintEngineSuspender suspender(this);
+            d->fallbackImage = d->bitmap->toImage();
+        } else {
+            d->fallbackImage = QImage(d->bitmap->size(), QImage::Format_ARGB32_Premultiplied);
+            d->fallbackImage.fill(Qt::transparent);
+        }
+    }
 
+    QImage &img = d->fallbackImage;
     QPainter p;
     QPaintEngine *engine = img.paintEngine();
 
@@ -1638,11 +1674,14 @@ void QWindowsDirect2DPaintEngine::rasterFill(const QVectorPath &path, const QBru
         if (!p.end())
             qWarning("%s: Paint Engine end returned false", __FUNCTION__);
 
-        d->updateClipEnabled(false);
-        d->updateTransform(QTransform());
-        drawImage(img.rect(), img, img.rect());
-        transformChanged();
-        clipEnabledChanged();
+        if (!d->flags.testFlag(EmulateComposition)) { // Emulated fallback will be flattened in end()
+            d->updateClipEnabled(false);
+            d->updateTransform(QTransform());
+            drawImage(img.rect(), img, img.rect());
+            d->fallbackImage = QImage();
+            transformChanged();
+            clipEnabledChanged();
+        }
     } else {
         qWarning("%s: Could not fall back to QImage", __FUNCTION__);
     }
@@ -1652,6 +1691,9 @@ bool QWindowsDirect2DPaintEngine::emulationRequired(EmulationType type) const
 {
     Q_D(const QWindowsDirect2DPaintEngine);
 
+    if (d->flags.testFlag(EmulateComposition))
+        return true;
+
     if (!state()->matrix.isAffine())
         return true;
 
@@ -1691,4 +1733,71 @@ void QWindowsDirect2DPaintEngine::adjustForAliasing(QPointF *point)
         (*point) += adjustment;
 }
 
+void QWindowsDirect2DPaintEngine::suspend()
+{
+    end();
+}
+
+void QWindowsDirect2DPaintEngine::resume()
+{
+    begin(paintDevice());
+    clipEnabledChanged();
+    penChanged();
+    brushChanged();
+    brushOriginChanged();
+    opacityChanged();
+    compositionModeChanged();
+    renderHintsChanged();
+    transformChanged();
+}
+
+class QWindowsDirect2DPaintEngineSuspenderImpl
+{
+    Q_DISABLE_COPY(QWindowsDirect2DPaintEngineSuspenderImpl)
+    QWindowsDirect2DPaintEngine *m_engine;
+    bool m_active;
+public:
+    QWindowsDirect2DPaintEngineSuspenderImpl(QWindowsDirect2DPaintEngine *engine)
+        : m_engine(engine)
+        , m_active(engine->isActive())
+    {
+        if (m_active)
+            m_engine->suspend();
+    }
+
+    ~QWindowsDirect2DPaintEngineSuspenderImpl()
+    {
+        if (m_active)
+            m_engine->resume();
+    }
+};
+
+class QWindowsDirect2DPaintEngineSuspenderPrivate
+{
+public:
+    QWindowsDirect2DPaintEngineSuspenderPrivate(QWindowsDirect2DPaintEngine *engine)
+        : engineSuspender(engine)
+        , dcSuspender(static_cast<QWindowsDirect2DPaintEnginePrivate *>(engine->d_ptr.data())->bitmap->deviceContext())
+    {
+    }
+
+    QWindowsDirect2DPaintEngineSuspenderImpl engineSuspender;
+    QWindowsDirect2DDeviceContextSuspender dcSuspender;
+};
+
+QWindowsDirect2DPaintEngineSuspender::QWindowsDirect2DPaintEngineSuspender(QWindowsDirect2DPaintEngine *engine)
+    : d_ptr(new QWindowsDirect2DPaintEngineSuspenderPrivate(engine))
+{
+
+}
+
+QWindowsDirect2DPaintEngineSuspender::~QWindowsDirect2DPaintEngineSuspender()
+{
+}
+
+void QWindowsDirect2DPaintEngineSuspender::resume()
+{
+    d_ptr.reset();
+}
+
 QT_END_NAMESPACE

src/plugins/platforms/direct2d/qwindowsdirect2dpaintengine.h

@@ -49,11 +49,13 @@ class QWindowsDirect2DBitmap;
 class QWindowsDirect2DPaintEngine : public QPaintEngineEx
 {
     Q_DECLARE_PRIVATE(QWindowsDirect2DPaintEngine)
-
+    friend class QWindowsDirect2DPaintEngineSuspenderImpl;
+    friend class QWindowsDirect2DPaintEngineSuspenderPrivate;
 public:
     enum Flag {
         NoFlag = 0,
-        TranslucentTopLevelWindow = 1
+        TranslucentTopLevelWindow = 1,
+        EmulateComposition = 2,
     };
     Q_DECLARE_FLAGS(Flags, Flag)
 
@@ -116,9 +118,24 @@ private:
     bool antiAliasingEnabled() const;
     void adjustForAliasing(QRectF *rect);
     void adjustForAliasing(QPointF *point);
+
+    void suspend();
+    void resume();
 };
 Q_DECLARE_OPERATORS_FOR_FLAGS(QWindowsDirect2DPaintEngine::Flags)
 
+class QWindowsDirect2DPaintEngineSuspenderPrivate;
+class QWindowsDirect2DPaintEngineSuspender
+{
+    Q_DISABLE_COPY(QWindowsDirect2DPaintEngineSuspender)
+    Q_DECLARE_PRIVATE(QWindowsDirect2DPaintEngineSuspender)
+    QScopedPointer<QWindowsDirect2DPaintEngineSuspenderPrivate> d_ptr;
+public:
+    QWindowsDirect2DPaintEngineSuspender(QWindowsDirect2DPaintEngine *engine);
+    ~QWindowsDirect2DPaintEngineSuspender();
+    void resume();
+};
+
 QT_END_NAMESPACE
 
 #endif // QWINDOWSDIRECT2DPAINTENGINE_H

src/plugins/platforms/direct2d/qwindowsdirect2dplatformpixmap.cpp

@@ -161,17 +161,8 @@ QImage QWindowsDirect2DPlatformPixmap::toImage(const QRect &rect) const
 {
     Q_D(const QWindowsDirect2DPlatformPixmap);
 
-    bool active = d->device->paintEngine()->isActive();
-
-    if (active)
-        d->device->paintEngine()->end();
-
-    QImage result = d->bitmap->toImage(rect);
-
-    if (active)
-        d->device->paintEngine()->begin(d->device.data());
-
-    return result;
+    QWindowsDirect2DPaintEngineSuspender suspender(static_cast<QWindowsDirect2DPaintEngine *>(d->device->paintEngine()));
+    return d->bitmap->toImage(rect);
 }
 
 QPaintEngine* QWindowsDirect2DPlatformPixmap::paintEngine() const