Add a private API to capture a graphics frame

This commit provides a minimal API for capturing a graphics
frame, save it and replay it later for debugging. The intention
here is to provide the basic need for future work to allow capturing
through tooling or programmatically from code. This API is
intended to be cross-platform by using Metal Capture Manager on
Apple devices and RenderDoc C++ API everywhere else.

Task-number: QTBUG-114067
Change-Id: If72d92bdef5e5985a0ec2e85e97fd1182da3c53c
Reviewed-by: Janne Koskinen <janne.p.koskinen@qt.io>

cmake/FindRenderDoc.cmake

@@ -0,0 +1,20 @@
+# Copyright (C) 2023 The Qt Company Ltd.
+# SPDX-License-Identifier: BSD-3-Clause
+
+if(WIN32 OR UNIX)
+    find_path(RenderDoc_INCLUDE_DIR
+        NAMES renderdoc_app.h)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(RenderDoc
+    DEFAULT_MSG
+    RenderDoc_INCLUDE_DIR)
+
+mark_as_advanced(RenderDoc_INCLUDE_DIR)
+
+if(RenderDoc_FOUND AND NOT TARGET RenderDoc::RenderDoc)
+    add_library(RenderDoc::RenderDoc INTERFACE IMPORTED)
+    set_target_properties(RenderDoc::RenderDoc PROPERTIES
+        INTERFACE_INCLUDE_DIRECTORIES "${RenderDoc_INCLUDE_DIR}")
+endif()

src/gui/CMakeLists.txt

@@ -428,6 +428,28 @@ qt_internal_extend_target(Gui CONDITION WIN32
         d3d12
 )
 
+if(QT_FEATURE_graphicsframecapture)
+    qt_internal_extend_target(Gui
+        SOURCES
+            util/qgraphicsframecapture_p.h util/qgraphicsframecapture.cpp
+            util/qgraphicsframecapture_p_p.h
+    )
+
+    qt_internal_extend_target(Gui CONDITION WIN32 OR (UNIX AND NOT APPLE)
+        LIBRARIES
+            RenderDoc::RenderDoc
+        SOURCES
+            util/qgraphicsframecapturerenderdoc_p_p.h util/qgraphicsframecapturerenderdoc.cpp
+    )
+
+    qt_internal_extend_target(Gui CONDITION IOS OR MACOS
+        SOURCES
+            util/qgraphicsframecapturemetal_p_p.h util/qgraphicsframecapturemetal.mm
+        PUBLIC_LIBRARIES
+            ${FWMetal}
+    )
+endif()
+
 if(QT_FEATURE_egl)
     qt_find_package(EGL)
 endif()

src/gui/configure.cmake

@@ -142,6 +142,7 @@ if((X11_SUPPORTED) OR QT_FIND_ALL_PACKAGES_ALWAYS)
 endif()
 qt_add_qmake_lib_dependency(xrender xlib)
 
+qt_find_package(RenderDoc)
 
 #### Tests
 
@@ -613,6 +614,20 @@ int main(int, char **)
 }
 ")
 
+qt_config_compile_test(renderdoc
+    LIBRARIES
+        RenderDoc::RenderDoc
+    LABEL "RenderDoc header check"
+    CODE
+"#include <renderdoc_app.h>
+int main(int, char **)
+{
+    if (RENDERDOC_Version::eRENDERDOC_API_Version_1_6_0)
+        return 0;
+    return 0;
+}
+")
+
 
 #### Features
 
@@ -1220,6 +1235,12 @@ qt_feature("undogroup" PUBLIC
     PURPOSE "Provides the ability to cluster QUndoCommands."
     CONDITION QT_FEATURE_undostack
 )
+qt_feature("graphicsframecapture" PRIVATE
+    SECTION "Utilities"
+    LABEL "QGraphicsFrameCapture"
+    PURPOSE "Provides a way to capture a graphic's API calls for a rendered frame."
+    CONDITION TEST_renderdoc OR (MACOS OR IOS)
+)
 qt_feature_definition("undogroup" "QT_NO_UNDOGROUP" NEGATE VALUE "1")
 qt_configure_add_summary_section(NAME "Qt Gui")
 qt_configure_add_summary_entry(ARGS "accessibility")

src/gui/util/qgraphicsframecapture.cpp

@@ -0,0 +1,91 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#include "qgraphicsframecapture_p.h"
+#if defined (Q_OS_WIN) || defined(Q_OS_LINUX)
+#include "qgraphicsframecapturerenderdoc_p_p.h"
+#elif defined(Q_OS_MACOS) || defined(Q_OS_IOS)
+#include "qgraphicsframecapturemetal_p_p.h"
+#endif
+
+#include <QtCore/qstandardpaths.h>
+#include <QtCore/qcoreapplication.h>
+
+QT_BEGIN_NAMESPACE
+
+QGraphicsFrameCapturePrivate::QGraphicsFrameCapturePrivate()
+    : m_capturePath(QStandardPaths::writableLocation(QStandardPaths::TempLocation) +
+                    QStringLiteral("/") + QCoreApplication::applicationName() +
+                    QStringLiteral("/captures")),
+      m_capturePrefix(QCoreApplication::applicationName())
+{
+
+}
+
+QGraphicsFrameCapture::QGraphicsFrameCapture()
+{
+#if defined (Q_OS_WIN) || defined(Q_OS_LINUX)
+    d.reset(new QGraphicsFrameCaptureRenderDoc);
+#elif defined(Q_OS_MACOS) || defined(Q_OS_IOS)
+    d.reset(new QGraphicsFrameCaptureMetal);
+#endif
+}
+
+QGraphicsFrameCapture::~QGraphicsFrameCapture()
+{
+
+}
+
+void QGraphicsFrameCapture::setRhi(QRhi *rhi)
+{
+    if (!d.isNull())
+        d->setRhi(rhi);
+}
+
+void QGraphicsFrameCapture::startCaptureFrame()
+{
+    if (!d.isNull())
+        d->startCaptureFrame();
+}
+
+void QGraphicsFrameCapture::endCaptureFrame()
+{
+    if (!d.isNull())
+        d->endCaptureFrame();
+}
+
+void QGraphicsFrameCapture::setCapturePath(const QString &path)
+{
+    if (!d.isNull())
+        d->setCapturePath(path);
+}
+
+void QGraphicsFrameCapture::setCapturePrefix(const QString &prefix)
+{
+    if (!d.isNull())
+        d->setCapturePrefix(prefix);
+}
+
+bool QGraphicsFrameCapture::isLoaded() const
+{
+    if (!d.isNull())
+        return d->initialized();
+
+    return false;
+}
+
+bool QGraphicsFrameCapture::isCapturing() const
+{
+    if (!d.isNull())
+        return d->isCapturing();
+
+    return false;
+}
+
+void QGraphicsFrameCapture::openCapture() const
+{
+    if (!d.isNull())
+        d->openCapture();
+}
+
+QT_END_NAMESPACE

src/gui/util/qgraphicsframecapture_p.h

@@ -0,0 +1,47 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef QGRAPHICSFRAMECAPTURE_P_H
+#define QGRAPHICSFRAMECAPTURE_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists purely as an
+// implementation detail.  This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include <QtCore/qscopedpointer.h>
+#include <QtGui/qtguiglobal.h>
+
+QT_BEGIN_NAMESPACE
+
+class QGraphicsFrameCapturePrivate;
+class QRhi;
+
+class Q_GUI_EXPORT QGraphicsFrameCapture
+{
+public:
+    QGraphicsFrameCapture();
+    ~QGraphicsFrameCapture();
+
+    void setRhi(QRhi *rhi);
+    void startCaptureFrame();
+    void endCaptureFrame();
+    void setCapturePath(const QString &path);
+    void setCapturePrefix(const QString &prefix);
+    bool isLoaded() const;
+    bool isCapturing() const;
+    void openCapture() const;
+
+private:
+    QScopedPointer<QGraphicsFrameCapturePrivate> d;
+};
+
+QT_END_NAMESPACE
+
+#endif // QGRAPHICSFRAMECAPTURE_P_H

src/gui/util/qgraphicsframecapture_p_p.h

@@ -0,0 +1,55 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef QGRAPHICSFRAMECAPTURE_P_P_H
+#define QGRAPHICSFRAMECAPTURE_P_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists purely as an
+// implementation detail.  This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include <QtCore/qnamespace.h>
+#include <QtCore/qstring.h>
+#include <QtCore/qloggingcategory.h>
+
+class QRhi;
+
+QT_BEGIN_NAMESPACE
+
+Q_DECLARE_LOGGING_CATEGORY(lcGraphicsFrameCapture)
+
+class QRhi;
+struct QRhiNativeHandles;
+class QGraphicsFrameCapturePrivate
+{
+public:
+    QGraphicsFrameCapturePrivate() ;
+    virtual ~QGraphicsFrameCapturePrivate() = default;
+
+    virtual void setRhi(QRhi *rhi) = 0;
+    virtual void startCaptureFrame() = 0;
+    virtual void endCaptureFrame() = 0;
+    virtual void setCapturePath(const QString &path) { m_capturePath = path; }
+    virtual void setCapturePrefix(const QString &prefix) { m_capturePrefix = prefix; }
+    virtual bool initialized() const = 0;
+    virtual bool isCapturing() const = 0;
+    virtual void openCapture() = 0;
+
+protected:
+    QRhi *m_rhi = nullptr;
+    QRhiNativeHandles *m_rhiHandles = nullptr;
+    void *m_nativeHandle = nullptr;
+    QString m_capturePath;
+    QString m_capturePrefix;
+};
+
+QT_END_NAMESPACE
+
+#endif // QGRAPHICSFRAMECAPTURE_P_P_H

src/gui/util/qgraphicsframecapturemetal.mm

@@ -0,0 +1,164 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#include "qgraphicsframecapturemetal_p_p.h"
+#include <QtCore/qurl.h>
+#include "Metal/Metal.h"
+#include "qglobal.h"
+#include <QtGui/rhi/qrhi.h>
+#include <QtGui/rhi/qrhi_platform.h>
+
+QT_BEGIN_NAMESPACE
+
+Q_LOGGING_CATEGORY(lcGraphicsFrameCapture, "qt.gui.graphicsframecapture")
+
+#if __has_feature(objc_arc)
+#error ARC not supported
+#endif
+
+uint QGraphicsFrameCaptureMetal::frameNumber = 0;
+
+QGraphicsFrameCaptureMetal::QGraphicsFrameCaptureMetal()
+{
+    qputenv("METAL_CAPTURE_ENABLED", QByteArrayLiteral("1"));
+
+    m_captureDescriptor = [MTLCaptureDescriptor new];
+}
+
+QGraphicsFrameCaptureMetal::~QGraphicsFrameCaptureMetal()
+{
+#ifdef Q_OS_MACOS
+    if (m_process) {
+        m_process->terminate();
+        delete m_process;
+    }
+#endif
+    [m_captureDescriptor release];
+}
+
+void QGraphicsFrameCaptureMetal::setRhi(QRhi *rhi)
+{
+    if (!rhi)
+        return;
+
+    QRhi::Implementation backend = rhi->backend();
+    const QRhiNativeHandles *nh = rhi->nativeHandles();
+
+    switch (backend) {
+    case QRhi::Implementation::Metal: {
+        const  QRhiMetalNativeHandles *mtlnh = static_cast<const QRhiMetalNativeHandles *>(nh);
+        if (mtlnh->cmdQueue) {
+            m_captureDescriptor.captureObject = mtlnh->cmdQueue;
+        } else if (mtlnh->dev) {
+            m_captureDescriptor.captureObject = mtlnh->dev;
+        } else {
+            qCWarning(lcGraphicsFrameCapture) << "No valid Metal Device or Metal Command Queue found";
+            m_initialized = false;
+            return;
+        }
+        break;
+    }
+    default: {
+        qCWarning(lcGraphicsFrameCapture) << "Invalid handles were provided. MTLCaptureManager works only with Metal API";
+        m_initialized = false;
+        return;
+        break;
+    }
+    }
+
+    if (!m_captureManager) {
+        m_captureManager = MTLCaptureManager.sharedCaptureManager;
+        bool supportDocs = [m_captureManager supportsDestination:MTLCaptureDestinationGPUTraceDocument];
+        if (supportDocs) {
+            m_captureDescriptor.destination = MTLCaptureDestinationGPUTraceDocument;
+            m_initialized = true;
+        }
+    }
+}
+
+void QGraphicsFrameCaptureMetal::startCaptureFrame()
+{
+    if (!initialized()) {
+        qCWarning(lcGraphicsFrameCapture) << "Capturing on Metal was not initialized. Starting capturing can not be done.";
+        return;
+    }
+
+    if (isCapturing()) {
+        qCWarning(lcGraphicsFrameCapture) << "A frame capture is already in progress,"
+                                             "will not initiate another one until QGraphicsFrameCapture::endCaptureFrame is called.";
+        return;
+    }
+
+    updateCaptureFileName();
+    NSError *error;
+    if (![m_captureManager startCaptureWithDescriptor:m_captureDescriptor error:&error]) {
+        QString errorMsg = QString::fromNSString(error.localizedDescription);
+        qCWarning(lcGraphicsFrameCapture, "Failed to start capture : %s", qPrintable(errorMsg));
+    }
+}
+
+void QGraphicsFrameCaptureMetal::endCaptureFrame()
+{
+    if (!initialized()) {
+        qCWarning(lcGraphicsFrameCapture) << "Capturing on Metal was not initialized. End capturing can not be done.";
+        return;
+    }
+
+    if (!isCapturing()) {
+        qCWarning(lcGraphicsFrameCapture) << "A call to QGraphicsFrameCapture::endCaptureFrame can not be done"
+                                             " without a call to QGraphicsFrameCapture::startCaptureFrame";
+        return;
+    }
+
+    [m_captureManager stopCapture];
+    frameNumber++;
+}
+
+bool QGraphicsFrameCaptureMetal::initialized() const
+{
+    return m_initialized;
+}
+
+bool QGraphicsFrameCaptureMetal::isCapturing() const
+{
+    if (!initialized()) {
+        qCWarning(lcGraphicsFrameCapture) << "Capturing on Metal was not initialized. Can not query if capturing is in progress or not.";
+        return false;
+    }
+
+    return [m_captureManager isCapturing];
+}
+
+void QGraphicsFrameCaptureMetal::openCapture()
+{
+#ifdef Q_OS_MACOS
+    if (!initialized()) {
+        qCWarning(lcGraphicsFrameCapture) << "Capturing on Metal was not initialized. Can not open XCode with a valid capture.";
+        return;
+    }
+
+    if (!m_process) {
+        m_process = new QProcess();
+        m_process->setProgram(QStringLiteral("xed"));
+        QStringList args;
+        args.append(QUrl::fromNSURL(m_traceURL).toLocalFile());
+        m_process->setArguments(args);
+    }
+
+    m_process->kill();
+    m_process->start();
+#endif
+}
+
+void QGraphicsFrameCaptureMetal::updateCaptureFileName()
+{
+    m_traceURL = QUrl::fromLocalFile(m_capturePath + "/" + m_capturePrefix + "_" + QString::number(frameNumber) + ".gputrace").toNSURL();
+    // We need to remove the trace file if it already existed else MTLCaptureManager
+    // will fail to.
+    if ([NSFileManager.defaultManager fileExistsAtPath:m_traceURL.path])
+        [NSFileManager.defaultManager removeItemAtURL:m_traceURL error:nil];
+
+    m_captureDescriptor.outputURL = m_traceURL;
+}
+
+QT_END_NAMESPACE

src/gui/util/qgraphicsframecapturemetal_p_p.h

@@ -0,0 +1,57 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef QGRAPHICSFRAMEDECAPTUREMETAL_P_P_H
+#define QGRAPHICSFRAMEDECAPTUREMETAL_P_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists purely as an
+// implementation detail.  This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include "qgraphicsframecapture_p_p.h"
+#include <QtCore/qmutex.h>
+#ifdef Q_OS_MACOS
+#include <QtCore/qprocess.h>
+#endif
+
+Q_FORWARD_DECLARE_OBJC_CLASS(MTLCaptureManager);
+Q_FORWARD_DECLARE_OBJC_CLASS(MTLCaptureDescriptor);
+Q_FORWARD_DECLARE_OBJC_CLASS(NSURL);
+
+QT_BEGIN_NAMESPACE
+
+class QGraphicsFrameCaptureMetal : public QGraphicsFrameCapturePrivate
+{
+public:
+    QGraphicsFrameCaptureMetal();
+    ~QGraphicsFrameCaptureMetal();
+
+    void setRhi(QRhi *rhi) override;
+    void startCaptureFrame() override;
+    void endCaptureFrame() override;
+    bool initialized() const override;
+    bool isCapturing() const override;
+    void openCapture() override;
+
+private:
+    void updateCaptureFileName();
+#ifdef Q_OS_MACOS
+    QProcess *m_process = nullptr;
+#endif
+    MTLCaptureManager *m_captureManager = nullptr;
+    MTLCaptureDescriptor *m_captureDescriptor = nullptr;
+    NSURL *m_traceURL = nullptr;
+    bool m_initialized = false;
+    static uint frameNumber;
+};
+
+QT_END_NAMESPACE
+
+#endif // QGRAPHICSFRAMEDECAPTUREMETAL_P_P_H

src/gui/util/qgraphicsframecapturerenderdoc.cpp

@@ -0,0 +1,287 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#include "qgraphicsframecapturerenderdoc_p_p.h"
+#include <QtCore/qcoreapplication.h>
+#include <QtCore/qfile.h>
+#include <QtCore/qlibrary.h>
+#include <QtCore/qmutex.h>
+#include "QtGui/rhi/qrhi.h"
+#include "QtGui/rhi/qrhi_platform.h"
+#include "QtGui/qopenglcontext.h"
+
+QT_BEGIN_NAMESPACE
+
+Q_LOGGING_CATEGORY(lcGraphicsFrameCapture, "qt.gui.graphicsframecapture")
+
+RENDERDOC_API_1_6_0 *QGraphicsFrameCaptureRenderDoc::s_rdocApi = nullptr;
+QBasicMutex QGraphicsFrameCaptureRenderDoc::s_frameCaptureMutex;
+
+static void *glNativeContext(QOpenGLContext *context) {
+    void *nctx = nullptr;
+    if (context != nullptr && context->isValid()) {
+#ifdef Q_OS_WIN
+        nctx = context->nativeInterface<QNativeInterface::QWGLContext>()->nativeContext();
+#endif
+
+#ifdef Q_OS_LINUX
+#if QT_CONFIG(egl)
+        QNativeInterface::QEGLContext *eglItf = context->nativeInterface<QNativeInterface::QEGLContext>();
+        if (eglItf)
+            nctx = eglItf->nativeContext();
+#endif
+
+#if QT_CONFIG(xcb_glx_plugin)
+        QNativeInterface::QGLXContext *glxItf = context->nativeInterface<QNativeInterface::QGLXContext>();
+        if (glxItf)
+            nctx = glxItf->nativeContext();
+#endif
+#endif
+
+#if defined(Q_OS_MACOS) || defined(Q_OS_IOS)
+        nctx = context->nativeInterface<QNativeInterface::QCocoaGLContext>()->nativeContext();
+#endif
+    }
+    return nctx;
+}
+/*!
+    \class QGraphicsFrameCaptureRenderDoc
+    \internal
+    \brief The QGraphicsFrameCaptureRenderDoc class provides a way to capture a record of draw calls
+           for different graphics APIs.
+    \since 6.6
+    \inmodule QtGui
+
+    For applications that render using graphics APIs like Vulkan or OpenGL, it would be
+    convenient to have a way to check the draw calls done by the application. Specially
+    for applications that make a large amount of draw calls and the output is different
+    from what is expected.
+
+    This class acts as a wrapper over \l {https://renderdoc.org/}{RenderDoc} that allows
+    applications to capture a rendered frame either programmatically, by clicking a key
+    on the keyboard or both. The captured frame could be viewed later using RenderDoc GUI.
+
+    Read the \l {https://renderdoc.org/docs/index.html} {RenderDoc Documentation}
+    for more information.
+
+    \section1 API device handle
+
+    The functions that capture a frame like QGraphicsFrameCaptureRenderDoc::startCaptureFrame takes a device
+    pointer as argument. This pointer is unique for each graphics API and is associated
+    with the window that will be captured. This pointer has a default value of \c nullptr.
+    If no value is passed to the function the underlying API will try to find the device to
+    use, but it is not guaranteed specially in a multi-window applications.
+
+    For OpenGL, the pointer should be the OpenGL context on the platform OpenGL is being
+    used. For example, on Windows it should be \c HGLRC.
+
+    For Vulkan, the pointer should point to the dispatch table within the \c VkInstance.
+*/
+
+
+
+/*!
+    Creates a new object of this class. The constructor will load RenderDoc library
+    from the default path.
+
+    Only one instance of RenderDoc library is loaded at runtime which means creating
+    several instances of this class will not affect the RenderDoc initialization.
+*/
+
+QGraphicsFrameCaptureRenderDoc::QGraphicsFrameCaptureRenderDoc()
+    : m_nativeHandlesSet(false)
+{
+    if (!s_rdocApi)
+        init();
+}
+
+void QGraphicsFrameCaptureRenderDoc::setRhi(QRhi *rhi)
+{
+    if (!rhi)
+        return;
+
+    QRhi::Implementation backend = rhi->backend();
+    const QRhiNativeHandles *nh = rhi->nativeHandles();
+
+    switch (backend) {
+    case QRhi::Implementation::D3D11: {
+#ifdef Q_OS_WIN
+        const  QRhiD3D11NativeHandles *d3d11nh = static_cast<const QRhiD3D11NativeHandles *>(nh);
+        m_nativeHandle = d3d11nh->dev;
+        break;
+#endif
+        qCWarning(lcGraphicsFrameCapture) << "Could not find valid handles for D3D11. Check platform support";
+        break;
+    }
+    case QRhi::Implementation::D3D12: {
+#ifdef Q_OS_WIN
+        const  QRhiD3D12NativeHandles *d3d12nh = static_cast<const QRhiD3D12NativeHandles *>(nh);
+        m_nativeHandle = d3d12nh->dev;
+        break;
+#endif
+        qCWarning(lcGraphicsFrameCapture) << "Could not find valid handles for D3D12. Check platform support";
+        break;
+    }
+    case QRhi::Implementation::Vulkan: {
+#if QT_CONFIG(vulkan)
+        const  QRhiVulkanNativeHandles *vknh = static_cast<const QRhiVulkanNativeHandles *>(nh);
+        m_nativeHandle = RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(vknh->inst->vkInstance());
+        break;
+#endif
+        qCWarning(lcGraphicsFrameCapture) << "Could not find valid handles for Vulkan. Check platform support";
+        break;
+    }
+    case QRhi::Implementation::OpenGLES2: {
+#ifndef QT_NO_OPENGL
+        const  QRhiGles2NativeHandles *glnh = static_cast<const QRhiGles2NativeHandles *>(nh);
+        m_nativeHandle = glNativeContext(glnh->context);
+        if (m_nativeHandle)
+            break;
+#endif
+        qCWarning(lcGraphicsFrameCapture) << "Could not find valid handles for OpenGL. Check platform support";
+        break;
+    }
+    case QRhi::Implementation::Metal:
+    case QRhi::Implementation::Null:
+        qCWarning(lcGraphicsFrameCapture) << "Invalid handles were provided."
+                                             " Metal and Null backends are not supported with RenderDoc";
+        break;
+    }
+
+    if (m_nativeHandle)
+        m_nativeHandlesSet = true;
+}
+
+/*!
+    Starts a frame capture using the set native handles provided through QGraphicsFrameCaptureRenderDoc::setRhi
+    \a device. This function must be called before QGraphicsFrameCaptureRenderDoc::endCaptureFrame.
+    \sa {API device handle}
+*/
+void QGraphicsFrameCaptureRenderDoc::startCaptureFrame()
+{
+
+    if (!initialized()) {
+        qCWarning(lcGraphicsFrameCapture) << "RenderDoc was not initialized."
+                                             " Starting capturing can not be done.";
+        return;
+    }
+
+    // There is a single instance of RenderDoc library and it needs mutex for multithreading access.
+    QMutexLocker locker(&s_frameCaptureMutex);
+    if (s_rdocApi->IsFrameCapturing()) {
+        qCWarning(lcGraphicsFrameCapture) << "A frame capture is already in progress, "
+                                             "will not initiate another one until"
+                                             " QGraphicsFrameCapture::endCaptureFrame is called.";
+        return;
+    }
+
+    qCInfo(lcGraphicsFrameCapture) << "A frame capture is going to start.";
+    updateCapturePathAndTemplate();
+    s_rdocApi->StartFrameCapture(m_nativeHandle, nullptr);
+}
+
+/*!
+    Ends a frame capture started by a call to QGraphicsFrameCaptureRenderDoc::startCaptureFrame
+    using the set native handles provided through QGraphicsFrameCaptureRenderDoc::setRhi.
+    This function must be called after QGraphicsFrameCaptureRenderDoc::startCaptureFrame.
+    Otherwise, a warning message will be printend and nothing will happen.
+    \sa {API device handle}
+*/
+void QGraphicsFrameCaptureRenderDoc::endCaptureFrame()
+{
+    if (!initialized()) {
+        qCWarning(lcGraphicsFrameCapture) << "RenderDoc was not initialized."
+                                             " End capturing can not be done.";
+        return;
+    }
+
+    // There is a single instance of RenderDoc library and it needs mutex for multithreading access.
+    QMutexLocker locker(&s_frameCaptureMutex);
+    if (!s_rdocApi->IsFrameCapturing()) {
+        qCWarning(lcGraphicsFrameCapture) << "A call to QGraphicsFrameCapture::endCaptureFrame can not be done"
+                                             " without a call to QGraphicsFrameCapture::startCaptureFrame";
+        return;
+    }
+
+    qCInfo(lcGraphicsFrameCapture) << "A frame capture is going to end.";
+    s_rdocApi->EndFrameCapture(m_nativeHandle, nullptr);
+}
+
+void QGraphicsFrameCaptureRenderDoc::updateCapturePathAndTemplate()
+{
+    if (!initialized()) {
+        qCWarning(lcGraphicsFrameCapture) << "RenderDoc was not initialized."
+                                             " Updating save location can not be done.";
+        return;
+    }
+
+
+    QString rdocFilePathTemplate = m_capturePath + QStringLiteral("/") + m_capturePrefix;
+    s_rdocApi->SetCaptureFilePathTemplate(rdocFilePathTemplate.toUtf8().constData());
+}
+
+/*!
+    Returns true if the API is loaded and can capture frames or not.
+*/
+bool QGraphicsFrameCaptureRenderDoc::initialized() const
+{
+    return s_rdocApi && m_nativeHandlesSet;
+}
+
+bool QGraphicsFrameCaptureRenderDoc::isCapturing() const
+{
+    if (!initialized()) {
+        qCWarning(lcGraphicsFrameCapture) << "RenderDoc was not initialized."
+                                             " Can not query if capturing is in progress or not.";
+        return false;
+    }
+
+    return s_rdocApi->IsFrameCapturing();
+}
+
+void QGraphicsFrameCaptureRenderDoc::openCapture()
+{
+    if (!initialized()) {
+        qCWarning(lcGraphicsFrameCapture) << "RenderDoc was not initialized."
+                                             " Can not open RenderDoc UI tool.";
+        return;
+    }
+
+    // There is a single instance of RenderDoc library and it needs mutex for multithreading access.
+    QMutexLocker locker(&s_frameCaptureMutex);
+    if (s_rdocApi->IsTargetControlConnected())
+        s_rdocApi->ShowReplayUI();
+    else
+        s_rdocApi->LaunchReplayUI(1, nullptr);
+}
+
+void QGraphicsFrameCaptureRenderDoc::init()
+{
+
+    // There is a single instance of RenderDoc library and it needs mutex for multithreading access.
+    QMutexLocker locker(&s_frameCaptureMutex);
+
+    QLibrary renderDocLib(QStringLiteral("renderdoc"));
+    pRENDERDOC_GetAPI RENDERDOC_GetAPI = (pRENDERDOC_GetAPI) renderDocLib.resolve("RENDERDOC_GetAPI");
+    if (!renderDocLib.isLoaded() || (RENDERDOC_GetAPI == nullptr)) {
+        qCWarning(lcGraphicsFrameCapture) << renderDocLib.errorString().toLatin1();
+        return;
+    }
+
+    int ret = RENDERDOC_GetAPI(eRENDERDOC_API_Version_1_6_0, static_cast<void **>(static_cast<void *>(&s_rdocApi)));
+
+    if (ret == 0) {
+        qCWarning(lcGraphicsFrameCapture) << "The requested RenderDoc API is invalid or not supported";
+        return;
+    }
+
+    s_rdocApi->MaskOverlayBits(RENDERDOC_OverlayBits::eRENDERDOC_Overlay_None,
+                               RENDERDOC_OverlayBits::eRENDERDOC_Overlay_None);
+    s_rdocApi->SetCaptureKeys(nullptr, 0);
+    s_rdocApi->SetFocusToggleKeys(nullptr, 0);
+
+    QString rdocFilePathTemplate = m_capturePath + QStringLiteral("/") + m_capturePrefix;
+    s_rdocApi->SetCaptureFilePathTemplate(rdocFilePathTemplate.toUtf8().constData());
+}
+
+QT_END_NAMESPACE

src/gui/util/qgraphicsframecapturerenderdoc_p_p.h

@@ -0,0 +1,49 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef QGRAPHICSFRAMECAPTURERENDERDOC_P_P_H
+#define QGRAPHICSFRAMECAPTURERENDERDOC_P_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists purely as an
+// implementation detail.  This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+
+#include <renderdoc_app.h>
+#include "qgraphicsframecapture_p_p.h"
+
+QT_BEGIN_NAMESPACE
+
+class QBasicMutex;
+
+class QGraphicsFrameCaptureRenderDoc : public QGraphicsFrameCapturePrivate
+{
+public:
+    QGraphicsFrameCaptureRenderDoc();
+    ~QGraphicsFrameCaptureRenderDoc() = default;
+
+    void setRhi(QRhi *rhi) override;
+    void startCaptureFrame() override;
+    void endCaptureFrame() override;
+    bool initialized() const override;
+    bool isCapturing() const override;
+    void openCapture() override;
+
+private:
+    void init();
+    void updateCapturePathAndTemplate();
+    static RENDERDOC_API_1_5_0 *s_rdocApi;
+    static QBasicMutex s_frameCaptureMutex;
+    bool m_nativeHandlesSet;
+};
+
+QT_END_NAMESPACE
+
+#endif // QGRAPHICSFRAMECAPTURERENDERDOC_P_P_H