wasm: begin work on accessibility backend

Implement a11y support by adding html elements of the
appropriate type and/or with the appropriate ARIA attribute
behind the canvas.

Also add a simple manual-test.

Change-Id: I2898fb038c1d326135a1341cdee323bc964420bb
Reviewed-by: Lorn Potter <lorn.potter@gmail.com>

src/plugins/platforms/wasm/CMakeLists.txt

@@ -11,6 +11,7 @@ qt_internal_add_plugin(QWasmIntegrationPlugin
     STATIC
     SOURCES
         main.cpp
+        qwasmaccessibility.cpp qwasmaccessibility.h
         qwasmclipboard.cpp qwasmclipboard.h
         qwasmcompositor.cpp qwasmcompositor.h
         qwasmcursor.cpp qwasmcursor.h

src/plugins/platforms/wasm/qwasmaccessibility.cpp

@@ -0,0 +1,250 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+
+#include "qwasmaccessibility.h"
+#include "qwasmscreen.h"
+
+// Qt WebAssembly a11y backend
+//
+// This backend implements accessibility support by creating "shadowing" html
+// elements for each Qt UI element. We access the DOM by using Emscripten's
+// val.h API.
+//
+// Currently, html elements are created in response to notifyAccessibilityUpdate
+// events. In addition or alternatively, we could also walk the accessibility tree
+// from setRootObject().
+
+
+QWasmAccessibility::QWasmAccessibility()
+{
+
+}
+
+QWasmAccessibility::~QWasmAccessibility()
+{
+
+}
+
+emscripten::val QWasmAccessibility::getContainer(QAccessibleInterface *iface)
+{
+    // Get to QWasmScreen::container(), return undefined element if unable to
+    QWindow *window = iface->window();
+    if (!window)
+        return emscripten::val::undefined();
+    QWasmScreen *screen = QWasmScreen::get(window->screen());
+    if (!screen)
+        return emscripten::val::undefined();
+    return screen->container();
+}
+
+emscripten::val QWasmAccessibility::getDocument(const emscripten::val &container)
+{
+    if (container.isUndefined())
+        return emscripten::val::undefined();
+    return container["ownerDocument"];
+}
+
+emscripten::val QWasmAccessibility::getDocument(QAccessibleInterface *iface)
+{
+    return getDocument(getContainer(iface));
+}
+
+emscripten::val QWasmAccessibility::createHtmlElement(QAccessibleInterface *iface)
+{
+    // Get the html container element for the interface; this depends on which
+    // QScreen it is on. If the interface is not on a screen yet we get an undefined
+    // container, and the code below handles that case as well.
+    emscripten::val container = getContainer(iface);
+
+    // Get the correct html document for the container, or fall back
+    // to the global document. TODO: Does using the correct document actually matter?
+    emscripten::val document = container.isUndefined() ? emscripten::val::global("document") : getDocument(container);
+
+    // Translate the Qt a11y elemen role into html element type + ARIA role.
+    // Here we can either create <div> elements with a spesific ARIA role,
+    // or create e.g. <button> elements which should have built-in accessibility.
+    emscripten::val element = [iface, document] {
+
+        emscripten::val element = emscripten::val::undefined();
+
+        switch (iface->role()) {
+
+        case QAccessible::Button: {
+            element = document.call<emscripten::val>("createElement", std::string("button"));
+        } break;
+
+        case QAccessible::CheckBox: {
+            element = document.call<emscripten::val>("createElement", std::string("input"));
+            element.call<void>("setAttribute", std::string("type"), std::string("checkbox"));
+        } break;
+
+        default:
+            qDebug() << "TODO: createHtmlElement() handle" << iface->role();
+            element = document.call<emscripten::val>("createElement", std::string("div"));
+            //element.set("AriaRole", "foo");
+        }
+
+        return element;
+
+    }();
+
+    // Add the html element to the container if we have one. If not there
+    // is a second chance when handling the ObjectShow event.
+    if (!container.isUndefined())
+        container.call<void>("appendChild", element);
+
+    return element;
+}
+
+void QWasmAccessibility::destroyHtmlElement(QAccessibleInterface *iface)
+{
+    Q_UNUSED(iface);
+    qDebug() << "TODO destroyHtmlElement";
+}
+
+emscripten::val QWasmAccessibility::ensureHtmlElement(QAccessibleInterface *iface)
+{
+    auto it = m_elements.find(iface);
+    if (it != m_elements.end())
+        return it.value();
+
+    emscripten::val element = createHtmlElement(iface);
+    m_elements.insert(iface, element);
+
+    return element;
+}
+
+void QWasmAccessibility::setHtmlElementVisibility(QAccessibleInterface *iface, bool visible)
+{
+    emscripten::val element = ensureHtmlElement(iface);
+    emscripten::val container = getContainer(iface);
+
+    if (container.isUndefined()) {
+        qDebug() << "TODO: setHtmlElementVisibility: unable to find html container for element" << iface;
+        return;
+    }
+
+    container.call<void>("appendChild", element);
+
+    element.set("ariaHidden", !visible); // ariaHidden mean completely hidden; maybe some sort of soft-hidden should be used.
+}
+
+void QWasmAccessibility::setHtmlElementGeometry(QAccessibleInterface *iface)
+{
+    emscripten::val element = ensureHtmlElement(iface);
+    setHtmlElementGeometry(iface, element);
+}
+
+void QWasmAccessibility::setHtmlElementGeometry(QAccessibleInterface *iface, emscripten::val element)
+{
+    // Position the element using "position: absolute" in order to place
+    // it under the corresponding Qt element on the canvas.
+    QRect geometry = iface->rect();
+    emscripten::val style = element["style"];
+    style.set("position", std::string("absolute"));
+    style.set("z-index", std::string("-1")); // FIXME: "0" should be sufficient to order beheind the canvas, but isn't
+    style.set("left", std::to_string(geometry.x()) + "px");
+    style.set("top", std::to_string(geometry.y()) + "px");
+    style.set("width", std::to_string(geometry.width()) + "px");
+    style.set("height", std::to_string(geometry.height()) + "px");
+}
+
+void QWasmAccessibility::setHtmlElementTextName(QAccessibleInterface *iface)
+{
+    emscripten::val element = ensureHtmlElement(iface);
+    QString text = iface->text(QAccessible::Name);
+    element.set("innerHTML", text.toStdString()); // FIXME: use something else than innerHTML
+}
+
+void QWasmAccessibility::handleStaticTextUpdate(QAccessibleEvent *event)
+{
+    switch (event->type()) {
+    case QAccessible::NameChanged: {
+        setHtmlElementTextName(event->accessibleInterface());
+    } break;
+    default:
+        qDebug() << "TODO: implement handleStaticTextUpdate for event" << event->type();
+    break;
+    }
+}
+
+void QWasmAccessibility::handleButtonUpdate(QAccessibleEvent *event)
+{
+    qDebug() << "TODO: implement handleButtonUpdate for event" << event->type();
+}
+
+void QWasmAccessibility::handleCheckBoxUpdate(QAccessibleEvent *event)
+{
+    switch (event->type()) {
+    case QAccessible::NameChanged: {
+        setHtmlElementTextName(event->accessibleInterface());
+    } break;
+    default:
+        qDebug() << "TODO: implement handleCheckBoxUpdate for event" << event->type();
+    break;
+    }
+}
+
+void QWasmAccessibility::notifyAccessibilityUpdate(QAccessibleEvent *event)
+{
+    QAccessibleInterface *iface = event->accessibleInterface();
+    if (!iface) {
+        qWarning() << "notifyAccessibilityUpdate with null a11y interface" ;
+        return;
+    }
+
+    // Handle some common event types. See
+    // https://doc.qt.io/qt-5/qaccessible.html#Event-enum
+    switch (event->type()) {
+    case QAccessible::ObjectShow:
+
+        setHtmlElementVisibility(iface, true);
+
+        // Sync up properties on show;
+        setHtmlElementGeometry(iface);
+        setHtmlElementTextName(iface);
+
+        return;
+    break;
+    case QAccessible::ObjectHide:
+        setHtmlElementVisibility(iface, false);
+        return;
+    break;
+    // TODO: maybe handle more types here
+    default:
+    break;
+    };
+
+    // Switch on interface role, see
+    // https://doc.qt.io/qt-5/qaccessibleinterface.html#role
+    switch (iface->role()) {
+    case QAccessible::StaticText:
+        handleStaticTextUpdate(event);
+    break;
+    case QAccessible::Button:
+        handleStaticTextUpdate(event);
+    break;
+    case QAccessible::CheckBox:
+        handleCheckBoxUpdate(event);
+    break;
+    default:
+        qDebug() << "TODO: implement notifyAccessibilityUpdate for role" << iface->role();
+    };
+}
+
+void QWasmAccessibility::setRootObject(QObject *o)
+{
+    qDebug() << "setRootObject" << o;
+    QAccessibleInterface *iface = QAccessible::queryAccessibleInterface(o);
+    Q_UNUSED(iface)
+}
+
+void QWasmAccessibility::initialize()
+{
+
+}
+
+void QWasmAccessibility::cleanup()
+{
+
+}

src/plugins/platforms/wasm/qwasmaccessibility.h

@@ -0,0 +1,44 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+
+#ifndef QWASMACCESIBILITY_H
+#define QWASMACCESIBILITY_H
+
+#include <QtCore/qhash.h>
+#include <qpa/qplatformaccessibility.h>
+
+#include <emscripten/val.h>
+
+class QWasmAccessibility : public QPlatformAccessibility
+{
+public:
+    QWasmAccessibility();
+    ~QWasmAccessibility();
+
+    static emscripten::val getContainer(QAccessibleInterface *iface);
+    static emscripten::val getDocument(const emscripten::val &container);
+    static emscripten::val getDocument(QAccessibleInterface *iface);
+
+    emscripten::val createHtmlElement(QAccessibleInterface *iface);
+    void destroyHtmlElement(QAccessibleInterface *iface);
+    emscripten::val ensureHtmlElement(QAccessibleInterface *iface);
+    void setHtmlElementVisibility(QAccessibleInterface *iface, bool visible);
+    void setHtmlElementGeometry(QAccessibleInterface *iface);
+    void setHtmlElementGeometry(QAccessibleInterface *iface, emscripten::val element);
+    void setHtmlElementTextName(QAccessibleInterface *iface);
+
+    void handleStaticTextUpdate(QAccessibleEvent *event);
+    void handleButtonUpdate(QAccessibleEvent *event);
+    void handleCheckBoxUpdate(QAccessibleEvent *event);
+
+    void notifyAccessibilityUpdate(QAccessibleEvent *event) override;
+    void setRootObject(QObject *o) override;
+    void initialize() override;
+    void cleanup() override;
+
+private:
+    QHash<QAccessibleInterface *, emscripten::val> m_elements;
+
+};
+
+#endif

src/plugins/platforms/wasm/qwasmintegration.cpp

@@ -8,6 +8,7 @@
 #include "qwasmopenglcontext.h"
 #include "qwasmtheme.h"
 #include "qwasmclipboard.h"
+#include "qwasmaccessibility.h"
 #include "qwasmservices.h"
 #include "qwasmoffscreensurface.h"
 #include "qwasmstring.h"
@@ -79,7 +80,8 @@ QWasmIntegration *QWasmIntegration::s_instance;
 QWasmIntegration::QWasmIntegration()
     : m_fontDb(nullptr),
       m_desktopServices(nullptr),
-      m_clipboard(new QWasmClipboard)
+      m_clipboard(new QWasmClipboard),
+      m_accessibility(new QWasmAccessibility)
 {
     s_instance = this;
 
@@ -170,6 +172,7 @@ QWasmIntegration::~QWasmIntegration()
     if (m_platformInputContext)
         delete m_platformInputContext;
     delete m_drag;
+    delete m_accessibility;
 
     for (const auto &elementAndScreen : m_screens)
         elementAndScreen.second->deleteScreen();
@@ -299,6 +302,14 @@ QPlatformClipboard* QWasmIntegration::clipboard() const
     return m_clipboard;
 }
 
+#ifndef QT_NO_ACCESSIBILITY
+QPlatformAccessibility *QWasmIntegration::accessibility() const
+{
+    return m_accessibility;
+}
+#endif
+
+
 void QWasmIntegration::addScreen(const emscripten::val &element)
 {
     QWasmScreen *screen = new QWasmScreen(element);

src/plugins/platforms/wasm/qwasmintegration.h

@@ -33,6 +33,7 @@ class QWasmScreen;
 class QWasmCompositor;
 class QWasmBackingStore;
 class QWasmClipboard;
+class QWasmAccessibility;
 class QWasmServices;
 
 class QWasmIntegration : public QObject, public QPlatformIntegration
@@ -66,6 +67,9 @@ public:
     QPlatformTheme *createPlatformTheme(const QString &name) const override;
     QPlatformServices *services() const override;
     QPlatformClipboard *clipboard() const override;
+#ifndef QT_NO_ACCESSIBILITY
+    QPlatformAccessibility *accessibility() const override;
+#endif
     void initialize() override;
     QPlatformInputContext *inputContext() const override;
 
@@ -94,6 +98,8 @@ private:
     mutable QHash<QWindow *, QWasmBackingStore *> m_backingStores;
     QList<QPair<emscripten::val, QWasmScreen *>> m_screens;
     mutable QWasmClipboard *m_clipboard;
+    mutable QWasmAccessibility *m_accessibility;
+
     qreal m_fontDpi = -1;
     mutable QScopedPointer<QPlatformInputContext> m_inputContext;
     static QWasmIntegration *s_instance;

src/plugins/platforms/wasm/qwasmscreen.cpp

@@ -51,8 +51,15 @@ QWasmScreen::QWasmScreen(const emscripten::val &containerOrCanvas)
         style.set("height", std::string("100%"));
     }
 
-    // Configure canvas
     emscripten::val style = m_canvas["style"];
+
+    // Configure container and canvas for accessibility support: set "position: relative"
+    // so that a11y child elements can be positioned with "position: absolute", and hide
+    // the canvas from screen readers.
+    m_container["style"].set("position", std::string("relative"));
+    m_canvas.call<void>("setAttribute", std::string("aria-hidden"), std::string("true")); // FIXME make the canvas non-focusable, as required by the aria-hidden role
+    style.set("z-index", std::string("1")); // a11y elements are at 0
+
     style.set("border", std::string("0px none"));
     style.set("background-color", std::string("white"));
 
@@ -122,6 +129,8 @@ QWasmScreen *QWasmScreen::get(QPlatformScreen *screen)
 
 QWasmScreen *QWasmScreen::get(QScreen *screen)
 {
+    if (!screen)
+        return nullptr;
     return get(screen->handle());
 }
 

tests/manual/wasm/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_subdirectory(eventloop)
 add_subdirectory(rasterwindow)
+add_subdirectory(a11y)
 if(QT_FEATURE_widgets)
 add_subdirectory(cursors)
 add_subdirectory(localfiles)

tests/manual/wasm/a11y/CMakeLists.txt

@@ -0,0 +1,3 @@
+if(QT_FEATURE_widgets)
+add_subdirectory(basic_widgets)
+endif()

tests/manual/wasm/a11y/basic_widgets/CMakeLists.txt

@@ -0,0 +1,17 @@
+qt_internal_add_manual_test(a11y_basic_widgets
+    GUI
+    SOURCES
+        main.cpp
+    PUBLIC_LIBRARIES
+        Qt::Core
+        Qt::Gui
+        Qt::Widgets
+)
+
+add_custom_command(
+    TARGET a11y_basic_widgets PRE_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy
+            ${CMAKE_CURRENT_SOURCE_DIR}/basic_widgets.html
+            ${CMAKE_CURRENT_BINARY_DIR}/basic_widgets.html
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/basic_widgets.html
+)

tests/manual/wasm/a11y/basic_widgets/basic_widgets.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+
+<script src="a11y_basic_widgets.js" async></script>
+
+<script>
+    window.onload = async () => {
+        let qt_instance = await createQtAppInstance({
+            qtContainerElements: [document.getElementById("qt_container")],
+        });
+    }
+</script>
+
+
+</head>
+
+<body>
+    <h1>Qt Accessibility Tester</H1>
+    <div id="qt_container" style="width:640px; height:640px"></div>
+</body>
+
+</html>

tests/manual/wasm/a11y/basic_widgets/main.cpp

@@ -0,0 +1,33 @@
+// Copyright (C) 2019 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR BSD-3-Clause
+
+#include <QtWidgets>
+
+class BasicA11yWidget: public QWidget
+{
+public:
+    BasicA11yWidget() {
+
+        QVBoxLayout *layout = new QVBoxLayout();
+
+        layout->addWidget(new QLabel("This is a text label"));
+        layout->addWidget(new QPushButton("This is a push button"));
+        layout->addWidget(new QCheckBox("This is a check box"));
+
+        // TODO: Add more widgets
+
+        layout->addStretch();
+
+        setLayout(layout);
+    }
+};
+
+int main(int argc, char **argv)
+{
+    QApplication app(argc, argv);
+
+    BasicA11yWidget a11yWidget;
+    a11yWidget.show();
+
+    return app.exec();
+}