Enhance the qtwasmtestlib with comparison functions and status reporting

Added the functionality to report text statuses from tests, reporting
file and line of assertion failures. Refactored the qtwasmtestlib.js
for improved stability.

Task-number: QTBUG-99611
Change-Id: I717e0cc38ac7f155fe870710f6b5e4bfb81b9317
Reviewed-by: Morten Johan Sørvig <morten.sorvig@qt.io>

tests/manual/wasm/qtwasmtestlib/README.md

@@ -14,7 +14,10 @@ Implementing a basic test case
 
 In the test cpp file, define the test functions as private slots. All test
 functions must call completeTestFunction() exactly once, or will time out
-otherwise. The can can be made after the test function itself has returned.
+otherwise. Subsequent calls to completeTestFunction will be disregarded.
+It is advised to use QWASMSUCCESS/QWASMFAIL for reporting the test execution
+status and QWASMCOMPARE/QWASMVERIFY to assert on test conditions. The call can
+be made after the test function itself has returned.
 
     class TestTest: public QObject
     {

tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.cpp

@@ -10,9 +10,11 @@
 #include <emscripten/threading.h>
 
 namespace QtWasmTest {
-
+namespace {
 QObject *g_testObject = nullptr;
+std::string g_currentTestName;
 std::function<void ()> g_cleanup;
+}
 
 void runOnMainThread(std::function<void(void)> fn);
 static bool isValidSlot(const QMetaMethod &sl);
@@ -30,24 +32,25 @@ void initTestCase(QObject *testObject, std::function<void ()> cleanup)
     g_cleanup = cleanup;
 }
 
+void verify(bool condition, std::string_view conditionString, std::string_view file, int line)
+{
+    if (!condition) {
+        completeTestFunction(
+            TestResult::Fail,
+            formatMessage(file, line, "Condition failed: " + std::string(conditionString)));
+    }
+}
+
 // Completes the currently running test function with a result. This function is
 // thread-safe and call be called from any thread.
-void completeTestFunction(TestResult result)
+void completeTestFunction(TestResult result, std::string message)
 {
+
     // Report test result to JavaScript test runner, on the main thread
-    runOnMainThread([result](){
-        const char *resultString;
-        switch (result) {
-            case TestResult::Pass:
-                resultString = "PASS";
-            break;
-            case TestResult::Fail:
-                resultString = "FAIL";
-            break;
-        };
+    runOnMainThread([resultString = result == TestResult::Pass ? "PASS" : "FAIL", message](){
         EM_ASM({
-            completeTestFunction(UTF8ToString($0));
-        }, resultString);
+            completeTestFunction(UTF8ToString($0), UTF8ToString($1), UTF8ToString($2));
+        }, g_currentTestName.c_str(), resultString, message.c_str());
     });
 }
 
@@ -55,6 +58,11 @@ void completeTestFunction(TestResult result)
 // Private API for the Javascript test runnner
 //
 
+std::string formatMessage(std::string_view file, int line, std::string_view message)
+{
+    return "[" + std::string(file) + ":" + QString::number(line).toStdString() + "] " + std::string(message);
+}
+
 void cleanupTestCase()
 {
     g_testObject = nullptr;
@@ -83,6 +91,8 @@ std::string getTestFunctions()
 
 void runTestFunction(std::string name)
 {
+    g_currentTestName = name;
+    QMetaObject::invokeMethod(g_testObject, "init");
     QMetaObject::invokeMethod(g_testObject, name.c_str());
 }
 

tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.h

@@ -14,10 +14,17 @@ enum TestResult {
     Pass,
     Fail,
 };
-void completeTestFunction(TestResult result = TestResult::Pass);
+
+std::string formatMessage(std::string_view file,
+                          int line,
+                          std::string_view message);
+
+void completeTestFunction(TestResult result, std::string message);
 void initTestCase(QObject *testObject, std::function<void ()> cleanup);
 template <typename App>
-void initTestCase(int argc, char **argv, std::shared_ptr<QObject> testObject)
+void initTestCase(int argc,
+                  char **argv,
+                std::shared_ptr<QObject> testObject)
 {
     auto app = std::make_shared<App>(argc, argv);
     auto cleanup = [testObject, app]() mutable {
@@ -28,8 +35,37 @@ void initTestCase(int argc, char **argv, std::shared_ptr<QObject> testObject)
     };
     initTestCase(testObject.get(), cleanup);
 }
+void verify(bool condition,
+            std::string_view conditionString,
+            std::string_view file,
+            int line);
 
+template<class L, class R>
+void compare(const L& lhs,
+             const R& rhs,
+             std::string_view lhsString,
+             std::string_view rhsString,
+             std::string_view file,
+             int line) {
+    if (lhs != rhs) {
+        completeTestFunction(
+            TestResult::Fail,
+            formatMessage(file, line, "Comparison failed: " + std::string(lhsString) + " == " + std::string(rhsString)));
+    }
 }
 
-#endif
+}  // namespace QtWasmTest
 
+#define QWASMVERIFY(condition) \
+    QtWasmTest::verify((condition), #condition, __FILE__, __LINE__);
+
+#define QWASMCOMPARE(left, right) \
+    QtWasmTest::compare((left), (right), #left, #right, __FILE__, __LINE__);
+
+#define QWASMSUCCESS() \
+    QtWasmTest::completeTestFunction(QtWasmTest::Pass, "")
+
+#define QWASMFAIL(message) \
+    QtWasmTest::completeTestFunction(QtWasmTest::Fail, QtWasmTest::formatMessage(__FILE__, __LINE__, message))
+
+#endif

tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.js

@@ -68,72 +68,80 @@
 // is the part which enables async testing). Test functions which fail
 // to call completeTestFunction() will time out after 2000ms.
 //
-let g_maxTime = 2000;
-var g_timeoutId = undefined;
-var g_testResolve = undefined;
-var g_testResult = undefined;
+const g_maxTime = 2000;
 
-function completeTestFunction(result)
-{
-    // Reset timeout
-    if (g_timeoutId !== undefined) {
-        clearTimeout(g_timeoutId);
-        g_timeoutId = undefined;
+class TestFunction {
+    constructor(instance, name) {
+        this.instance = instance;
+        this.name = name;
+        this.resolve = undefined;
+        this.reject = undefined;
+        this.timeoutId = undefined;
     }
 
-    // Set test result directy, or resolve the pending promise
-    if (g_testResolve === undefined) {
-        g_testResult = result
-    } else {
-        g_testResolve(result);
-        g_testResolve = undefined;
+    complete(result, details) {
+        // Reset timeout
+        clearTimeout(this.timeoutId);
+        this.timeoutId = undefined;
+
+        const callback = result.startsWith('FAIL') ? this.reject : this.resolve;
+        callback(`${result}${details ? ': ' + details : ''}`);
     }
-}
 
-function runTestFunction(instance, name)
-{
-    if (g_timeoutId !== undefined)
-        console.log("existing timer found");
+    run() {
+        // Set timer which will catch test functions
+        // which fail to call completeTestFunction()
+        this.timeoutId = setTimeout(() => {
+            completeTestFunction(this.name, 'FAIL', `Timeout after ${g_maxTime} ms`)
+        }, g_maxTime);
 
-    // Set timer which will catch test functions
-    // which fail to call completeTestFunction()
-    g_timeoutId = setTimeout( () => {
-        if (g_timeoutId === undefined)
-            return;
-        g_timeoutId = undefined;
-        completeTestFunction("FAIL")
-    }, g_maxTime);
+        return new Promise((resolve, reject) => {
+            this.resolve = resolve;
+            this.reject = reject;
 
-    instance.runTestFunction(name);
-
-    // If the test function completed with a result immediately then return
-    // the result directly, otherwise return a Promise to the result.
-    if (g_testResult !== undefined) {
-        let result = g_testResult;
-        g_testResult = undefined;
-        return result;
-    } else {
-        return new Promise((resolve) => {
-            g_testResolve = resolve;
+            this.instance.runTestFunction(this.name);
         });
     }
+};
+
+function completeTestFunction(testFunctionName, result, details) {
+    if (!window.currentTestFunction || testFunctionName !== window.currentTestFunction.name)
+        return;
+
+    window.currentTestFunction.complete(result, details);
 }
 
-async function runTestCaseImpl(testFunctionStarted, testFunctionCompleted, qtContainers)
-{
-    // Create test case instance
-    let config = {
-        qtContainerElements : qtContainers || []
+async function runTestFunction(instance, name) {
+    if (window.currentTestFunction) {
+        throw new Error(`While trying to run ${name}: Last function hasn't yet finished`);
     }
-    let instance = await createQtAppInstance(config);
+    window.currentTestFunction = new TestFunction(instance, name);
+    try {
+        const result = await window.currentTestFunction.run();
+        return result;
+    } finally {
+        delete window.currentTestFunction;
+    }
+}
+
+async function runTestCaseImpl(testFunctionStarted, testFunctionCompleted, qtContainers) {
+    // Create test case instance
+    const config = {
+        qtContainerElements: qtContainers || []
+    }
+    const instance = await createQtAppInstance(config);
 
     // Run all test functions
-    let functionsString = instance.getTestFunctions();
-    let functions = functionsString.split(" ").filter(Boolean);
-    for (name of functions) {
+    const functionsString = instance.getTestFunctions();
+    const functions = functionsString.split(" ").filter(Boolean);
+    for (const name of functions) {
         testFunctionStarted(name);
-        let result = await runTestFunction(instance, name);
-        testFunctionCompleted(name, result);
+        try {
+            const result = await runTestFunction(instance, name);
+            testFunctionCompleted(result);
+        } catch (err) {
+            testFunctionCompleted(err.message ?? err);
+        }
     }
 
     // Cleanup
@@ -147,22 +155,13 @@ function testFunctionStarted(name) {
     g_htmlLogElement.innerHTML += line;
 }
 
-function testFunctionCompleted(name, status) {
-    var color = "black";
-    switch (status) {
-        case "PASS":
-            color = "green";
-        break;
-        case "FAIL":
-            color = "red";
-        break;
-    }
-    let line = "<text style='color:" + color + ";'>" + status + "</text><br>";
+function testFunctionCompleted(status) {
+    const color = status.startsWith("PASS") ? "green" : status.startsWith("FAIL") ? "red" : "black";
+    let line = `<span style='color: ${color};'>${status}</text><br>`;
     g_htmlLogElement.innerHTML += line;
 }
 
-async function runTestCase(htmlLogElement, qtContainers)
-{
+async function runTestCase(htmlLogElement, qtContainers) {
     g_htmlLogElement = htmlLogElement;
     try {
         await runTestCaseImpl(testFunctionStarted, testFunctionCompleted, qtContainers);