testlib: add QFINDTESTDATA macro for finding testdata files

Automated tests often need to load some data from external files.
Currently, a wide variety of approaches for this have been used in Qt
autotests, including:

  - embed the source directory into the test binary at compile time, and
    find the testdata relative to that; this fails when the source tree
    is no longer available (e.g. when the tests are deployed to a device).

  - use a path relative to the current working directory, and trust that
    the caller always sets the current working directory such that the
    testdata can be found; this fails when the caller uses a different
    working directory than expected.

  - use a path relative to QCoreApplication::applicationDirPath();
    this fails when source tree != build tree (since testdata is not
    automatically copied into the build tree).

  - compile the files into the binary using the Qt resource system; this
    should work, but does not allow for testing of code which genuinely
    needs external files.

It seems that there is not a simple method for determining the testdata
path which can be reliably used in all circumstances, so various tests
have reinvented the testdata location method in different ways.
Therefore, this is a good candidate for an addition to the testlib API.

The current implementation of QFINDTESTDATA is able to find testdata
in all three of (build tree, install tree, source tree), in that order.

Change-Id: Ib2fed860723ccf437240da3b00db22dfe1a6b56c
Reviewed-by: Jason McDonald <jason.mcdonald@nokia.com>

doc/src/snippets/code/src_qtestlib_qtestcase.cpp

@@ -236,5 +236,18 @@ QTest::touchEvent(&widget)
     .release(1);
 //! [25]
 
+
+//! [26]
+// Source: /home/user/sources/myxmlparser/tests/tst_myxmlparser/tst_myxmlparser.cpp
+// Build:  /home/user/build/myxmlparser/tests/tst_myxmlparser
+// Qt:     /usr/local/Qt-5.0.0
+void tst_MyXmlParser::parse()
+{
+    MyXmlParser parser;
+    QString input = QFINDTESTDATA("testxml/simple1.xml");
+    QVERIFY(parser.parse(input));
+}
+//! [26]
+
 }
 

src/modules/qt_testlib.pri

@@ -13,4 +13,4 @@ QT.testlib.plugins = $$QT_MODULE_PLUGIN_BASE
 QT.testlib.imports = $$QT_MODULE_IMPORT_BASE
 QT.testlib.depends = core
 QT.testlib.CONFIG = console
-QT.testlib.DEFINES = QT_TESTLIB_LIB
+QT.testlib.DEFINES = QT_TESTLIB_LIB QT_TESTCASE_BUILDDIR=\\\"$$OUT_PWD\\\"

src/testlib/qtestcase.cpp

@@ -54,6 +54,7 @@
 #include <QtCore/qdir.h>
 #include <QtCore/qprocess.h>
 #include <QtCore/qdebug.h>
+#include <QtCore/qlibraryinfo.h>
 
 #include <QtTest/private/qtestlog_p.h>
 #include <QtTest/private/qtesttable_p.h>
@@ -335,6 +336,54 @@ QT_BEGIN_NAMESPACE
    \sa QTest::TestFailMode, QVERIFY(), QCOMPARE()
 */
 
+/*! \macro QFINDTESTDATA(filename)
+
+   \relates QTest
+
+   Returns a QString for the testdata file referred to by \a filename, or an
+   empty QString if the testdata file could not be found.
+
+   This macro allows the test to load data from an external file without
+   hardcoding an absolute filename into the test, or using relative paths
+   which may be error prone.
+
+   The returned path will be the first path from the following list which
+   resolves to an existing file or directory:
+
+   \list
+   \o \a filename relative to QCoreApplication::applicationDirPath()
+      (only if a QCoreApplication or QApplication object has been created).
+   \o \a filename relative to the test's standard install directory
+      (QLibraryInfo::TestsPath with the lowercased testcase name appended).
+   \o \a filename relative to the directory containing the source file from which
+      QFINDTESTDATA is invoked.
+   \endlist
+
+   If the named file/directory does not exist at any of these locations,
+   a warning is printed to the test log.
+
+   For example, in this code:
+   \snippet doc/src/snippets/code/src_qtestlib_qtestcase.cpp 26
+
+   The testdata file will be resolved as the first existing file from:
+
+   \list
+   \o \c{/home/user/build/myxmlparser/tests/tst_myxmlparser/testxml/simple1.xml}
+   \o \c{/usr/local/Qt-5.0.0/tests/tst_myxmlparser/testxml/simple1.xml}
+   \o \c{/home/user/sources/myxmlparser/tests/tst_myxmlparser/testxml/simple1.xml}
+   \endlist
+
+   This allows the test to find its testdata regardless of whether the
+   test has been installed, and regardless of whether the test's build tree
+   is equal to the test's source tree.
+
+   \bold {Note:} reliable detection of testdata from the source directory requires
+   either that qmake is used, or the \c{QT_TESTCASE_BUILDDIR} macro is defined to
+   point to the working directory from which the compiler is invoked, or only
+   absolute paths to the source files are passed to the compiler. Otherwise, the
+   absolute path of the source directory cannot be determined.
+*/
+
 /*! \macro QTEST_MAIN(TestClass)
 
     \relates QTest
@@ -2081,6 +2130,96 @@ void QTest::ignoreMessage(QtMsgType type, const char *message)
     QTestResult::ignoreMessage(type, message);
 }
 
+/*! \internal
+ */
+QString QTest::qFindTestData(const QString& base, const char *file, int line, const char *builddir)
+{
+    QString found;
+
+    // Testdata priorities:
+
+    //  1. relative to test binary.
+    if (qApp) {
+        QString binpath = QCoreApplication::applicationDirPath();
+        QString candidate = QString::fromLatin1("%1/%2").arg(binpath).arg(base);
+        if (QFileInfo(candidate).exists()) {
+            found = candidate;
+        }
+        else if (QTestLog::verboseLevel() >= 2) {
+            QTestLog::info(qPrintable(
+                QString::fromLatin1("testdata %1 not found relative to test binary [%2]; "
+                                    "checking next location")
+                    .arg(base).arg(candidate)),
+                file, line);
+        }
+    }
+
+    //  2. installed path.
+    if (found.isEmpty()) {
+        const char *testObjectName = QTestResult::currentTestObjectName();
+        if (testObjectName) {
+            QString testsPath = QLibraryInfo::location(QLibraryInfo::TestsPath);
+            QString candidate = QString::fromLatin1("%1/%2/%3")
+                .arg(testsPath)
+                .arg(QFile::decodeName(testObjectName).toLower())
+                .arg(base);
+            if (QFileInfo(candidate).exists()) {
+                found = candidate;
+            }
+            else if (QTestLog::verboseLevel() >= 2) {
+                QTestLog::info(qPrintable(
+                    QString::fromLatin1("testdata %1 not found in tests install path [%2]; "
+                                        "checking next location")
+                        .arg(base).arg(candidate)),
+                    file, line);
+            }
+        }
+    }
+
+    //  3. relative to test source.
+    if (found.isEmpty()) {
+        // srcdir is the directory containing the calling source file.
+        QFileInfo srcdir = QFileInfo(QFile::decodeName(file)).path();
+
+        // If the srcdir is relative, that means it is relative to the current working
+        // directory of the compiler at compile time, which should be passed in as `builddir'.
+        if (!srcdir.isAbsolute() && builddir) {
+            srcdir.setFile(QFile::decodeName(builddir) + QLatin1String("/") + srcdir.filePath());
+        }
+
+        QString candidate = QString::fromLatin1("%1/%2").arg(srcdir.canonicalFilePath()).arg(base);
+        if (QFileInfo(candidate).exists()) {
+            found = candidate;
+        }
+        else if (QTestLog::verboseLevel() >= 2) {
+            QTestLog::info(qPrintable(
+                QString::fromLatin1("testdata %1 not found relative to source path [%2]")
+                    .arg(base).arg(candidate)),
+                file, line);
+        }
+    }
+
+    if (found.isEmpty()) {
+        QTest::qWarn(qPrintable(
+            QString::fromLatin1("testdata %1 could not be located!").arg(base)),
+            file, line);
+    }
+    else if (QTestLog::verboseLevel() >= 1) {
+        QTestLog::info(qPrintable(
+            QString::fromLatin1("testdata %1 was located at %2").arg(base).arg(found)),
+            file, line);
+    }
+
+    return found;
+}
+
+/*! \internal
+ */
+QString QTest::qFindTestData(const char *base, const char *file, int line, const char *builddir)
+{
+    return qFindTestData(QFile::decodeName(base), file, line, builddir);
+}
+
 /*! \internal
  */
 void *QTest::qData(const char *tagName, int typeId)

src/testlib/qtestcase.h

@@ -44,6 +44,7 @@
 
 #include <QtTest/qtest_global.h>
 
+#include <QtCore/qstring.h>
 #include <QtCore/qnamespace.h>
 #include <QtCore/qmetatype.h>
 
@@ -151,6 +152,14 @@ do {\
 #define QWARN(msg)\
     QTest::qWarn(msg, __FILE__, __LINE__)
 
+#ifdef QT_TESTCASE_BUILDDIR
+# define QFINDTESTDATA(basepath)\
+    QTest::qFindTestData(basepath, __FILE__, __LINE__, QT_TESTCASE_BUILDDIR)
+#else
+# define QFINDTESTDATA(basepath)\
+    QTest::qFindTestData(basepath, __FILE__, __LINE__)
+#endif
+
 class QObject;
 class QTestData;
 
@@ -182,6 +191,9 @@ namespace QTest
     Q_TESTLIB_EXPORT void qWarn(const char *message, const char *file = 0, int line = 0);
     Q_TESTLIB_EXPORT void ignoreMessage(QtMsgType type, const char *message);
 
+    Q_TESTLIB_EXPORT QString qFindTestData(const char* basepath, const char* file = 0, int line = 0, const char* builddir = 0);
+    Q_TESTLIB_EXPORT QString qFindTestData(const QString& basepath, const char* file = 0, int line = 0, const char* builddir = 0);
+
     Q_TESTLIB_EXPORT void *qData(const char *tagName, int typeId);
     Q_TESTLIB_EXPORT void *qGlobalData(const char *tagName, int typeId);
     Q_TESTLIB_EXPORT void *qElementData(const char *elementName, int metaTypeId);

tests/auto/testlib/selftests/expected_findtestdata.lightxml

@@ -0,0 +1,16 @@
+<Environment>
+    <QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
+    <QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
+</Environment>
+<TestFunction name="initTestCase">
+<Incident type="pass" file="" line="0" />
+</TestFunction>
+<TestFunction name="paths">
+<Message type="warn" file="findtestdata.cpp" line="131">
+    <Description><![CDATA[testdata testfile could not be located!]]></Description>
+</Message>
+<Incident type="pass" file="" line="0" />
+</TestFunction>
+<TestFunction name="cleanupTestCase">
+<Incident type="pass" file="" line="0" />
+</TestFunction>

tests/auto/testlib/selftests/expected_findtestdata.txt

@@ -0,0 +1,9 @@
+********* Start testing of FindTestData *********
+Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@
+PASS   : FindTestData::initTestCase()
+WARNING: FindTestData::paths() testdata testfile could not be located!
+   Loc: [findtestdata.cpp(131)]
+PASS   : FindTestData::paths()
+PASS   : FindTestData::cleanupTestCase()
+Totals: 3 passed, 0 failed, 0 skipped
+********* Finished testing of FindTestData *********

tests/auto/testlib/selftests/expected_findtestdata.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<TestCase name="FindTestData">
+<Environment>
+    <QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
+    <QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
+</Environment>
+<TestFunction name="initTestCase">
+<Incident type="pass" file="" line="0" />
+</TestFunction>
+<TestFunction name="paths">
+<Message type="warn" file="findtestdata.cpp" line="131">
+    <Description><![CDATA[testdata testfile could not be located!]]></Description>
+</Message>
+<Incident type="pass" file="" line="0" />
+</TestFunction>
+<TestFunction name="cleanupTestCase">
+<Incident type="pass" file="" line="0" />
+</TestFunction>
+</TestCase>

tests/auto/testlib/selftests/expected_findtestdata.xunitxml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<testsuite errors="1" failures="0" tests="3" name="FindTestData">
+  <properties>
+    <property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/>
+    <property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
+  </properties>
+  <testcase result="pass" name="initTestCase"/>
+  <testcase result="pass" name="paths">
+    <!-- message="testdata testfile could not be located!" type="warn" -->
+  </testcase>
+  <testcase result="pass" name="cleanupTestCase"/>
+  <system-err>
+<![CDATA[testdata testfile could not be located!]]>
+  </system-err>
+</testsuite>

tests/auto/testlib/selftests/findtestdata/findtestdata.cpp

@@ -0,0 +1,144 @@
+/****************************************************************************
+**
+** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** GNU Lesser General Public License Usage
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this
+** file. Please review the following information to ensure the GNU Lesser
+** General Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU General
+** Public License version 3.0 as published by the Free Software Foundation
+** and appearing in the file LICENSE.GPL included in the packaging of this
+** file. Please review the following information to ensure the GNU General
+** Public License version 3.0 requirements will be met:
+** http://www.gnu.org/copyleft/gpl.html.
+**
+** Other Usage
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+
+#include <QtCore/QtCore>
+#include <QtTest/QtTest>
+
+#define TESTFILE "testfile"
+
+class FindTestData: public QObject
+{
+    Q_OBJECT
+private slots:
+    void initTestCase();
+    void cleanupTestCase();
+
+    void paths();
+
+private:
+    bool touch(QString const&);
+};
+
+void FindTestData::initTestCase()
+{
+    // verify that our qt.conf is working as expected.
+    QString app_path = QCoreApplication::applicationDirPath();
+    QString install_path = app_path + "/tests";
+    QVERIFY(QDir("/").mkpath(install_path));
+    QVERIFY(QDir("/").mkpath(install_path + "/findtestdata"));
+    QCOMPARE(QLibraryInfo::location(QLibraryInfo::TestsPath), install_path);
+
+    // make fake source and build directories
+    QVERIFY(QDir("/").mkpath(app_path + "/fakesrc"));
+    QVERIFY(QDir("/").mkpath(app_path + "/fakebuild"));
+}
+
+void FindTestData::cleanupTestCase()
+{
+    QString app_path = QCoreApplication::applicationDirPath();
+    QFile(app_path + "/tests/findtestdata/" TESTFILE).remove();
+    QFile(app_path + "/tests/" TESTFILE).remove();
+    QFile(app_path + "/fakesrc/" TESTFILE).remove();
+    QDir("/").rmpath(app_path + "/tests/findtestdata");
+    QDir("/").rmpath(app_path + "/fakesrc");
+    QDir("/").rmpath(app_path + "/fakebuild");
+}
+
+// Create an empty file at the specified path (or return false)
+bool FindTestData::touch(QString const& path)
+{
+    QFile file(path);
+    if (!file.open(QIODevice::WriteOnly)) {
+        qWarning("Failed to create %s: %s", qPrintable(path), qPrintable(file.errorString()));
+        return false;
+    }
+    return true;
+}
+
+void FindTestData::paths()
+{
+    // There are three possible locations for the testdata.
+    // In this test, we will put the testdata at all three locations,
+    // then remove it from one location at a time and verify that
+    // QFINDTESTDATA correctly falls back each time.
+
+    // 1. relative to test binary.
+    QString app_path = QCoreApplication::applicationDirPath();
+    QString testfile_path1 = app_path + "/" TESTFILE;
+    QVERIFY(touch(testfile_path1));
+
+    // 2. at the test install path (faked via qt.conf)
+    QString testfile_path2 = app_path + "/tests/findtestdata/" TESTFILE;
+    QVERIFY(touch(testfile_path2));
+
+    // 3. at the source path (which we will fake later on)
+    QString testfile_path3 = app_path + "/fakesrc/" TESTFILE;
+    QVERIFY(touch(testfile_path3));
+
+    // OK, all testdata created.  Verify that they are found in correct priority order.
+
+    QCOMPARE(QFINDTESTDATA(TESTFILE), testfile_path1);
+    QVERIFY(QFile(testfile_path1).remove());
+
+    QCOMPARE(QFINDTESTDATA(TESTFILE), testfile_path2);
+    QVERIFY(QFile(testfile_path2).remove());
+
+    // We cannot reasonably redefine __FILE__, so we call the underlying function instead.
+    // __FILE__ may be absolute or relative path; test both.
+
+    // absolute:
+    QCOMPARE(QTest::qFindTestData(TESTFILE, qPrintable(app_path + "/fakesrc/fakefile.cpp"), __LINE__), testfile_path3);
+
+    // relative: (pretend that we were compiled within fakebuild directory, pointing at files in ../fakesrc)
+    QCOMPARE(QTest::qFindTestData(TESTFILE, "../fakesrc/fakefile.cpp", __LINE__, qPrintable(app_path + "/fakebuild")), testfile_path3);
+
+    QVERIFY(QFile(testfile_path3).remove());
+
+    // Note, this is expected to generate a warning.
+    // We can't use ignoreMessage, because the warning comes from testlib,
+    // not via a "normal" qWarning.  But it's OK, our caller (tst_selftests)
+    // will verify that the warning is printed.
+    QCOMPARE(QFINDTESTDATA(TESTFILE), QString());
+}
+
+QTEST_MAIN(FindTestData)
+#include "findtestdata.moc"

tests/auto/testlib/selftests/findtestdata/findtestdata.pro

@@ -0,0 +1,8 @@
+SOURCES += findtestdata.cpp
+QT = core testlib
+
+mac:CONFIG -= app_bundle
+CONFIG -= debug_and_release_target
+
+
+TARGET = findtestdata

tests/auto/testlib/selftests/findtestdata/qt.conf

@@ -0,0 +1,4 @@
+# This file does not need to contain anything.
+# Its mere presence will cause this directory to be treated as the
+# Qt install prefix.  The findtestdata test uses this to "fake" the
+# tests install path.

tests/auto/testlib/selftests/selftests.pro

@@ -6,7 +6,7 @@ SUBDIRS = subtest test warnings maxwarnings cmptest globaldata skip \
           exceptionthrow qexecstringlist datatable commandlinedata\
           benchlibwalltime benchlibcallgrind benchlibeventcounter benchlibtickcounter \
           benchliboptions xunit badxml longstring float printdatatags \
-          printdatatagswithglobaltags
+          printdatatagswithglobaltags findtestdata
 
 INSTALLS =
 

tests/auto/testlib/selftests/selftests.qrc

@@ -124,6 +124,10 @@
         <file>expected_subtest.txt</file>
         <file>expected_subtest.xml</file>
         <file>expected_subtest.xunitxml</file>
+        <file>expected_findtestdata.lightxml</file>
+        <file>expected_findtestdata.txt</file>
+        <file>expected_findtestdata.xml</file>
+        <file>expected_findtestdata.xunitxml</file>
         <file>expected_warnings.lightxml</file>
         <file>expected_warnings.txt</file>
         <file>expected_warnings.xml</file>

tests/auto/testlib/selftests/tst_selftests.cpp

@@ -335,6 +335,7 @@ void tst_Selftests::runSubTest_data()
         << "sleep"
         << "strcmp"
         << "subtest"
+        << "findtestdata"
         << "warnings"
         << "xunit"
     ;