589 lines
17 KiB
C++
589 lines
17 KiB
C++
/****************************************************************************
|
|
**
|
|
** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
|
|
** All rights reserved.
|
|
** Contact: Nokia Corporation (qt-info@nokia.com)
|
|
**
|
|
** This file is part of the test suite of the Qt Toolkit.
|
|
**
|
|
** $QT_BEGIN_LICENSE:LGPL$
|
|
** No Commercial Usage
|
|
** This file contains pre-release code and may not be distributed.
|
|
** You may use this file in accordance with the terms and conditions
|
|
** contained in the Technology Preview License Agreement accompanying
|
|
** this package.
|
|
**
|
|
** GNU Lesser General Public License Usage
|
|
** Alternatively, this file may be used under the terms of the GNU Lesser
|
|
** General Public License version 2.1 as published by the Free Software
|
|
** Foundation and appearing in the file LICENSE.LGPL included in the
|
|
** packaging of this file. Please review the following information to
|
|
** ensure the GNU Lesser General Public License version 2.1 requirements
|
|
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
|
|
**
|
|
** In addition, as a special exception, Nokia gives you certain additional
|
|
** rights. These rights are described in the Nokia Qt LGPL Exception
|
|
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
|
|
**
|
|
** If you have questions regarding the use of this file, please contact
|
|
** Nokia at qt-info@nokia.com.
|
|
**
|
|
**
|
|
**
|
|
**
|
|
**
|
|
**
|
|
**
|
|
**
|
|
** $QT_END_LICENSE$
|
|
**
|
|
****************************************************************************/
|
|
|
|
#include <QDir>
|
|
#include <QFile>
|
|
#include <QRegExp>
|
|
#include <QStringList>
|
|
#include <QTest>
|
|
#include <QSet>
|
|
#include <QProcess>
|
|
#include <QDebug>
|
|
|
|
enum FindSubdirsMode {
|
|
Flat = 0,
|
|
Recursive
|
|
};
|
|
|
|
class tst_MakeTestSelfTest: public QObject
|
|
{
|
|
Q_OBJECT
|
|
|
|
private slots:
|
|
void tests_auto_pro();
|
|
|
|
void tests_pro_files();
|
|
void tests_pro_files_data();
|
|
|
|
void naming_convention();
|
|
void naming_convention_data();
|
|
|
|
void make_check();
|
|
|
|
private:
|
|
QStringList find_subdirs(QString const&, FindSubdirsMode, QString const& = QString());
|
|
|
|
QSet<QString> all_test_classes;
|
|
};
|
|
|
|
bool looks_like_testcase(QString const&,QString*);
|
|
bool looks_like_subdirs(QString const&);
|
|
QStringList find_test_class(QString const&);
|
|
|
|
/*
|
|
Verify that auto.pro only contains other .pro files (and not directories).
|
|
We enforce this so that we can process every .pro file other than auto.pro
|
|
independently and get all the tests.
|
|
If tests were allowed to appear directly in auto.pro, we'd have the problem
|
|
that we need to somehow run these tests from auto.pro while preventing
|
|
recursion into the other .pro files.
|
|
*/
|
|
void tst_MakeTestSelfTest::tests_auto_pro()
|
|
{
|
|
QStringList subdirsList = find_subdirs(SRCDIR "/../auto.pro", Flat);
|
|
if (QTest::currentTestFailed()) {
|
|
return;
|
|
}
|
|
|
|
foreach (QString const& subdir, subdirsList) {
|
|
QVERIFY2(subdir.endsWith(".pro"), qPrintable(QString(
|
|
"auto.pro contains a subdir `%1'.\n"
|
|
"auto.pro must _only_ contain other .pro files, not actual subdirs.\n"
|
|
"Please move `%1' into some other .pro file referenced by auto.pro."
|
|
).arg(subdir)));
|
|
}
|
|
}
|
|
|
|
/* Verify that all tests are listed somewhere in one of the autotest .pro files */
|
|
void tst_MakeTestSelfTest::tests_pro_files()
|
|
{
|
|
static QStringList lines;
|
|
|
|
if (lines.isEmpty()) {
|
|
QDir dir(SRCDIR "/..");
|
|
QStringList proFiles = dir.entryList(QStringList() << "*.pro");
|
|
foreach (QString const& proFile, proFiles) {
|
|
QString filename = QString("%1/../%2").arg(SRCDIR).arg(proFile);
|
|
QFile file(filename);
|
|
if (!file.open(QIODevice::ReadOnly)) {
|
|
QFAIL(qPrintable(QString("open %1: %2").arg(filename).arg(file.errorString())));
|
|
}
|
|
while (!file.atEnd()) {
|
|
lines << file.readLine().trimmed();
|
|
}
|
|
}
|
|
}
|
|
|
|
QFETCH(QString, subdir);
|
|
QRegExp re(QString("( |=|^|#)%1( |\\\\|$)").arg(QRegExp::escape(subdir)));
|
|
foreach (const QString& line, lines) {
|
|
if (re.indexIn(line) != -1) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
QFAIL(qPrintable(QString(
|
|
"Subdir `%1' is missing from tests/auto/*.pro\n"
|
|
"This means the test won't be compiled or run on any platform.\n"
|
|
"If this is intentional, please put the test name in a comment in one of the .pro files.").arg(subdir))
|
|
);
|
|
|
|
}
|
|
|
|
void tst_MakeTestSelfTest::tests_pro_files_data()
|
|
{
|
|
QTest::addColumn<QString>("subdir");
|
|
QDir dir(SRCDIR "/..");
|
|
QStringList subdirs = dir.entryList(QDir::AllDirs|QDir::NoDotAndDotDot);
|
|
|
|
foreach (const QString& subdir, subdirs) {
|
|
if (subdir == QString::fromLatin1("tmp")
|
|
|| subdir.startsWith(".")
|
|
|| !dir.exists(subdir + "/" + subdir + ".pro"))
|
|
{
|
|
continue;
|
|
}
|
|
QTest::newRow(qPrintable(subdir)) << subdir;
|
|
}
|
|
}
|
|
|
|
QString format_list(QStringList const& list)
|
|
{
|
|
if (list.count() == 1) {
|
|
return list.at(0);
|
|
}
|
|
return QString("one of (%1)").arg(list.join(", "));
|
|
}
|
|
|
|
void tst_MakeTestSelfTest::naming_convention()
|
|
{
|
|
QFETCH(QString, subdir);
|
|
QFETCH(QString, target);
|
|
|
|
QDir dir(SRCDIR "/../" + subdir);
|
|
|
|
QStringList cppfiles = dir.entryList(QStringList() << "*.h" << "*.cpp");
|
|
if (cppfiles.isEmpty()) {
|
|
// Common convention is to have test/test.pro and source files in parent dir
|
|
if (dir.dirName() == "test") {
|
|
dir.cdUp();
|
|
cppfiles = dir.entryList(QStringList() << "*.h" << "*.cpp");
|
|
}
|
|
|
|
if (cppfiles.isEmpty()) {
|
|
QSKIP("Couldn't locate source files for test", SkipSingle);
|
|
}
|
|
}
|
|
|
|
QStringList possible_test_classes;
|
|
foreach (QString const& file, cppfiles) {
|
|
possible_test_classes << find_test_class(dir.path() + "/" + file);
|
|
}
|
|
|
|
if (possible_test_classes.isEmpty()) {
|
|
QSKIP(qPrintable(QString("Couldn't locate test class in %1").arg(format_list(cppfiles))), SkipSingle);
|
|
}
|
|
|
|
QVERIFY2(possible_test_classes.contains(target), qPrintable(QString(
|
|
"TARGET is %1, while test class appears to be %2.\n"
|
|
"TARGET and test class _must_ match so that all testcase names can be accurately "
|
|
"determined even if a test fails to compile or run.")
|
|
.arg(target)
|
|
.arg(format_list(possible_test_classes))
|
|
));
|
|
|
|
QVERIFY2(!all_test_classes.contains(target), qPrintable(QString(
|
|
"It looks like there are multiple tests named %1.\n"
|
|
"This makes it impossible to separate results for these tests.\n"
|
|
"Please ensure all tests are uniquely named.")
|
|
.arg(target)
|
|
));
|
|
|
|
all_test_classes << target;
|
|
}
|
|
|
|
void tst_MakeTestSelfTest::naming_convention_data()
|
|
{
|
|
QTest::addColumn<QString>("subdir");
|
|
QTest::addColumn<QString>("target");
|
|
|
|
foreach (const QString& subdir, find_subdirs(SRCDIR "/../auto.pro", Recursive)) {
|
|
if (QFileInfo(SRCDIR "/../" + subdir).isDir()) {
|
|
QString target;
|
|
if (looks_like_testcase(SRCDIR "/../" + subdir + "/" + QFileInfo(subdir).baseName() + ".pro", &target)) {
|
|
QTest::newRow(qPrintable(subdir)) << subdir << target.toLower();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
Returns true if a .pro file seems to be for an autotest.
|
|
Running qmake to figure this out takes too long.
|
|
*/
|
|
bool looks_like_testcase(QString const& pro_file, QString* target)
|
|
{
|
|
QFile file(pro_file);
|
|
if (!file.open(QIODevice::ReadOnly)) {
|
|
return false;
|
|
}
|
|
|
|
*target = QString();
|
|
|
|
bool loaded_qttest = false;
|
|
|
|
do {
|
|
QByteArray line = file.readLine();
|
|
if (line.isEmpty()) {
|
|
break;
|
|
}
|
|
|
|
line = line.trimmed();
|
|
line.replace(' ', "");
|
|
|
|
if (line == "load(qttest_p4)") {
|
|
loaded_qttest = true;
|
|
}
|
|
|
|
if (line.startsWith("TARGET=")) {
|
|
*target = QString::fromLatin1(line.mid(sizeof("TARGET=")-1));
|
|
if (target->contains('/')) {
|
|
*target = target->right(target->lastIndexOf('/')+1);
|
|
}
|
|
}
|
|
|
|
if (loaded_qttest && !target->isEmpty()) {
|
|
break;
|
|
}
|
|
} while(1);
|
|
|
|
if (!loaded_qttest) {
|
|
return false;
|
|
}
|
|
|
|
if (!target->isEmpty() && !target->startsWith("tst_")) {
|
|
return false;
|
|
}
|
|
|
|
// If no target was set, default to tst_<dirname>
|
|
if (target->isEmpty()) {
|
|
*target = "tst_" + QFileInfo(pro_file).baseName();
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/*
|
|
Returns true if a .pro file seems to be a subdirs project.
|
|
Running qmake to figure this out takes too long.
|
|
*/
|
|
bool looks_like_subdirs(QString const& pro_file)
|
|
{
|
|
QFile file(pro_file);
|
|
if (!file.open(QIODevice::ReadOnly)) {
|
|
return false;
|
|
}
|
|
|
|
do {
|
|
QByteArray line = file.readLine();
|
|
if (line.isEmpty()) {
|
|
break;
|
|
}
|
|
|
|
line = line.trimmed();
|
|
line.replace(' ', "");
|
|
|
|
if (line == "TEMPLATE=subdirs") {
|
|
return true;
|
|
}
|
|
} while(1);
|
|
|
|
return false;
|
|
}
|
|
|
|
/*
|
|
Returns a list of all subdirs in a given .pro file
|
|
*/
|
|
QStringList tst_MakeTestSelfTest::find_subdirs(QString const& pro_file, FindSubdirsMode mode, QString const& prefix)
|
|
{
|
|
QStringList out;
|
|
|
|
QByteArray features = qgetenv("QMAKEFEATURES");
|
|
|
|
if (features.isEmpty()) {
|
|
features = SRCDIR "/features";
|
|
}
|
|
else {
|
|
features.prepend(SRCDIR "/features"
|
|
#ifdef Q_OS_WIN32
|
|
";"
|
|
#else
|
|
":"
|
|
#endif
|
|
);
|
|
}
|
|
|
|
QStringList args;
|
|
args << pro_file << "-o" << SRCDIR "/dummy_output" << "CONFIG+=dump_subdirs";
|
|
|
|
/* Turn on every option there is, to ensure we process every single directory */
|
|
args
|
|
<< "QT_CONFIG+=dbus"
|
|
<< "QT_CONFIG+=declarative"
|
|
<< "QT_CONFIG+=egl"
|
|
<< "QT_CONFIG+=multimedia"
|
|
<< "QT_CONFIG+=OdfWriter"
|
|
<< "QT_CONFIG+=opengl"
|
|
<< "QT_CONFIG+=openvg"
|
|
<< "QT_CONFIG+=phonon"
|
|
<< "QT_CONFIG+=private_tests"
|
|
<< "QT_CONFIG+=pulseaudio"
|
|
<< "QT_CONFIG+=qt3support"
|
|
<< "QT_CONFIG+=script"
|
|
<< "QT_CONFIG+=svg"
|
|
<< "QT_CONFIG+=webkit"
|
|
<< "QT_CONFIG+=xmlpatterns"
|
|
<< "CONFIG+=mac"
|
|
<< "CONFIG+=embedded"
|
|
<< "CONFIG+=symbian"
|
|
;
|
|
|
|
|
|
|
|
QString cmd_with_args = QString("qmake %1").arg(args.join(" "));
|
|
|
|
QProcess proc;
|
|
|
|
proc.setProcessChannelMode(QProcess::MergedChannels);
|
|
|
|
QProcessEnvironment env = QProcessEnvironment::systemEnvironment();
|
|
env.insert("QMAKEFEATURES", features);
|
|
proc.setProcessEnvironment(env);
|
|
|
|
proc.start("qmake", args);
|
|
if (!proc.waitForStarted(10000)) {
|
|
QTest::qFail(qPrintable(QString("Failed to run qmake: %1\nCommand: %2")
|
|
.arg(proc.errorString())
|
|
.arg(cmd_with_args)),
|
|
__FILE__, __LINE__
|
|
);
|
|
return out;
|
|
}
|
|
if (!proc.waitForFinished(30000)) {
|
|
QTest::qFail(qPrintable(QString("qmake did not finish within 30 seconds\nCommand: %1\nOutput: %2")
|
|
.arg(proc.errorString())
|
|
.arg(cmd_with_args)
|
|
.arg(QString::fromLocal8Bit(proc.readAll()))),
|
|
__FILE__, __LINE__
|
|
);
|
|
return out;
|
|
}
|
|
|
|
if (proc.exitStatus() != QProcess::NormalExit) {
|
|
QTest::qFail(qPrintable(QString("qmake crashed\nCommand: %1\nOutput: %2")
|
|
.arg(cmd_with_args)
|
|
.arg(QString::fromLocal8Bit(proc.readAll()))),
|
|
__FILE__, __LINE__
|
|
);
|
|
return out;
|
|
}
|
|
|
|
if (proc.exitCode() != 0) {
|
|
QTest::qFail(qPrintable(QString("qmake exited with code %1\nCommand: %2\nOutput: %3")
|
|
.arg(proc.exitCode())
|
|
.arg(cmd_with_args)
|
|
.arg(QString::fromLocal8Bit(proc.readAll()))),
|
|
__FILE__, __LINE__
|
|
);
|
|
return out;
|
|
}
|
|
|
|
QList<QByteArray> lines = proc.readAll().split('\n');
|
|
if (!lines.count()) {
|
|
QTest::qFail(qPrintable(QString("qmake seems to have not output anything\nCommand: %1\n")
|
|
.arg(cmd_with_args)),
|
|
__FILE__, __LINE__
|
|
);
|
|
return out;
|
|
}
|
|
|
|
foreach (QByteArray const& line, lines) {
|
|
static const QByteArray marker = "Project MESSAGE: subdir: ";
|
|
if (line.startsWith(marker)) {
|
|
QString subdir = QString::fromLocal8Bit(line.mid(marker.size()).trimmed());
|
|
out << prefix + subdir;
|
|
|
|
if (mode == Flat) {
|
|
continue;
|
|
}
|
|
|
|
// Need full path to subdir
|
|
QString subdir_filepath = subdir;
|
|
subdir_filepath.prepend(QFileInfo(pro_file).path() + "/");
|
|
|
|
// Add subdirs recursively
|
|
if (subdir.endsWith(".pro") && looks_like_subdirs(subdir_filepath)) {
|
|
// Need full path to .pro file
|
|
out << find_subdirs(subdir_filepath, mode, prefix);
|
|
}
|
|
|
|
if (QFileInfo(subdir_filepath).isDir()) {
|
|
subdir_filepath += "/" + subdir + ".pro";
|
|
if (looks_like_subdirs(subdir_filepath)) {
|
|
out << find_subdirs(subdir_filepath, mode, prefix + subdir + "/");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
void tst_MakeTestSelfTest::make_check()
|
|
{
|
|
/*
|
|
Run `make check' over the whole tests tree with a custom TESTRUNNER,
|
|
to verify that the TESTRUNNER mechanism works right.
|
|
*/
|
|
QString testsDir(SRCDIR "/..");
|
|
QString checktest(SRCDIR "/checktest/checktest");
|
|
|
|
#if defined(Q_OS_WIN32) || defined(Q_OS_MAC)
|
|
if (qgetenv("RUN_SLOW_TESTS").isEmpty()) {
|
|
QSKIP("This test is too slow to run by default on this OS. Set RUN_SLOW_TESTS=1 to run it.", SkipAll);
|
|
}
|
|
#endif
|
|
|
|
#ifdef Q_OS_WIN32
|
|
checktest.replace("/", "\\");
|
|
checktest += ".exe";
|
|
#endif
|
|
|
|
QProcess make;
|
|
make.setWorkingDirectory(testsDir);
|
|
|
|
QStringList arguments;
|
|
arguments << "-k";
|
|
arguments << "check";
|
|
arguments << QString("TESTRUNNER=%1").arg(checktest);
|
|
|
|
// find the right make; from externaltests.cpp
|
|
static const char makes[] =
|
|
"nmake.exe\0"
|
|
"mingw32-make.exe\0"
|
|
"gmake\0"
|
|
"make\0"
|
|
;
|
|
|
|
bool ok = false;
|
|
for (const char *p = makes; *p; p += strlen(p) + 1) {
|
|
make.start(p, arguments);
|
|
if (make.waitForStarted()) {
|
|
ok = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!ok) {
|
|
QFAIL("Could not find the right make tool in PATH");
|
|
}
|
|
|
|
QVERIFY(make.waitForFinished(1000 * 60 * 10));
|
|
QCOMPARE(make.exitStatus(), QProcess::NormalExit);
|
|
|
|
int pass = 0;
|
|
QList<QByteArray> out = make.readAllStandardOutput().split('\n');
|
|
QStringList fails;
|
|
foreach (QByteArray line, out) {
|
|
while (line.endsWith("\r")) {
|
|
line.chop(1);
|
|
}
|
|
if (line.startsWith("CHECKTEST FAIL")) {
|
|
fails << QString::fromLocal8Bit(line);
|
|
}
|
|
if (line.startsWith("CHECKTEST PASS")) {
|
|
++pass;
|
|
}
|
|
}
|
|
|
|
// We can't check that the exit code of make is 0, because some tests
|
|
// may have failed to compile, but that doesn't mean `make check' is broken.
|
|
// We do assume there are at least this many unbroken tests, though.
|
|
QVERIFY2(fails.count() == 0,
|
|
qPrintable(QString("`make check' doesn't work for %1 tests:\n%2")
|
|
.arg(fails.count()).arg(fails.join("\n")))
|
|
);
|
|
QVERIFY(pass > 50);
|
|
}
|
|
|
|
QStringList find_test_class(QString const& filename)
|
|
{
|
|
QStringList out;
|
|
|
|
QFile file(filename);
|
|
if (!file.open(QIODevice::ReadOnly)) {
|
|
return out;
|
|
}
|
|
|
|
static char const* klass_indicators[] = {
|
|
"QTEST_MAIN(",
|
|
"QTEST_APPLESS_MAIN(",
|
|
"class",
|
|
"staticconstcharklass[]=\"", /* hax0r tests which define their own metaobject */
|
|
0
|
|
};
|
|
|
|
do {
|
|
QByteArray line = file.readLine();
|
|
if (line.isEmpty()) {
|
|
break;
|
|
}
|
|
|
|
line = line.trimmed();
|
|
line.replace(' ', "");
|
|
|
|
for (int i = 0; klass_indicators[i]; ++i) {
|
|
char const* prefix = klass_indicators[i];
|
|
if (!line.startsWith(prefix)) {
|
|
continue;
|
|
}
|
|
QByteArray klass = line.mid(strlen(prefix));
|
|
if (!klass.startsWith("tst_")) {
|
|
continue;
|
|
}
|
|
for (int j = 0; j < klass.size(); ++j) {
|
|
char c = klass[j];
|
|
if (c == '_'
|
|
|| (c >= '0' && c <= '9')
|
|
|| (c >= 'A' && c <= 'Z')
|
|
|| (c >= 'a' && c <= 'z')) {
|
|
continue;
|
|
}
|
|
else {
|
|
klass.truncate(j);
|
|
break;
|
|
}
|
|
}
|
|
QString klass_str = QString::fromLocal8Bit(klass).toLower();
|
|
if (!out.contains(klass_str))
|
|
out << klass_str;
|
|
break;
|
|
}
|
|
} while(1);
|
|
|
|
return out;
|
|
}
|
|
|
|
QTEST_MAIN(tst_MakeTestSelfTest)
|
|
#include "tst_maketestselftest.moc"
|