AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add a feature to enable CodeCoverage analysis of testlib

Browse Source 
Based on Asmo Saarela's advice (QTPM-686), adapted on advice from
FrogLogic support and converted to a feature so that the selftest and
testlib qmake config can be co-ordinated.

Task-number: QTPM-1385
Change-Id: Icd706f086009e1e08b3f8c5cd553f792402e28c0
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>
This commit is contained in:
Edward Welbourne 2018-08-06 17:07:07 +02:00
parent f3de22b211
commit 3a5f86d784
62 changed files with 177 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/testlib/configure.json
View File

@ -5,6 +5,12 @@
],
"features": {
"testlib_selfcover": {
"label": "Coverage testing of testlib itself",
"purpose": "Gauges how thoroughly testlib's selftest exercises testlib's code",
"autoDetect": false,
"output": [ "publicFeature" ]
},
"itemmodeltester": {
"label": "Tester for item models",
"purpose": "Provides a utility to test item models.",

25
src/testlib/qtest.h
View File

@ -374,9 +374,30 @@ QT_END_NAMESPACE
# define QTEST_SET_MAIN_SOURCE_PATH QTest::setMainSourcePath(__FILE__);
#endif
// Hooks for coverage-testing of QTestLib itself:
#if QT_CONFIG(testlib_selfcover) && defined(__COVERAGESCANNER__)
struct QtCoverageScanner
{
QtCoverageScanner(const char *name)
{
__coveragescanner_clear();
__coveragescanner_testname(name);
}
~QtCoverageScanner()
{
__coveragescanner_save();
__coveragescanner_testname("");
}
};
#define TESTLIB_SELFCOVERAGE_START(name) QtCoverageScanner _qtCoverageScanner(name);
#else
#define TESTLIB_SELFCOVERAGE_START(name)
#endif
#define QTEST_APPLESS_MAIN(TestObject) \
int main(int argc, char *argv[]) \
{ \
TESTLIB_SELFCOVERAGE_START(TestObject) \
TestObject tc; \
QTEST_SET_MAIN_SOURCE_PATH \
return QTest::qExec(&tc, argc, argv); \
@ -406,6 +427,7 @@ int main(int argc, char *argv[]) \
#define QTEST_MAIN(TestObject) \
int main(int argc, char *argv[]) \
{ \
TESTLIB_SELFCOVERAGE_START(#TestObject) \
QApplication app(argc, argv); \
app.setAttribute(Qt::AA_Use96Dpi, true); \
QTEST_DISABLE_KEYPAD_NAVIGATION \
@ -421,6 +443,7 @@ int main(int argc, char *argv[]) \
#define QTEST_MAIN(TestObject) \
int main(int argc, char *argv[]) \
{ \
TESTLIB_SELFCOVERAGE_START(#TestObject) \
QGuiApplication app(argc, argv); \
app.setAttribute(Qt::AA_Use96Dpi, true); \
TestObject tc; \
@ -433,6 +456,7 @@ int main(int argc, char *argv[]) \
#define QTEST_MAIN(TestObject) \
int main(int argc, char *argv[]) \
{ \
TESTLIB_SELFCOVERAGE_START(#TestObject) \
QCoreApplication app(argc, argv); \
app.setAttribute(Qt::AA_Use96Dpi, true); \
TestObject tc; \
@ -445,6 +469,7 @@ int main(int argc, char *argv[]) \
#define QTEST_GUILESS_MAIN(TestObject) \
int main(int argc, char *argv[]) \
{ \
TESTLIB_SELFCOVERAGE_START(#TestObject) \
QCoreApplication app(argc, argv); \
app.setAttribute(Qt::AA_Use96Dpi, true); \
TestObject tc; \

2
src/testlib/qtestcase.cpp
View File

@ -246,7 +246,7 @@ static void stackTrace()
static bool installCoverageTool(const char * appname, const char * testname)
{
#ifdef __COVERAGESCANNER__
#if defined(__COVERAGESCANNER__) && !QT_CONFIG(testlib_selfcover)
if (!qEnvironmentVariableIsEmpty("QT_TESTCOCOON_ACTIVE"))
return false;
// Set environment variable QT_TESTCOCOON_ACTIVE to prevent an eventual subtest from

5
src/testlib/qtestlog.cpp
View File

@ -73,6 +73,10 @@ QT_BEGIN_NAMESPACE
static void saveCoverageTool(const char * appname, bool testfailed, bool installedTestCoverage)
{
#ifdef __COVERAGESCANNER__
# if QT_CONFIG(testlib_selfcover)
__coveragescanner_teststate(QTestLog::failCount() > 0 ? "FAILED" :
QTestLog::passCount() > 0 ? "PASSED" : "SKIPPED");
# else
if (!installedTestCoverage)
return;
// install again to make sure the filename is correct.
@ -83,6 +87,7 @@ static void saveCoverageTool(const char * appname, bool testfailed, bool install
__coveragescanner_testname("");
__coveragescanner_clear();
unsetenv("QT_TESTCOCOON_ACTIVE");
# endif // testlib_selfcover
#else
Q_UNUSED(appname);
Q_UNUSED(testfailed);

28
src/testlib/selfcover.pri Normal file
View File

@ -0,0 +1,28 @@
# Configuration for testlib and its tests, to instrument with
# FrogLogic's Squish CoCo (cf. testcocoon.prf, which handles similar
# for general code; but testlib needs special handling).
# Only for use when feature testlib_selfcover is enabled:
!qtConfig(testlib_selfcover): return()
# This enables verification that testlib itself is adequately tested,
# as a grounds for trusting that testing with it is useful.
# Exclude all non-testlib source from coverage instrumentation:
COVERAGE_OPTIONS = --cs-exclude-file-abs-wildcard=$$QT_SOURCE_TREE/*
COVERAGE_OPTIONS += --cs-include-file-abs-wildcard=*/src/testlib/*
COVERAGE_OPTIONS += --cs-mcc # enable Multiple Condition Coverage
COVERAGE_OPTIONS += --cs-mcdc # enable Multiple Condition / Decision Coverage
# (recommended for ISO 26262 ASIL A, B and C -- highly recommended for ASIL D)
# https://doc.froglogic.com/squish-coco/4.1/codecoverage.html#sec%3Amcdc
QMAKE_CFLAGS += $$COVERAGE_OPTIONS
QMAKE_CXXFLAGS += $$COVERAGE_OPTIONS
QMAKE_LFLAGS += $$COVERAGE_OPTIONS
# FIXME: relies on QMAKE_* being just the command-names, with no path prefix
QMAKE_CC = cs$$QMAKE_CC
QMAKE_CXX = cs$$QMAKE_CXX
QMAKE_LINK = cs$$QMAKE_LINK
QMAKE_LINK_SHLIB = cs$$QMAKE_LINK_SHLIB
QMAKE_AR = cs$$QMAKE_AR
QMAKE_LIB = cs$$QMAKE_LIB

1
src/testlib/testlib.pro
View File

@ -146,4 +146,5 @@ mac {
!qtHaveModule(network): HEADERSCLEAN_EXCLUDE += qtest_network.h
include(selfcover.pri)
load(qt_module)

2
tests/auto/testlib/qabstractitemmodeltester/qabstractitemmodeltester.pro
View File

@ -11,3 +11,5 @@ SOURCES += \
HEADERS += \
$${mtdir}/dynamictreemodel.h
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/qsignalspy/qsignalspy.pro
View File

@ -2,3 +2,5 @@ CONFIG += testcase
TARGET = tst_qsignalspy
SOURCES += tst_qsignalspy.cpp
QT = core testlib
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/alive/alive.pro
View File

@ -6,3 +6,5 @@ CONFIG -= debug_and_release_target
TARGET = alive
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/assert/assert.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = assert
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/badxml/badxml.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = badxml
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/benchlibcallgrind/benchlibcallgrind.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = benchlibcallgrind
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/benchlibcounting/benchlibcounting.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = benchlibcounting
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/benchlibeventcounter/benchlibeventcounter.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = benchlibeventcounter
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/benchliboptions/benchliboptions.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = benchliboptions
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/benchlibtickcounter/benchlibtickcounter.pro
View File

@ -6,3 +6,5 @@ CONFIG -= debug_and_release_target
TARGET = benchlibtickcounter
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/benchlibwalltime/benchlibwalltime.pro
View File

@ -6,3 +6,5 @@ CONFIG -= debug_and_release_target
TARGET = benchlibwalltime
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/blacklisted/blacklisted.pro
View File

@ -5,3 +5,5 @@ mac: CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = blacklisted
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/cmptest/cmptest.pro
View File

@ -6,3 +6,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = cmptest
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/commandlinedata/commandlinedata.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = commandlinedata
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/counting/counting.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = counting
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/crashes/crashes.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = crashes
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/datatable/datatable.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = datatable
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/datetime/datetime.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = datetime
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/deleteLater/deleteLater.pro
View File

@ -5,3 +5,5 @@ CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = tst_deleteLater
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/deleteLater_noApp/deleteLater_noApp.pro
View File

@ -5,3 +5,5 @@ CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = tst_deleteLater_noApp
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/differentexec/differentexec.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = differentexec
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/exceptionthrow/exceptionthrow.pro
View File

@ -6,3 +6,5 @@ CONFIG -= debug_and_release_target
CONFIG += exceptions
TARGET = exceptionthrow
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/expectfail/expectfail.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = expectfail
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/failcleanup/failcleanup.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = failcleanup
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/failinit/failinit.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = failinit
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/failinitdata/failinitdata.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = failinitdata
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/fetchbogus/fetchbogus.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = fetchbogus
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/findtestdata/findtestdata.pro
View File

@ -7,3 +7,5 @@ CONFIG -= debug_and_release_target
RESOURCES = findtestdata.qrc
TARGET = findtestdata
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/float/float.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = float
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/globaldata/globaldata.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = globaldata
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/keyboard/keyboard.pro
View File

@ -5,3 +5,5 @@ macos:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = keyboard
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/longstring/longstring.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = longstring
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/maxwarnings/maxwarnings.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = maxwarnings
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/mouse/mouse.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = mouse
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/multiexec/multiexec.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = multiexec
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/pairdiagnostics/pairdiagnostics.pro
View File

@ -4,3 +4,5 @@ QT = core testlib
CONFIG -= app_bundle debug_and_release_target
TARGET = pairdiagnostics
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/printdatatags/printdatatags.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = printdatatags
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/printdatatagswithglobaltags/printdatatagswithglobaltags.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = printdatatagswithglobaltags
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/qexecstringlist/qexecstringlist.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = qexecstringlist
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/silent/silent.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = silent
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/singleskip/singleskip.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = singleskip
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/skip/skip.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = skip
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/skipcleanup/skipcleanup.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = skipcleanup
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/skipinit/skipinit.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = skipinit
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/skipinitdata/skipinitdata.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = skipinitdata
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/sleep/sleep.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = sleep
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/strcmp/strcmp.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = strcmp
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/subtest/subtest.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = subtest
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

1
tests/auto/testlib/selftests/test/test.pro
View File

@ -19,3 +19,4 @@ RESOURCES += expected_files
include(../selftests.pri)
!android:!winrt: for(file, SUBPROGRAMS): TEST_HELPER_INSTALLS += "../$${file}/$${file}"
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/testlib/testlib.pro
View File

@ -5,3 +5,5 @@ darwin: CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = testlib
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/tuplediagnostics/tuplediagnostics.pro
View File

@ -4,3 +4,5 @@ QT = core testlib
CONFIG -= app_bundle debug_and_release_target
TARGET = tuplediagnostics
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/verbose1/verbose1.pro
View File

@ -8,3 +8,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = verbose1
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/verbose2/verbose2.pro
View File

@ -8,3 +8,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = verbose2
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/verifyexceptionthrown/verifyexceptionthrown.pro
View File

@ -6,3 +6,5 @@ CONFIG -= debug_and_release_target
CONFIG += exceptions
TARGET = verifyexceptionthrown
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/warnings/warnings.pro
View File

@ -5,3 +5,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = warnings
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)

2
tests/auto/testlib/selftests/xunit/xunit.pro
View File

@ -6,3 +6,5 @@ mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
TARGET = xunit
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)