Add a CSV logging feature to the benchlib

This is only useful for logging benchmarks, since it won't print test
passes, failures, etc. It's useful for importing to spreadsheets to do
number-crunching.

[ChangeLog][QtTest]Added a CSV logging mode that is suitable for
importing benchmark results into spreadsheets. This can be enabled by
the -csv option on the command-line. The CSV logging mode will not print
test failures, debug messages, warnings, etc.

Change-Id: I245d6f86bb380645c9bc0d748cf474b3ed42cab8
Reviewed-by: Sergio Ahumada <sahumada@blackberry.com>
Reviewed-by: Jason McDonald <macadder1@gmail.com>

src/testlib/doc/src/qttestlib-manual.qdoc

@@ -186,6 +186,9 @@
     Outputs results as a stream of XML tags.
     \li \c -xunitxml \br
     Outputs results as an Xunit XML document.
+    \li \c -csv \br
+    Outputs results as comma-separated values (CSV). This mode is only suitable for
+    benchmarks, since it suppresses normal pass/fail messages.
     \endlist
 
     The first version of the \c -o option may be repeated in order to log

src/testlib/qcsvbenchmarklogger.cpp

@@ -0,0 +1,103 @@
+/****************************************************************************
+**
+** Copyright (C) 2013 Intel Corporation
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the QtTest module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qcsvbenchmarklogger_p.h"
+#include "qtestresult_p.h"
+#include "qbenchmark_p.h"
+
+QCsvBenchmarkLogger::QCsvBenchmarkLogger(const char *filename)
+    : QAbstractTestLogger(filename)
+{
+}
+
+QCsvBenchmarkLogger::~QCsvBenchmarkLogger()
+{
+}
+
+void QCsvBenchmarkLogger::startLogging()
+{
+    // don't print anything
+}
+
+void QCsvBenchmarkLogger::stopLogging()
+{
+    // don't print anything
+}
+
+void QCsvBenchmarkLogger::enterTestFunction(const char *)
+{
+    // don't print anything
+}
+
+void QCsvBenchmarkLogger::leaveTestFunction()
+{
+    // don't print anything
+}
+
+void QCsvBenchmarkLogger::addIncident(QAbstractTestLogger::IncidentTypes, const char *, const char *, int)
+{
+    // don't print anything
+}
+
+void QCsvBenchmarkLogger::addBenchmarkResult(const QBenchmarkResult &result)
+{
+    const char *fn = QTestResult::currentTestFunction() ? QTestResult::currentTestFunction()
+        : "UnknownTestFunc";
+    const char *tag = QTestResult::currentDataTag() ? QTestResult::currentDataTag() : "";
+    const char *gtag = QTestResult::currentGlobalDataTag()
+                     ? QTestResult::currentGlobalDataTag()
+                     : "";
+    const char *filler = (tag[0] && gtag[0]) ? ":" : "";
+
+    const char *metric = QTest::benchmarkMetricName(result.metric);
+
+    char buf[1024];
+    // "function","[globaltag:]tag","metric",value_per_iteration,total,iterations
+    qsnprintf(buf, sizeof(buf), "\"%s\",\"%s%s%s\",\"%s\",%.13g,%.13g,%u\n",
+              fn, gtag, filler, tag, metric,
+              result.value / result.iterations, result.value, result.iterations);
+    outputString(buf);
+}
+
+void QCsvBenchmarkLogger::addMessage(QAbstractTestLogger::MessageTypes, const QString &, const char *, int)
+{
+    // don't print anything
+}

src/testlib/qcsvbenchmarklogger_p.h

@@ -0,0 +1,82 @@
+/****************************************************************************
+**
+** Copyright (C) 2013 Intel Corporation
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the QtTest module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QCSVBENCHMARKLOGGER_P_H
+#define QCSVBENCHMARKLOGGER_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists purely as an
+// implementation detail.  This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include "qabstracttestlogger_p.h"
+
+QT_BEGIN_NAMESPACE
+
+class QCsvBenchmarkLogger : public QAbstractTestLogger
+{
+public:
+    QCsvBenchmarkLogger(const char *filename);
+    ~QCsvBenchmarkLogger();
+
+    void startLogging() Q_DECL_OVERRIDE;
+    void stopLogging() Q_DECL_OVERRIDE;
+
+    void enterTestFunction(const char *function) Q_DECL_OVERRIDE;
+    void leaveTestFunction() Q_DECL_OVERRIDE;
+
+    void addIncident(IncidentTypes type, const char *description,
+                     const char *file = 0, int line = 0) Q_DECL_OVERRIDE;
+    void addBenchmarkResult(const QBenchmarkResult &result) Q_DECL_OVERRIDE;
+
+    void addMessage(MessageTypes type, const QString &message,
+                            const char *file = 0, int line = 0) Q_DECL_OVERRIDE;
+};
+
+QT_END_NAMESPACE
+
+#endif // QCSVBENCHMARKLOGGER_P_H

src/testlib/qtestcase.cpp

@@ -1320,6 +1320,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
          "                       Use - to output to stdout\n"
          "                       Valid formats are:\n"
          "                         txt      : Plain text\n"
+         "                         csv      : CSV format (suitable for benchmarks)\n"
          "                         xunitxml : XML XUnit document\n"
          "                         xml      : XML document\n"
          "                         lightxml : A stream of XML tags\n"
@@ -1329,6 +1330,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
          " Old-style logging options:\n"
          " -o filename         : Write the output into file\n"
          " -txt                : Output results in Plain Text\n"
+         " -csv                : Output results in a CSV format (suitable for benchmarks)\n"
          " -xunitxml           : Output results as XML XUnit document\n"
          " -xml                : Output results as XML document\n"
          " -lightxml           : Output results as stream of XML tags\n"
@@ -1408,6 +1410,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
             }
         } else if (strcmp(argv[i], "-txt") == 0) {
             logFormat = QTestLog::Plain;
+        } else if (strcmp(argv[i], "-csv") == 0) {
+            logFormat = QTestLog::CSV;
         } else if (strcmp(argv[i], "-xunitxml") == 0) {
             logFormat = QTestLog::XunitXML;
         } else if (strcmp(argv[i], "-xml") == 0) {
@@ -1438,6 +1442,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
                 // New-style
                 if (strcmp(format, "txt") == 0)
                     logFormat = QTestLog::Plain;
+                else if (strcmp(format, "csv") == 0)
+                    logFormat = QTestLog::CSV;
                 else if (strcmp(format, "lightxml") == 0)
                     logFormat = QTestLog::LightXML;
                 else if (strcmp(format, "xml") == 0)
@@ -1445,7 +1451,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
                 else if (strcmp(format, "xunitxml") == 0)
                     logFormat = QTestLog::XunitXML;
                 else {
-                    fprintf(stderr, "output format must be one of txt, lightxml, xml or xunitxml\n");
+                    fprintf(stderr, "output format must be one of txt, csv, lightxml, xml or xunitxml\n");
                     exit(1);
                 }
                 if (strcmp(filename, "-") == 0 && QTestLog::loggerUsingStdout()) {

src/testlib/qtestlog.cpp

@@ -45,6 +45,7 @@
 #include <QtTest/private/qtestresult_p.h>
 #include <QtTest/private/qabstracttestlogger_p.h>
 #include <QtTest/private/qplaintestlogger_p.h>
+#include <QtTest/private/qcsvbenchmarklogger_p.h>
 #include <QtTest/private/qxunittestlogger_p.h>
 #include <QtTest/private/qxmltestlogger_p.h>
 #include <QtCore/qatomic.h>
@@ -452,6 +453,9 @@ void QTestLog::addLogger(LogMode mode, const char *filename)
     case QTestLog::Plain:
         logger = new QPlainTestLogger(filename);
         break;
+    case QTestLog::CSV:
+        logger = new QCsvBenchmarkLogger(filename);
+        break;
     case QTestLog::XML:
         logger = new QXmlTestLogger(QXmlTestLogger::Complete, filename);
         break;

src/testlib/qtestlog_p.h

@@ -63,7 +63,7 @@ class QRegularExpression;
 class Q_TESTLIB_EXPORT QTestLog
 {
 public:
-    enum LogMode { Plain = 0, XML, LightXML, XunitXML };
+    enum LogMode { Plain = 0, XML, LightXML, XunitXML, CSV };
 
     static void enterTestFunction(const char* function);
     static void leaveTestFunction();

src/testlib/testlib.pro

@@ -33,7 +33,7 @@ HEADERS = qbenchmark.h \
     qtestmouse.h \
     qtestspontaneevent.h \
     qtestsystem.h \
-    qtesttouch.h \
+    qtesttouch.h
 
 SOURCES = qtestcase.cpp \
     qtestlog.cpp \
@@ -51,6 +51,7 @@ SOURCES = qtestcase.cpp \
     qbenchmarkevent.cpp \
     qbenchmarkperfevents.cpp \
     qbenchmarkmetric.cpp \
+    qcsvbenchmarklogger.cpp \
     qtestelement.cpp \
     qtestelementattribute.cpp \
     qtestxunitstreamer.cpp \

tests/auto/testlib/selftests/expected_benchlibcallgrind.csv

@@ -0,0 +1 @@
+"twoHundredMillionInstructions","","InstructionReads",200000158,200000158,1

tests/auto/testlib/selftests/expected_benchlibcounting.csv

@@ -0,0 +1 @@
+"passingBenchmark","","Events",0,0,1

tests/auto/testlib/selftests/expected_benchlibeventcounter.csv

@@ -0,0 +1,7 @@
+"events","0","Events",0,0,1
+"events","1","Events",1,1,1
+"events","10","Events",10,10,1
+"events","100","Events",100,100,1
+"events","500","Events",500,500,1
+"events","5000","Events",5000,5000,1
+"events","100000","Events",100000,100000,1

tests/auto/testlib/selftests/expected_benchliboptions.csv

@@ -0,0 +1,3 @@
+"threeEvents","","Events",3,3,1
+"threeEvents","","Events",3,45,15
+"threeEvents","","Events",3,3,1

tests/auto/testlib/selftests/expected_benchlibtickcounter.csv

@@ -0,0 +1 @@
+"threeBillionTicks","","CPUTicks",3000000000,3000000000,1

tests/auto/testlib/selftests/expected_benchlibwalltime.csv

@@ -0,0 +1,3 @@
+"waitForOneThousand","","WalltimeMilliseconds",1000,1000,1
+"waitForFourThousand","","WalltimeMilliseconds",4000,4000,1
+"qbenchmark_once","","WalltimeMilliseconds",0,0,1

tests/auto/testlib/selftests/selftests.qrc

@@ -10,23 +10,28 @@
         <file>expected_badxml.xml</file>
         <file>expected_badxml.xunitxml</file>
         <file>expected_benchlibcallgrind.txt</file>
+        <file>expected_benchlibcallgrind.csv</file>
         <file>expected_benchlibcounting.lightxml</file>
         <file>expected_benchlibcounting.txt</file>
         <file>expected_benchlibcounting.xml</file>
         <file>expected_benchlibcounting.xunitxml</file>
+        <file>expected_benchlibcounting.csv</file>
         <file>expected_benchlibeventcounter.lightxml</file>
         <file>expected_benchlibeventcounter.txt</file>
         <file>expected_benchlibeventcounter.xml</file>
         <file>expected_benchlibeventcounter.xunitxml</file>
+        <file>expected_benchlibeventcounter.csv</file>
         <file>expected_benchliboptions.txt</file>
         <file>expected_benchlibtickcounter.lightxml</file>
         <file>expected_benchlibtickcounter.txt</file>
         <file>expected_benchlibtickcounter.xml</file>
         <file>expected_benchlibtickcounter.xunitxml</file>
+        <file>expected_benchlibtickcounter.csv</file>
         <file>expected_benchlibwalltime.lightxml</file>
         <file>expected_benchlibwalltime.txt</file>
         <file>expected_benchlibwalltime.xml</file>
         <file>expected_benchlibwalltime.xunitxml</file>
+        <file>expected_benchlibwalltime.csv</file>
         <file>expected_cmptest.lightxml</file>
         <file>expected_cmptest.txt</file>
         <file>expected_cmptest.xml</file>

tests/auto/testlib/selftests/tst_selftests.cpp

@@ -234,6 +234,12 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
                      QStringList() << "lightxml",
                      QStringList() << "-lightxml" << "-o" << logName("lightxml")
                     )
+        << LoggerSet("old stdout csv", // benchmarks only
+                     QStringList() << "stdout csv",
+                     QStringList() << "-csv")
+        << LoggerSet("old csv", // benchmarks only
+                     QStringList() << "csv",
+                     QStringList() << "-csv" << "-o" << logName("csv"))
         // Test with new-style options for a single logger
         << LoggerSet("new stdout txt",
                      QStringList() << "stdout txt",
@@ -267,6 +273,12 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
                      QStringList() << "lightxml",
                      QStringList() << "-o" << logName("lightxml")+",lightxml"
                     )
+        << LoggerSet("new stdout csv", // benchmarks only
+                     QStringList() << "stdout csv",
+                     QStringList() << "-o" << "-,csv")
+        << LoggerSet("new csv", // benchmarks only
+                     QStringList() << "csv",
+                     QStringList() << "-o" << logName("csv")+",csv")
         // Test with two loggers (don't test all 32 combinations, just a sample)
         << LoggerSet("stdout txt + txt",
                      QStringList() << "stdout txt" << "txt",
@@ -288,7 +300,7 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
                      QStringList() << "-o" << logName("lightxml")+",lightxml"
                                    << "-o" << "-,xunitxml"
                     )
-        // All loggers at the same time
+        // All loggers at the same time (except csv)
         << LoggerSet("all loggers",
                      QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml",
                      QStringList() << "-o" << logName("txt")+",txt"
@@ -488,6 +500,10 @@ void tst_Selftests::runSubTest_data()
             }
             if (subtest == "badxml" && (loggerSet.name == "all loggers" || loggerSet.name.contains("txt")))
                 continue; // XML only, do not mix txt and XML for encoding test.
+
+            if (loggerSet.name.contains("csv") && !subtest.startsWith("benchlib"))
+                continue;
+
             const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow")
                 || subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate")
                 || subtest == QLatin1String("crashes") || subtest == QLatin1String("silent");
@@ -706,7 +722,7 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
             else if (expected.startsWith(QLatin1String("FAIL!  : tst_Exception::throwException() Caught unhandled exce")) && expected != output)
                 // On some platforms we compile without RTTI, and as a result we never throw an exception.
                 QCOMPARE(output.simplified(), QString::fromLatin1("tst_Exception::throwException()").simplified());
-            else if (benchmark || line.startsWith("<BenchmarkResult")) {
+            else if (benchmark || line.startsWith("<BenchmarkResult") || (logFormat(logger) == "csv" && line.startsWith('"'))) {
                 // Don't do a literal comparison for benchmark results, since
                 // results have some natural variance.
                 QString error;
@@ -813,6 +829,35 @@ BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error)
         out.iterations = iterations;
         return out;
     }
+
+    if (line.startsWith('"')) {
+        // CSV result
+        // format:
+        //  "function","[globaltag:]tag","metric",value_per_iteration,total,iterations
+        QStringList split = line.split(',');
+        if (split.count() != 6) {
+            if (error) *error = QString("Wrong number of columns (%1)").arg(split.count());
+            return out;
+        }
+
+        bool ok;
+        double total = split.at(4).toDouble(&ok);
+        if (!ok) {
+            if (error) *error = split.at(4) + " is not a valid number";
+            return out;
+        }
+        double iterations = split.at(5).toDouble(&ok);
+        if (!ok) {
+            if (error) *error = split.at(5) + " is not a valid number";
+            return out;
+        }
+
+        out.unit = split.at(2);
+        out.total = total;
+        out.iterations = iterations;
+        return out;
+    }
+
     // Text result
     // This code avoids using a QRegExp because QRegExp might be broken.
     // Sample format: 4,000 msec per iteration (total: 4,000, iterations: 1)