5635823e17
As in the past, to avoid rewriting various autotests that contain line-number information, an extra blank line has been inserted at the end of the license text to ensure that this commit does not change the total number of lines in the license header. Change-Id: I311e001373776812699d6efc045b5f742890c689 Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>
649 lines
17 KiB
C++
649 lines
17 KiB
C++
/****************************************************************************
|
|
**
|
|
** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
|
|
** Contact: http://www.qt-project.org/
|
|
**
|
|
** This file is part of the test suite of the Qt Toolkit.
|
|
**
|
|
** $QT_BEGIN_LICENSE:LGPL$
|
|
** GNU Lesser General Public License Usage
|
|
** This file may be used under the terms of the GNU Lesser General Public
|
|
** License version 2.1 as published by the Free Software Foundation and
|
|
** appearing in the file LICENSE.LGPL included in the packaging of this
|
|
** file. Please review the following information to ensure the GNU Lesser
|
|
** General Public License version 2.1 requirements will be met:
|
|
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
|
|
**
|
|
** In addition, as a special exception, Nokia gives you certain additional
|
|
** rights. These rights are described in the Nokia Qt LGPL Exception
|
|
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
|
|
**
|
|
** GNU General Public License Usage
|
|
** Alternatively, this file may be used under the terms of the GNU General
|
|
** Public License version 3.0 as published by the Free Software Foundation
|
|
** and appearing in the file LICENSE.GPL included in the packaging of this
|
|
** file. Please review the following information to ensure the GNU General
|
|
** Public License version 3.0 requirements will be met:
|
|
** http://www.gnu.org/copyleft/gpl.html.
|
|
**
|
|
** Other Usage
|
|
** Alternatively, this file may be used in accordance with the terms and
|
|
** conditions contained in a signed written agreement between you and Nokia.
|
|
**
|
|
**
|
|
**
|
|
**
|
|
**
|
|
**
|
|
** $QT_END_LICENSE$
|
|
**
|
|
****************************************************************************/
|
|
|
|
|
|
#include <atWrapper.h>
|
|
#include <datagenerator/datagenerator.h>
|
|
|
|
#include <QString>
|
|
#include <QHash>
|
|
#include <QFile>
|
|
#include <QFtp>
|
|
#include <QObject>
|
|
#include <QHostInfo>
|
|
#include <QWidget>
|
|
#include <QImage>
|
|
#include <QtTest/QSignalSpy>
|
|
#include <QLibraryInfo>
|
|
|
|
static const char *ArthurDir = "../../arthur";
|
|
|
|
#include <string.h>
|
|
|
|
atWrapper::atWrapper()
|
|
{
|
|
|
|
// initTests();
|
|
|
|
}
|
|
|
|
bool atWrapper::initTests(bool *haveBaseline)
|
|
{
|
|
qDebug() << "Running test on buildkey:" << QLibraryInfo::buildKey() << " qt version:" << qVersion();
|
|
|
|
qDebug( "Initializing tests..." );
|
|
|
|
if (!loadConfig( QHostInfo::localHostName().split( "." ).first() + ".ini" ))
|
|
return false;
|
|
|
|
//Reset the FTP environment where the results are stored
|
|
*haveBaseline = setupFTP();
|
|
|
|
// Retrieve the latest test result baseline from the FTP server.
|
|
downloadBaseline();
|
|
return true;
|
|
}
|
|
|
|
void atWrapper::downloadBaseline()
|
|
{
|
|
|
|
qDebug() << "Now downloading baseline...";
|
|
|
|
QFtp ftp;
|
|
|
|
QObject::connect( &ftp, SIGNAL( listInfo( const QUrlInfo & ) ), this, SLOT( ftpMgetAddToList(const QUrlInfo & ) ) );
|
|
|
|
//Making sure that the needed local directories exist.
|
|
|
|
QHashIterator<QString, QString> j(enginesToTest);
|
|
|
|
while ( j.hasNext() )
|
|
{
|
|
j.next();
|
|
|
|
QDir dir( output );
|
|
|
|
if ( !dir.cd( j.key() + ".baseline" ) )
|
|
dir.mkdir( j.key() + ".baseline" );
|
|
|
|
}
|
|
|
|
//FTP to the host specified in the config file, and retrieve the test result baseline.
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
|
|
ftp.cd( ftpBaseDir );
|
|
|
|
QHashIterator<QString, QString> i(enginesToTest);
|
|
while ( i.hasNext() )
|
|
{
|
|
i.next();
|
|
mgetDirList.clear();
|
|
mgetDirList << i.key() + ".baseline";
|
|
ftp.cd( i.key() + ".baseline" );
|
|
ftp.list();
|
|
ftp.cd( ".." );
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
|
|
ftpMgetDone( true );
|
|
}
|
|
|
|
ftp.close();
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
|
|
}
|
|
|
|
void atWrapper::ftpMgetAddToList( const QUrlInfo &urlInfo )
|
|
{
|
|
//Simply adding to the list of files to download.
|
|
mgetDirList << urlInfo.name();
|
|
|
|
}
|
|
|
|
void atWrapper::ftpMgetDone( bool error)
|
|
{
|
|
Q_UNUSED( error );
|
|
|
|
//Downloading the files listed in mgetDirList...
|
|
QFtp ftp;
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
|
|
QFile* file;
|
|
|
|
if ( mgetDirList.size() > 1 )
|
|
for ( int i = 1; i < mgetDirList.size(); ++i )
|
|
{
|
|
file = new QFile( QString( output ) + "/" + mgetDirList.at( 0 ) + "/" + mgetDirList.at( i ) );
|
|
if (file->open(QIODevice::WriteOnly)) {
|
|
ftp.get( ftpBaseDir + "/" + mgetDirList.at( 0 ) + "/" + mgetDirList.at( i ), file );
|
|
ftp.list(); //Only there to fill up a slot in the pendingCommands queue.
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
file->close();
|
|
} else {
|
|
qDebug() << "Couldn't open file for writing: " << file->fileName();
|
|
}
|
|
}
|
|
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
}
|
|
|
|
void atWrapper::uploadFailed( QString dir, QString filename, QByteArray filedata )
|
|
{
|
|
//Upload a failed test case image to the FTP server.
|
|
QFtp ftp;
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
|
|
ftp.cd( ftpBaseDir );
|
|
ftp.cd( dir );
|
|
|
|
ftp.put( filedata, filename, QFtp::Binary );
|
|
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
}
|
|
|
|
// returns false if no baseline exists
|
|
bool atWrapper::setupFTP()
|
|
{
|
|
qDebug( "Setting up FTP environment" );
|
|
|
|
QString dir = "";
|
|
ftpMkDir( ftpBaseDir );
|
|
|
|
ftpBaseDir += "/" + QLibraryInfo::buildKey();
|
|
|
|
ftpMkDir( ftpBaseDir );
|
|
|
|
ftpBaseDir += "/" + QString( qVersion() );
|
|
|
|
ftpMkDir( ftpBaseDir );
|
|
|
|
QHashIterator<QString, QString> i(enginesToTest);
|
|
QHashIterator<QString, QString> j(enginesToTest);
|
|
|
|
bool haveBaseline = true;
|
|
//Creating the baseline directories for each engine
|
|
while ( i.hasNext() )
|
|
{
|
|
i.next();
|
|
//qDebug() << "Creating dir with key:" << i.key();
|
|
ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".failed" );
|
|
ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".diff" );
|
|
if (!ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".baseline" ))
|
|
haveBaseline = false;
|
|
}
|
|
|
|
|
|
QFtp ftp;
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
|
|
ftp.cd( ftpBaseDir );
|
|
//Deleting previous failed directory and all the files in it, then recreating it.
|
|
while ( j.hasNext() )
|
|
{
|
|
j.next();
|
|
rmDirList.clear();
|
|
rmDirList << ftpBaseDir + "/" + j.key() + ".failed" + "/";
|
|
ftpRmDir( j.key() + ".failed" );
|
|
ftp.rmdir( j.key() + ".failed" );
|
|
ftp.mkdir( j.key() + ".failed" );
|
|
ftp.list();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
|
|
rmDirList.clear();
|
|
rmDirList << ftpBaseDir + "/" + j.key() + ".diff" + "/";
|
|
ftpRmDir( j.key() + ".diff" );
|
|
ftp.rmdir( j.key() + ".diff" );
|
|
ftp.mkdir( j.key() + ".diff" );
|
|
ftp.list();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
|
|
}
|
|
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
|
|
return haveBaseline;
|
|
}
|
|
|
|
void atWrapper::ftpRmDir( QString dir )
|
|
{
|
|
//Hack to remove a populated directory. (caveat: containing only files and empty dirs, not recursive!)
|
|
qDebug() << "Now removing directory: " << dir;
|
|
QFtp ftp;
|
|
QObject::connect( &ftp, SIGNAL( listInfo( const QUrlInfo & ) ), this, SLOT( ftpRmDirAddToList(const QUrlInfo & ) ) );
|
|
QObject::connect( &ftp, SIGNAL( done( bool ) ), this, SLOT( ftpRmDirDone( bool ) ) );
|
|
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
|
|
ftp.list( ftpBaseDir + "/" + dir );
|
|
ftp.close();
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
}
|
|
|
|
void atWrapper::ftpRmDirDone( bool error )
|
|
{
|
|
//Deleting each file in the directory listning, rmDirList.
|
|
Q_UNUSED( error );
|
|
|
|
QFtp ftp;
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
|
|
if ( rmDirList.size() > 1 )
|
|
for (int i = 1; i < rmDirList.size(); ++i)
|
|
ftp.remove( rmDirList.at(0) + rmDirList.at( i ) );
|
|
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
}
|
|
|
|
// returns false if the directory already exists
|
|
bool atWrapper::ftpMkDir( QString dir )
|
|
{
|
|
//Simply used to avoid QFTP from bailing out and loosing a queue of commands.
|
|
// IE: conveniance.
|
|
QFtp ftp;
|
|
|
|
QSignalSpy commandSpy(&ftp, SIGNAL(commandFinished(int, bool)));
|
|
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
const int command = ftp.mkdir( dir );
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
|
|
for (int i = 0; i < commandSpy.count(); ++i)
|
|
if (commandSpy.at(i).at(0) == command)
|
|
return commandSpy.at(i).at(1).toBool();
|
|
|
|
return false;
|
|
}
|
|
|
|
|
|
void atWrapper::ftpRmDirAddToList( const QUrlInfo &urlInfo )
|
|
{
|
|
//Just adding the file to the list for deletion
|
|
rmDirList << urlInfo.name();
|
|
}
|
|
|
|
|
|
bool atWrapper::executeTests()
|
|
{
|
|
qDebug("Executing the tests...");
|
|
|
|
QHashIterator<QString, QString> i(enginesToTest);
|
|
|
|
DataGenerator generator;
|
|
|
|
//Running datagenerator against all the frameworks specified in the config file.
|
|
while ( i.hasNext() )
|
|
{
|
|
|
|
i.next();
|
|
|
|
qDebug( "Now testing: " + i.key().toLatin1() );
|
|
|
|
char* params[13];
|
|
//./bin/datagenerator -framework data/framework.ini -engine OpenGL -suite 1.1 -output outtest
|
|
|
|
|
|
QByteArray eng = i.key().toLatin1();
|
|
QByteArray fwk = framework.toLatin1();
|
|
QByteArray sut = suite.toLatin1();
|
|
QByteArray out = output.toLatin1();
|
|
QByteArray siz = size.toLatin1();
|
|
QByteArray fill = fillColor.toLatin1();
|
|
|
|
params[1] = "-framework";
|
|
params[2] = fwk.data();
|
|
params[3] = "-engine";
|
|
params[4] = eng.data();
|
|
params[5] = "-suite";
|
|
params[6] = sut.data();
|
|
params[7] = "-output";
|
|
params[8] = out.data();
|
|
params[9] = "-size";
|
|
params[10] = siz.data();
|
|
params[11] = "-fill";
|
|
params[12] = fill.data();
|
|
|
|
generator.run( 13, params );
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void atWrapper::createBaseline()
|
|
{
|
|
qDebug( "Now uploading a baseline of only the latest test values" );
|
|
|
|
QHashIterator<QString, QString> i(enginesToTest);
|
|
|
|
QDir dir( output );
|
|
QFtp ftp;
|
|
ftp.connectToHost( ftpHost );
|
|
ftp.login( ftpUser, ftpPass );
|
|
ftp.cd( ftpBaseDir );
|
|
//Upload all the latest test results to the FTP server's baseline directory.
|
|
while ( i.hasNext() )
|
|
{
|
|
|
|
i.next();
|
|
dir.cd( i.key() );
|
|
ftp.cd( i.key() + ".baseline" );
|
|
dir.setFilter(QDir::Files | QDir::Hidden | QDir::NoSymLinks);
|
|
dir.setNameFilters( QStringList() << "*.png" );
|
|
QFileInfoList list = dir.entryInfoList();
|
|
dir.cd( ".." );
|
|
for (int n = 0; n < list.size(); n++)
|
|
{
|
|
QFileInfo fileInfo = list.at( n );
|
|
QFile file( QString( output ) + "/" + i.key() + "/" + fileInfo.fileName() );
|
|
file.open( QIODevice::ReadOnly );
|
|
QByteArray fileData = file.readAll();
|
|
//qDebug() << "Sending up:" << fileInfo.fileName() << "with file size" << fileData.size();
|
|
file.close();
|
|
ftp.put( fileData, fileInfo.fileName(), QFtp::Binary );
|
|
}
|
|
|
|
ftp.cd( ".." );
|
|
}
|
|
|
|
ftp.close();
|
|
|
|
while ( ftp.hasPendingCommands() )
|
|
QCoreApplication::instance()->processEvents();
|
|
}
|
|
|
|
bool atWrapper::compare()
|
|
{
|
|
qDebug( "Now comparing the results to the baseline" );
|
|
|
|
QHashIterator<QString, QString> i(enginesToTest);
|
|
|
|
while ( i.hasNext() )
|
|
{
|
|
i.next();
|
|
|
|
compareDirs( output , i.key() );
|
|
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void atWrapper::compareDirs( QString basedir, QString target )
|
|
{
|
|
|
|
QDir dir( basedir );
|
|
|
|
/* The following should be redundant now.
|
|
|
|
if ( !dir.cd( target + ".failed" ) )
|
|
dir.mkdir( target + ".failed" );
|
|
else
|
|
dir.cdUp();
|
|
|
|
*/
|
|
|
|
if ( !dir.cd( target + ".diff" ) )
|
|
dir.mkdir( target + ".diff" );
|
|
else
|
|
dir.cdUp();
|
|
|
|
|
|
|
|
//Perform comparisons between the two directories.
|
|
|
|
dir.setFilter(QDir::Files | QDir::Hidden | QDir::NoSymLinks);
|
|
dir.setNameFilters( QStringList() << "*.png" );
|
|
dir.cd( target + ".baseline" );
|
|
QFileInfoList list = dir.entryInfoList();
|
|
|
|
for (int i = 0; i < list.size(); ++i)
|
|
{
|
|
QFileInfo fileInfo = list.at(i);
|
|
diff ( basedir, target, fileInfo.fileName() );
|
|
}
|
|
}
|
|
|
|
bool atWrapper::diff( QString basedir, QString dir, QString target )
|
|
{
|
|
//Comparing the two specified files, and then uploading them to
|
|
//the ftp server if they differ
|
|
|
|
basedir += "/" + dir;
|
|
QString one = basedir + ".baseline/" + target;
|
|
QString two = basedir + "/" + target;
|
|
|
|
QFile file( one );
|
|
|
|
file.open( QIODevice::ReadOnly );
|
|
QByteArray contentsOfOne = file.readAll();
|
|
file.close();
|
|
|
|
file.setFileName( two );
|
|
|
|
file.open( QIODevice::ReadOnly );
|
|
QByteArray contentsOfTwo = file.readAll();
|
|
file.close();
|
|
|
|
if ( contentsOfTwo.size() == 0 )
|
|
{
|
|
qDebug() << "No test result found for baseline: " << one;
|
|
file.setFileName( one );
|
|
file.open( QIODevice::ReadOnly );
|
|
file.copy( basedir + ".failed/" + target + "_missing" );
|
|
uploadFailed( dir + ".failed", target + "_missing", contentsOfTwo );
|
|
return false;
|
|
}
|
|
|
|
|
|
if ( ( memcmp( contentsOfOne, contentsOfTwo, contentsOfOne.size() ) ) == 0 )
|
|
return true;
|
|
else
|
|
{
|
|
qDebug() << "Sorry, the result did not match: " << one;
|
|
file.setFileName( two );
|
|
file.open( QIODevice::ReadOnly );
|
|
file.copy( basedir + ".failed/" + target );
|
|
file.close();
|
|
uploadFailed( dir + ".failed", target, contentsOfTwo );
|
|
uploadDiff( basedir, dir, target );
|
|
return false;
|
|
}
|
|
}
|
|
|
|
void atWrapper::uploadDiff( QString basedir, QString dir, QString filename )
|
|
{
|
|
|
|
qDebug() << basedir;
|
|
QImage im1( basedir + ".baseline/" + filename );
|
|
QImage im2( basedir + "/" + filename );
|
|
|
|
QImage im3(im1.size(), QImage::Format_ARGB32);
|
|
|
|
im1 = im1.convertToFormat(QImage::Format_ARGB32);
|
|
im2 = im2.convertToFormat(QImage::Format_ARGB32);
|
|
|
|
for ( int y=0; y<im1.height(); ++y )
|
|
{
|
|
uint *s = (uint *) im1.scanLine(y);
|
|
uint *d = (uint *) im2.scanLine(y);
|
|
uint *w = (uint *) im3.scanLine(y);
|
|
|
|
for ( int x=0; x<im1.width(); ++x )
|
|
{
|
|
if (*s != *d)
|
|
*w = 0xff000000;
|
|
else
|
|
*w = 0xffffffff;
|
|
w++;
|
|
s++;
|
|
d++;
|
|
}
|
|
}
|
|
|
|
im3.save( basedir + ".diff/" + filename ,"PNG");
|
|
|
|
QFile file( basedir + ".diff/" + filename );
|
|
file.open( QIODevice::ReadOnly );
|
|
QByteArray contents = file.readAll();
|
|
file.close();
|
|
|
|
uploadFailed( dir + ".diff", filename, contents );
|
|
|
|
}
|
|
|
|
bool atWrapper::loadConfig( QString path )
|
|
{
|
|
qDebug() << "Loading config file from ... " << path;
|
|
configPath = path;
|
|
//If there is no config file, don't proceed;
|
|
if ( !QFile::exists( path ) )
|
|
{
|
|
return false;
|
|
}
|
|
|
|
|
|
QSettings settings( path, QSettings::IniFormat, this );
|
|
|
|
|
|
//FIXME: Switch to QStringList or something, hash is not needed!
|
|
int numEngines = settings.beginReadArray("engines");
|
|
|
|
for ( int i = 0; i < numEngines; ++i )
|
|
{
|
|
settings.setArrayIndex(i);
|
|
enginesToTest.insert( settings.value( "engine" ).toString(), "Info here please :p" );
|
|
}
|
|
|
|
settings.endArray();
|
|
|
|
framework = QString(ArthurDir) + QDir::separator() + settings.value( "framework" ).toString();
|
|
suite = settings.value( "suite" ).toString();
|
|
output = settings.value( "output" ).toString();
|
|
size = settings.value( "size", "480,360" ).toString();
|
|
fillColor = settings.value( "fill", "white" ).toString();
|
|
ftpUser = settings.value( "ftpUser" ).toString();
|
|
ftpPass = settings.value( "ftpPass" ).toString();
|
|
ftpHost = settings.value( "ftpHost" ).toString();
|
|
ftpBaseDir = settings.value( "ftpBaseDir" ).toString();
|
|
|
|
|
|
QDir::current().mkdir( output );
|
|
|
|
output += "/" + QLibraryInfo::buildKey();
|
|
|
|
QDir::current().mkdir( output );
|
|
|
|
output += "/" + QString( qVersion() );
|
|
|
|
QDir::current().mkdir( output );
|
|
|
|
|
|
ftpBaseDir += "/" + QHostInfo::localHostName().split( "." ).first();
|
|
|
|
|
|
/*
|
|
framework = "data/framework.ini";
|
|
suite = "1.1";
|
|
output = "testresults";
|
|
ftpUser = "anonymous";
|
|
ftpPass = "anonymouspass";
|
|
ftpHost = "kramer.troll.no";
|
|
ftpBaseDir = "/arthurtest";
|
|
*/
|
|
return true;
|
|
}
|
|
|
|
bool atWrapper::runAutoTests()
|
|
{
|
|
//SVG needs this widget...
|
|
QWidget dummy;
|
|
|
|
bool haveBaseline = false;
|
|
|
|
if (!initTests(&haveBaseline))
|
|
return false;
|
|
executeTests();
|
|
|
|
if ( !haveBaseline )
|
|
{
|
|
qDebug( " First run! Creating baseline..." );
|
|
createBaseline();
|
|
}
|
|
else
|
|
{
|
|
qDebug( " Comparing results..." );
|
|
compare();
|
|
}
|
|
return true;
|
|
}
|