Add support for PUT & POST to qget test

Due to some bugs that are not reproducable with a normal HTTP GET
This patch also adds the option to process multiple URLs
serially (using application level queuing) rather than the default
parallel (using QNetworkAccessManager queuing on 6 TCP connections)
& renames the authentication command line options to match wget.

Change-Id: I10915feb3bba23abbd7a72f9844c03f347f9bff5
Reviewed-by: Richard J. Moore <rich@kde.org>
Reviewed-by: Martin Petersson <Martin.Petersson@nokia.com>
This commit is contained in:
Shane Kearns 2012-03-22 16:24:17 +00:00 committed by Qt by Nokia
parent 565da6030f
commit b08f74f089
5 changed files with 424 additions and 189 deletions

View File

@ -0,0 +1,166 @@
/****************************************************************************
**
** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
** Contact: http://www.qt-project.org/
**
** This file is part of the test suite of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** This file may be used under the terms of the GNU Lesser General Public
** License version 2.1 as published by the Free Software Foundation and
** appearing in the file LICENSE.LGPL included in the packaging of this
** file. Please review the following information to ensure the GNU Lesser
** General Public License version 2.1 requirements will be met:
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Nokia gives you certain additional
** rights. These rights are described in the Nokia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU General
** Public License version 3.0 as published by the Free Software Foundation
** and appearing in the file LICENSE.GPL included in the packaging of this
** file. Please review the following information to ensure the GNU General
** Public License version 3.0 requirements will be met:
** http://www.gnu.org/copyleft/gpl.html.
**
** Other Usage
** Alternatively, this file may be used in accordance with the terms and
** conditions contained in a signed written agreement between you and Nokia.
**
**
**
**
**
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include "qget.h"
#include <QAuthenticator>
#include <QCoreApplication>
#include <QDebug>
#include <QSslError>
DownloadManager::DownloadManager()
: queueMode (Parallel)
{
connect(&nam, SIGNAL(finished(QNetworkReply*)), this, SLOT(finished(QNetworkReply*)));
connect(&nam, SIGNAL(authenticationRequired(QNetworkReply*, QAuthenticator*)), this, SLOT(authenticationRequired(QNetworkReply*, QAuthenticator*)));
connect(&nam, SIGNAL(proxyAuthenticationRequired(const QNetworkProxy&, QAuthenticator*)), this, SLOT(proxyAuthenticationRequired(const QNetworkProxy&, QAuthenticator*)));
#ifndef QT_NO_SSL
connect(&nam, SIGNAL(sslErrors(QNetworkReply*, const QList<QSslError>&)), this, SLOT(sslErrors(QNetworkReply*, const QList<QSslError>&)));
#endif
}
DownloadManager::~DownloadManager()
{
}
void DownloadManager::get(const QUrl &url, const QString &user, const QString &password)
{
DownloadItem *dl = new DownloadItem(QNetworkRequest(url), user, password, nam);
transfers.append(dl);
connect(dl, SIGNAL(downloadFinished(TransferItem*)), SLOT(downloadFinished(TransferItem*)));
}
void DownloadManager::upload(const QUrl &url, const QString &user, const QString &password, const QString &filename, const QString &contentType, TransferItem::Method method)
{
QScopedPointer<QFile> file(new QFile(filename));
if (!file->open(QFile::ReadOnly)) {
qDebug() << "Can't open input file" << file->fileName() << file->errorString();
return;
}
QNetworkRequest request(url);
if (!contentType.isEmpty())
request.setHeader(QNetworkRequest::ContentTypeHeader, contentType);
UploadItem *ul = new UploadItem(request, user, password, nam, file.take(), method);
transfers.append(ul);
connect(ul, SIGNAL(downloadFinished(TransferItem*)), SLOT(downloadFinished(TransferItem*)));
}
void DownloadManager::finished(QNetworkReply *reply)
{
}
void DownloadManager::downloadFinished(TransferItem *item)
{
qDebug() << "finished " << item->reply->url() << " with http status: " << item->reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if (item->reply->error() != QNetworkReply::NoError)
qDebug() << "and error: " << item->reply->error() << item->reply->errorString();
transfers.removeOne(item);
item->deleteLater();
checkForAllDone();
}
void DownloadManager::checkForAllDone()
{
if (transfers.isEmpty()) {
qDebug() << "All Done.";
QCoreApplication::quit();
}
foreach (TransferItem *item, transfers) {
if (!item->reply) {
item->start();
//by default multiple downloads are processed in parallel.
//but in serial mode, only start one transfer at a time.
if (queueMode == Serial)
break;
}
}
}
void DownloadManager::authenticationRequired(QNetworkReply *reply, QAuthenticator *auth)
{
qDebug() << "authenticationRequired" << reply;
TransferItem *transfer = findTransfer(reply);
//provide the credentials exactly once, so that it fails if credentials are incorrect.
if (transfer && !transfer->user.isEmpty() || !transfer->password.isEmpty()) {
auth->setUser(transfer->user);
auth->setPassword(transfer->password);
transfer->user.clear();
transfer->password.clear();
}
}
void DownloadManager::proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator *auth)
{
//provide the credentials exactly once, so that it fails if credentials are incorrect.
if (!proxyUser.isEmpty() || !proxyPassword.isEmpty()) {
auth->setUser(proxyUser);
auth->setPassword(proxyPassword);
proxyUser.clear();
proxyPassword.clear();
}
}
#ifndef QT_NO_SSL
void DownloadManager::sslErrors(QNetworkReply *reply, const QList<QSslError> &errors)
{
qDebug() << "sslErrors";
foreach (const QSslError &error, errors) {
qDebug() << error.errorString();
qDebug() << error.certificate().toPem();
}
}
#endif
TransferItem *DownloadManager::findTransfer(QNetworkReply *reply)
{
foreach (TransferItem *item, transfers) {
if (item->reply == reply)
return item;
}
return 0;
}
void DownloadManager::setQueueMode(QueueMode mode)
{
queueMode = mode;
}

View File

@ -41,9 +41,7 @@
#include "qget.h"
#include <QSslError>
#include <QNetworkProxy>
#include <QAuthenticator>
#include <QDebug>
#include <QCoreApplication>
#include <QList>
@ -52,156 +50,6 @@
#include <QNetworkConfigurationManager>
#include <QNetworkSession>
DownloadManager::DownloadManager()
{
connect(&nam, SIGNAL(finished(QNetworkReply*)), this, SLOT(finished(QNetworkReply*)));
connect(&nam, SIGNAL(authenticationRequired(QNetworkReply*, QAuthenticator*)), this, SLOT(authenticationRequired(QNetworkReply*, QAuthenticator*)));
connect(&nam, SIGNAL(proxyAuthenticationRequired(const QNetworkProxy&, QAuthenticator*)), this, SLOT(proxyAuthenticationRequired(const QNetworkProxy&, QAuthenticator*)));
#ifndef QT_NO_SSL
connect(&nam, SIGNAL(sslErrors(QNetworkReply*, const QList<QSslError>&)), this, SLOT(sslErrors(QNetworkReply*, const QList<QSslError>&)));
#endif
}
DownloadManager::~DownloadManager()
{
}
void DownloadManager::get(const QUrl& url)
{
//currently multiple downloads are processed in parallel.
//could add an option for serial using the downloads list as a queue
//which would require DownloadItem to hold a request rather than a reply
QNetworkReply* reply = nam.get(QNetworkRequest(url));
DownloadItem *dl = new DownloadItem(reply, nam);
downloads.append(dl);
connect(dl, SIGNAL(downloadFinished(DownloadItem*)), SLOT(downloadFinished(DownloadItem*)));
}
void DownloadManager::finished(QNetworkReply* reply)
{
}
void DownloadManager::downloadFinished(DownloadItem *item)
{
downloads.removeOne(item);
item->deleteLater();
checkForAllDone();
}
void DownloadManager::checkForAllDone()
{
if (downloads.isEmpty()) {
qDebug() << "All Done.";
QCoreApplication::quit();
}
}
void DownloadManager::authenticationRequired(QNetworkReply* reply, QAuthenticator* auth)
{
//provide the credentials exactly once, so that it fails if credentials are incorrect.
if (!httpUser.isEmpty() || !httpPassword.isEmpty()) {
auth->setUser(httpUser);
auth->setPassword(httpPassword);
httpUser.clear();
httpPassword.clear();
}
}
void DownloadManager::proxyAuthenticationRequired(const QNetworkProxy& proxy, QAuthenticator* auth)
{
//provide the credentials exactly once, so that it fails if credentials are incorrect.
if (!proxyUser.isEmpty() || !proxyPassword.isEmpty()) {
auth->setUser(proxyUser);
auth->setPassword(proxyPassword);
proxyUser.clear();
proxyPassword.clear();
}
}
#ifndef QT_NO_SSL
void DownloadManager::sslErrors(QNetworkReply* reply, const QList<QSslError>& errors)
{
qDebug() << "sslErrors";
foreach (const QSslError& error, errors) {
qDebug() << error.errorString();
qDebug() << error.certificate().toPem();
}
}
#endif
DownloadItem::DownloadItem(QNetworkReply* r, QNetworkAccessManager& manager) : reply(r), nam(manager)
{
reply->setParent(this);
connect(reply, SIGNAL(readyRead()), this, SLOT(readyRead()));
connect(reply, SIGNAL(finished()), this, SLOT(finished()));
}
DownloadItem::~DownloadItem()
{
}
void DownloadItem::readyRead()
{
if (!file.isOpen()) {
qDebug() << reply->header(QNetworkRequest::ContentTypeHeader) << reply->header(QNetworkRequest::ContentLengthHeader);
QString path = reply->url().path();
path = path.mid(path.lastIndexOf('/') + 1);
if (path.isEmpty())
path = QLatin1String("index.html");
file.setFileName(path);
for (int i=1;i<1000;i++) {
if (!file.exists() && file.open(QIODevice::WriteOnly | QIODevice::Truncate))
break;
file.setFileName(QString(QLatin1String("%1.%2")).arg(path).arg(i));
}
if (!file.isOpen()) {
qDebug() << "couldn't open output file";
reply->abort();
return;
}
qDebug() << reply->url() << " -> " << file.fileName();
}
file.write(reply->readAll());
}
void DownloadItem::finished()
{
if (reply->attribute(QNetworkRequest::RedirectionTargetAttribute).isValid()) {
QUrl url = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
url = reply->url().resolved(url);
qDebug() << reply->url() << "redirected to " << url;
if (redirects.contains(url)) {
qDebug() << "redirect loop detected";
} else if (redirects.count() > 10) {
qDebug() << "too many redirects";
} else {
//follow redirect
if (file.isOpen()) {
if (!file.seek(0) || !file.resize(0)) {
file.close();
file.remove();
}
}
reply->deleteLater();
reply = nam.get(QNetworkRequest(url));
reply->setParent(this);
connect(reply, SIGNAL(readyRead()), this, SLOT(readyRead()));
connect(reply, SIGNAL(finished()), this, SLOT(finished()));
redirects.append(url);
return;
}
}
if (file.isOpen()) {
file.write(reply->readAll());
file.close();
}
qDebug() << "finished " << reply->url() << " with http status: " << reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if (reply->error() != QNetworkReply::NoError)
qDebug() << "and error: " << reply->error() << reply->errorString();
emit downloadFinished(this);
}
void printShortUsage()
{
qDebug() << QCoreApplication::applicationName() << " [options] [list of urls]" << endl
@ -215,8 +63,8 @@ void printUsage()
<< "Get one or more urls using QNetworkAccessManager" << endl
<< "Options:"
<< "--help This message" << endl
<< "--http-user=<username> Set username to use for http 401 challenges" << endl
<< "--http-password=<password> Set password to use for http 401 challenges" << endl
<< "--user=<username> Set username to use for authentication" << endl
<< "--password=<password> Set password to use for authentication" << endl
<< "--proxy-user=<username> Set username to use for proxy authentication" << endl
<< "--proxy-password=<password> Set password to use for proxy authentication" << endl
<< "--proxy=on Use system proxy (default)" << endl
@ -225,7 +73,11 @@ void printUsage()
<< " ,http HTTP proxy (default)" << endl
<< " ,socks SOCKS5 proxy" << endl
<< " ,ftp FTP proxy" << endl
<< " ,httpcaching HTTP caching proxy (no CONNECT method)" << endl;
<< " ,httpcaching HTTP caching proxy (no CONNECT method)" << endl
<< "--post=filename upload the file to the next url using HTTP POST" << endl
<< "--put=filename upload the file to the next url using HTTP PUT" << endl
<< "--content-type=<MIME> set content-type header for upload" << endl
<< "--serial don't run requests in parallel" << endl;
}
int main(int argc, char *argv[])
@ -240,18 +92,23 @@ int main(int argc, char *argv[])
QNetworkProxyFactory::setUseSystemConfiguration(true);
DownloadManager dl;
QString uploadFileName;
QString contentType;
QString httpUser;
QString httpPassword;
TransferItem::Method method = TransferItem::Get;
//arguments match wget where possible
foreach (QString str, app.arguments().mid(1)) {
if (str == "--help")
printUsage();
else if (str.startsWith("--http-user="))
dl.setHttpUser(str.mid(12));
else if (str.startsWith("--http-passwd="))
dl.setHttpPassword(str.mid(14));
else if (str.startsWith("--user="))
httpUser = str.mid(7);
else if (str.startsWith("--password="))
httpPassword = str.mid(11);
else if (str.startsWith("--proxy-user="))
dl.setProxyUser(str.mid(13));
else if (str.startsWith("--proxy-passwd="))
dl.setProxyPassword(str.mid(15));
else if (str.startsWith("--proxy-password="))
dl.setProxyPassword(str.mid(17));
else if (str == "--proxy=off")
QNetworkProxyFactory::setUseSystemConfiguration(false);
else if (str == "--proxy=on")
@ -294,10 +151,33 @@ int main(int argc, char *argv[])
qDebug() << "proxy:" << proxy.hostName() << proxy.port() << proxy.type();
dl.setProxy(proxy);
}
else if (str.startsWith("--put=")) {
method = TransferItem::Put;
uploadFileName = str.mid(6);
}
else if (str.startsWith("--post=")) {
method = TransferItem::Post;
uploadFileName = str.mid(7);
}
else if (str.startsWith("--content-type="))
contentType=str.mid(15);
else if (str == "--serial")
dl.setQueueMode(DownloadManager::Serial);
else if (str.startsWith("-"))
qDebug() << "unsupported option" << str;
else
dl.get(QUrl::fromUserInput(str));
else {
QUrl url(QUrl::fromUserInput(str));
switch (method) {
case TransferItem::Put:
case TransferItem::Post:
dl.upload(url, httpUser, httpPassword, uploadFileName, contentType, method);
break;
case TransferItem::Get:
dl.get(url, httpUser, httpPassword);
break;
}
method = TransferItem::Get; //default for urls without a request type before it
}
}
QMetaObject::invokeMethod(&dl, "checkForAllDone", Qt::QueuedConnection);
return app.exec();

View File

@ -46,25 +46,50 @@
#include <QNetworkReply>
#include <QFile>
class DownloadItem : public QObject
class TransferItem : public QObject
{
Q_OBJECT
public:
DownloadItem(QNetworkReply* r, QNetworkAccessManager& nam);
enum Method {Get,Put,Post};
TransferItem(const QNetworkRequest &r, const QString &u, const QString &p, QNetworkAccessManager &n, Method m);
void start();
signals:
void downloadFinished(TransferItem *self);
public slots:
void progress(qint64,qint64);
public:
Method method;
QNetworkRequest request;
QNetworkReply *reply;
QNetworkAccessManager &nam;
QFile *inputFile;
QFile *outputFile;
QList<QUrl> redirects;
QString user;
QString password;
};
class DownloadItem : public TransferItem
{
Q_OBJECT
public:
DownloadItem(const QNetworkRequest &r, const QString &user, const QString &password, QNetworkAccessManager &nam);
~DownloadItem();
QNetworkReply* networkReply() { return reply; }
signals:
void downloadFinished(DownloadItem *self);
private slots:
void readyRead();
void finished();
private:
QNetworkAccessManager& nam;
QNetworkReply* reply;
QFile file;
QList<QUrl> redirects;
};
class UploadItem : public TransferItem
{
Q_OBJECT
public:
UploadItem(const QNetworkRequest &r, const QString &user, const QString &password, QNetworkAccessManager &nam, QFile *f, TransferItem::Method method);
~UploadItem();
private slots:
void finished();
};
class DownloadManager : public QObject
@ -73,31 +98,31 @@ class DownloadManager : public QObject
public:
DownloadManager();
~DownloadManager();
void get(const QUrl& url);
void setProxy(const QNetworkProxy& proxy) { nam.setProxy(proxy); }
void setHttpUser(const QString& user) { httpUser = user; }
void setHttpPassword(const QString& password) { httpPassword = password; }
void setProxyUser(const QString& user) { proxyUser = user; }
void setProxyPassword(const QString& password) { proxyPassword = password; }
void get(const QUrl &url, const QString &user, const QString &password);
void upload(const QUrl &url, const QString &user, const QString &password, const QString &filename, const QString &contentType, TransferItem::Method method);
void setProxy(const QNetworkProxy &proxy) { nam.setProxy(proxy); }
void setProxyUser(const QString &user) { proxyUser = user; }
void setProxyPassword(const QString &password) { proxyPassword = password; }
enum QueueMode { Parallel, Serial };
void setQueueMode(QueueMode mode);
public slots:
void checkForAllDone();
private slots:
void finished(QNetworkReply* reply);
void authenticationRequired(QNetworkReply* reply, QAuthenticator* authenticator);
void proxyAuthenticationRequired(const QNetworkProxy& proxy, QAuthenticator* authenticator);
#ifndef QT_NO_SSL
void sslErrors(QNetworkReply* reply, const QList<QSslError>& errors);
#endif
void downloadFinished(DownloadItem *item);
void finished(QNetworkReply *reply);
void authenticationRequired(QNetworkReply *reply, QAuthenticator *authenticator);
void proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator *authenticator);
void sslErrors(QNetworkReply *reply, const QList<QSslError> &errors);
void downloadFinished(TransferItem *item);
private:
TransferItem *findTransfer(QNetworkReply *reply);
QNetworkAccessManager nam;
QList<DownloadItem*> downloads;
QString httpUser;
QString httpPassword;
QList<TransferItem*> transfers;
QString proxyUser;
QString proxyPassword;
QueueMode queueMode;
};
#endif // QGET_H

View File

@ -3,4 +3,6 @@ QT = core network
CONFIG += console
SOURCES += qget.cpp
SOURCES += transferitem.cpp
SOURCES += downloadmanager.cpp
HEADERS += qget.h

View File

@ -0,0 +1,162 @@
/****************************************************************************
**
** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
** Contact: http://www.qt-project.org/
**
** This file is part of the test suite of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** This file may be used under the terms of the GNU Lesser General Public
** License version 2.1 as published by the Free Software Foundation and
** appearing in the file LICENSE.LGPL included in the packaging of this
** file. Please review the following information to ensure the GNU Lesser
** General Public License version 2.1 requirements will be met:
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Nokia gives you certain additional
** rights. These rights are described in the Nokia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU General
** Public License version 3.0 as published by the Free Software Foundation
** and appearing in the file LICENSE.GPL included in the packaging of this
** file. Please review the following information to ensure the GNU General
** Public License version 3.0 requirements will be met:
** http://www.gnu.org/copyleft/gpl.html.
**
** Other Usage
** Alternatively, this file may be used in accordance with the terms and
** conditions contained in a signed written agreement between you and Nokia.
**
**
**
**
**
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include "qget.h"
#include <QDebug>
TransferItem::TransferItem(const QNetworkRequest &r, const QString &u, const QString &p, QNetworkAccessManager &n, Method m)
: method(m), request(r), reply(0), nam(n), inputFile(0), outputFile(0), user(u), password(p)
{
}
void TransferItem::progress(qint64 sent, qint64 total)
{
if (total > 0)
qDebug() << (sent*100/total) << "%";
else
qDebug() << sent << "B";
}
void TransferItem::start()
{
switch (method) {
case Get:
reply = nam.get(request);
connect(reply, SIGNAL(readyRead()), this, SLOT(readyRead()));
connect(reply, SIGNAL(downloadProgress(qint64,qint64)), this, SLOT(progress(qint64,qint64)));
break;
case Put:
reply = nam.put(request, inputFile);
connect(reply, SIGNAL(uploadProgress(qint64,qint64)), this, SLOT(progress(qint64,qint64)));
break;
case Post:
reply = nam.post(request, inputFile);
connect(reply, SIGNAL(uploadProgress(qint64,qint64)), this, SLOT(progress(qint64,qint64)));
break;
}
connect(reply, SIGNAL(finished()), this, SLOT(finished()));
}
DownloadItem::DownloadItem(const QNetworkRequest &r, const QString &user, const QString &password, QNetworkAccessManager &manager)
: TransferItem(r, user, password, manager, Get)
{
}
DownloadItem::~DownloadItem()
{
}
void DownloadItem::readyRead()
{
if (!outputFile)
outputFile = new QFile(this);
if (!outputFile->isOpen()) {
qDebug() << reply->header(QNetworkRequest::ContentTypeHeader) << reply->header(QNetworkRequest::ContentLengthHeader);
QString path = reply->url().path();
path = path.mid(path.lastIndexOf('/') + 1);
if (path.isEmpty())
path = QLatin1String("index.html");
outputFile->setFileName(path);
for (int i=1;i<1000;i++) {
if (!outputFile->exists() && outputFile->open(QIODevice::WriteOnly | QIODevice::Truncate))
break;
outputFile->setFileName(QString(QLatin1String("%1.%2")).arg(path).arg(i));
}
if (!outputFile->isOpen()) {
qDebug() << "couldn't open output file";
reply->abort();
return;
}
qDebug() << reply->url() << " -> " << outputFile->fileName();
}
outputFile->write(reply->readAll());
}
void DownloadItem::finished()
{
if (reply->attribute(QNetworkRequest::RedirectionTargetAttribute).isValid()) {
QUrl url = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
url = reply->url().resolved(url);
qDebug() << reply->url() << "redirected to " << url;
if (redirects.contains(url)) {
qDebug() << "redirect loop detected";
} else if (redirects.count() > 10) {
qDebug() << "too many redirects";
} else {
//follow redirect
if (outputFile->isOpen()) {
if (!outputFile->seek(0) || !outputFile->resize(0)) {
outputFile->close();
outputFile->remove();
}
}
reply->deleteLater();
reply = nam.get(QNetworkRequest(url));
reply->setParent(this);
connect(reply, SIGNAL(readyRead()), this, SLOT(readyRead()));
connect(reply, SIGNAL(finished()), this, SLOT(finished()));
redirects.append(url);
return;
}
}
if (outputFile && outputFile->isOpen()) {
outputFile->write(reply->readAll());
outputFile->close();
}
emit downloadFinished(this);
}
UploadItem::UploadItem(const QNetworkRequest &r, const QString &user, const QString &password, QNetworkAccessManager &n, QFile *f, TransferItem::Method method)
: TransferItem(r,user, password, n,method)
{
inputFile = f;
f->setParent(this);
qDebug() << f->fileName() << f->isOpen() << f->size();
}
UploadItem::~UploadItem()
{
}
void UploadItem::finished()
{
emit downloadFinished(this);
}