Async open file support in QNetworkAccessManager

This change adds support for BackgroundRequestAttribute to local file request. It is useful
when opening files located in network drives where the file open operation could take several
seconds to complete. When this attribute is activated the QNetworkAccessManager::get call
returns the reply immediately and the user has to wait until QNetworkReply::finished signal
is emitted or QNetworkReply::isFinished function returns true.

Task-number: QTBUG-45925
Change-Id: Ie2019dd94fe04253d1ef6874811d7e749a5aad93
Reviewed-by: Edward Welbourne <edward.welbourne@theqtcompany.com>
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@theqtcompany.com>
This commit is contained in:
Jesus Fernandez 2016-02-15 17:18:12 +01:00 committed by Edward Welbourne
parent 95ea7552a1
commit 8c86d57e32
10 changed files with 316 additions and 88 deletions

View File

@ -37,7 +37,8 @@ HEADERS += \
access/qnetworkdiskcache.h \
access/qhttpthreaddelegate_p.h \
access/qhttpmultipart.h \
access/qhttpmultipart_p.h
access/qhttpmultipart_p.h \
access/qnetworkfile_p.h
SOURCES += \
access/qftp.cpp \
@ -68,7 +69,8 @@ SOURCES += \
access/qabstractnetworkcache.cpp \
access/qnetworkdiskcache.cpp \
access/qhttpthreaddelegate.cpp \
access/qhttpmultipart.cpp
access/qhttpmultipart.cpp \
access/qnetworkfile.cpp
mac: LIBS_PRIVATE += -framework Security

View File

@ -1525,27 +1525,35 @@ void QNetworkAccessManagerPrivate::clearCache(QNetworkAccessManager *manager)
manager->d_func()->objectCache.clear();
manager->d_func()->authenticationManager->clearCache();
if (manager->d_func()->httpThread) {
manager->d_func()->httpThread->quit();
manager->d_func()->httpThread->wait(5000);
if (manager->d_func()->httpThread->isFinished())
delete manager->d_func()->httpThread;
else
QObject::connect(manager->d_func()->httpThread, SIGNAL(finished()), manager->d_func()->httpThread, SLOT(deleteLater()));
manager->d_func()->httpThread = 0;
}
manager->d_func()->destroyThread();
}
QNetworkAccessManagerPrivate::~QNetworkAccessManagerPrivate()
{
if (httpThread) {
httpThread->quit();
httpThread->wait(5000);
if (httpThread->isFinished())
delete httpThread;
destroyThread();
}
QThread * QNetworkAccessManagerPrivate::createThread()
{
if (!thread) {
thread = new QThread;
thread->setObjectName(QStringLiteral("QNetworkAccessManager thread"));
thread->start();
}
Q_ASSERT(thread);
return thread;
}
void QNetworkAccessManagerPrivate::destroyThread()
{
if (thread) {
thread->quit();
thread->wait(5000);
if (thread->isFinished())
delete thread;
else
QObject::connect(httpThread, SIGNAL(finished()), httpThread, SLOT(deleteLater()));
httpThread = 0;
QObject::connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater()));
thread = 0;
}
}

View File

@ -172,6 +172,7 @@ private:
friend class QNetworkReplyImplPrivate;
friend class QNetworkReplyHttpImpl;
friend class QNetworkReplyHttpImplPrivate;
friend class QNetworkReplyFileImpl;
Q_DECLARE_PRIVATE(QNetworkAccessManager)
Q_PRIVATE_SLOT(d_func(), void _q_replyFinished())

View File

@ -74,7 +74,7 @@ class QNetworkAccessManagerPrivate: public QObjectPrivate
public:
QNetworkAccessManagerPrivate()
: networkCache(0), cookieJar(0),
httpThread(0),
thread(0),
#ifndef QT_NO_NETWORKPROXY
proxyFactory(0),
#endif
@ -107,6 +107,9 @@ public:
}
~QNetworkAccessManagerPrivate();
QThread * createThread();
void destroyThread();
void _q_replyFinished();
void _q_replyEncrypted();
void _q_replySslErrors(const QList<QSslError> &errors);
@ -158,7 +161,7 @@ public:
QNetworkCookieJar *cookieJar;
QThread *httpThread;
QThread *thread;
#ifndef QT_NO_NETWORKPROXY

View File

@ -0,0 +1,91 @@
/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the QtNetwork module of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 3 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL3 included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 3 requirements
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 2.0 or (at your option) the GNU General
** Public license version 3 or any later version approved by the KDE Free
** Qt Foundation. The licenses are as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
** https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include "qnetworkfile_p.h"
#include <QtCore/QDebug>
#include <QNetworkReply>
#include <QtCore/QFileInfo>
#include <QtCore/QMetaObject>
#include <QtCore/QCoreApplication>
QT_BEGIN_NAMESPACE
QNetworkFile::QNetworkFile()
: QFile()
{
}
QNetworkFile::QNetworkFile(const QString &name)
: QFile(name)
{
}
void QNetworkFile::open()
{
bool opened = false;
QFileInfo fi(fileName());
if (fi.isDir()) {
QString msg = QCoreApplication::translate("QNetworkAccessFileBackend",
"Cannot open %1: Path is a directory").arg(fileName());
error(QNetworkReply::ContentOperationNotPermittedError, msg);
} else {
headerRead(QNetworkRequest::LastModifiedHeader, QVariant::fromValue(fi.lastModified()));
headerRead(QNetworkRequest::ContentLengthHeader, QVariant::fromValue(fi.size()));
opened = QFile::open(QIODevice::ReadOnly | QIODevice::Unbuffered);
if (!opened) {
QString msg = QCoreApplication::translate("QNetworkAccessFileBackend",
"Error opening %1: %2").arg(fileName(), errorString());
if (exists())
error(QNetworkReply::ContentAccessDenied, msg);
else
error(QNetworkReply::ContentNotFoundError, msg);
}
}
finished(opened);
}
void QNetworkFile::close()
{
// This override is needed because 'using' keyword cannot be used for slots. And the base
// function is not an invokable/slot function.
QFile::close();
}
QT_END_NAMESPACE

View File

@ -0,0 +1,68 @@
/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the QtNetwork module of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 3 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL3 included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 3 requirements
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 2.0 or (at your option) the GNU General
** Public license version 3 or any later version approved by the KDE Free
** Qt Foundation. The licenses are as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
** https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#ifndef QNETWORKFILE_H
#define QNETWORKFILE_H
#include <QFile>
#include <qnetworkreply.h>
QT_BEGIN_NAMESPACE
class QNetworkFile : public QFile
{
Q_OBJECT
public:
QNetworkFile();
QNetworkFile(const QString &name);
using QFile::open;
public Q_SLOTS:
void open();
void close() Q_DECL_OVERRIDE;
Q_SIGNALS:
void finished(bool ok);
void headerRead(QNetworkRequest::KnownHeaders header, const QVariant &value);
void error(QNetworkReply::NetworkError error, const QString &message);
};
QT_END_NAMESPACE
#endif // QNETWORKFILE_H

View File

@ -40,31 +40,45 @@
#include "qnetworkreplyfileimpl_p.h"
#include "QtCore/qdatetime.h"
#include "qnetworkaccessmanager_p.h"
#include <QtCore/QCoreApplication>
#include <QtCore/QFileInfo>
#include <QtCore/QThread>
#include "qnetworkfile_p.h"
#include "qnetworkrequest.h"
QT_BEGIN_NAMESPACE
QNetworkReplyFileImplPrivate::QNetworkReplyFileImplPrivate()
: QNetworkReplyPrivate(), realFileSize(0)
: QNetworkReplyPrivate(), managerPrivate(0), realFile(0)
{
qRegisterMetaType<QNetworkRequest::KnownHeaders>();
qRegisterMetaType<QNetworkReply::NetworkError>();
}
QNetworkReplyFileImpl::~QNetworkReplyFileImpl()
{
QNetworkReplyFileImplPrivate *d = (QNetworkReplyFileImplPrivate*) d_func();
if (d->realFile) {
if (d->realFile->thread() == QThread::currentThread())
delete d->realFile;
else
QMetaObject::invokeMethod(d->realFile, "deleteLater", Qt::QueuedConnection);
}
}
QNetworkReplyFileImpl::QNetworkReplyFileImpl(QObject *parent, const QNetworkRequest &req, const QNetworkAccessManager::Operation op)
: QNetworkReply(*new QNetworkReplyFileImplPrivate(), parent)
QNetworkReplyFileImpl::QNetworkReplyFileImpl(QNetworkAccessManager *manager, const QNetworkRequest &req, const QNetworkAccessManager::Operation op)
: QNetworkReply(*new QNetworkReplyFileImplPrivate(), manager)
{
setRequest(req);
setUrl(req.url());
setOperation(op);
setFinished(true);
QNetworkReply::open(QIODevice::ReadOnly);
QNetworkReplyFileImplPrivate *d = (QNetworkReplyFileImplPrivate*) d_func();
d->managerPrivate = manager->d_func();
QUrl url = req.url();
if (url.host() == QLatin1String("localhost"))
url.setHost(QString());
@ -77,7 +91,7 @@ QNetworkReplyFileImpl::QNetworkReplyFileImpl(QObject *parent, const QNetworkRequ
setError(QNetworkReply::ProtocolInvalidOperationError, msg);
QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection,
Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ProtocolInvalidOperationError));
QMetaObject::invokeMethod(this, "finished", Qt::QueuedConnection);
fileOpenFinished(false);
return;
}
#endif
@ -85,7 +99,6 @@ QNetworkReplyFileImpl::QNetworkReplyFileImpl(QObject *parent, const QNetworkRequ
url.setPath(QLatin1String("/"));
setUrl(url);
QString fileName = url.toLocalFile();
if (fileName.isEmpty()) {
if (url.scheme() == QLatin1String("qrc")) {
@ -100,68 +113,85 @@ QNetworkReplyFileImpl::QNetworkReplyFileImpl(QObject *parent, const QNetworkRequ
}
}
QFileInfo fi(fileName);
if (fi.isDir()) {
QString msg = QCoreApplication::translate("QNetworkAccessFileBackend", "Cannot open %1: Path is a directory").arg(url.toString());
setError(QNetworkReply::ContentOperationNotPermittedError, msg);
QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection,
Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ContentOperationNotPermittedError));
QMetaObject::invokeMethod(this, "finished", Qt::QueuedConnection);
return;
}
if (req.attribute(QNetworkRequest::BackgroundRequestAttribute).toBool()) { // Asynchronous open
auto realFile = new QNetworkFile(fileName);
connect(realFile, &QNetworkFile::headerRead, this, &QNetworkReplyFileImpl::setHeader,
Qt::QueuedConnection);
connect(realFile, &QNetworkFile::error, this, &QNetworkReplyFileImpl::setError,
Qt::QueuedConnection);
connect(realFile, SIGNAL(finished(bool)), SLOT(fileOpenFinished(bool)),
Qt::QueuedConnection);
d->realFile.setFileName(fileName);
bool opened = d->realFile.open(QIODevice::ReadOnly | QIODevice::Unbuffered);
realFile->moveToThread(d->managerPrivate->createThread());
QMetaObject::invokeMethod(realFile, "open", Qt::QueuedConnection);
// could we open the file?
if (!opened) {
QString msg = QCoreApplication::translate("QNetworkAccessFileBackend", "Error opening %1: %2")
.arg(d->realFile.fileName(), d->realFile.errorString());
d->realFile = realFile;
} else { // Synch open
setFinished(true);
if (d->realFile.exists()) {
setError(QNetworkReply::ContentAccessDenied, msg);
QFileInfo fi(fileName);
if (fi.isDir()) {
QString msg = QCoreApplication::translate("QNetworkAccessFileBackend", "Cannot open %1: Path is a directory").arg(url.toString());
setError(QNetworkReply::ContentOperationNotPermittedError, msg);
QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection,
Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ContentAccessDenied));
} else {
setError(QNetworkReply::ContentNotFoundError, msg);
QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection,
Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ContentNotFoundError));
Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ContentOperationNotPermittedError));
QMetaObject::invokeMethod(this, "finished", Qt::QueuedConnection);
return;
}
d->realFile = new QFile(fileName, this);
bool opened = d->realFile->open(QIODevice::ReadOnly | QIODevice::Unbuffered);
// could we open the file?
if (!opened) {
QString msg = QCoreApplication::translate("QNetworkAccessFileBackend", "Error opening %1: %2")
.arg(d->realFile->fileName(), d->realFile->errorString());
if (fi.exists()) {
setError(QNetworkReply::ContentAccessDenied, msg);
QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection,
Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ContentAccessDenied));
} else {
setError(QNetworkReply::ContentNotFoundError, msg);
QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection,
Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ContentNotFoundError));
}
QMetaObject::invokeMethod(this, "finished", Qt::QueuedConnection);
return;
}
setHeader(QNetworkRequest::LastModifiedHeader, fi.lastModified());
setHeader(QNetworkRequest::ContentLengthHeader, fi.size());
QMetaObject::invokeMethod(this, "metaDataChanged", Qt::QueuedConnection);
QMetaObject::invokeMethod(this, "downloadProgress", Qt::QueuedConnection,
Q_ARG(qint64, fi.size()), Q_ARG(qint64, fi.size()));
QMetaObject::invokeMethod(this, "readyRead", Qt::QueuedConnection);
QMetaObject::invokeMethod(this, "finished", Qt::QueuedConnection);
return;
}
setHeader(QNetworkRequest::LastModifiedHeader, fi.lastModified());
d->realFileSize = fi.size();
setHeader(QNetworkRequest::ContentLengthHeader, d->realFileSize);
QMetaObject::invokeMethod(this, "metaDataChanged", Qt::QueuedConnection);
QMetaObject::invokeMethod(this, "downloadProgress", Qt::QueuedConnection,
Q_ARG(qint64, d->realFileSize), Q_ARG(qint64, d->realFileSize));
QMetaObject::invokeMethod(this, "readyRead", Qt::QueuedConnection);
QMetaObject::invokeMethod(this, "finished", Qt::QueuedConnection);
}
void QNetworkReplyFileImpl::close()
{
Q_D(QNetworkReplyFileImpl);
QNetworkReply::close();
d->realFile.close();
if (d->realFile) {
if (d->realFile->thread() == thread())
d->realFile->close();
else
QMetaObject::invokeMethod(d->realFile, "close", Qt::QueuedConnection);
}
}
void QNetworkReplyFileImpl::abort()
{
Q_D(QNetworkReplyFileImpl);
QNetworkReply::close();
d->realFile.close();
close();
}
qint64 QNetworkReplyFileImpl::bytesAvailable() const
{
Q_D(const QNetworkReplyFileImpl);
if (!d->realFile.isOpen())
if (!d->isFinished || !d->realFile || !d->realFile->isOpen())
return QNetworkReply::bytesAvailable();
return QNetworkReply::bytesAvailable() + d->realFile.bytesAvailable();
return QNetworkReply::bytesAvailable() + d->realFile->bytesAvailable();
}
bool QNetworkReplyFileImpl::isSequential () const
@ -171,8 +201,9 @@ bool QNetworkReplyFileImpl::isSequential () const
qint64 QNetworkReplyFileImpl::size() const
{
Q_D(const QNetworkReplyFileImpl);
return d->realFileSize;
bool ok;
int size = header(QNetworkRequest::ContentLengthHeader).toInt(&ok);
return ok ? size : 0;
}
/*!
@ -181,11 +212,11 @@ qint64 QNetworkReplyFileImpl::size() const
qint64 QNetworkReplyFileImpl::readData(char *data, qint64 maxlen)
{
Q_D(QNetworkReplyFileImpl);
if (!d->realFile.isOpen())
if (!d->isFinished || !d->realFile || !d->realFile->isOpen())
return -1;
qint64 ret = d->realFile.read(data, maxlen);
if (bytesAvailable() == 0 && d->realFile.isOpen())
d->realFile.close();
qint64 ret = d->realFile->read(data, maxlen);
if (bytesAvailable() == 0)
d->realFile->close();
if (ret == 0 && bytesAvailable() == 0)
return -1;
else {
@ -195,6 +226,17 @@ qint64 QNetworkReplyFileImpl::readData(char *data, qint64 maxlen)
}
}
void QNetworkReplyFileImpl::fileOpenFinished(bool isOpen)
{
setFinished(true);
if (isOpen) {
const auto fileSize = size();
Q_EMIT metaDataChanged();
Q_EMIT downloadProgress(fileSize, fileSize);
Q_EMIT readyRead();
}
Q_EMIT finished();
}
QT_END_NAMESPACE

View File

@ -59,13 +59,12 @@
QT_BEGIN_NAMESPACE
class QNetworkReplyFileImplPrivate;
class QNetworkReplyFileImpl: public QNetworkReply
{
Q_OBJECT
public:
QNetworkReplyFileImpl(QObject *parent, const QNetworkRequest &req, const QNetworkAccessManager::Operation op);
QNetworkReplyFileImpl(QNetworkAccessManager *manager, const QNetworkRequest &req, const QNetworkAccessManager::Operation op);
~QNetworkReplyFileImpl();
virtual void abort() Q_DECL_OVERRIDE;
@ -77,6 +76,9 @@ public:
virtual qint64 readData(char *data, qint64 maxlen) Q_DECL_OVERRIDE;
private Q_SLOTS:
void fileOpenFinished(bool isOpen);
Q_DECLARE_PRIVATE(QNetworkReplyFileImpl)
};
@ -85,12 +87,14 @@ class QNetworkReplyFileImplPrivate: public QNetworkReplyPrivate
public:
QNetworkReplyFileImplPrivate();
QFile realFile;
qint64 realFileSize;
QNetworkAccessManagerPrivate *managerPrivate;
QPointer<QFile> realFile;
Q_DECLARE_PUBLIC(QNetworkReplyFileImpl)
};
QT_END_NAMESPACE
Q_DECLARE_METATYPE(QNetworkRequest::KnownHeaders)
#endif // QNETWORKREPLYFILEIMPL_H

View File

@ -617,17 +617,10 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq
thread->setObjectName(QStringLiteral("Qt HTTP synchronous thread"));
QObject::connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater()));
thread->start();
} else if (!managerPrivate->httpThread) {
} else {
// We use the manager-global thread.
// At some point we could switch to having multiple threads if it makes sense.
managerPrivate->httpThread = new QThread();
managerPrivate->httpThread->setObjectName(QStringLiteral("Qt HTTP thread"));
managerPrivate->httpThread->start();
thread = managerPrivate->httpThread;
} else {
// Asynchronous request, thread already exists
thread = managerPrivate->httpThread;
thread = managerPrivate->createThread();
}
QUrl url = newHttpRequest.url();
@ -1139,8 +1132,8 @@ void QNetworkReplyHttpImplPrivate::onRedirected(const QUrl &redirectUrl, int htt
cookedHeaders.clear();
if (managerPrivate->httpThread)
managerPrivate->httpThread->disconnect();
if (managerPrivate->thread)
managerPrivate->thread->disconnect();
// Recurse
QMetaObject::invokeMethod(q, "start", Qt::QueuedConnection,

View File

@ -205,6 +205,7 @@ private Q_SLOTS:
void invalidProtocol();
void getFromData_data();
void getFromData();
void getFromFile_data();
void getFromFile();
void getFromFileSpecial_data();
void getFromFileSpecial();
@ -650,8 +651,10 @@ private slots:
#endif
void slotError(QAbstractSocket::SocketError err)
{
Q_ASSERT(!client.isNull());
qDebug() << "slotError" << err << client->errorString();
if (client.isNull())
qDebug() << "slotError" << err;
else
qDebug() << "slotError" << err << client->errorString();
}
public slots:
@ -1674,14 +1677,26 @@ void tst_QNetworkReply::getFromData()
QCOMPARE(reply->readAll(), expected);
}
void tst_QNetworkReply::getFromFile_data()
{
QTest::addColumn<bool>("backgroundAttribute");
QTest::newRow("no-background-attribute") << false;
QTest::newRow("background-attribute") << true;
}
void tst_QNetworkReply::getFromFile()
{
QFETCH(bool, backgroundAttribute);
// create the file:
QTemporaryFile file(QDir::currentPath() + "/temp-XXXXXX");
file.setAutoRemove(true);
QVERIFY2(file.open(), qPrintable(file.errorString()));
QNetworkRequest request(QUrl::fromLocalFile(file.fileName()));
if (backgroundAttribute)
request.setAttribute(QNetworkRequest::BackgroundRequestAttribute, QVariant::fromValue(true));
QNetworkReplyPtr reply;
static const char fileData[] = "This is some data that is in the file.\r\n";
@ -1691,6 +1706,7 @@ void tst_QNetworkReply::getFromFile()
QCOMPARE(file.size(), qint64(data.size()));
RUN_REQUEST(runSimpleRequest(QNetworkAccessManager::GetOperation, request, reply));
QVERIFY(waitForFinish(reply) != Timeout);
QCOMPARE(reply->url(), request.url());
QCOMPARE(reply->error(), QNetworkReply::NoError);